diff --git a/README.md b/README.md index 4eae8243..fc5c42cb 100644 --- a/README.md +++ b/README.md @@ -919,6 +919,22 @@ svr.set_keep_alive_max_count(2); // Default is 100, 0 for no limit svr.set_keep_alive_timeout(10); // Default is 5 ``` +#### Keep-alive parking + +```cpp +svr.set_keep_alive_parking(true); // Call it before listen() +``` + +By default, a kept-alive connection holds a worker thread while it waits for its next request, so connections beyond the thread count wait for one to be closed. With keep-alive parking, a connection waiting for its next request is handed to an event loop instead, and goes back to a worker when the request arrives. An idle connection then costs a socket and a few kilobytes instead of a thread, and connections that outnumber the threads take turns. + +With it: + +- Consecutive requests on a connection may run on different threads. +- When the [max queued requests](#max-queued-requests) limit is reached, only a connection that has not been served yet is closed. An established one waits for room. +- Each request pays for handing the connection over, so with no more connections than threads and very light handlers, it can serve fewer requests per second. +- A handler, a content provider and a WebSocket still hold their thread while they run. +- `SSLServer` ignores it for now, and an override of `process_and_close_socket()` is not called. + ### Client address behind a reverse proxy ```cpp @@ -962,7 +978,7 @@ Please see [Server example](https://github.com/yhirose/cpp-httplib/blob/master/e When a new task arrives and the queued tasks outnumber the idle threads, a temporary thread is spawned (up to the maximum). A dynamic thread exits automatically once it has waited for the idle timeout without receiving a task. The idle timeout defaults to 3 seconds, configurable via `CPPHTTPLIB_THREAD_POOL_IDLE_TIMEOUT`. -A connection holds its thread while it is served: between requests up to the keep-alive timeout, through a request body sent slowly but within the read timeout, and for the whole life of a WebSocket. Enough of them leave no thread for anyone else, so in front of untrusted clients, put a reverse proxy that limits connections per client, and reject unwanted WebSocket upgrades in the pre-request handler. +A connection holds its thread while it is served: between requests up to the keep-alive timeout (unless [keep-alive parking](#keep-alive-parking) is on), through a request body sent slowly but within the read timeout, and for the whole life of a WebSocket. Enough of them leave no thread for anyone else, so in front of untrusted clients, put a reverse proxy that limits connections per client, and reject unwanted WebSocket upgrades in the pre-request handler. If you want to set the thread counts at runtime: diff --git a/httplib.h b/httplib.h index d9a4ba59..ccaeabaf 100644 --- a/httplib.h +++ b/httplib.h @@ -2384,6 +2384,10 @@ public: template Server &set_idle_interval(const std::chrono::duration &duration); + // Keeps idle keep-alive connections in an event loop instead of a worker + // thread each. Call it before listen(). See README for what changes. + Server &set_keep_alive_parking(bool on); + Server &set_payload_max_length(size_t length); Server &set_compress(bool on); @@ -2454,6 +2458,13 @@ protected: // stop() takes the listening socket away; listen() then waits for handlers bool is_stopping() const { return svr_sock_ == INVALID_SOCKET; } + // See set_keep_alive_parking(). Defined in the implementation part. + struct KeepAliveConnection; + struct ParkingState; + std::unique_ptr parking_; + bool keep_alive_parking_ = false; + bool idle_interval_set_ = false; + std::vector trusted_proxies_; size_t keep_alive_max_count_ = CPPHTTPLIB_KEEPALIVE_MAX_COUNT; @@ -2513,6 +2524,11 @@ private: SocketOptions socket_options) const; int bind_internal(const std::string &host, int port, int socket_flags); bool listen_internal(); + bool accept_loop(TaskQueue &task_queue); + bool parking_loop(TaskQueue &task_queue); + void prepare_accepted_socket(socket_t sock) const; + void resume_connection(std::shared_ptr conn); + bool abandon_listen_socket(); bool routing(Request &req, Response &res, Stream &strm, bool skip_pre_routing); @@ -2560,6 +2576,7 @@ private: ContentReceiver multipart_receiver) const; virtual bool process_and_close_socket(socket_t sock); + virtual bool supports_keep_alive_parking() const; void output_log(const Request &req, const Response &res) const; void output_pre_compression_log(const Request &req, @@ -3520,6 +3537,7 @@ public: private: bool process_and_close_socket(socket_t sock) override; + bool supports_keep_alive_parking() const override; tls::ctx_t ctx_ = nullptr; // update_certs_pem() replaces the certificates and CAs that handshakes read @@ -7178,30 +7196,27 @@ inline bool keep_alive(const std::atomic &svr_sock, socket_t sock, return false; } -// `has_buffered_request` reports whether the connection's stream already holds -// bytes of the next request. A client may pipeline its requests (RFC 9112 -// 9.3.2), so reading one request can pull the start of the next one into the -// stream's buffer; that request must be served without waiting for the socket -// to become readable again, since its bytes are no longer on the socket. -// Otherwise `wait_for_next_request` waits for the socket, and returning false -// ends the loop. `callback` is told whether the socket has just been seen -// readable. `served` counts the requests served on this connection so far. -template -inline bool process_server_socket_core(size_t &served, +// Serves the requests of one connection while they come. A client may +// pipeline its requests (RFC 9112 9.3.2), so reading one request can pull the +// start of the next one into the stream's buffer; that request must be served +// without waiting for the socket to become readable again, since its bytes are +// no longer on the socket. Otherwise `wait_for_next_request` waits for the +// socket, and returning false ends the loop. `served` counts the requests +// served on this connection so far. +template +inline bool process_server_socket_core(S &strm, size_t &served, size_t keep_alive_max_count, - W wait_for_next_request, - P has_buffered_request, T callback) { + W wait_for_next_request, T callback) { auto ret = false; // A keep_alive_max_count of 0 sets no limit. for (; keep_alive_max_count == 0 || served < keep_alive_max_count; served++) { - auto socket_readable = false; - if (!has_buffered_request()) { + if (!strm.is_readable()) { if (!wait_for_next_request()) { break; } - socket_readable = true; + strm.set_readable_hint(); } auto close_connection = served + 1 == keep_alive_max_count; auto connection_closed = false; - ret = callback(socket_readable, close_connection, connection_closed); + ret = callback(strm, close_connection, connection_closed); if (!ret || connection_closed) { break; } } return ret; @@ -7220,14 +7235,9 @@ process_server_socket(const std::atomic &svr_sock, socket_t sock, write_timeout_sec, write_timeout_usec); size_t served = 0; return process_server_socket_core( - served, keep_alive_max_count, + strm, served, keep_alive_max_count, [&]() { return keep_alive(svr_sock, sock, keep_alive_timeout_sec); }, - [&]() { return strm.is_readable(); }, - [&](bool socket_readable, bool close_connection, - bool &connection_closed) { - if (socket_readable) { strm.set_readable_hint(); } - return callback(strm, close_connection, connection_closed); - }); + callback); } inline bool process_client_socket( @@ -7254,8 +7264,10 @@ inline int shutdown_socket(socket_t sock) noexcept { // the final shutdown+close. Closing with unread data in the receive queue // (or bytes arriving after the receive side is closed) makes the stack send // an abortive RST instead of a graceful FIN, which can make the peer see the -// response as a failed read even though it was fully written. -inline void drain_and_close_socket(socket_t sock) noexcept { +// response as a failed read even though it was fully written. With a +// `wait_msec` of 0, only what has already arrived is read. +inline void drain_and_close_socket(socket_t sock, + time_t wait_msec = 100) noexcept { #ifdef _WIN32 shutdown(sock, SD_SEND); #else @@ -7265,15 +7277,18 @@ inline void drain_and_close_socket(socket_t sock) noexcept { char buf[CPPHTTPLIB_RECV_BUFSIZ]; size_t total = 0; const auto deadline = std::chrono::steady_clock::now() + - std::chrono::milliseconds(100); // bound #1 + std::chrono::milliseconds(wait_msec); // bound #1 - while (total < size_t(1024u * 1024u)) { // bound #2 + for (auto first = true; total < size_t(1024u * 1024u); // bound #2 + first = false) { const auto remaining = std::chrono::duration_cast( deadline - std::chrono::steady_clock::now()) .count(); - if (remaining <= 0) { break; } - if (select_read(sock, 0, static_cast(remaining)) <= 0) { break; } + if (remaining <= 0 && !first) { break; } + const auto wait_usec = + (std::max)(remaining, std::chrono::microseconds::rep(0)); + if (select_read(sock, 0, static_cast(wait_usec)) <= 0) { break; } const auto n = read_socket(sock, buf, sizeof(buf), CPPHTTPLIB_RECV_FLAGS); if (n <= 0) { break; } total += static_cast(n); @@ -7828,6 +7843,26 @@ inline bool is_accept_transient_error() { #endif } +inline socket_t accept_socket(socket_t svr_sock) { +#if defined _WIN32 + // sockets connected via WASAccept inherit flags NO_HANDLE_INHERIT, + // OVERLAPPED + return WSAAccept(svr_sock, nullptr, nullptr, nullptr, 0); +#elif defined SOCK_CLOEXEC + return accept4(svr_sock, nullptr, nullptr, SOCK_CLOEXEC); +#else + return accept(svr_sock, nullptr, nullptr); +#endif +} + +inline bool is_would_block_error() { +#ifdef _WIN32 + return WSAGetLastError() == WSAEWOULDBLOCK; +#else + return errno == EAGAIN || errno == EWOULDBLOCK; +#endif +} + inline bool create_socket_pair(socket_t sv[2]) { #ifdef _WIN32 auto listener = create_socket( @@ -11412,14 +11447,9 @@ inline bool process_server_socket_ssl( write_timeout_sec, write_timeout_usec); size_t served = 0; return process_server_socket_core( - served, keep_alive_max_count, + strm, served, keep_alive_max_count, [&]() { return keep_alive(svr_sock, sock, keep_alive_timeout_sec); }, - [&]() { return strm.is_readable(); }, - [&](bool socket_readable, bool close_connection, - bool &connection_closed) { - if (socket_readable) { strm.set_readable_hint(); } - return callback(strm, close_connection, connection_closed); - }); + callback); } template @@ -14054,6 +14084,117 @@ WebSocketSSLStream::set_read_interrupt(std::function interrupted) { * Group 4: Server implementation */ +// A kept-alive connection that the event loop of set_keep_alive_parking() +// owns while it is idle, and a worker owns while it serves a request. +struct Server::KeepAliveConnection { + KeepAliveConnection(socket_t s, const Server &svr) + : sock(s), strm(s, svr.read_timeout_sec_, svr.read_timeout_usec_, + svr.write_timeout_sec_, svr.write_timeout_usec_) {} + ~KeepAliveConnection() { + if (sock != INVALID_SOCKET) { detail::drain_and_close_socket(sock, 0); } + } + + socket_t sock; + // Kept across requests: it may hold the start of a pipelined request. + detail::SocketStream strm; + size_t served = 0; + + bool addresses_known = false; + std::string remote_addr; + int remote_port = 0; + std::string local_addr; + int local_port = 0; + + // While parked + std::list>::iterator parked_at; + std::chrono::steady_clock::time_point deadline; +}; + +struct Server::ParkingState { + using Clock = std::chrono::steady_clock; + using Token = detail::Poller::Token; + using Connection = std::shared_ptr; + + // Kept for the Server's life, so that stop() can wake() it without a lock. + std::unique_ptr poller; + + // The listening socket while parking_loop() watches it. Whichever of stop() + // and parking_loop() takes it out of here decides who closes it, so that it + // is never closed while the poller still watches it. + std::atomic listen_sock{INVALID_SOCKET}; + + // From stop(): whether parking_loop() takes over closing `sock`. + bool release_listen(socket_t sock) { + if (!listen_sock.compare_exchange_strong(sock, INVALID_SOCKET)) { + return false; + } + poller->wake(); + return true; + } + + // From a worker: hands an idle connection back, unless the loop has ended. + bool give_back(Connection conn) { + { + std::lock_guard guard(mutex); + if (closed) { return false; } + returned.push_back(std::move(conn)); + } + poller->wake(); + return true; + } + + // The rest belongs to parking_loop(). A parked connection's token is its + // address, and `parked` lists them in the order they expire. + void park(Connection conn, Clock::time_point deadline) { + auto &c = *conn; + c.deadline = deadline; + c.parked_at = parked.insert(parked.end(), std::move(conn)); + poller->arm(c.sock, reinterpret_cast(&c)); + } + + Connection unpark(Token token) { + auto &c = *reinterpret_cast(token); + auto conn = std::move(*c.parked_at); + parked.erase(c.parked_at); + return conn; + } + + Clock::time_point next_deadline() const { + return parked.empty() ? (Clock::time_point::max)() + : parked.front()->deadline; + } + + void expire(Clock::time_point now) { + while (!parked.empty() && parked.front()->deadline <= now) { + poller->disarm(parked.front()->sock); + parked.pop_front(); // closes it + } + } + + void close_all() { + std::vector left; + { + std::lock_guard guard(mutex); + closed = true; + left.swap(returned); + } + for (auto &conn : parked) { + poller->disarm(conn->sock); + } + parked.clear(); + backlog.clear(); + backlog_waiting = false; + } + + std::mutex mutex; + std::vector returned; + bool closed = false; + std::atomic backlog_waiting{false}; + + std::list parked; + std::list backlog; +}; + // HTTP server implementation inline Server::Server() : new_task_queue([] { @@ -14398,6 +14539,20 @@ inline Server &Server::set_write_timeout(time_t sec, time_t usec) { inline Server &Server::set_idle_interval(time_t sec, time_t usec) { idle_interval_sec_ = sec; idle_interval_usec_ = usec; + idle_interval_set_ = true; + return *this; +} + +inline Server &Server::set_keep_alive_parking(bool on) { + // Never destroyed once made: stop() may use it from another thread. + if (on && !parking_) { + auto poller = detail::make_poller(); + if (poller) { + parking_.reset(new ParkingState()); + parking_->poller = std::move(poller); + } + } + keep_alive_parking_ = on; return *this; } @@ -14477,7 +14632,7 @@ inline void Server::stop() noexcept { // bind_to_port() without listen_after_bind() still owns the descriptor. The // exchange is what makes this safe to call concurrently with the accept loop. socket_t sock = svr_sock_.exchange(INVALID_SOCKET); - if (sock != INVALID_SOCKET) { + if (sock != INVALID_SOCKET && !(parking_ && parking_->release_listen(sock))) { detail::shutdown_socket(sock); detail::close_socket(sock); } @@ -15099,73 +15254,9 @@ inline bool Server::listen_internal() { { std::unique_ptr task_queue(new_task_queue()); - while (svr_sock_ != INVALID_SOCKET) { -#ifndef _WIN32 - if (idle_interval_sec_ > 0 || idle_interval_usec_ > 0) { -#endif - auto val = detail::select_read(svr_sock_, idle_interval_sec_, - idle_interval_usec_); - if (val == 0) { // Timeout - task_queue->on_idle(); - continue; - } -#ifndef _WIN32 - } -#endif - -#if defined _WIN32 - // sockets connected via WASAccept inherit flags NO_HANDLE_INHERIT, - // OVERLAPPED - socket_t sock = WSAAccept(svr_sock_, nullptr, nullptr, nullptr, 0); -#elif defined SOCK_CLOEXEC - socket_t sock = accept4(svr_sock_, nullptr, nullptr, SOCK_CLOEXEC); -#else - socket_t sock = accept(svr_sock_, nullptr, nullptr); -#endif - - if (sock == INVALID_SOCKET) { - // NOTE: Winsock reports failures through WSAGetLastError() and never - // touches the CRT errno, so the two have to be asked platform by - // platform rather than by testing errno here. - if (detail::is_accept_resource_error()) { - // The per-process descriptor limit or the network stack's buffer - // space has been reached. Try to accept new connections after a - // short sleep. - std::this_thread::sleep_for(std::chrono::microseconds{1}); - continue; - } else if (detail::is_accept_transient_error()) { - continue; - } - // Take the descriptor out of svr_sock_ before closing it: a later - // stop() would otherwise shutdown()/close() a value the OS may have - // reused, and keep_alive() watches svr_sock_ to notice the server is - // gone. The exchange also settles the race with a concurrent stop(), - // since whichever side takes the descriptor closes it exactly once. - auto listen_sock = svr_sock_.exchange(INVALID_SOCKET); - if (listen_sock != INVALID_SOCKET) { - detail::close_socket(listen_sock); - ret = false; - output_error_log(Error::Connection, nullptr); - } else { - ; // The server socket was closed by user. - } - break; - } - - detail::set_socket_opt_time(sock, SOL_SOCKET, SO_RCVTIMEO, - read_timeout_sec_, read_timeout_usec_); - detail::set_socket_opt_time(sock, SOL_SOCKET, SO_SNDTIMEO, - write_timeout_sec_, write_timeout_usec_); - - if (tcp_nodelay_) { set_socket_opt(sock, IPPROTO_TCP, TCP_NODELAY, 1); } - - if (!task_queue->enqueue( - [this, sock]() { process_and_close_socket(sock); })) { - output_error_log(Error::ResourceExhaustion, nullptr); - detail::shutdown_socket(sock); - detail::close_socket(sock); - } - } + ret = keep_alive_parking_ && parking_ && supports_keep_alive_parking() + ? parking_loop(*task_queue) + : accept_loop(*task_queue); task_queue->shutdown(); } @@ -15174,6 +15265,62 @@ inline bool Server::listen_internal() { return ret; } +inline void Server::prepare_accepted_socket(socket_t sock) const { + detail::set_socket_opt_time(sock, SOL_SOCKET, SO_RCVTIMEO, read_timeout_sec_, + read_timeout_usec_); + detail::set_socket_opt_time(sock, SOL_SOCKET, SO_SNDTIMEO, write_timeout_sec_, + write_timeout_usec_); + if (tcp_nodelay_) { set_socket_opt(sock, IPPROTO_TCP, TCP_NODELAY, 1); } +} + +// Serves each connection on one worker thread for its whole life. +inline bool Server::accept_loop(TaskQueue &task_queue) { + auto ret = true; + while (svr_sock_ != INVALID_SOCKET) { +#ifndef _WIN32 + if (idle_interval_sec_ > 0 || idle_interval_usec_ > 0) { +#endif + auto val = detail::select_read(svr_sock_, idle_interval_sec_, + idle_interval_usec_); + if (val == 0) { // Timeout + task_queue.on_idle(); + continue; + } +#ifndef _WIN32 + } +#endif + + auto sock = detail::accept_socket(svr_sock_); + + if (sock == INVALID_SOCKET) { + // NOTE: Winsock reports failures through WSAGetLastError() and never + // touches the CRT errno, so the two have to be asked platform by + // platform rather than by testing errno here. + if (detail::is_accept_resource_error()) { + // The per-process descriptor limit or the network stack's buffer + // space has been reached. Try to accept new connections after a + // short sleep. + std::this_thread::sleep_for(std::chrono::microseconds{1}); + continue; + } else if (detail::is_accept_transient_error()) { + continue; + } + if (abandon_listen_socket()) { ret = false; } + break; + } + + prepare_accepted_socket(sock); + + if (!task_queue.enqueue( + [this, sock]() { process_and_close_socket(sock); })) { + output_error_log(Error::ResourceExhaustion, nullptr); + detail::shutdown_socket(sock); + detail::close_socket(sock); + } + } + return ret; +} + #ifndef CPPHTTPLIB_NO_EXCEPTIONS // Call from a catch block. The response is a 500 unless the exception handler // sets another status. Returns whether the exception handler ran. @@ -15988,6 +16135,199 @@ inline bool Server::process_and_close_socket(socket_t sock) { return ret; } +// After accept() failed for good. Takes the descriptor out of svr_sock_ +// before closing it: a later stop() would otherwise shutdown()/close() a value +// the OS may have reused, and keep_alive() watches svr_sock_ to notice the +// server is gone. The exchange also settles the race with a concurrent stop(), +// since whichever side takes the descriptor closes it exactly once. Returns +// false when stop() took it first. +inline bool Server::abandon_listen_socket() { + auto sock = svr_sock_.exchange(INVALID_SOCKET); + if (sock == INVALID_SOCKET) { return false; } + detail::close_socket(sock); + output_error_log(Error::Connection, nullptr); + return true; +} + +inline bool Server::supports_keep_alive_parking() const { return true; } + +// Like accept_loop(), but a connection waiting for its next request sits in +// the poller instead of keeping a worker. A worker serves a connection while it +// has requests to read, then gives it back to this loop. +inline bool Server::parking_loop(TaskQueue &task_queue) { + using Clock = ParkingState::Clock; + using Connection = ParkingState::Connection; + const ParkingState::Token listen_token = 0; + const auto never = (Clock::time_point::max)(); + + auto &st = *parking_; + auto &poller = *st.poller; + + std::vector ready; + poller.wait(0, ready); // drops a wake() left from an earlier stop() + { + std::lock_guard guard(st.mutex); + st.closed = false; + } + + // Publish the listening socket before checking that stop() has not taken it, + // so that a stop() from now on leaves closing it to this loop. + auto listen_sock = svr_sock_.load(); + st.listen_sock = listen_sock; + auto listening = listen_sock != INVALID_SOCKET && svr_sock_ == listen_sock; + if (listening) { detail::set_nonblocking(listen_sock, true); } + auto listen_rearm_at = Clock::now(); // `never` while it is armed + auto fatal = false; + + const auto keep_alive = std::chrono::seconds(keep_alive_timeout_sec_); + // Only an interval the user asked for: Windows' default exists for the sake + // of stop(), which wake() already serves here. + const auto idle = std::chrono::microseconds( + idle_interval_set_ ? idle_interval_sec_ * 1000000 + idle_interval_usec_ + : 0); + auto next_idle = idle.count() > 0 ? Clock::now() + idle : never; + + auto dispatch = [&](const Connection &conn) { + return task_queue.enqueue([this, conn]() { resume_connection(conn); }); + }; + + std::vector returned; + while (listening && svr_sock_ == listen_sock) { + auto now = Clock::now(); + if (now >= listen_rearm_at) { + poller.arm(listen_sock, listen_token); + listen_rearm_at = never; + } + + auto until = (std::min)({st.next_deadline(), listen_rearm_at, next_idle, + now + std::chrono::seconds(60)}); + // Rounded up, so that the wait does not end just short of the deadline. + auto timeout = std::chrono::duration_cast( + until - now + std::chrono::microseconds(999)); + poller.wait(static_cast((std::max)(timeout.count(), + std::chrono::milliseconds::rep(0))), + ready); + if (svr_sock_ != listen_sock) { break; } + now = Clock::now(); + + { + std::lock_guard guard(st.mutex); + returned.swap(st.returned); + } + for (auto &conn : returned) { + st.park(std::move(conn), now + keep_alive); + } + returned.clear(); + + // Connections that found the queue full go first, in their order. + while (!st.backlog.empty() && dispatch(st.backlog.front())) { + st.backlog.pop_front(); + } + + for (auto token : ready) { + if (token == listen_token) { + listen_rearm_at = now; + for (auto i = 0; i < 64; i++) { + auto sock = detail::accept_socket(listen_sock); + if (sock == INVALID_SOCKET) { + if (detail::is_would_block_error()) { break; } + if (detail::is_accept_resource_error()) { + // Out of descriptors: let the deadlines free some first. + listen_rearm_at = now + std::chrono::milliseconds(10); + break; + } + if (detail::is_accept_transient_error()) { continue; } + fatal = true; + break; + } + // Some platforms make it non-blocking like the listening socket. + detail::set_nonblocking(sock, false); + prepare_accepted_socket(sock); + st.park(std::make_shared(sock, *this), + now + keep_alive); + if (idle.count() > 0) { next_idle = now + idle; } + } + continue; + } + + auto conn = st.unpark(token); + if (conn->served == 0) { + // A new connection is turned away when the queue is full, as + // accept_loop() does. It is closed when `conn` goes away. + if (!dispatch(conn)) { + output_error_log(Error::ResourceExhaustion, nullptr); + } + } else if (!st.backlog.empty() || !dispatch(conn)) { + // Closing an established connection would lose its request. + st.backlog.push_back(std::move(conn)); + } + } + st.backlog_waiting = !st.backlog.empty(); + if (fatal) { break; } + + st.expire(now); + if (now >= next_idle) { + task_queue.on_idle(); + next_idle = now + idle; + } + } + + // A worker still serving a request closes its connection itself. + st.close_all(); + poller.disarm(listen_sock); + + // See ParkingState::listen_sock. + if (st.listen_sock.exchange(INVALID_SOCKET) == INVALID_SOCKET) { + // stop() handed it over. + detail::shutdown_socket(listen_sock); + detail::close_socket(listen_sock); + return true; + } + return !(fatal && abandon_listen_socket()); +} + +inline void +Server::resume_connection(std::shared_ptr conn) { + auto &c = *conn; + if (!c.addresses_known) { + detail::get_remote_ip_and_port(c.sock, c.remote_addr, c.remote_port); + detail::get_local_ip_and_port(c.sock, c.local_addr, c.local_port); + c.addresses_known = true; + } + + auto woken_by_poller = true; + auto park = false; + auto websocket_upgraded = false; + serve_guarded([&]() { + return detail::process_server_socket_core( + c.strm, c.served, keep_alive_max_count_, + [&]() { + // parking_loop() saw the socket readable just before this ran. + if (woken_by_poller) { + woken_by_poller = false; + return true; + } + if (is_stopping()) { return false; } + // Waiting here for a slow client would keep the worker from others. + park = detail::select_read(c.sock, 0, 0) <= 0; + return !park; + }, + [&](Stream &strm, bool close_connection, bool &connection_closed) { + return process_request(strm, c.remote_addr, c.remote_port, + c.local_addr, c.local_port, close_connection, + connection_closed, nullptr, + &websocket_upgraded); + }); + }); + + auto &st = *parking_; + // A backlog waits for a free worker, and this one is about to be free. + if (st.backlog_waiting) { st.poller->wake(); } + if (park && st.give_back(conn)) { return; } + detail::drain_and_close_socket(c.sock); + c.sock = INVALID_SOCKET; +} + inline void Server::output_log(const Request &req, const Response &res) const { if (logger_) { std::lock_guard guard(logger_mutex_); @@ -19585,6 +19925,9 @@ inline bool SSLServer::process_and_close_socket(socket_t sock) { return ret; } +// Not yet: the TLS session would have to live in the parked connection. +inline bool SSLServer::supports_keep_alive_parking() const { return false; } + inline bool SSLServer::update_certs_pem(const char *cert_pem, const char *key_pem, const char *client_ca_pem, diff --git a/test/test.cc b/test/test.cc index cdde9d2a..c899a5ef 100644 --- a/test/test.cc +++ b/test/test.cc @@ -34144,3 +34144,274 @@ TEST(PollerFactoryTest, MakePollerWorks) { std::vector ready; EXPECT_TRUE(poller->wait(1000, ready)); } + +// --------------------------------------------------------------------------- +// Keep-alive parking +// --------------------------------------------------------------------------- + +namespace parking_test { + +// Runs `svr` on a free port and stops it when it goes away. +struct RunningServer { + explicit RunningServer(Server &svr) : svr_(svr) { + port = svr_.bind_to_any_port("127.0.0.1"); + thread = std::thread([this] { svr_.listen_after_bind(); }); + svr_.wait_until_ready(); + } + ~RunningServer() { + svr_.stop(); + if (thread.joinable()) { thread.join(); } + } + + Server &svr_; + int port = 0; + std::thread thread; +}; + +struct Sockets { + std::vector socks; + ~Sockets() { + for (auto s : socks) { + detail::close_socket(s); + } + } + socket_t connect(int port) { + auto error = Error::Success; + auto s = detail::create_client_socket("127.0.0.1", "", port, AF_UNSPEC, + false, false, nullptr, 2, 0, 2, 0, 2, + 0, std::string(), error); + if (s != INVALID_SOCKET) { socks.push_back(s); } + return s; + } +}; + +std::chrono::steady_clock::time_point in_ms(int ms) { + return std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); +} + +bool readable_by(socket_t sock, + std::chrono::steady_clock::time_point deadline) { + auto left = std::chrono::duration_cast( + deadline - std::chrono::steady_clock::now()) + .count(); + return left > 0 && detail::select_read(sock, 0, left) > 0; +} + +bool send_get(socket_t sock, const std::string &path = "/") { + auto req = "GET " + path + " HTTP/1.1\r\nHost: localhost\r\n\r\n"; + return detail::send_socket(sock, req.data(), req.size(), 0) == + static_cast(req.size()); +} + +// Reads one response whose body is "ok". +bool receives_ok(socket_t sock, int timeout_ms) { + auto deadline = in_ms(timeout_ms); + std::string buf; + while (readable_by(sock, deadline)) { + char tmp[1024]; + auto n = detail::read_socket(sock, tmp, sizeof(tmp), 0); + if (n <= 0) { return false; } + buf.append(tmp, static_cast(n)); + auto end = buf.find("\r\n\r\n"); + if (end != std::string::npos && buf.size() >= end + 4 + 2) { + return buf.compare(0, 12, "HTTP/1.1 200") == 0 && + buf.compare(end + 4, 2, "ok") == 0; + } + } + return false; +} + +bool get_ok(socket_t sock, int timeout_ms) { + return send_get(sock) && receives_ok(sock, timeout_ms); +} + +// Whether the server closes `sock` by `deadline`. +bool sees_eof(socket_t sock, std::chrono::steady_clock::time_point deadline) { + char c; + return readable_by(sock, deadline) && + detail::read_socket(sock, &c, 1, 0) <= 0; +} + +bool wait_until(const std::function &cond, int timeout_ms) { + auto deadline = in_ms(timeout_ms); + while (!cond()) { + if (std::chrono::steady_clock::now() > deadline) { return false; } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + return true; +} + +void serve_ok(Server &svr, std::function new_task_queue) { + svr.new_task_queue = std::move(new_task_queue); + svr.set_keep_alive_parking(true); + svr.Get("/", [](const Request &, Response &res) { + res.set_content("ok", "text/plain"); + }); +} + +std::function threads(size_t n) { + return [n] { return new ThreadPool(n); }; +} + +// Counts what the event loop managed to queue, so that a test can wait for it. +class CountingTaskQueue final : public TaskQueue { +public: + CountingTaskQueue(size_t threads, size_t max_queued, std::atomic &ok, + std::atomic &full) + : pool_(threads, threads, max_queued), ok_(ok), full_(full) {} + + bool enqueue(std::function fn) override { + auto queued = pool_.enqueue(std::move(fn)); + (queued ? ok_ : full_)++; + return queued; + } + void shutdown() override { pool_.shutdown(); } + +private: + ThreadPool pool_; + std::atomic &ok_; + std::atomic &full_; +}; + +} // namespace parking_test + +TEST(KeepAliveParkingTest, IdleConnectionsDoNotHoldTheWorkers) { + using namespace parking_test; + Server svr; + serve_ok(svr, threads(1)); + RunningServer running(svr); + Sockets socks; + + // One kept-alive connection that has been served, and many that have not + // sent anything yet. None of them may keep the only worker. + auto served = socks.connect(running.port); + ASSERT_TRUE(get_ok(served, 2000)); + for (int i = 0; i < 49; i++) { + ASSERT_NE(INVALID_SOCKET, socks.connect(running.port)); + } + + EXPECT_TRUE(get_ok(socks.connect(running.port), 1000)); + EXPECT_TRUE(get_ok(served, 1000)); +} + +TEST(KeepAliveParkingTest, IdleConnectionsTimeOutTogether) { + using namespace parking_test; + Server svr; + serve_ok(svr, threads(1)); + svr.set_keep_alive_timeout(1); + RunningServer running(svr); + Sockets socks; + + for (int i = 0; i < 3; i++) { + ASSERT_NE(INVALID_SOCKET, socks.connect(running.port)); + } + // One worker waiting on each in turn would take about 3 seconds. + auto deadline = in_ms(2000); + for (auto s : socks.socks) { + EXPECT_TRUE(sees_eof(s, deadline)); + } +} + +TEST(KeepAliveParkingTest, EstablishedConnectionWaitsForAFullQueue) { + using namespace parking_test; + std::atomic queued{0}, full{0}; + Server svr; + // One worker and room for one queued task. + serve_ok(svr, [&] { return new CountingTaskQueue(1, 1, queued, full); }); + + std::mutex m; + std::condition_variable cv; + auto entered = false; + auto released = false; + svr.Get("/block", [&](const Request &, Response &res) { + std::unique_lock lock(m); + entered = true; + cv.notify_all(); + cv.wait_for(lock, std::chrono::seconds(3), [&] { return released; }); + res.set_content("ok", "text/plain"); + }); + auto release = [&] { + std::lock_guard guard(m); + released = true; + cv.notify_all(); + }; + + RunningServer running(svr); + // Before the server stops, so that a failed run does not wait for the + // handler's timeout. + auto release_on_exit = detail::scope_exit([&] { release(); }); + Sockets socks; + + auto a = socks.connect(running.port); + ASSERT_TRUE(get_ok(a, 2000)); + + // B keeps the only worker busy. + auto b = socks.connect(running.port); + ASSERT_TRUE(send_get(b, "/block")); + ASSERT_TRUE(wait_until( + [&] { + std::lock_guard guard(m); + return entered; + }, + 2000)); + + // A new connection C fills the queue. + auto c = socks.connect(running.port); + ASSERT_TRUE(send_get(c)); + ASSERT_TRUE(wait_until([&] { return queued == 3; }, 2000)); + + // A's next request finds the queue full, but A has been served before, so it + // waits instead of being cut off. + ASSERT_TRUE(send_get(a)); + ASSERT_TRUE(wait_until([&] { return full >= 1; }, 2000)); + + release(); + EXPECT_TRUE(receives_ok(b, 2000)); + EXPECT_TRUE(receives_ok(c, 2000)); + EXPECT_TRUE(receives_ok(a, 2000)); +} + +TEST(KeepAliveParkingTest, StopClosesParkedConnectionsPromptly) { + using namespace parking_test; + for (int round = 0; round < 5; round++) { + SCOPED_TRACE(round); + Server svr; + serve_ok(svr, threads(2)); + RunningServer running(svr); + Sockets socks; + + for (int i = 0; i < 20; i++) { + ASSERT_TRUE(get_ok(socks.connect(running.port), 1000)); + } + + auto start = std::chrono::steady_clock::now(); + svr.stop(); + running.thread.join(); + EXPECT_LT(std::chrono::steady_clock::now() - start, + std::chrono::milliseconds(500)); + + auto deadline = in_ms(1000); + for (auto s : socks.socks) { + EXPECT_TRUE(sees_eof(s, deadline)); + } + } +} + +#ifdef CPPHTTPLIB_SSL_ENABLED +TEST(KeepAliveParkingTest, SSLServerServesAsBefore) { + using namespace parking_test; + SSLServer svr(SERVER_CERT_FILE, SERVER_PRIVATE_KEY_FILE); + ASSERT_TRUE(svr.is_valid()); + serve_ok(svr, threads(2)); + RunningServer running(svr); + + SSLClient cli("127.0.0.1", running.port); + cli.enable_server_certificate_verification(false); + cli.set_keep_alive(true); + for (int i = 0; i < 2; i++) { + auto res = cli.Get("/"); + ASSERT_TRUE(res) << "Error: " << to_string(res.error()); + EXPECT_EQ("ok", res->body); + } +} +#endif