From 58bd9a190058805642935fa8874e99a5ddf47b35 Mon Sep 17 00:00:00 2001 From: yhirose Date: Fri, 20 Nov 2026 23:37:44 -0500 Subject: [PATCH] Add Server::set_keep_alive_parking() A kept-alive connection holds a worker thread while it waits for its next request, so connections beyond the thread count wait for one to be closed, and with a pool smaller than the number of clients some clients get nothing until the keep-alive timeout or the max count ends another connection. With keep-alive parking, listen() runs an event loop on the detail::Poller instead of the blocking accept loop. A connection waiting for its next request is parked there, and goes back to a worker through the TaskQueue when the request arrives. The handlers, content providers and WebSocket keep their blocking API and still hold a thread while they run. It is opt-in and off by default. With it on: - consecutive requests on a connection may run on different threads; - when the queue is full, a connection that has not been served yet is closed as before, but an established one waits in a backlog instead of losing its request; - the loop owns the listening socket while it watches it, and stop() hands it over instead of closing it under the poller; - SSLServer ignores the setting for now, and an override of process_and_close_socket() is not called. On two CPUs with two workers, 50 keep-alive clients of a 2.7 ms handler were all served evenly (the slowest response took 70 ms instead of 5 s with 10 clients getting nothing), light requests from 50 clients went from 82k to 248k per second, and 10 clients were served after 10000 idle connections that had stopped the old loop. With no more clients than workers, handing the connection over costs about half of the requests per second of a very light handler. The old accept loop moved unchanged into accept_loop(). process_server_socket_core() now takes the stream, and drain_and_close_socket() takes how long to wait for the peer. --- README.md | 18 +- httplib.h | 551 +++++++++++++++++++++++++++++++++++++++++---------- test/test.cc | 271 +++++++++++++++++++++++++ 3 files changed, 735 insertions(+), 105 deletions(-) diff --git a/README.md b/README.md index 4eae8243..fc5c42cb 100644 --- a/README.md +++ b/README.md @@ -919,6 +919,22 @@ svr.set_keep_alive_max_count(2); // Default is 100, 0 for no limit svr.set_keep_alive_timeout(10); // Default is 5 ``` +#### Keep-alive parking + +```cpp +svr.set_keep_alive_parking(true); // Call it before listen() +``` + +By default, a kept-alive connection holds a worker thread while it waits for its next request, so connections beyond the thread count wait for one to be closed. With keep-alive parking, a connection waiting for its next request is handed to an event loop instead, and goes back to a worker when the request arrives. An idle connection then costs a socket and a few kilobytes instead of a thread, and connections that outnumber the threads take turns. + +With it: + +- Consecutive requests on a connection may run on different threads. +- When the [max queued requests](#max-queued-requests) limit is reached, only a connection that has not been served yet is closed. An established one waits for room. +- Each request pays for handing the connection over, so with no more connections than threads and very light handlers, it can serve fewer requests per second. +- A handler, a content provider and a WebSocket still hold their thread while they run. +- `SSLServer` ignores it for now, and an override of `process_and_close_socket()` is not called. + ### Client address behind a reverse proxy ```cpp @@ -962,7 +978,7 @@ Please see [Server example](https://github.com/yhirose/cpp-httplib/blob/master/e When a new task arrives and the queued tasks outnumber the idle threads, a temporary thread is spawned (up to the maximum). A dynamic thread exits automatically once it has waited for the idle timeout without receiving a task. The idle timeout defaults to 3 seconds, configurable via `CPPHTTPLIB_THREAD_POOL_IDLE_TIMEOUT`. -A connection holds its thread while it is served: between requests up to the keep-alive timeout, through a request body sent slowly but within the read timeout, and for the whole life of a WebSocket. Enough of them leave no thread for anyone else, so in front of untrusted clients, put a reverse proxy that limits connections per client, and reject unwanted WebSocket upgrades in the pre-request handler. +A connection holds its thread while it is served: between requests up to the keep-alive timeout (unless [keep-alive parking](#keep-alive-parking) is on), through a request body sent slowly but within the read timeout, and for the whole life of a WebSocket. Enough of them leave no thread for anyone else, so in front of untrusted clients, put a reverse proxy that limits connections per client, and reject unwanted WebSocket upgrades in the pre-request handler. If you want to set the thread counts at runtime: diff --git a/httplib.h b/httplib.h index d9a4ba59..ccaeabaf 100644 --- a/httplib.h +++ b/httplib.h @@ -2384,6 +2384,10 @@ public: template Server &set_idle_interval(const std::chrono::duration &duration); + // Keeps idle keep-alive connections in an event loop instead of a worker + // thread each. Call it before listen(). See README for what changes. + Server &set_keep_alive_parking(bool on); + Server &set_payload_max_length(size_t length); Server &set_compress(bool on); @@ -2454,6 +2458,13 @@ protected: // stop() takes the listening socket away; listen() then waits for handlers bool is_stopping() const { return svr_sock_ == INVALID_SOCKET; } + // See set_keep_alive_parking(). Defined in the implementation part. + struct KeepAliveConnection; + struct ParkingState; + std::unique_ptr parking_; + bool keep_alive_parking_ = false; + bool idle_interval_set_ = false; + std::vector trusted_proxies_; size_t keep_alive_max_count_ = CPPHTTPLIB_KEEPALIVE_MAX_COUNT; @@ -2513,6 +2524,11 @@ private: SocketOptions socket_options) const; int bind_internal(const std::string &host, int port, int socket_flags); bool listen_internal(); + bool accept_loop(TaskQueue &task_queue); + bool parking_loop(TaskQueue &task_queue); + void prepare_accepted_socket(socket_t sock) const; + void resume_connection(std::shared_ptr conn); + bool abandon_listen_socket(); bool routing(Request &req, Response &res, Stream &strm, bool skip_pre_routing); @@ -2560,6 +2576,7 @@ private: ContentReceiver multipart_receiver) const; virtual bool process_and_close_socket(socket_t sock); + virtual bool supports_keep_alive_parking() const; void output_log(const Request &req, const Response &res) const; void output_pre_compression_log(const Request &req, @@ -3520,6 +3537,7 @@ public: private: bool process_and_close_socket(socket_t sock) override; + bool supports_keep_alive_parking() const override; tls::ctx_t ctx_ = nullptr; // update_certs_pem() replaces the certificates and CAs that handshakes read @@ -7178,30 +7196,27 @@ inline bool keep_alive(const std::atomic &svr_sock, socket_t sock, return false; } -// `has_buffered_request` reports whether the connection's stream already holds -// bytes of the next request. A client may pipeline its requests (RFC 9112 -// 9.3.2), so reading one request can pull the start of the next one into the -// stream's buffer; that request must be served without waiting for the socket -// to become readable again, since its bytes are no longer on the socket. -// Otherwise `wait_for_next_request` waits for the socket, and returning false -// ends the loop. `callback` is told whether the socket has just been seen -// readable. `served` counts the requests served on this connection so far. -template -inline bool process_server_socket_core(size_t &served, +// Serves the requests of one connection while they come. A client may +// pipeline its requests (RFC 9112 9.3.2), so reading one request can pull the +// start of the next one into the stream's buffer; that request must be served +// without waiting for the socket to become readable again, since its bytes are +// no longer on the socket. Otherwise `wait_for_next_request` waits for the +// socket, and returning false ends the loop. `served` counts the requests +// served on this connection so far. +template +inline bool process_server_socket_core(S &strm, size_t &served, size_t keep_alive_max_count, - W wait_for_next_request, - P has_buffered_request, T callback) { + W wait_for_next_request, T callback) { auto ret = false; // A keep_alive_max_count of 0 sets no limit. for (; keep_alive_max_count == 0 || served < keep_alive_max_count; served++) { - auto socket_readable = false; - if (!has_buffered_request()) { + if (!strm.is_readable()) { if (!wait_for_next_request()) { break; } - socket_readable = true; + strm.set_readable_hint(); } auto close_connection = served + 1 == keep_alive_max_count; auto connection_closed = false; - ret = callback(socket_readable, close_connection, connection_closed); + ret = callback(strm, close_connection, connection_closed); if (!ret || connection_closed) { break; } } return ret; @@ -7220,14 +7235,9 @@ process_server_socket(const std::atomic &svr_sock, socket_t sock, write_timeout_sec, write_timeout_usec); size_t served = 0; return process_server_socket_core( - served, keep_alive_max_count, + strm, served, keep_alive_max_count, [&]() { return keep_alive(svr_sock, sock, keep_alive_timeout_sec); }, - [&]() { return strm.is_readable(); }, - [&](bool socket_readable, bool close_connection, - bool &connection_closed) { - if (socket_readable) { strm.set_readable_hint(); } - return callback(strm, close_connection, connection_closed); - }); + callback); } inline bool process_client_socket( @@ -7254,8 +7264,10 @@ inline int shutdown_socket(socket_t sock) noexcept { // the final shutdown+close. Closing with unread data in the receive queue // (or bytes arriving after the receive side is closed) makes the stack send // an abortive RST instead of a graceful FIN, which can make the peer see the -// response as a failed read even though it was fully written. -inline void drain_and_close_socket(socket_t sock) noexcept { +// response as a failed read even though it was fully written. With a +// `wait_msec` of 0, only what has already arrived is read. +inline void drain_and_close_socket(socket_t sock, + time_t wait_msec = 100) noexcept { #ifdef _WIN32 shutdown(sock, SD_SEND); #else @@ -7265,15 +7277,18 @@ inline void drain_and_close_socket(socket_t sock) noexcept { char buf[CPPHTTPLIB_RECV_BUFSIZ]; size_t total = 0; const auto deadline = std::chrono::steady_clock::now() + - std::chrono::milliseconds(100); // bound #1 + std::chrono::milliseconds(wait_msec); // bound #1 - while (total < size_t(1024u * 1024u)) { // bound #2 + for (auto first = true; total < size_t(1024u * 1024u); // bound #2 + first = false) { const auto remaining = std::chrono::duration_cast( deadline - std::chrono::steady_clock::now()) .count(); - if (remaining <= 0) { break; } - if (select_read(sock, 0, static_cast(remaining)) <= 0) { break; } + if (remaining <= 0 && !first) { break; } + const auto wait_usec = + (std::max)(remaining, std::chrono::microseconds::rep(0)); + if (select_read(sock, 0, static_cast(wait_usec)) <= 0) { break; } const auto n = read_socket(sock, buf, sizeof(buf), CPPHTTPLIB_RECV_FLAGS); if (n <= 0) { break; } total += static_cast(n); @@ -7828,6 +7843,26 @@ inline bool is_accept_transient_error() { #endif } +inline socket_t accept_socket(socket_t svr_sock) { +#if defined _WIN32 + // sockets connected via WASAccept inherit flags NO_HANDLE_INHERIT, + // OVERLAPPED + return WSAAccept(svr_sock, nullptr, nullptr, nullptr, 0); +#elif defined SOCK_CLOEXEC + return accept4(svr_sock, nullptr, nullptr, SOCK_CLOEXEC); +#else + return accept(svr_sock, nullptr, nullptr); +#endif +} + +inline bool is_would_block_error() { +#ifdef _WIN32 + return WSAGetLastError() == WSAEWOULDBLOCK; +#else + return errno == EAGAIN || errno == EWOULDBLOCK; +#endif +} + inline bool create_socket_pair(socket_t sv[2]) { #ifdef _WIN32 auto listener = create_socket( @@ -11412,14 +11447,9 @@ inline bool process_server_socket_ssl( write_timeout_sec, write_timeout_usec); size_t served = 0; return process_server_socket_core( - served, keep_alive_max_count, + strm, served, keep_alive_max_count, [&]() { return keep_alive(svr_sock, sock, keep_alive_timeout_sec); }, - [&]() { return strm.is_readable(); }, - [&](bool socket_readable, bool close_connection, - bool &connection_closed) { - if (socket_readable) { strm.set_readable_hint(); } - return callback(strm, close_connection, connection_closed); - }); + callback); } template @@ -14054,6 +14084,117 @@ WebSocketSSLStream::set_read_interrupt(std::function interrupted) { * Group 4: Server implementation */ +// A kept-alive connection that the event loop of set_keep_alive_parking() +// owns while it is idle, and a worker owns while it serves a request. +struct Server::KeepAliveConnection { + KeepAliveConnection(socket_t s, const Server &svr) + : sock(s), strm(s, svr.read_timeout_sec_, svr.read_timeout_usec_, + svr.write_timeout_sec_, svr.write_timeout_usec_) {} + ~KeepAliveConnection() { + if (sock != INVALID_SOCKET) { detail::drain_and_close_socket(sock, 0); } + } + + socket_t sock; + // Kept across requests: it may hold the start of a pipelined request. + detail::SocketStream strm; + size_t served = 0; + + bool addresses_known = false; + std::string remote_addr; + int remote_port = 0; + std::string local_addr; + int local_port = 0; + + // While parked + std::list>::iterator parked_at; + std::chrono::steady_clock::time_point deadline; +}; + +struct Server::ParkingState { + using Clock = std::chrono::steady_clock; + using Token = detail::Poller::Token; + using Connection = std::shared_ptr; + + // Kept for the Server's life, so that stop() can wake() it without a lock. + std::unique_ptr poller; + + // The listening socket while parking_loop() watches it. Whichever of stop() + // and parking_loop() takes it out of here decides who closes it, so that it + // is never closed while the poller still watches it. + std::atomic listen_sock{INVALID_SOCKET}; + + // From stop(): whether parking_loop() takes over closing `sock`. + bool release_listen(socket_t sock) { + if (!listen_sock.compare_exchange_strong(sock, INVALID_SOCKET)) { + return false; + } + poller->wake(); + return true; + } + + // From a worker: hands an idle connection back, unless the loop has ended. + bool give_back(Connection conn) { + { + std::lock_guard guard(mutex); + if (closed) { return false; } + returned.push_back(std::move(conn)); + } + poller->wake(); + return true; + } + + // The rest belongs to parking_loop(). A parked connection's token is its + // address, and `parked` lists them in the order they expire. + void park(Connection conn, Clock::time_point deadline) { + auto &c = *conn; + c.deadline = deadline; + c.parked_at = parked.insert(parked.end(), std::move(conn)); + poller->arm(c.sock, reinterpret_cast(&c)); + } + + Connection unpark(Token token) { + auto &c = *reinterpret_cast(token); + auto conn = std::move(*c.parked_at); + parked.erase(c.parked_at); + return conn; + } + + Clock::time_point next_deadline() const { + return parked.empty() ? (Clock::time_point::max)() + : parked.front()->deadline; + } + + void expire(Clock::time_point now) { + while (!parked.empty() && parked.front()->deadline <= now) { + poller->disarm(parked.front()->sock); + parked.pop_front(); // closes it + } + } + + void close_all() { + std::vector left; + { + std::lock_guard guard(mutex); + closed = true; + left.swap(returned); + } + for (auto &conn : parked) { + poller->disarm(conn->sock); + } + parked.clear(); + backlog.clear(); + backlog_waiting = false; + } + + std::mutex mutex; + std::vector returned; + bool closed = false; + std::atomic backlog_waiting{false}; + + std::list parked; + std::list backlog; +}; + // HTTP server implementation inline Server::Server() : new_task_queue([] { @@ -14398,6 +14539,20 @@ inline Server &Server::set_write_timeout(time_t sec, time_t usec) { inline Server &Server::set_idle_interval(time_t sec, time_t usec) { idle_interval_sec_ = sec; idle_interval_usec_ = usec; + idle_interval_set_ = true; + return *this; +} + +inline Server &Server::set_keep_alive_parking(bool on) { + // Never destroyed once made: stop() may use it from another thread. + if (on && !parking_) { + auto poller = detail::make_poller(); + if (poller) { + parking_.reset(new ParkingState()); + parking_->poller = std::move(poller); + } + } + keep_alive_parking_ = on; return *this; } @@ -14477,7 +14632,7 @@ inline void Server::stop() noexcept { // bind_to_port() without listen_after_bind() still owns the descriptor. The // exchange is what makes this safe to call concurrently with the accept loop. socket_t sock = svr_sock_.exchange(INVALID_SOCKET); - if (sock != INVALID_SOCKET) { + if (sock != INVALID_SOCKET && !(parking_ && parking_->release_listen(sock))) { detail::shutdown_socket(sock); detail::close_socket(sock); } @@ -15099,73 +15254,9 @@ inline bool Server::listen_internal() { { std::unique_ptr task_queue(new_task_queue()); - while (svr_sock_ != INVALID_SOCKET) { -#ifndef _WIN32 - if (idle_interval_sec_ > 0 || idle_interval_usec_ > 0) { -#endif - auto val = detail::select_read(svr_sock_, idle_interval_sec_, - idle_interval_usec_); - if (val == 0) { // Timeout - task_queue->on_idle(); - continue; - } -#ifndef _WIN32 - } -#endif - -#if defined _WIN32 - // sockets connected via WASAccept inherit flags NO_HANDLE_INHERIT, - // OVERLAPPED - socket_t sock = WSAAccept(svr_sock_, nullptr, nullptr, nullptr, 0); -#elif defined SOCK_CLOEXEC - socket_t sock = accept4(svr_sock_, nullptr, nullptr, SOCK_CLOEXEC); -#else - socket_t sock = accept(svr_sock_, nullptr, nullptr); -#endif - - if (sock == INVALID_SOCKET) { - // NOTE: Winsock reports failures through WSAGetLastError() and never - // touches the CRT errno, so the two have to be asked platform by - // platform rather than by testing errno here. - if (detail::is_accept_resource_error()) { - // The per-process descriptor limit or the network stack's buffer - // space has been reached. Try to accept new connections after a - // short sleep. - std::this_thread::sleep_for(std::chrono::microseconds{1}); - continue; - } else if (detail::is_accept_transient_error()) { - continue; - } - // Take the descriptor out of svr_sock_ before closing it: a later - // stop() would otherwise shutdown()/close() a value the OS may have - // reused, and keep_alive() watches svr_sock_ to notice the server is - // gone. The exchange also settles the race with a concurrent stop(), - // since whichever side takes the descriptor closes it exactly once. - auto listen_sock = svr_sock_.exchange(INVALID_SOCKET); - if (listen_sock != INVALID_SOCKET) { - detail::close_socket(listen_sock); - ret = false; - output_error_log(Error::Connection, nullptr); - } else { - ; // The server socket was closed by user. - } - break; - } - - detail::set_socket_opt_time(sock, SOL_SOCKET, SO_RCVTIMEO, - read_timeout_sec_, read_timeout_usec_); - detail::set_socket_opt_time(sock, SOL_SOCKET, SO_SNDTIMEO, - write_timeout_sec_, write_timeout_usec_); - - if (tcp_nodelay_) { set_socket_opt(sock, IPPROTO_TCP, TCP_NODELAY, 1); } - - if (!task_queue->enqueue( - [this, sock]() { process_and_close_socket(sock); })) { - output_error_log(Error::ResourceExhaustion, nullptr); - detail::shutdown_socket(sock); - detail::close_socket(sock); - } - } + ret = keep_alive_parking_ && parking_ && supports_keep_alive_parking() + ? parking_loop(*task_queue) + : accept_loop(*task_queue); task_queue->shutdown(); } @@ -15174,6 +15265,62 @@ inline bool Server::listen_internal() { return ret; } +inline void Server::prepare_accepted_socket(socket_t sock) const { + detail::set_socket_opt_time(sock, SOL_SOCKET, SO_RCVTIMEO, read_timeout_sec_, + read_timeout_usec_); + detail::set_socket_opt_time(sock, SOL_SOCKET, SO_SNDTIMEO, write_timeout_sec_, + write_timeout_usec_); + if (tcp_nodelay_) { set_socket_opt(sock, IPPROTO_TCP, TCP_NODELAY, 1); } +} + +// Serves each connection on one worker thread for its whole life. +inline bool Server::accept_loop(TaskQueue &task_queue) { + auto ret = true; + while (svr_sock_ != INVALID_SOCKET) { +#ifndef _WIN32 + if (idle_interval_sec_ > 0 || idle_interval_usec_ > 0) { +#endif + auto val = detail::select_read(svr_sock_, idle_interval_sec_, + idle_interval_usec_); + if (val == 0) { // Timeout + task_queue.on_idle(); + continue; + } +#ifndef _WIN32 + } +#endif + + auto sock = detail::accept_socket(svr_sock_); + + if (sock == INVALID_SOCKET) { + // NOTE: Winsock reports failures through WSAGetLastError() and never + // touches the CRT errno, so the two have to be asked platform by + // platform rather than by testing errno here. + if (detail::is_accept_resource_error()) { + // The per-process descriptor limit or the network stack's buffer + // space has been reached. Try to accept new connections after a + // short sleep. + std::this_thread::sleep_for(std::chrono::microseconds{1}); + continue; + } else if (detail::is_accept_transient_error()) { + continue; + } + if (abandon_listen_socket()) { ret = false; } + break; + } + + prepare_accepted_socket(sock); + + if (!task_queue.enqueue( + [this, sock]() { process_and_close_socket(sock); })) { + output_error_log(Error::ResourceExhaustion, nullptr); + detail::shutdown_socket(sock); + detail::close_socket(sock); + } + } + return ret; +} + #ifndef CPPHTTPLIB_NO_EXCEPTIONS // Call from a catch block. The response is a 500 unless the exception handler // sets another status. Returns whether the exception handler ran. @@ -15988,6 +16135,199 @@ inline bool Server::process_and_close_socket(socket_t sock) { return ret; } +// After accept() failed for good. Takes the descriptor out of svr_sock_ +// before closing it: a later stop() would otherwise shutdown()/close() a value +// the OS may have reused, and keep_alive() watches svr_sock_ to notice the +// server is gone. The exchange also settles the race with a concurrent stop(), +// since whichever side takes the descriptor closes it exactly once. Returns +// false when stop() took it first. +inline bool Server::abandon_listen_socket() { + auto sock = svr_sock_.exchange(INVALID_SOCKET); + if (sock == INVALID_SOCKET) { return false; } + detail::close_socket(sock); + output_error_log(Error::Connection, nullptr); + return true; +} + +inline bool Server::supports_keep_alive_parking() const { return true; } + +// Like accept_loop(), but a connection waiting for its next request sits in +// the poller instead of keeping a worker. A worker serves a connection while it +// has requests to read, then gives it back to this loop. +inline bool Server::parking_loop(TaskQueue &task_queue) { + using Clock = ParkingState::Clock; + using Connection = ParkingState::Connection; + const ParkingState::Token listen_token = 0; + const auto never = (Clock::time_point::max)(); + + auto &st = *parking_; + auto &poller = *st.poller; + + std::vector ready; + poller.wait(0, ready); // drops a wake() left from an earlier stop() + { + std::lock_guard guard(st.mutex); + st.closed = false; + } + + // Publish the listening socket before checking that stop() has not taken it, + // so that a stop() from now on leaves closing it to this loop. + auto listen_sock = svr_sock_.load(); + st.listen_sock = listen_sock; + auto listening = listen_sock != INVALID_SOCKET && svr_sock_ == listen_sock; + if (listening) { detail::set_nonblocking(listen_sock, true); } + auto listen_rearm_at = Clock::now(); // `never` while it is armed + auto fatal = false; + + const auto keep_alive = std::chrono::seconds(keep_alive_timeout_sec_); + // Only an interval the user asked for: Windows' default exists for the sake + // of stop(), which wake() already serves here. + const auto idle = std::chrono::microseconds( + idle_interval_set_ ? idle_interval_sec_ * 1000000 + idle_interval_usec_ + : 0); + auto next_idle = idle.count() > 0 ? Clock::now() + idle : never; + + auto dispatch = [&](const Connection &conn) { + return task_queue.enqueue([this, conn]() { resume_connection(conn); }); + }; + + std::vector returned; + while (listening && svr_sock_ == listen_sock) { + auto now = Clock::now(); + if (now >= listen_rearm_at) { + poller.arm(listen_sock, listen_token); + listen_rearm_at = never; + } + + auto until = (std::min)({st.next_deadline(), listen_rearm_at, next_idle, + now + std::chrono::seconds(60)}); + // Rounded up, so that the wait does not end just short of the deadline. + auto timeout = std::chrono::duration_cast( + until - now + std::chrono::microseconds(999)); + poller.wait(static_cast((std::max)(timeout.count(), + std::chrono::milliseconds::rep(0))), + ready); + if (svr_sock_ != listen_sock) { break; } + now = Clock::now(); + + { + std::lock_guard guard(st.mutex); + returned.swap(st.returned); + } + for (auto &conn : returned) { + st.park(std::move(conn), now + keep_alive); + } + returned.clear(); + + // Connections that found the queue full go first, in their order. + while (!st.backlog.empty() && dispatch(st.backlog.front())) { + st.backlog.pop_front(); + } + + for (auto token : ready) { + if (token == listen_token) { + listen_rearm_at = now; + for (auto i = 0; i < 64; i++) { + auto sock = detail::accept_socket(listen_sock); + if (sock == INVALID_SOCKET) { + if (detail::is_would_block_error()) { break; } + if (detail::is_accept_resource_error()) { + // Out of descriptors: let the deadlines free some first. + listen_rearm_at = now + std::chrono::milliseconds(10); + break; + } + if (detail::is_accept_transient_error()) { continue; } + fatal = true; + break; + } + // Some platforms make it non-blocking like the listening socket. + detail::set_nonblocking(sock, false); + prepare_accepted_socket(sock); + st.park(std::make_shared(sock, *this), + now + keep_alive); + if (idle.count() > 0) { next_idle = now + idle; } + } + continue; + } + + auto conn = st.unpark(token); + if (conn->served == 0) { + // A new connection is turned away when the queue is full, as + // accept_loop() does. It is closed when `conn` goes away. + if (!dispatch(conn)) { + output_error_log(Error::ResourceExhaustion, nullptr); + } + } else if (!st.backlog.empty() || !dispatch(conn)) { + // Closing an established connection would lose its request. + st.backlog.push_back(std::move(conn)); + } + } + st.backlog_waiting = !st.backlog.empty(); + if (fatal) { break; } + + st.expire(now); + if (now >= next_idle) { + task_queue.on_idle(); + next_idle = now + idle; + } + } + + // A worker still serving a request closes its connection itself. + st.close_all(); + poller.disarm(listen_sock); + + // See ParkingState::listen_sock. + if (st.listen_sock.exchange(INVALID_SOCKET) == INVALID_SOCKET) { + // stop() handed it over. + detail::shutdown_socket(listen_sock); + detail::close_socket(listen_sock); + return true; + } + return !(fatal && abandon_listen_socket()); +} + +inline void +Server::resume_connection(std::shared_ptr conn) { + auto &c = *conn; + if (!c.addresses_known) { + detail::get_remote_ip_and_port(c.sock, c.remote_addr, c.remote_port); + detail::get_local_ip_and_port(c.sock, c.local_addr, c.local_port); + c.addresses_known = true; + } + + auto woken_by_poller = true; + auto park = false; + auto websocket_upgraded = false; + serve_guarded([&]() { + return detail::process_server_socket_core( + c.strm, c.served, keep_alive_max_count_, + [&]() { + // parking_loop() saw the socket readable just before this ran. + if (woken_by_poller) { + woken_by_poller = false; + return true; + } + if (is_stopping()) { return false; } + // Waiting here for a slow client would keep the worker from others. + park = detail::select_read(c.sock, 0, 0) <= 0; + return !park; + }, + [&](Stream &strm, bool close_connection, bool &connection_closed) { + return process_request(strm, c.remote_addr, c.remote_port, + c.local_addr, c.local_port, close_connection, + connection_closed, nullptr, + &websocket_upgraded); + }); + }); + + auto &st = *parking_; + // A backlog waits for a free worker, and this one is about to be free. + if (st.backlog_waiting) { st.poller->wake(); } + if (park && st.give_back(conn)) { return; } + detail::drain_and_close_socket(c.sock); + c.sock = INVALID_SOCKET; +} + inline void Server::output_log(const Request &req, const Response &res) const { if (logger_) { std::lock_guard guard(logger_mutex_); @@ -19585,6 +19925,9 @@ inline bool SSLServer::process_and_close_socket(socket_t sock) { return ret; } +// Not yet: the TLS session would have to live in the parked connection. +inline bool SSLServer::supports_keep_alive_parking() const { return false; } + inline bool SSLServer::update_certs_pem(const char *cert_pem, const char *key_pem, const char *client_ca_pem, diff --git a/test/test.cc b/test/test.cc index cdde9d2a..c899a5ef 100644 --- a/test/test.cc +++ b/test/test.cc @@ -34144,3 +34144,274 @@ TEST(PollerFactoryTest, MakePollerWorks) { std::vector ready; EXPECT_TRUE(poller->wait(1000, ready)); } + +// --------------------------------------------------------------------------- +// Keep-alive parking +// --------------------------------------------------------------------------- + +namespace parking_test { + +// Runs `svr` on a free port and stops it when it goes away. +struct RunningServer { + explicit RunningServer(Server &svr) : svr_(svr) { + port = svr_.bind_to_any_port("127.0.0.1"); + thread = std::thread([this] { svr_.listen_after_bind(); }); + svr_.wait_until_ready(); + } + ~RunningServer() { + svr_.stop(); + if (thread.joinable()) { thread.join(); } + } + + Server &svr_; + int port = 0; + std::thread thread; +}; + +struct Sockets { + std::vector socks; + ~Sockets() { + for (auto s : socks) { + detail::close_socket(s); + } + } + socket_t connect(int port) { + auto error = Error::Success; + auto s = detail::create_client_socket("127.0.0.1", "", port, AF_UNSPEC, + false, false, nullptr, 2, 0, 2, 0, 2, + 0, std::string(), error); + if (s != INVALID_SOCKET) { socks.push_back(s); } + return s; + } +}; + +std::chrono::steady_clock::time_point in_ms(int ms) { + return std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); +} + +bool readable_by(socket_t sock, + std::chrono::steady_clock::time_point deadline) { + auto left = std::chrono::duration_cast( + deadline - std::chrono::steady_clock::now()) + .count(); + return left > 0 && detail::select_read(sock, 0, left) > 0; +} + +bool send_get(socket_t sock, const std::string &path = "/") { + auto req = "GET " + path + " HTTP/1.1\r\nHost: localhost\r\n\r\n"; + return detail::send_socket(sock, req.data(), req.size(), 0) == + static_cast(req.size()); +} + +// Reads one response whose body is "ok". +bool receives_ok(socket_t sock, int timeout_ms) { + auto deadline = in_ms(timeout_ms); + std::string buf; + while (readable_by(sock, deadline)) { + char tmp[1024]; + auto n = detail::read_socket(sock, tmp, sizeof(tmp), 0); + if (n <= 0) { return false; } + buf.append(tmp, static_cast(n)); + auto end = buf.find("\r\n\r\n"); + if (end != std::string::npos && buf.size() >= end + 4 + 2) { + return buf.compare(0, 12, "HTTP/1.1 200") == 0 && + buf.compare(end + 4, 2, "ok") == 0; + } + } + return false; +} + +bool get_ok(socket_t sock, int timeout_ms) { + return send_get(sock) && receives_ok(sock, timeout_ms); +} + +// Whether the server closes `sock` by `deadline`. +bool sees_eof(socket_t sock, std::chrono::steady_clock::time_point deadline) { + char c; + return readable_by(sock, deadline) && + detail::read_socket(sock, &c, 1, 0) <= 0; +} + +bool wait_until(const std::function &cond, int timeout_ms) { + auto deadline = in_ms(timeout_ms); + while (!cond()) { + if (std::chrono::steady_clock::now() > deadline) { return false; } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + return true; +} + +void serve_ok(Server &svr, std::function new_task_queue) { + svr.new_task_queue = std::move(new_task_queue); + svr.set_keep_alive_parking(true); + svr.Get("/", [](const Request &, Response &res) { + res.set_content("ok", "text/plain"); + }); +} + +std::function threads(size_t n) { + return [n] { return new ThreadPool(n); }; +} + +// Counts what the event loop managed to queue, so that a test can wait for it. +class CountingTaskQueue final : public TaskQueue { +public: + CountingTaskQueue(size_t threads, size_t max_queued, std::atomic &ok, + std::atomic &full) + : pool_(threads, threads, max_queued), ok_(ok), full_(full) {} + + bool enqueue(std::function fn) override { + auto queued = pool_.enqueue(std::move(fn)); + (queued ? ok_ : full_)++; + return queued; + } + void shutdown() override { pool_.shutdown(); } + +private: + ThreadPool pool_; + std::atomic &ok_; + std::atomic &full_; +}; + +} // namespace parking_test + +TEST(KeepAliveParkingTest, IdleConnectionsDoNotHoldTheWorkers) { + using namespace parking_test; + Server svr; + serve_ok(svr, threads(1)); + RunningServer running(svr); + Sockets socks; + + // One kept-alive connection that has been served, and many that have not + // sent anything yet. None of them may keep the only worker. + auto served = socks.connect(running.port); + ASSERT_TRUE(get_ok(served, 2000)); + for (int i = 0; i < 49; i++) { + ASSERT_NE(INVALID_SOCKET, socks.connect(running.port)); + } + + EXPECT_TRUE(get_ok(socks.connect(running.port), 1000)); + EXPECT_TRUE(get_ok(served, 1000)); +} + +TEST(KeepAliveParkingTest, IdleConnectionsTimeOutTogether) { + using namespace parking_test; + Server svr; + serve_ok(svr, threads(1)); + svr.set_keep_alive_timeout(1); + RunningServer running(svr); + Sockets socks; + + for (int i = 0; i < 3; i++) { + ASSERT_NE(INVALID_SOCKET, socks.connect(running.port)); + } + // One worker waiting on each in turn would take about 3 seconds. + auto deadline = in_ms(2000); + for (auto s : socks.socks) { + EXPECT_TRUE(sees_eof(s, deadline)); + } +} + +TEST(KeepAliveParkingTest, EstablishedConnectionWaitsForAFullQueue) { + using namespace parking_test; + std::atomic queued{0}, full{0}; + Server svr; + // One worker and room for one queued task. + serve_ok(svr, [&] { return new CountingTaskQueue(1, 1, queued, full); }); + + std::mutex m; + std::condition_variable cv; + auto entered = false; + auto released = false; + svr.Get("/block", [&](const Request &, Response &res) { + std::unique_lock lock(m); + entered = true; + cv.notify_all(); + cv.wait_for(lock, std::chrono::seconds(3), [&] { return released; }); + res.set_content("ok", "text/plain"); + }); + auto release = [&] { + std::lock_guard guard(m); + released = true; + cv.notify_all(); + }; + + RunningServer running(svr); + // Before the server stops, so that a failed run does not wait for the + // handler's timeout. + auto release_on_exit = detail::scope_exit([&] { release(); }); + Sockets socks; + + auto a = socks.connect(running.port); + ASSERT_TRUE(get_ok(a, 2000)); + + // B keeps the only worker busy. + auto b = socks.connect(running.port); + ASSERT_TRUE(send_get(b, "/block")); + ASSERT_TRUE(wait_until( + [&] { + std::lock_guard guard(m); + return entered; + }, + 2000)); + + // A new connection C fills the queue. + auto c = socks.connect(running.port); + ASSERT_TRUE(send_get(c)); + ASSERT_TRUE(wait_until([&] { return queued == 3; }, 2000)); + + // A's next request finds the queue full, but A has been served before, so it + // waits instead of being cut off. + ASSERT_TRUE(send_get(a)); + ASSERT_TRUE(wait_until([&] { return full >= 1; }, 2000)); + + release(); + EXPECT_TRUE(receives_ok(b, 2000)); + EXPECT_TRUE(receives_ok(c, 2000)); + EXPECT_TRUE(receives_ok(a, 2000)); +} + +TEST(KeepAliveParkingTest, StopClosesParkedConnectionsPromptly) { + using namespace parking_test; + for (int round = 0; round < 5; round++) { + SCOPED_TRACE(round); + Server svr; + serve_ok(svr, threads(2)); + RunningServer running(svr); + Sockets socks; + + for (int i = 0; i < 20; i++) { + ASSERT_TRUE(get_ok(socks.connect(running.port), 1000)); + } + + auto start = std::chrono::steady_clock::now(); + svr.stop(); + running.thread.join(); + EXPECT_LT(std::chrono::steady_clock::now() - start, + std::chrono::milliseconds(500)); + + auto deadline = in_ms(1000); + for (auto s : socks.socks) { + EXPECT_TRUE(sees_eof(s, deadline)); + } + } +} + +#ifdef CPPHTTPLIB_SSL_ENABLED +TEST(KeepAliveParkingTest, SSLServerServesAsBefore) { + using namespace parking_test; + SSLServer svr(SERVER_CERT_FILE, SERVER_PRIVATE_KEY_FILE); + ASSERT_TRUE(svr.is_valid()); + serve_ok(svr, threads(2)); + RunningServer running(svr); + + SSLClient cli("127.0.0.1", running.port); + cli.enable_server_certificate_verification(false); + cli.set_keep_alive(true); + for (int i = 0; i < 2; i++) { + auto res = cli.Get("/"); + ASSERT_TRUE(res) << "Error: " << to_string(res.error()); + EXPECT_EQ("ok", res->body); + } +} +#endif