Add Server::set_keep_alive_parking()

A kept-alive connection holds a worker thread while it waits for its
next request, so connections beyond the thread count wait for one to be
closed, and with a pool smaller than the number of clients some clients
get nothing until the keep-alive timeout or the max count ends another
connection.

With keep-alive parking, listen() runs an event loop on the detail::Poller
instead of the blocking accept loop. A connection waiting for its next
request is parked there, and goes back to a worker through the TaskQueue
when the request arrives. The handlers, content providers and WebSocket
keep their blocking API and still hold a thread while they run.

It is opt-in and off by default. With it on:
- consecutive requests on a connection may run on different threads;
- when the queue is full, a connection that has not been served yet is
  closed as before, but an established one waits in a backlog instead of
  losing its request;
- the loop owns the listening socket while it watches it, and stop()
  hands it over instead of closing it under the poller;
- SSLServer ignores the setting for now, and an override of
  process_and_close_socket() is not called.

On two CPUs with two workers, 50 keep-alive clients of a 2.7 ms handler
were all served evenly (the slowest response took 70 ms instead of 5 s
with 10 clients getting nothing), light requests from 50 clients went
from 82k to 248k per second, and 10 clients were served after 10000
idle connections that had stopped the old loop. With no more clients
than workers, handing the connection over costs about half of the
requests per second of a very light handler.

The old accept loop moved unchanged into accept_loop().
process_server_socket_core() now takes the stream, and
drain_and_close_socket() takes how long to wait for the peer.
This commit is contained in:
yhirose committed 2026-11-20 23:39:33 -05:00
1 parent 1b914e1597
commit 58bd9a1900
3 files changed
+735 -105

No files matched your search

+271
View File
@@ -34144,3 +34144,274 @@ TEST(PollerFactoryTest, MakePollerWorks) {
std::vector<detail::Poller::Token> ready;
EXPECT_TRUE(poller->wait(1000, ready));
}
// ---------------------------------------------------------------------------
// Keep-alive parking
// ---------------------------------------------------------------------------
namespace parking_test {
// Runs `svr` on a free port and stops it when it goes away.
struct RunningServer {
explicit RunningServer(Server &svr) : svr_(svr) {
port = svr_.bind_to_any_port("127.0.0.1");
thread = std::thread([this] { svr_.listen_after_bind(); });
svr_.wait_until_ready();
}
~RunningServer() {
svr_.stop();
if (thread.joinable()) { thread.join(); }
}
Server &svr_;
int port = 0;
std::thread thread;
};
struct Sockets {
std::vector<socket_t> socks;
~Sockets() {
for (auto s : socks) {
detail::close_socket(s);
}
}
socket_t connect(int port) {
auto error = Error::Success;
auto s = detail::create_client_socket("127.0.0.1", "", port, AF_UNSPEC,
false, false, nullptr, 2, 0, 2, 0, 2,
0, std::string(), error);
if (s != INVALID_SOCKET) { socks.push_back(s); }
return s;
}
};
std::chrono::steady_clock::time_point in_ms(int ms) {
return std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
}
bool readable_by(socket_t sock,
std::chrono::steady_clock::time_point deadline) {
auto left = std::chrono::duration_cast<std::chrono::microseconds>(
deadline - std::chrono::steady_clock::now())
.count();
return left > 0 && detail::select_read(sock, 0, left) > 0;
}
bool send_get(socket_t sock, const std::string &path = "/") {
auto req = "GET " + path + " HTTP/1.1\r\nHost: localhost\r\n\r\n";
return detail::send_socket(sock, req.data(), req.size(), 0) ==
static_cast<ssize_t>(req.size());
}
// Reads one response whose body is "ok".
bool receives_ok(socket_t sock, int timeout_ms) {
auto deadline = in_ms(timeout_ms);
std::string buf;
while (readable_by(sock, deadline)) {
char tmp[1024];
auto n = detail::read_socket(sock, tmp, sizeof(tmp), 0);
if (n <= 0) { return false; }
buf.append(tmp, static_cast<size_t>(n));
auto end = buf.find("\r\n\r\n");
if (end != std::string::npos && buf.size() >= end + 4 + 2) {
return buf.compare(0, 12, "HTTP/1.1 200") == 0 &&
buf.compare(end + 4, 2, "ok") == 0;
}
}
return false;
}
bool get_ok(socket_t sock, int timeout_ms) {
return send_get(sock) && receives_ok(sock, timeout_ms);
}
// Whether the server closes `sock` by `deadline`.
bool sees_eof(socket_t sock, std::chrono::steady_clock::time_point deadline) {
char c;
return readable_by(sock, deadline) &&
detail::read_socket(sock, &c, 1, 0) <= 0;
}
bool wait_until(const std::function<bool()> &cond, int timeout_ms) {
auto deadline = in_ms(timeout_ms);
while (!cond()) {
if (std::chrono::steady_clock::now() > deadline) { return false; }
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
return true;
}
void serve_ok(Server &svr, std::function<TaskQueue *(void)> new_task_queue) {
svr.new_task_queue = std::move(new_task_queue);
svr.set_keep_alive_parking(true);
svr.Get("/", [](const Request &, Response &res) {
res.set_content("ok", "text/plain");
});
}
std::function<TaskQueue *(void)> threads(size_t n) {
return [n] { return new ThreadPool(n); };
}
// Counts what the event loop managed to queue, so that a test can wait for it.
class CountingTaskQueue final : public TaskQueue {
public:
CountingTaskQueue(size_t threads, size_t max_queued, std::atomic<int> &ok,
std::atomic<int> &full)
: pool_(threads, threads, max_queued), ok_(ok), full_(full) {}
bool enqueue(std::function<void()> fn) override {
auto queued = pool_.enqueue(std::move(fn));
(queued ? ok_ : full_)++;
return queued;
}
void shutdown() override { pool_.shutdown(); }
private:
ThreadPool pool_;
std::atomic<int> &ok_;
std::atomic<int> &full_;
};
} // namespace parking_test
TEST(KeepAliveParkingTest, IdleConnectionsDoNotHoldTheWorkers) {
using namespace parking_test;
Server svr;
serve_ok(svr, threads(1));
RunningServer running(svr);
Sockets socks;
// One kept-alive connection that has been served, and many that have not
// sent anything yet. None of them may keep the only worker.
auto served = socks.connect(running.port);
ASSERT_TRUE(get_ok(served, 2000));
for (int i = 0; i < 49; i++) {
ASSERT_NE(INVALID_SOCKET, socks.connect(running.port));
}
EXPECT_TRUE(get_ok(socks.connect(running.port), 1000));
EXPECT_TRUE(get_ok(served, 1000));
}
TEST(KeepAliveParkingTest, IdleConnectionsTimeOutTogether) {
using namespace parking_test;
Server svr;
serve_ok(svr, threads(1));
svr.set_keep_alive_timeout(1);
RunningServer running(svr);
Sockets socks;
for (int i = 0; i < 3; i++) {
ASSERT_NE(INVALID_SOCKET, socks.connect(running.port));
}
// One worker waiting on each in turn would take about 3 seconds.
auto deadline = in_ms(2000);
for (auto s : socks.socks) {
EXPECT_TRUE(sees_eof(s, deadline));
}
}
TEST(KeepAliveParkingTest, EstablishedConnectionWaitsForAFullQueue) {
using namespace parking_test;
std::atomic<int> queued{0}, full{0};
Server svr;
// One worker and room for one queued task.
serve_ok(svr, [&] { return new CountingTaskQueue(1, 1, queued, full); });
std::mutex m;
std::condition_variable cv;
auto entered = false;
auto released = false;
svr.Get("/block", [&](const Request &, Response &res) {
std::unique_lock<std::mutex> lock(m);
entered = true;
cv.notify_all();
cv.wait_for(lock, std::chrono::seconds(3), [&] { return released; });
res.set_content("ok", "text/plain");
});
auto release = [&] {
std::lock_guard<std::mutex> guard(m);
released = true;
cv.notify_all();
};
RunningServer running(svr);
// Before the server stops, so that a failed run does not wait for the
// handler's timeout.
auto release_on_exit = detail::scope_exit([&] { release(); });
Sockets socks;
auto a = socks.connect(running.port);
ASSERT_TRUE(get_ok(a, 2000));
// B keeps the only worker busy.
auto b = socks.connect(running.port);
ASSERT_TRUE(send_get(b, "/block"));
ASSERT_TRUE(wait_until(
[&] {
std::lock_guard<std::mutex> guard(m);
return entered;
},
2000));
// A new connection C fills the queue.
auto c = socks.connect(running.port);
ASSERT_TRUE(send_get(c));
ASSERT_TRUE(wait_until([&] { return queued == 3; }, 2000));
// A's next request finds the queue full, but A has been served before, so it
// waits instead of being cut off.
ASSERT_TRUE(send_get(a));
ASSERT_TRUE(wait_until([&] { return full >= 1; }, 2000));
release();
EXPECT_TRUE(receives_ok(b, 2000));
EXPECT_TRUE(receives_ok(c, 2000));
EXPECT_TRUE(receives_ok(a, 2000));
}
TEST(KeepAliveParkingTest, StopClosesParkedConnectionsPromptly) {
using namespace parking_test;
for (int round = 0; round < 5; round++) {
SCOPED_TRACE(round);
Server svr;
serve_ok(svr, threads(2));
RunningServer running(svr);
Sockets socks;
for (int i = 0; i < 20; i++) {
ASSERT_TRUE(get_ok(socks.connect(running.port), 1000));
}
auto start = std::chrono::steady_clock::now();
svr.stop();
running.thread.join();
EXPECT_LT(std::chrono::steady_clock::now() - start,
std::chrono::milliseconds(500));
auto deadline = in_ms(1000);
for (auto s : socks.socks) {
EXPECT_TRUE(sees_eof(s, deadline));
}
}
}
#ifdef CPPHTTPLIB_SSL_ENABLED
TEST(KeepAliveParkingTest, SSLServerServesAsBefore) {
using namespace parking_test;
SSLServer svr(SERVER_CERT_FILE, SERVER_PRIVATE_KEY_FILE);
ASSERT_TRUE(svr.is_valid());
serve_ok(svr, threads(2));
RunningServer running(svr);
SSLClient cli("127.0.0.1", running.port);
cli.enable_server_certificate_verification(false);
cli.set_keep_alive(true);
for (int i = 0; i < 2; i++) {
auto res = cli.Get("/");
ASSERT_TRUE(res) << "Error: " << to_string(res.error());
EXPECT_EQ("ok", res->body);
}
}
#endif