From a694b8c306b94a66809d9bbddaf843b4f8ba17b7 Mon Sep 17 00:00:00 2001 From: yhirose Date: Fri, 20 Nov 2026 20:57:31 -0500 Subject: [PATCH] Add a one-shot Poller with poll, epoll and kqueue backends Groundwork for keep-alive parking, an opt-in event loop that parks idle keep-alive connections instead of blocking a worker thread on each. detail::Poller delivers readiness once per arm(), so a descriptor is either armed (owned by the poller) or owned by whoever got the event. wake() ends a wait() from any thread, and an armed descriptor must be disarmed before it is closed. - PollPoller (poll() / WSAPoll()) works everywhere. - EpollPoller (Linux) uses EPOLLONESHOT. - KqueuePoller (macOS, FreeBSD) uses EV_ONESHOT. - PollerWaker carries wake() for all three: an eventfd on Linux, a socket pair elsewhere. detail::create_socket_pair() emulates socketpair() on Windows with sockets that child processes do not inherit. make_poller() picks the best one and falls back to poll(). The contract tests run on every backend that builds. Nothing uses the poller yet. --- httplib.h | 439 +++++++++++++++++++++++++++++++++++++++++++++++++++ test/test.cc | 236 +++++++++++++++++++++++++++ 2 files changed, 675 insertions(+) diff --git a/httplib.h b/httplib.h index 845366e3..9bfae501 100644 --- a/httplib.h +++ b/httplib.h @@ -374,6 +374,8 @@ using socklen_t = int; #ifdef __linux__ #include #undef _res // Undefine _res macro to avoid conflicts with user code (#2278) +#include +#include #endif #include #include @@ -393,6 +395,11 @@ using socket_t = int; #include #endif +#if defined(__APPLE__) || defined(__FreeBSD__) +#define CPPHTTPLIB_KQUEUE_POLLER +#include +#endif + #include #include #include @@ -3930,6 +3937,126 @@ EncodingType encoding_type(const Request &req, const Response &res, EncodingType encoding_type(const Request &req, const Response &res); +// Creates a connected pair of stream sockets that child processes do not +// inherit. Windows has no socketpair(), so it is emulated there. +bool create_socket_pair(socket_t sv[2]); + +// Ends a Poller's wait() from another thread. The poller watches fd() for +// readability, level-triggered, and calls drain() when it is reported. +class PollerWaker { +public: + ~PollerWaker(); + + // An eventfd on Linux, a socket pair elsewhere. + bool open(); + socket_t fd() const { return fd_; } + // An address inside the poller, which no caller's token can be. + uintptr_t token() const { return reinterpret_cast(this); } + + // From any thread. While one is pending, later calls make no syscall. + void wake() noexcept; + + void drain() noexcept; + +private: + socket_t fd_ = INVALID_SOCKET; + socket_t pair_fd_ = INVALID_SOCKET; // the end wake() writes; none on Linux + std::atomic pending_{false}; +}; + +// One-shot readiness notification for a reactor: +// - A delivered event disarms the descriptor; arm() it again to wait again. +// - A hang-up or an error counts as readable. +// - Only an unarmed descriptor may be armed, and an armed one must be +// disarmed before it is closed. +// - wake() may be called from any thread; the rest belongs to one thread. +class Poller { +public: + using Token = uintptr_t; + + virtual ~Poller() = default; + + virtual bool arm(socket_t fd, Token token) = 0; + virtual void disarm(socket_t fd) = 0; + + // Fills `ready` in no particular order and returns whether wake() was + // called. A negative timeout waits until either happens. + virtual bool wait(int timeout_ms, std::vector &ready) = 0; + + virtual void wake() noexcept = 0; +}; + +// poll() on POSIX, WSAPoll() on Windows. Works everywhere. +class PollPoller final : public Poller { +public: + // nullptr when the descriptors cannot be created. + static std::unique_ptr create(); + + bool arm(socket_t fd, Token token) override; + void disarm(socket_t fd) override; + bool wait(int timeout_ms, std::vector &ready) override; + void wake() noexcept override { waker_.wake(); } + +private: + PollPoller() = default; + void remove_at(size_t index); + + PollerWaker waker_; + + // fds_[0] is the waker's, which is always armed; tokens_[i] belongs to + // fds_[i]. + std::vector fds_; + std::vector tokens_; + std::unordered_map index_; +}; + +#ifdef __linux__ +// epoll with EPOLLONESHOT. Waits in O(ready) instead of O(armed). +class EpollPoller final : public Poller { +public: + // nullptr when the descriptors cannot be created. + static std::unique_ptr create(); + ~EpollPoller() override; + + bool arm(socket_t fd, Token token) override; + void disarm(socket_t fd) override; + bool wait(int timeout_ms, std::vector &ready) override; + void wake() noexcept override { waker_.wake(); } + +private: + EpollPoller() = default; + + int epfd_ = -1; + PollerWaker waker_; // always armed + std::vector events_; +}; +#endif + +#ifdef CPPHTTPLIB_KQUEUE_POLLER +// kqueue with EV_ONESHOT. Waits in O(ready) instead of O(armed). +class KqueuePoller final : public Poller { +public: + // nullptr when the descriptors cannot be created. + static std::unique_ptr create(); + ~KqueuePoller() override; + + bool arm(socket_t fd, Token token) override; + void disarm(socket_t fd) override; + bool wait(int timeout_ms, std::vector &ready) override; + void wake() noexcept override { waker_.wake(); } + +private: + KqueuePoller() = default; + + int kq_ = -1; + PollerWaker waker_; // always armed + std::vector events_; +}; +#endif + +// The best poller for this platform, or nullptr when it cannot be created. +std::unique_ptr make_poller(); + class BufferStream final : public Stream { public: BufferStream() = default; @@ -7770,6 +7897,318 @@ inline bool is_accept_transient_error() { #endif } +inline bool create_socket_pair(socket_t sv[2]) { +#ifdef _WIN32 + auto listener = create_socket( + std::string(), "127.0.0.1", 0, AF_INET, 0, false, false, nullptr, + [](socket_t sock, struct addrinfo &ai, bool &) { + return ::bind(sock, ai.ai_addr, static_cast(ai.ai_addrlen)) == 0 && + ::listen(sock, 1) == 0; + }); + if (listener == INVALID_SOCKET) { return false; } + + socket_t a = INVALID_SOCKET; + socket_t b = INVALID_SOCKET; + sockaddr_in addr{}; + auto addr_len = static_cast(sizeof(addr)); + if (::getsockname(listener, reinterpret_cast(&addr), &addr_len) == + 0) { + a = create_socket(std::string(), "127.0.0.1", ntohs(addr.sin_port), AF_INET, + 0, true, false, nullptr, + [](socket_t sock, struct addrinfo &ai, bool &) { + return ::connect(sock, ai.ai_addr, + static_cast(ai.ai_addrlen)) == 0; + }); + // Unlike accept(), this keeps the listener's no-inherit flag. + if (a != INVALID_SOCKET) { + b = WSAAccept(listener, nullptr, nullptr, nullptr, 0); + } + } + close_socket(listener); + + // Any local process could connect while the listener was open, so make sure + // that the connection we accepted is the one we made. + if (b != INVALID_SOCKET) { + sockaddr_in local{}, peer{}; + auto local_len = static_cast(sizeof(local)); + auto peer_len = static_cast(sizeof(peer)); + if (::getsockname(a, reinterpret_cast(&local), &local_len) != + 0 || + ::getpeername(b, reinterpret_cast(&peer), &peer_len) != 0 || + local.sin_port != peer.sin_port || + local.sin_addr.s_addr != peer.sin_addr.s_addr) { + close_socket(b); + b = INVALID_SOCKET; + } + } + + if (b == INVALID_SOCKET) { + if (a != INVALID_SOCKET) { close_socket(a); } + return false; + } + sv[0] = a; + sv[1] = b; + return true; +#else +#ifdef SOCK_CLOEXEC + return ::socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, sv) == 0; +#else + if (::socketpair(AF_UNIX, SOCK_STREAM, 0, sv) != 0) { return false; } + fcntl(sv[0], F_SETFD, FD_CLOEXEC); + fcntl(sv[1], F_SETFD, FD_CLOEXEC); + return true; +#endif +#endif +} + +inline PollerWaker::~PollerWaker() { + if (pair_fd_ != INVALID_SOCKET) { close_socket(pair_fd_); } + if (fd_ != INVALID_SOCKET) { close_socket(fd_); } +} + +inline bool PollerWaker::open() { + // wake() must not block its caller, and drain() must not block the poller. +#ifdef __linux__ + fd_ = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + return fd_ != INVALID_SOCKET; +#else + socket_t sv[2]; + if (!create_socket_pair(sv)) { return false; } + fd_ = sv[0]; + pair_fd_ = sv[1]; + set_nonblocking(fd_, true); + set_nonblocking(pair_fd_, true); + return true; +#endif +} + +inline void PollerWaker::wake() noexcept { + if (pending_.exchange(true)) { return; } +#ifdef __linux__ + const uint64_t one = 1; + auto r = write(fd_, &one, sizeof(one)); + (void)r; +#else + const char c = 1; + send_socket(pair_fd_, &c, 1, CPPHTTPLIB_SEND_FLAGS); +#endif +} + +inline void PollerWaker::drain() noexcept { + // While pending_ is set, at most one wake is unread, and one read takes it. +#ifdef __linux__ + uint64_t count; + auto r = read(fd_, &count, sizeof(count)); + (void)r; +#else + char buf[64]; + read_socket(fd_, buf, sizeof(buf), CPPHTTPLIB_RECV_FLAGS); +#endif + // Only after the read: cleared before it, a wake() in between would be read + // here and leave pending_ set with nothing to read. One skipped during the + // read is fine, as the caller checks what it was woken for afterwards. + pending_ = false; +} + +inline std::unique_ptr PollPoller::create() { + std::unique_ptr poller(new PollPoller()); + if (!poller->waker_.open()) { return nullptr; } + + struct pollfd pfd; + pfd.fd = poller->waker_.fd(); + pfd.events = POLLIN; + pfd.revents = 0; + poller->fds_.push_back(pfd); + poller->tokens_.push_back(0); + return std::unique_ptr(poller.release()); +} + +inline bool PollPoller::arm(socket_t fd, Token token) { + assert(index_.find(fd) == index_.end()); + + struct pollfd pfd; + pfd.fd = fd; + pfd.events = POLLIN; + pfd.revents = 0; + index_[fd] = fds_.size(); + fds_.push_back(pfd); + tokens_.push_back(token); + return true; +} + +inline void PollPoller::disarm(socket_t fd) { + auto it = index_.find(fd); + if (it != index_.end()) { remove_at(it->second); } +} + +// Fills the hole with the last entry, so that arm() and disarm() are O(1). +inline void PollPoller::remove_at(size_t index) { + assert(index > 0 && index < fds_.size()); + auto last = fds_.size() - 1; + index_.erase(fds_[index].fd); + if (index != last) { + fds_[index] = fds_[last]; + tokens_[index] = tokens_[last]; + index_[fds_[index].fd] = index; + } + fds_.pop_back(); + tokens_.pop_back(); +} + +inline bool PollPoller::wait(int timeout_ms, std::vector &ready) { + ready.clear(); + + auto n = handle_EINTR([&]() { + return poll_wrapper(fds_.data(), static_cast(fds_.size()), + timeout_ms); + }); + if (n <= 0) { return false; } + + auto woken = fds_[0].revents != 0; + if (woken) { + n--; + waker_.drain(); + } + + // Walk backwards: remove_at() moves the last entry, already seen, into the + // freed slot. + for (auto i = fds_.size(); n > 0 && i-- > 1;) { + auto revents = fds_[i].revents; + if (revents == 0) { continue; } + n--; + assert(!(revents & POLLNVAL)); // closed while armed + if (!(revents & POLLNVAL)) { ready.push_back(tokens_[i]); } + remove_at(i); + } + return woken; +} + +#ifdef __linux__ +inline std::unique_ptr EpollPoller::create() { + std::unique_ptr poller(new EpollPoller()); + poller->epfd_ = epoll_create1(EPOLL_CLOEXEC); + if (poller->epfd_ < 0 || !poller->waker_.open()) { return nullptr; } + + epoll_event ev{}; + ev.events = EPOLLIN; // level-triggered: it stays readable until drained + ev.data.u64 = poller->waker_.token(); + if (epoll_ctl(poller->epfd_, EPOLL_CTL_ADD, poller->waker_.fd(), &ev) != 0) { + return nullptr; + } + poller->events_.resize(256); + return std::unique_ptr(poller.release()); +} + +inline EpollPoller::~EpollPoller() { + if (epfd_ >= 0) { close(epfd_); } +} + +inline bool EpollPoller::arm(socket_t fd, Token token) { + epoll_event ev{}; + ev.events = EPOLLIN | EPOLLONESHOT; + ev.data.u64 = token; + // A delivered descriptor stays registered, only disabled, so modify it. + // One that was closed in between, or never armed, has to be added. + return epoll_ctl(epfd_, EPOLL_CTL_MOD, fd, &ev) == 0 || + (errno == ENOENT && epoll_ctl(epfd_, EPOLL_CTL_ADD, fd, &ev) == 0); +} + +inline void EpollPoller::disarm(socket_t fd) { + epoll_ctl(epfd_, EPOLL_CTL_DEL, fd, nullptr); +} + +inline bool EpollPoller::wait(int timeout_ms, std::vector &ready) { + ready.clear(); + auto n = handle_EINTR([&]() { + return epoll_wait(epfd_, events_.data(), static_cast(events_.size()), + timeout_ms); + }); + + auto woken = false; + for (auto i = 0; i < n; i++) { + auto token = static_cast(events_[static_cast(i)].data.u64); + if (token == waker_.token()) { + woken = true; + waker_.drain(); + } else { + ready.push_back(token); + } + } + return woken; +} +#endif + +#ifdef CPPHTTPLIB_KQUEUE_POLLER +inline std::unique_ptr KqueuePoller::create() { + std::unique_ptr poller(new KqueuePoller()); + poller->kq_ = kqueue(); + if (poller->kq_ < 0) { return nullptr; } + fcntl(poller->kq_, F_SETFD, FD_CLOEXEC); + if (!poller->waker_.open()) { return nullptr; } + + // Level-triggered: it stays readable until drained. + struct kevent ev; + EV_SET(&ev, poller->waker_.fd(), EVFILT_READ, EV_ADD, 0, 0, + reinterpret_cast(poller->waker_.token())); + if (kevent(poller->kq_, &ev, 1, nullptr, 0, nullptr) != 0) { return nullptr; } + poller->events_.resize(256); + return std::unique_ptr(poller.release()); +} + +inline KqueuePoller::~KqueuePoller() { + if (kq_ >= 0) { close(kq_); } +} + +inline bool KqueuePoller::arm(socket_t fd, Token token) { + // EV_ONESHOT deletes the event once it is delivered. + struct kevent ev; + EV_SET(&ev, fd, EVFILT_READ, EV_ADD | EV_ONESHOT, 0, 0, + reinterpret_cast(token)); + return kevent(kq_, &ev, 1, nullptr, 0, nullptr) == 0; +} + +inline void KqueuePoller::disarm(socket_t fd) { + struct kevent ev; + EV_SET(&ev, fd, EVFILT_READ, EV_DELETE, 0, 0, nullptr); + kevent(kq_, &ev, 1, nullptr, 0, nullptr); +} + +inline bool KqueuePoller::wait(int timeout_ms, std::vector &ready) { + ready.clear(); + struct timespec ts; + ts.tv_sec = timeout_ms / 1000; + ts.tv_nsec = static_cast(timeout_ms % 1000) * 1000000; + auto n = handle_EINTR([&]() { + return kevent(kq_, nullptr, 0, events_.data(), + static_cast(events_.size()), + timeout_ms < 0 ? nullptr : &ts); + }); + + auto woken = false; + for (auto i = 0; i < n; i++) { + auto token = reinterpret_cast(events_[static_cast(i)].udata); + if (token == waker_.token()) { + woken = true; + waker_.drain(); + } else { + ready.push_back(token); + } + } + return woken; +} +#endif + +inline std::unique_ptr make_poller() { +#if defined(__linux__) + auto poller = EpollPoller::create(); + if (poller) { return poller; } +#elif defined(CPPHTTPLIB_KQUEUE_POLLER) + auto poller = KqueuePoller::create(); + if (poller) { return poller; } +#endif + return PollPoller::create(); +} + inline bool bind_ip_address(socket_t sock, const std::string &host) { struct addrinfo hints; struct addrinfo *result; diff --git a/test/test.cc b/test/test.cc index 64f37dee..d3b92c11 100644 --- a/test/test.cc +++ b/test/test.cc @@ -33916,3 +33916,239 @@ TEST(WebSocketProxyTest, WssTunnelsThroughConnect) { EXPECT_EQ(1, proxy.connect_hits()); } #endif + +// --------------------------------------------------------------------------- +// Poller +// --------------------------------------------------------------------------- + +namespace poller_test { + +struct PollerBackend { + const char *name; + std::unique_ptr (*create)(); +}; + +void PrintTo(const PollerBackend &backend, std::ostream *os) { + *os << backend.name; +} + +// Every backend runs the same contract tests. A new one is added here only. +std::vector poller_backends() { + return { + {"poll", &detail::PollPoller::create}, +#ifdef __linux__ + {"epoll", &detail::EpollPoller::create}, +#endif +#ifdef CPPHTTPLIB_KQUEUE_POLLER + {"kqueue", &detail::KqueuePoller::create}, +#endif + }; +} + +// A descriptor whose readability the test controls. +struct TestSocketPair { + socket_t fd[2] = {INVALID_SOCKET, INVALID_SOCKET}; + + TestSocketPair() { detail::create_socket_pair(fd); } + ~TestSocketPair() { + for (auto s : fd) { + if (s != INVALID_SOCKET) { detail::close_socket(s); } + } + } + + bool ok() const { return fd[0] != INVALID_SOCKET; } + socket_t reader() const { return fd[0]; } + void make_readable() { + const char c = 1; + detail::send_socket(fd[1], &c, 1, 0); + } + void close_writer() { + detail::close_socket(fd[1]); + fd[1] = INVALID_SOCKET; + } +}; + +using Tokens = std::vector; + +class PollerTest : public ::testing::TestWithParam { +protected: + void SetUp() override { + poller_ = GetParam().create(); + ASSERT_TRUE(poller_ != nullptr); + } + + Tokens wait_for(int timeout_ms) { + Tokens ready; + poller_->wait(timeout_ms, ready); + std::sort(ready.begin(), ready.end()); + return ready; + } + + std::unique_ptr poller_; +}; + +} // namespace poller_test + +using poller_test::PollerTest; +using poller_test::TestSocketPair; +using poller_test::Tokens; + +INSTANTIATE_TEST_SUITE_P( + Backends, PollerTest, ::testing::ValuesIn(poller_test::poller_backends()), + [](const ::testing::TestParamInfo &p) { + return std::string(p.param.name); + }); + +TEST_P(PollerTest, DeliversOnceThenStaysDisarmedUntilRearmed) { + TestSocketPair sp; + ASSERT_TRUE(sp.ok()); + ASSERT_TRUE(poller_->arm(sp.reader(), 7)); + EXPECT_EQ(Tokens{}, wait_for(0)); + + sp.make_readable(); + EXPECT_EQ(Tokens{7}, wait_for(1000)); + EXPECT_EQ(Tokens{}, wait_for(0)); // still readable, but disarmed + + ASSERT_TRUE(poller_->arm(sp.reader(), 8)); + EXPECT_EQ(Tokens{8}, wait_for(1000)); +} + +TEST_P(PollerTest, DisarmedDescriptorIsNotDelivered) { + TestSocketPair sp; + ASSERT_TRUE(sp.ok()); + ASSERT_TRUE(poller_->arm(sp.reader(), 1)); + poller_->disarm(sp.reader()); + poller_->disarm(sp.reader()); // disarming twice is harmless + sp.make_readable(); + EXPECT_EQ(Tokens{}, wait_for(0)); +} + +TEST_P(PollerTest, WaitHonoursItsTimeout) { + TestSocketPair sp; + ASSERT_TRUE(sp.ok()); + ASSERT_TRUE(poller_->arm(sp.reader(), 1)); + + auto start = std::chrono::steady_clock::now(); + EXPECT_EQ(Tokens{}, wait_for(30)); + auto elapsed = std::chrono::steady_clock::now() - start; + EXPECT_GE(elapsed, std::chrono::milliseconds(25)); + EXPECT_LT(elapsed, std::chrono::seconds(1)); +} + +TEST_P(PollerTest, OnlyTheReadyDescriptorsAreDelivered) { + std::vector> sps; + for (detail::Poller::Token i = 0; i < 8; i++) { + sps.emplace_back(new TestSocketPair()); + ASSERT_TRUE(sps.back()->ok()); + ASSERT_TRUE(poller_->arm(sps.back()->reader(), i)); + } + + sps[0]->make_readable(); + sps[3]->make_readable(); + sps[7]->make_readable(); + EXPECT_EQ((Tokens{0, 3, 7}), wait_for(1000)); + + // The others are still armed, the delivered ones are not. + sps[1]->make_readable(); + EXPECT_EQ(Tokens{1}, wait_for(1000)); + EXPECT_EQ(Tokens{}, wait_for(0)); +} + +TEST_P(PollerTest, StaysConsistentAfterOtherDescriptorsWereDelivered) { + TestSocketPair s0, s1, s2; + ASSERT_TRUE(s0.ok() && s1.ok() && s2.ok()); + ASSERT_TRUE(poller_->arm(s0.reader(), 0)); + ASSERT_TRUE(poller_->arm(s1.reader(), 1)); + ASSERT_TRUE(poller_->arm(s2.reader(), 2)); + + // A delivery may move another entry inside the poller. + s0.make_readable(); + EXPECT_EQ(Tokens{0}, wait_for(1000)); + + // The entries left must still be found by descriptor. + poller_->disarm(s2.reader()); + s1.make_readable(); + s2.make_readable(); + EXPECT_EQ(Tokens{1}, wait_for(1000)); +} + +TEST_P(PollerTest, PeerHangUpCountsAsReadable) { + TestSocketPair sp; + ASSERT_TRUE(sp.ok()); + ASSERT_TRUE(poller_->arm(sp.reader(), 5)); + sp.close_writer(); + EXPECT_EQ(Tokens{5}, wait_for(1000)); + + char c; + EXPECT_EQ(0, detail::read_socket(sp.reader(), &c, 1, 0)); // EOF +} + +TEST_P(PollerTest, WakeEndsABlockedWait) { + std::thread waker([&] { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + poller_->wake(); + }); + + Tokens ready; + auto start = std::chrono::steady_clock::now(); + EXPECT_TRUE(poller_->wait(5000, ready)); // only the waker ends this early + waker.join(); + EXPECT_TRUE(ready.empty()); + EXPECT_LT(std::chrono::steady_clock::now() - start, std::chrono::seconds(1)); +} + +TEST_P(PollerTest, WakeBeforeWaitIsNotLost) { + poller_->wake(); // e.g. stop() running before the reactor reaches wait() + Tokens ready; + EXPECT_TRUE(poller_->wait(1000, ready)); +} + +TEST_P(PollerTest, WakesCoalesceAndAreConsumed) { + for (int i = 0; i < 1000; i++) { + poller_->wake(); + } + Tokens ready; + EXPECT_TRUE(poller_->wait(0, ready)); + EXPECT_FALSE(poller_->wait(0, ready)); + + poller_->wake(); + EXPECT_TRUE(poller_->wait(0, ready)); +} + +TEST_P(PollerTest, ConcurrentWakesAreNeverLost) { + // Two threads publish changes and wake() after each, as a reactor's workers + // do. Every change must be seen without a wait() timing out. A lost wake() + // needs an unlucky interleaving, hence the rounds. + for (int round = 0; round < 5; round++) { + SCOPED_TRACE(round); + const int per_thread = 20000; + std::atomic posted{0}; + auto post = [&] { + for (int i = 0; i < per_thread; i++) { + posted++; + poller_->wake(); + } + }; + std::thread t1(post), t2(post); + + auto lost = false; + Tokens ready; + for (auto seen = 0; seen < 2 * per_thread; seen = posted) { + if (!poller_->wait(1000, ready)) { + lost = true; + break; + } + } + t1.join(); + t2.join(); + ASSERT_FALSE(lost); + } +} + +TEST(PollerFactoryTest, MakePollerWorks) { + auto poller = detail::make_poller(); + ASSERT_TRUE(poller != nullptr); + poller->wake(); + std::vector ready; + EXPECT_TRUE(poller->wait(1000, ready)); +}