Add a one-shot Poller with poll, epoll and kqueue backends

Groundwork for keep-alive parking, an opt-in event loop that parks idle
keep-alive connections instead of blocking a worker thread on each.

detail::Poller delivers readiness once per arm(), so a descriptor is
either armed (owned by the poller) or owned by whoever got the event.
wake() ends a wait() from any thread, and an armed descriptor must be
disarmed before it is closed.

- PollPoller (poll() / WSAPoll()) works everywhere.
- EpollPoller (Linux) uses EPOLLONESHOT.
- KqueuePoller (macOS, FreeBSD) uses EV_ONESHOT.
- PollerWaker carries wake() for all three: an eventfd on Linux, a socket
  pair elsewhere. detail::create_socket_pair() emulates socketpair() on
  Windows with sockets that child processes do not inherit.

make_poller() picks the best one and falls back to poll(). The contract
tests run on every backend that builds. Nothing uses the poller yet.
This commit is contained in:
yhirose committed 2026-11-21 20:22:05 -05:00
1 parent 726e4583ef
commit a694b8c306
2 files changed
+675

No files matched your search

+439
View File
@@ -374,6 +374,8 @@ using socklen_t = int;
#ifdef __linux__
#include <resolv.h>
#undef _res // Undefine _res macro to avoid conflicts with user code (#2278)
#include <sys/epoll.h>
#include <sys/eventfd.h>
#endif
#include <csignal>
#include <netinet/tcp.h>
@@ -393,6 +395,11 @@ using socket_t = int;
#include <TargetConditionals.h>
#endif
#if defined(__APPLE__) || defined(__FreeBSD__)
#define CPPHTTPLIB_KQUEUE_POLLER
#include <sys/event.h>
#endif
#include <algorithm>
#include <array>
#include <atomic>
@@ -3930,6 +3937,126 @@ EncodingType encoding_type(const Request &req, const Response &res,
EncodingType encoding_type(const Request &req, const Response &res);
// Creates a connected pair of stream sockets that child processes do not
// inherit. Windows has no socketpair(), so it is emulated there.
bool create_socket_pair(socket_t sv[2]);
// Ends a Poller's wait() from another thread. The poller watches fd() for
// readability, level-triggered, and calls drain() when it is reported.
class PollerWaker {
public:
~PollerWaker();
// An eventfd on Linux, a socket pair elsewhere.
bool open();
socket_t fd() const { return fd_; }
// An address inside the poller, which no caller's token can be.
uintptr_t token() const { return reinterpret_cast<uintptr_t>(this); }
// From any thread. While one is pending, later calls make no syscall.
void wake() noexcept;
void drain() noexcept;
private:
socket_t fd_ = INVALID_SOCKET;
socket_t pair_fd_ = INVALID_SOCKET; // the end wake() writes; none on Linux
std::atomic<bool> pending_{false};
};
// One-shot readiness notification for a reactor:
// - A delivered event disarms the descriptor; arm() it again to wait again.
// - A hang-up or an error counts as readable.
// - Only an unarmed descriptor may be armed, and an armed one must be
// disarmed before it is closed.
// - wake() may be called from any thread; the rest belongs to one thread.
class Poller {
public:
using Token = uintptr_t;
virtual ~Poller() = default;
virtual bool arm(socket_t fd, Token token) = 0;
virtual void disarm(socket_t fd) = 0;
// Fills `ready` in no particular order and returns whether wake() was
// called. A negative timeout waits until either happens.
virtual bool wait(int timeout_ms, std::vector<Token> &ready) = 0;
virtual void wake() noexcept = 0;
};
// poll() on POSIX, WSAPoll() on Windows. Works everywhere.
class PollPoller final : public Poller {
public:
// nullptr when the descriptors cannot be created.
static std::unique_ptr<Poller> create();
bool arm(socket_t fd, Token token) override;
void disarm(socket_t fd) override;
bool wait(int timeout_ms, std::vector<Token> &ready) override;
void wake() noexcept override { waker_.wake(); }
private:
PollPoller() = default;
void remove_at(size_t index);
PollerWaker waker_;
// fds_[0] is the waker's, which is always armed; tokens_[i] belongs to
// fds_[i].
std::vector<struct pollfd> fds_;
std::vector<Token> tokens_;
std::unordered_map<socket_t, size_t> index_;
};
#ifdef __linux__
// epoll with EPOLLONESHOT. Waits in O(ready) instead of O(armed).
class EpollPoller final : public Poller {
public:
// nullptr when the descriptors cannot be created.
static std::unique_ptr<Poller> create();
~EpollPoller() override;
bool arm(socket_t fd, Token token) override;
void disarm(socket_t fd) override;
bool wait(int timeout_ms, std::vector<Token> &ready) override;
void wake() noexcept override { waker_.wake(); }
private:
EpollPoller() = default;
int epfd_ = -1;
PollerWaker waker_; // always armed
std::vector<struct epoll_event> events_;
};
#endif
#ifdef CPPHTTPLIB_KQUEUE_POLLER
// kqueue with EV_ONESHOT. Waits in O(ready) instead of O(armed).
class KqueuePoller final : public Poller {
public:
// nullptr when the descriptors cannot be created.
static std::unique_ptr<Poller> create();
~KqueuePoller() override;
bool arm(socket_t fd, Token token) override;
void disarm(socket_t fd) override;
bool wait(int timeout_ms, std::vector<Token> &ready) override;
void wake() noexcept override { waker_.wake(); }
private:
KqueuePoller() = default;
int kq_ = -1;
PollerWaker waker_; // always armed
std::vector<struct kevent> events_;
};
#endif
// The best poller for this platform, or nullptr when it cannot be created.
std::unique_ptr<Poller> make_poller();
class BufferStream final : public Stream {
public:
BufferStream() = default;
@@ -7770,6 +7897,318 @@ inline bool is_accept_transient_error() {
#endif
}
inline bool create_socket_pair(socket_t sv[2]) {
#ifdef _WIN32
auto listener = create_socket(
std::string(), "127.0.0.1", 0, AF_INET, 0, false, false, nullptr,
[](socket_t sock, struct addrinfo &ai, bool &) {
return ::bind(sock, ai.ai_addr, static_cast<int>(ai.ai_addrlen)) == 0 &&
::listen(sock, 1) == 0;
});
if (listener == INVALID_SOCKET) { return false; }
socket_t a = INVALID_SOCKET;
socket_t b = INVALID_SOCKET;
sockaddr_in addr{};
auto addr_len = static_cast<int>(sizeof(addr));
if (::getsockname(listener, reinterpret_cast<sockaddr *>(&addr), &addr_len) ==
0) {
a = create_socket(std::string(), "127.0.0.1", ntohs(addr.sin_port), AF_INET,
0, true, false, nullptr,
[](socket_t sock, struct addrinfo &ai, bool &) {
return ::connect(sock, ai.ai_addr,
static_cast<int>(ai.ai_addrlen)) == 0;
});
// Unlike accept(), this keeps the listener's no-inherit flag.
if (a != INVALID_SOCKET) {
b = WSAAccept(listener, nullptr, nullptr, nullptr, 0);
}
}
close_socket(listener);
// Any local process could connect while the listener was open, so make sure
// that the connection we accepted is the one we made.
if (b != INVALID_SOCKET) {
sockaddr_in local{}, peer{};
auto local_len = static_cast<int>(sizeof(local));
auto peer_len = static_cast<int>(sizeof(peer));
if (::getsockname(a, reinterpret_cast<sockaddr *>(&local), &local_len) !=
0 ||
::getpeername(b, reinterpret_cast<sockaddr *>(&peer), &peer_len) != 0 ||
local.sin_port != peer.sin_port ||
local.sin_addr.s_addr != peer.sin_addr.s_addr) {
close_socket(b);
b = INVALID_SOCKET;
}
}
if (b == INVALID_SOCKET) {
if (a != INVALID_SOCKET) { close_socket(a); }
return false;
}
sv[0] = a;
sv[1] = b;
return true;
#else
#ifdef SOCK_CLOEXEC
return ::socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, sv) == 0;
#else
if (::socketpair(AF_UNIX, SOCK_STREAM, 0, sv) != 0) { return false; }
fcntl(sv[0], F_SETFD, FD_CLOEXEC);
fcntl(sv[1], F_SETFD, FD_CLOEXEC);
return true;
#endif
#endif
}
inline PollerWaker::~PollerWaker() {
if (pair_fd_ != INVALID_SOCKET) { close_socket(pair_fd_); }
if (fd_ != INVALID_SOCKET) { close_socket(fd_); }
}
inline bool PollerWaker::open() {
// wake() must not block its caller, and drain() must not block the poller.
#ifdef __linux__
fd_ = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
return fd_ != INVALID_SOCKET;
#else
socket_t sv[2];
if (!create_socket_pair(sv)) { return false; }
fd_ = sv[0];
pair_fd_ = sv[1];
set_nonblocking(fd_, true);
set_nonblocking(pair_fd_, true);
return true;
#endif
}
inline void PollerWaker::wake() noexcept {
if (pending_.exchange(true)) { return; }
#ifdef __linux__
const uint64_t one = 1;
auto r = write(fd_, &one, sizeof(one));
(void)r;
#else
const char c = 1;
send_socket(pair_fd_, &c, 1, CPPHTTPLIB_SEND_FLAGS);
#endif
}
inline void PollerWaker::drain() noexcept {
// While pending_ is set, at most one wake is unread, and one read takes it.
#ifdef __linux__
uint64_t count;
auto r = read(fd_, &count, sizeof(count));
(void)r;
#else
char buf[64];
read_socket(fd_, buf, sizeof(buf), CPPHTTPLIB_RECV_FLAGS);
#endif
// Only after the read: cleared before it, a wake() in between would be read
// here and leave pending_ set with nothing to read. One skipped during the
// read is fine, as the caller checks what it was woken for afterwards.
pending_ = false;
}
inline std::unique_ptr<Poller> PollPoller::create() {
std::unique_ptr<PollPoller> poller(new PollPoller());
if (!poller->waker_.open()) { return nullptr; }
struct pollfd pfd;
pfd.fd = poller->waker_.fd();
pfd.events = POLLIN;
pfd.revents = 0;
poller->fds_.push_back(pfd);
poller->tokens_.push_back(0);
return std::unique_ptr<Poller>(poller.release());
}
inline bool PollPoller::arm(socket_t fd, Token token) {
assert(index_.find(fd) == index_.end());
struct pollfd pfd;
pfd.fd = fd;
pfd.events = POLLIN;
pfd.revents = 0;
index_[fd] = fds_.size();
fds_.push_back(pfd);
tokens_.push_back(token);
return true;
}
inline void PollPoller::disarm(socket_t fd) {
auto it = index_.find(fd);
if (it != index_.end()) { remove_at(it->second); }
}
// Fills the hole with the last entry, so that arm() and disarm() are O(1).
inline void PollPoller::remove_at(size_t index) {
assert(index > 0 && index < fds_.size());
auto last = fds_.size() - 1;
index_.erase(fds_[index].fd);
if (index != last) {
fds_[index] = fds_[last];
tokens_[index] = tokens_[last];
index_[fds_[index].fd] = index;
}
fds_.pop_back();
tokens_.pop_back();
}
inline bool PollPoller::wait(int timeout_ms, std::vector<Token> &ready) {
ready.clear();
auto n = handle_EINTR([&]() {
return poll_wrapper(fds_.data(), static_cast<nfds_t>(fds_.size()),
timeout_ms);
});
if (n <= 0) { return false; }
auto woken = fds_[0].revents != 0;
if (woken) {
n--;
waker_.drain();
}
// Walk backwards: remove_at() moves the last entry, already seen, into the
// freed slot.
for (auto i = fds_.size(); n > 0 && i-- > 1;) {
auto revents = fds_[i].revents;
if (revents == 0) { continue; }
n--;
assert(!(revents & POLLNVAL)); // closed while armed
if (!(revents & POLLNVAL)) { ready.push_back(tokens_[i]); }
remove_at(i);
}
return woken;
}
#ifdef __linux__
inline std::unique_ptr<Poller> EpollPoller::create() {
std::unique_ptr<EpollPoller> poller(new EpollPoller());
poller->epfd_ = epoll_create1(EPOLL_CLOEXEC);
if (poller->epfd_ < 0 || !poller->waker_.open()) { return nullptr; }
epoll_event ev{};
ev.events = EPOLLIN; // level-triggered: it stays readable until drained
ev.data.u64 = poller->waker_.token();
if (epoll_ctl(poller->epfd_, EPOLL_CTL_ADD, poller->waker_.fd(), &ev) != 0) {
return nullptr;
}
poller->events_.resize(256);
return std::unique_ptr<Poller>(poller.release());
}
inline EpollPoller::~EpollPoller() {
if (epfd_ >= 0) { close(epfd_); }
}
inline bool EpollPoller::arm(socket_t fd, Token token) {
epoll_event ev{};
ev.events = EPOLLIN | EPOLLONESHOT;
ev.data.u64 = token;
// A delivered descriptor stays registered, only disabled, so modify it.
// One that was closed in between, or never armed, has to be added.
return epoll_ctl(epfd_, EPOLL_CTL_MOD, fd, &ev) == 0 ||
(errno == ENOENT && epoll_ctl(epfd_, EPOLL_CTL_ADD, fd, &ev) == 0);
}
inline void EpollPoller::disarm(socket_t fd) {
epoll_ctl(epfd_, EPOLL_CTL_DEL, fd, nullptr);
}
inline bool EpollPoller::wait(int timeout_ms, std::vector<Token> &ready) {
ready.clear();
auto n = handle_EINTR([&]() {
return epoll_wait(epfd_, events_.data(), static_cast<int>(events_.size()),
timeout_ms);
});
auto woken = false;
for (auto i = 0; i < n; i++) {
auto token = static_cast<Token>(events_[static_cast<size_t>(i)].data.u64);
if (token == waker_.token()) {
woken = true;
waker_.drain();
} else {
ready.push_back(token);
}
}
return woken;
}
#endif
#ifdef CPPHTTPLIB_KQUEUE_POLLER
inline std::unique_ptr<Poller> KqueuePoller::create() {
std::unique_ptr<KqueuePoller> poller(new KqueuePoller());
poller->kq_ = kqueue();
if (poller->kq_ < 0) { return nullptr; }
fcntl(poller->kq_, F_SETFD, FD_CLOEXEC);
if (!poller->waker_.open()) { return nullptr; }
// Level-triggered: it stays readable until drained.
struct kevent ev;
EV_SET(&ev, poller->waker_.fd(), EVFILT_READ, EV_ADD, 0, 0,
reinterpret_cast<void *>(poller->waker_.token()));
if (kevent(poller->kq_, &ev, 1, nullptr, 0, nullptr) != 0) { return nullptr; }
poller->events_.resize(256);
return std::unique_ptr<Poller>(poller.release());
}
inline KqueuePoller::~KqueuePoller() {
if (kq_ >= 0) { close(kq_); }
}
inline bool KqueuePoller::arm(socket_t fd, Token token) {
// EV_ONESHOT deletes the event once it is delivered.
struct kevent ev;
EV_SET(&ev, fd, EVFILT_READ, EV_ADD | EV_ONESHOT, 0, 0,
reinterpret_cast<void *>(token));
return kevent(kq_, &ev, 1, nullptr, 0, nullptr) == 0;
}
inline void KqueuePoller::disarm(socket_t fd) {
struct kevent ev;
EV_SET(&ev, fd, EVFILT_READ, EV_DELETE, 0, 0, nullptr);
kevent(kq_, &ev, 1, nullptr, 0, nullptr);
}
inline bool KqueuePoller::wait(int timeout_ms, std::vector<Token> &ready) {
ready.clear();
struct timespec ts;
ts.tv_sec = timeout_ms / 1000;
ts.tv_nsec = static_cast<long>(timeout_ms % 1000) * 1000000;
auto n = handle_EINTR([&]() {
return kevent(kq_, nullptr, 0, events_.data(),
static_cast<int>(events_.size()),
timeout_ms < 0 ? nullptr : &ts);
});
auto woken = false;
for (auto i = 0; i < n; i++) {
auto token = reinterpret_cast<Token>(events_[static_cast<size_t>(i)].udata);
if (token == waker_.token()) {
woken = true;
waker_.drain();
} else {
ready.push_back(token);
}
}
return woken;
}
#endif
inline std::unique_ptr<Poller> make_poller() {
#if defined(__linux__)
auto poller = EpollPoller::create();
if (poller) { return poller; }
#elif defined(CPPHTTPLIB_KQUEUE_POLLER)
auto poller = KqueuePoller::create();
if (poller) { return poller; }
#endif
return PollPoller::create();
}
inline bool bind_ip_address(socket_t sock, const std::string &host) {
struct addrinfo hints;
struct addrinfo *result;