mirror of
https://github.com/yhirose/cpp-httplib.git
synced 2026-10-19 11:19:36 +00:00
Fix README and docs pages that no longer match the code
Bring the READMEs and the docs pages (en and ja) in line with the current header: - Fix the client sample that does not compile, the Transfer-Encoding statement for default headers, the verifier callback comment, the thread pool spawn rule, the sanitize_filename description, and the inverted Accept-Encoding sentence. - Describe recent behaviour: TLS errors behind Read/Write failures, Host validation, Connection: close for a body without length, content_reader rules, refused CONNECT, WebSocketClient proxies, CPPHTTPLIB_INTERIM_RESPONSE_MAX_COUNT, the SSE event size limit, and the payload limit of the Stream API. - Complete the API references of WebSocket, WebSocketClient and the stream:: helpers, and the SSE reconnect rules. - Fix cookbook pages: keychain loading on every backend, received Close status, a multipart sample that trusted the client's file name, an echo sample that sent binary as text, SNI, pre-request handler scope, stop() with WebSocket handlers, and the thread pool notes. - Restore the mutex in the LLM app's Chapter 5 code.
This commit is contained in:
1 parent
94cf1b5c9a
commit
9bfd2ddf1c
77 files changed
+378
-145
No files matched your search
@@ -76,7 +76,7 @@ Defining `CPPHTTPLIB_OPENSSL_SUPPORT` enables `httplib::Client("https://...")` t
|
||||
|
||||
## 4.2 Defining the Model List
|
||||
|
||||
Let's define the list of models that the app can handle. Here are four models we've verified for translation tasks.
|
||||
Let's define the list of models that the app can handle. Here are three models we've verified for translation tasks.
|
||||
|
||||
```cpp
|
||||
struct ModelInfo {
|
||||
|
||||
@@ -875,6 +875,7 @@ The only server-side change is the single `set_mount_point` line. Add it before
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <mutex>
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
@@ -934,11 +935,11 @@ std::filesystem::path get_models_dir() {
|
||||
// Model download
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// Abort the download if progress_cb returns false
|
||||
// If progress_cb returns false, the download is aborted
|
||||
bool download_model(const ModelInfo &model,
|
||||
std::function<bool(int)> progress_cb) {
|
||||
httplib::Client cli("https://huggingface.co");
|
||||
cli.set_follow_location(true); // Hugging Face redirects to CDN
|
||||
cli.set_follow_location(true); // Hugging Face redirects to a CDN
|
||||
|
||||
auto url = "/" + model.repo + "/resolve/main/" + model.filename;
|
||||
auto path = get_models_dir() / model.filename;
|
||||
@@ -948,12 +949,12 @@ bool download_model(const ModelInfo &model,
|
||||
if (!ofs) { return false; }
|
||||
|
||||
auto res = cli.Get(url,
|
||||
// content_receiver: receive chunks and write to file
|
||||
// content_receiver: receive data chunk by chunk and write to file
|
||||
[&](const char *data, size_t len) {
|
||||
ofs.write(data, len);
|
||||
return ofs.good();
|
||||
},
|
||||
// progress: report download progress (return false to abort)
|
||||
// progress: report download progress (returning false aborts)
|
||||
[&, last_pct = -1](size_t current, size_t total) mutable {
|
||||
int pct = total ? (int)(current * 100 / total) : 0;
|
||||
if (pct == last_pct) return true; // Skip if same value
|
||||
@@ -987,11 +988,11 @@ void signal_handler(int sig) {
|
||||
}
|
||||
|
||||
int main() {
|
||||
// Create model storage directory
|
||||
// Create the model storage directory
|
||||
auto models_dir = get_models_dir();
|
||||
std::filesystem::create_directories(models_dir);
|
||||
|
||||
// Auto-download default model if not present
|
||||
// Automatically download the default model if not yet present
|
||||
std::string selected_model = MODELS[0].filename;
|
||||
auto path = models_dir / selected_model;
|
||||
if (!std::filesystem::exists(path)) {
|
||||
@@ -1006,6 +1007,7 @@ int main() {
|
||||
std::cout << std::endl;
|
||||
}
|
||||
auto llm = llamalib::Llama{path};
|
||||
std::mutex llm_mutex; // Protect access during model switching
|
||||
|
||||
svr.set_logger([](const auto &req, const auto &res) {
|
||||
std::cout << req.method << " " << req.path << " -> " << res.status
|
||||
@@ -1020,6 +1022,7 @@ int main() {
|
||||
|
||||
svr.Post("/translate",
|
||||
[&](const httplib::Request &req, httplib::Response &res) {
|
||||
// JSON parsing and validation (see Chapter 2 for details)
|
||||
auto input = json::parse(req.body, nullptr, false);
|
||||
if (input.is_discarded()) {
|
||||
res.status = 400;
|
||||
@@ -1043,6 +1046,7 @@ int main() {
|
||||
". Output only the translation, nothing else.\n\n" + text;
|
||||
|
||||
try {
|
||||
std::lock_guard<std::mutex> lock(llm_mutex);
|
||||
auto translation = llm.chat(prompt);
|
||||
res.set_content(json{{"translation", translation}}.dump(),
|
||||
"application/json");
|
||||
@@ -1052,7 +1056,7 @@ int main() {
|
||||
}
|
||||
});
|
||||
|
||||
// --- SSE streaming translation (Chapter 3) --------------------------------
|
||||
// --- SSE streaming translation (Chapter 3) -------------------------------
|
||||
|
||||
svr.Post("/translate/stream",
|
||||
[&](const httplib::Request &req, httplib::Response &res) {
|
||||
@@ -1081,6 +1085,7 @@ int main() {
|
||||
res.set_chunked_content_provider(
|
||||
"text/event-stream",
|
||||
[&, prompt](size_t, httplib::DataSink &sink) {
|
||||
std::lock_guard<std::mutex> lock(llm_mutex);
|
||||
try {
|
||||
llm.chat(prompt, [&](std::string_view token) {
|
||||
sink.os << "data: "
|
||||
@@ -1098,7 +1103,7 @@ int main() {
|
||||
});
|
||||
});
|
||||
|
||||
// --- Model list (Chapter 4) -----------------------------------------------
|
||||
// --- Model list (Chapter 4) ----------------------------------------------
|
||||
|
||||
svr.Get("/models",
|
||||
[&](const httplib::Request &, httplib::Response &res) {
|
||||
@@ -1117,7 +1122,7 @@ int main() {
|
||||
res.set_content(json{{"models", arr}}.dump(), "application/json");
|
||||
});
|
||||
|
||||
// --- Model selection (Chapter 4) ------------------------------------------
|
||||
// --- Model selection (Chapter 4) -----------------------------------------
|
||||
|
||||
svr.Post("/models/select",
|
||||
[&](const httplib::Request &req, httplib::Response &res) {
|
||||
@@ -1143,16 +1148,16 @@ int main() {
|
||||
|
||||
const auto &model = *it;
|
||||
|
||||
// Always respond with SSE (same format whether downloaded or not)
|
||||
// Always respond with SSE (same format whether already downloaded or not)
|
||||
res.set_chunked_content_provider(
|
||||
"text/event-stream",
|
||||
[&, model](size_t, httplib::DataSink &sink) {
|
||||
// SSE event send helper
|
||||
// SSE event sending helper
|
||||
auto send = [&](const json &event) {
|
||||
sink.os << "data: " << event.dump() << "\n\n";
|
||||
};
|
||||
|
||||
// Download if not yet downloaded (report progress via SSE)
|
||||
// Download if not yet present (report progress via SSE)
|
||||
auto path = get_models_dir() / model.filename;
|
||||
if (!std::filesystem::exists(path)) {
|
||||
bool ok = download_model(model, [&](int pct) {
|
||||
@@ -1168,8 +1173,11 @@ int main() {
|
||||
|
||||
// Load and switch to the model
|
||||
send({{"status", "loading"}});
|
||||
llm = llamalib::Llama{path};
|
||||
selected_model = model.filename;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(llm_mutex);
|
||||
llm = llamalib::Llama{path};
|
||||
selected_model = model.filename;
|
||||
}
|
||||
|
||||
send({{"status", "ready"}});
|
||||
sink.done();
|
||||
@@ -1177,11 +1185,11 @@ int main() {
|
||||
});
|
||||
});
|
||||
|
||||
// --- Static file serving (Chapter 5) --------------------------------------
|
||||
// --- Static file serving (Chapter 5) -------------------------------------
|
||||
|
||||
svr.set_mount_point("/", "./public");
|
||||
|
||||
// Allow graceful shutdown via `Ctrl+C` (`SIGINT`) or `kill` (`SIGTERM`)
|
||||
// Allow the server to be stopped with `Ctrl+C` (`SIGINT`) or `kill` (`SIGTERM`)
|
||||
signal(SIGINT, signal_handler);
|
||||
signal(SIGTERM, signal_handler);
|
||||
|
||||
|
||||
@@ -675,9 +675,10 @@ int main() {
|
||||
To summarize the changes from Chapter 5:
|
||||
|
||||
- `#include <csignal>` replaced with `#include <thread>`, `<cpp-embedlib-httplib.h>`, `"WebAssets.h"`, `"webview/webview.h"`
|
||||
- Removed the `signal_handler` function
|
||||
- Removed the `signal_handler` function and moved `svr` into `main()`
|
||||
- `svr.set_mount_point("/", "./public")` replaced with `httplib::mount(svr, Web::FS)`
|
||||
- `svr.listen("127.0.0.1", 8080)` replaced with `bind_to_any_port` + `listen_after_bind` + WebView event loop
|
||||
- Added `setup_macos_edit_menu()` and its `<objc/objc-runtime.h>` include (macOS only)
|
||||
|
||||
Not a single line of handler code has changed. The REST API, SSE streaming, and model management built through Chapter 5 all work as-is.
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ Compatibility with existing tools and libraries is a big design decision. We des
|
||||
|
||||
## 7.3 Concurrent Request Handling
|
||||
|
||||
Our server processes requests one at a time. If another request arrives while a translation is in progress, it waits until the previous inference finishes. This is fine for a desktop app used by one person, but it becomes a problem for a server shared by multiple users.
|
||||
Our server runs inference one request at a time. If another request arrives while a translation is in progress, it waits until the previous inference finishes. This is fine for a desktop app used by one person, but it becomes a problem for a server shared by multiple users.
|
||||
|
||||
`llama-server` handles concurrent requests through a mechanism called **slots**.
|
||||
|
||||
@@ -67,7 +67,7 @@ Our server processes requests one at a time. If another request arrives while a
|
||||
|
||||
The key point is that tokens from each slot are not inferred **one by one in sequence**, but rather **all at once in a single batch**. GPUs excel at parallel processing, so processing two users simultaneously takes almost the same time as processing one. This is called "continuous batching."
|
||||
|
||||
In our server, cpp-httplib's thread pool assigns one thread per request, but the inference itself runs single-threaded inside `llm.chat()`. `llama-server` consolidates this inference step into a shared batch processing loop.
|
||||
In our server, cpp-httplib's thread pool serves each connection on its own worker thread, so requests on different connections are handled concurrently. Inference, however, is serialized by `llm_mutex`, so only one `llm.chat()` runs at a time. `llama-server` consolidates this inference step into a shared batch processing loop.
|
||||
|
||||
## 7.4 Differences in SSE Format
|
||||
|
||||
|
||||
Reference in new issue
Block a user