From 9e634310b438c98ce963c9af399898d0687487a7 Mon Sep 17 00:00:00 2001 From: Sergey Lyubka Date: Wed, 9 Feb 2022 12:24:06 +0000 Subject: [PATCH] Use MG_EV_HTTP_CHUNK for streaming upload --- docs/README.md | 55 ------------------------ examples/file-upload/main.c | 31 +++++++++---- examples/file-upload/web_root/app.js | 41 ------------------ examples/file-upload/web_root/index.html | 48 ++++++++++++++++----- mongoose.c | 33 ++++++++------ src/http.c | 33 ++++++++------ 6 files changed, 99 insertions(+), 142 deletions(-) delete mode 100644 examples/file-upload/web_root/app.js diff --git a/docs/README.md b/docs/README.md index fc6ed344..302ec11b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1176,61 +1176,6 @@ void fn(struct mg_connection *c, int ev, void *ev_data, void *fn_data) { } ``` -### mg\_http\_upload() - -```c -int mg_http_upload(struct mg_connection *c, struct mg_http_message *hm, - struct mg_fs *fs, const char *dir); -``` - -Handle file upload. See [file upload example](https://github.com/cesanta/mongoose/tree/master/examples/file-uploads/). - -This function expects a series of POST requests with file data. POST requests -should have `name` and `offset` query string parameters set: - -```text -POST /whatever_uri?name=myfile.txt&offset=1234 HTTP/1.0 -Content-Length: 5 - -hello -``` - -- `name` - A mandatory query string parameter, specifies a file name. It it - created in the `dir` directory -- `offset` - An optional parameter, default `0`. If it set to `0`, or omitted, - then a file gets truncated before write. Otherwise, the body of - the POST request gets appended to the file -- Server must call `mg_http_upload()` when `/whatever_uri` is hit - -The expected usage of this API function follows: -- A client splits a file into small enough chunks, to ensure that a chunk - fits into the server's RAM -- Then, each chunk is POST-ed to the server with using URI, like - `/some_uri?name=FILENAME&offset=OFFSET` -- Initial OFFSET is `0`, and subsequent offsets are non-zero -- Each chunk gets appended to the file -- When the last chunk is POST-ed, upload finishes -- POST data must not be encoded in any way; it it saved as-is - -Parameters: -- `c` - Connection to use -- `hm` - POST message, containing parameters described above -- `fs` - Filesystem to use -- `dir` - Path to directory - -Return value: Request body length or negative value on error - -Usage example: - -```c -// Mongoose events handler -void fn(struct mg_connection *c, int ev, void *ev_data, void *fn_data) { - if (ev == MG_EV_HTTP_MSG) { - struct mg_http_message *hm = (struct mg_http_message *) ev_data; - mg_http_upload(c, hm, &mg_fs_posix, "."); // Upload to current folder - } -``` - ### mg\_http\_bauth() ```c diff --git a/examples/file-upload/main.c b/examples/file-upload/main.c index 8e277755..dbd6c35b 100644 --- a/examples/file-upload/main.c +++ b/examples/file-upload/main.c @@ -1,5 +1,11 @@ // Copyright (c) 2020 Cesanta Software Limited // All rights reserved +// +// Streaming upload example. Demonstrates how to use chunked encoding +// to send large payload in smaller chunks. To test, use curl utility: +// +// curl http://localhost:8000/upload \ +// --data-binary @my_large_file.txt -H 'Transfer-Encoding: chunked' #include "mongoose.h" @@ -7,20 +13,29 @@ // /upload - Saves the next file chunk // all other URI - serves web_root/ directory static void cb(struct mg_connection *c, int ev, void *ev_data, void *fn_data) { - if (ev == MG_EV_HTTP_MSG) { - struct mg_http_message *hm = (struct mg_http_message *) ev_data; - if (mg_http_match_uri(hm, "/upload")) { - mg_http_upload(c, hm, &mg_fs_posix, "/tmp"); - } else { - struct mg_http_serve_opts opts = {.root_dir = "web_root"}; - mg_http_serve_dir(c, hm, &opts); + struct mg_http_message *hm = (struct mg_http_message *) ev_data; + if (ev == MG_EV_HTTP_CHUNK && mg_http_match_uri(hm, "/upload")) { + LOG(LL_INFO, ("Got chunk len %lu", (unsigned long) hm->chunk.len)); + LOG(LL_INFO, ("Query string: [%.*s]", (int) hm->query.len, hm->query.ptr)); + LOG(LL_INFO, ("Chunk data:\n%.*s", (int) hm->chunk.len, hm->chunk.ptr)); + mg_http_delete_chunk(c, hm); + if (hm->chunk.len == 0) { + LOG(LL_INFO, ("Last chunk received, sending response")); + mg_http_reply(c, 200, "", "ok (chunked)\n"); } + } else if (ev == MG_EV_HTTP_MSG && mg_http_match_uri(hm, "/upload")) { + LOG(LL_INFO, ("Got all %lu bytes!", (unsigned long) hm->body.len)); + LOG(LL_INFO, ("Query string: [%.*s]", (int) hm->query.len, hm->query.ptr)); + LOG(LL_INFO, ("Body:\n%.*s", (int) hm->body.len, hm->body.ptr)); + mg_http_reply(c, 200, "", "ok (%lu)\n", (unsigned long) hm->body.len); + } else if (ev == MG_EV_HTTP_MSG) { + struct mg_http_serve_opts opts = {.root_dir = "web_root"}; + mg_http_serve_dir(c, hm, &opts); } } int main(void) { struct mg_mgr mgr; - struct mg_timer t1; mg_mgr_init(&mgr); mg_log_set("3"); diff --git a/examples/file-upload/web_root/app.js b/examples/file-upload/web_root/app.js deleted file mode 100644 index f90c01d6..00000000 --- a/examples/file-upload/web_root/app.js +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) 2020 Cesanta Software Limited -// All rights reserved - -// Helper function to display upload status -var setStatus = function(text) { - document.getElementById('el3').innerText = text; -}; - -// When user clicks on a button, trigger file selection dialog -var button = document.getElementById('el2'); -button.onclick = function(ev) { - input.click(); -}; - -// Send a large blob of data chunk by chunk -var sendFileData = function(name, data, chunkSize) { - var sendChunk = function(offset) { - var chunk = data.subarray(offset, offset + chunkSize) || ''; - var opts = {method: 'POST', body: chunk}; - var url = '/upload?offset=' + offset + '&name=' + encodeURIComponent(name); - setStatus( - 'sending bytes ' + offset + '..' + (offset + chunk.length) + ' of ' + - data.length); - fetch(url, opts).then(function(res) { - if (chunk.length > 0) sendChunk(offset + chunk.length); - }); - }; - sendChunk(0); -}; - -// If user selected a file, read it into memory and trigger sendFileData() -var input = document.getElementById('el1'); -input.onchange = function(ev) { - if (!ev.target.files[0]) return; - var f = ev.target.files[0], r = new FileReader(); - r.readAsArrayBuffer(f); - r.onload = function() { - ev.target.value = ''; - sendFileData(f.name, new Uint8Array(r.result), 4096); - }; -}; diff --git a/examples/file-upload/web_root/index.html b/examples/file-upload/web_root/index.html index 92142cb9..72c48871 100644 --- a/examples/file-upload/web_root/index.html +++ b/examples/file-upload/web_root/index.html @@ -16,21 +16,49 @@
Mongoose always buffers a full HTTP message before invoking MG_EV_HTTP_MSG event. Big POST request require of lot - of RAM to buffer everything. Therefore, in order to upload large - files on memory-constrained system, a large file should be send - in small chunks. -

- In this example, JavaScript code on this page sends uploaded - file in 2k chunks using an /upload endpoint. - Uploaded file is stored in /tmp directory by the helper - API function mg_http_upload() + of RAM to buffer everything. +

+ In order to upload large files to a memory-constrained system, use + MG_EV_HTTP_CHUNK on a server side. It fires when + a partial HTTP message has been received (or a chunk-encoded chunk). + Use mg_http_delete_chunk() to release chunk memory. + When 0-sized chunk is received, that's the end of the message. + Use MG_MAX_RECV_BUF_SIZE build constant to limit + maximum chunk size on a server side. +

+ In this example, JavaScript code uses "fetch()" browser API. + Uploaded file is not saved, but rather printed by server side.
-
+
- + diff --git a/mongoose.c b/mongoose.c index 25a0b188..73d7dd38 100644 --- a/mongoose.c +++ b/mongoose.c @@ -1407,7 +1407,7 @@ static struct mg_str guess_content_type(struct mg_str path, const char *extra) { static int getrange(struct mg_str *s, int64_t *a, int64_t *b) { size_t i, numparsed = 0; - LOG(LL_INFO, ("%.*s", (int) s->len, s->ptr)); + // LOG(LL_INFO, ("%.*s", (int) s->len, s->ptr)); for (i = 0; i + 6 < s->len; i++) { if (memcmp(&s->ptr[i], "bytes=", 6) == 0) { struct mg_str p = mg_str_n(s->ptr + i + 6, s->len - i - 6); @@ -1573,7 +1573,7 @@ static void listdir(struct mg_connection *c, struct mg_http_message *hm, "Index of %.*s%s%s" "" - "

Index of %.*s

" + "

Innex of %.*s

" "" "" @@ -1821,21 +1821,17 @@ static bool mg_is_chunked(struct mg_http_message *hm) { void mg_http_delete_chunk(struct mg_connection *c, struct mg_http_message *hm) { struct mg_str ch = hm->chunk; - if (mg_is_chunked(hm)) { - ch.len += 4; // \r\n before and after the chunk - ch.ptr -= 2; + const char *end = (char *) &c->recv.buf[c->recv.len], *ce; + bool chunked = mg_is_chunked(hm); + if (!mg_is_chunked(hm)) return; + if (chunked) { + ch.len += 4, ch.ptr -= 2; // \r\n before and after the chunk while (ch.ptr > hm->body.ptr && *ch.ptr != '\n') ch.ptr--, ch.len++; } - { - const char *end = &ch.ptr[ch.len]; - size_t n = (size_t) (end - (char *) c->recv.buf); - if (c->recv.len > n) { - memmove((char *) ch.ptr, end, (size_t) (c->recv.len - n)); - } - // LOG(LL_INFO, ("DELETING CHUNK: %zu %zu %zu\n%.*s", c->recv.len, n, - // ch.len, (int) ch.len, ch.ptr)); - } + ce = &ch.ptr[ch.len]; + if (ce < end) memmove((void *) ch.ptr, ce, (size_t) (end - ce)); c->recv.len -= ch.len; + if (c->pfn_data == NULL) c->pfn_data = (char *) c->pfn_data - ch.len; } int mg_http_upload(struct mg_connection *c, struct mg_http_message *hm, @@ -1889,7 +1885,16 @@ static void http_cb(struct mg_connection *c, int ev, void *evd, void *fnd) { if (n > 0 && !is_chunked) { hm.chunk = mg_str_n((char *) &c->recv.buf[n], c->recv.len - (size_t) n); + // Store remaining body length in c->pfn_data + if (c->pfn_data == NULL) + c->pfn_data = (void *) (hm.message.len - (size_t) n); mg_call(c, MG_EV_HTTP_CHUNK, &hm); + if (c->pfn_data == NULL) { + hm.chunk.len = 0; // Last chunk! + mg_call(c, MG_EV_HTTP_CHUNK, &hm); // Lest user know + memmove(c->recv.buf, c->recv.buf + n, c->recv.len - (size_t) n); + c->recv.len -= (size_t) n; + } } break; } diff --git a/src/http.c b/src/http.c index 3b528d41..0bd43228 100644 --- a/src/http.c +++ b/src/http.c @@ -455,7 +455,7 @@ static struct mg_str guess_content_type(struct mg_str path, const char *extra) { static int getrange(struct mg_str *s, int64_t *a, int64_t *b) { size_t i, numparsed = 0; - LOG(LL_INFO, ("%.*s", (int) s->len, s->ptr)); + // LOG(LL_INFO, ("%.*s", (int) s->len, s->ptr)); for (i = 0; i + 6 < s->len; i++) { if (memcmp(&s->ptr[i], "bytes=", 6) == 0) { struct mg_str p = mg_str_n(s->ptr + i + 6, s->len - i - 6); @@ -621,7 +621,7 @@ static void listdir(struct mg_connection *c, struct mg_http_message *hm, "Index of %.*s%s%s" "" - "

Index of %.*s

Name" "ModifiedSize
" + "

Innex of %.*s

" "" "" @@ -869,21 +869,17 @@ static bool mg_is_chunked(struct mg_http_message *hm) { void mg_http_delete_chunk(struct mg_connection *c, struct mg_http_message *hm) { struct mg_str ch = hm->chunk; - if (mg_is_chunked(hm)) { - ch.len += 4; // \r\n before and after the chunk - ch.ptr -= 2; + const char *end = (char *) &c->recv.buf[c->recv.len], *ce; + bool chunked = mg_is_chunked(hm); + if (!mg_is_chunked(hm)) return; + if (chunked) { + ch.len += 4, ch.ptr -= 2; // \r\n before and after the chunk while (ch.ptr > hm->body.ptr && *ch.ptr != '\n') ch.ptr--, ch.len++; } - { - const char *end = &ch.ptr[ch.len]; - size_t n = (size_t) (end - (char *) c->recv.buf); - if (c->recv.len > n) { - memmove((char *) ch.ptr, end, (size_t) (c->recv.len - n)); - } - // LOG(LL_INFO, ("DELETING CHUNK: %zu %zu %zu\n%.*s", c->recv.len, n, - // ch.len, (int) ch.len, ch.ptr)); - } + ce = &ch.ptr[ch.len]; + if (ce < end) memmove((void *) ch.ptr, ce, (size_t) (end - ce)); c->recv.len -= ch.len; + if (c->pfn_data == NULL) c->pfn_data = (char *) c->pfn_data - ch.len; } int mg_http_upload(struct mg_connection *c, struct mg_http_message *hm, @@ -937,7 +933,16 @@ static void http_cb(struct mg_connection *c, int ev, void *evd, void *fnd) { if (n > 0 && !is_chunked) { hm.chunk = mg_str_n((char *) &c->recv.buf[n], c->recv.len - (size_t) n); + // Store remaining body length in c->pfn_data + if (c->pfn_data == NULL) + c->pfn_data = (void *) (hm.message.len - (size_t) n); mg_call(c, MG_EV_HTTP_CHUNK, &hm); + if (c->pfn_data == NULL) { + hm.chunk.len = 0; // Last chunk! + mg_call(c, MG_EV_HTTP_CHUNK, &hm); // Lest user know + memmove(c->recv.buf, c->recv.buf + n, c->recv.len - (size_t) n); + c->recv.len -= (size_t) n; + } } break; }
Name" "ModifiedSize