Get rid of MG_EV_HTTP_CHUNK

examples/file-upload-single-post/main.c

@@ -1,8 +1,8 @@
 // Copyright (c) 2020 Cesanta Software Limited
 // All rights reserved
 //
-// Streaming upload example. Demonstrates how to use MG_EV_HTTP_CHUNK events
+// Streaming upload example. Demonstrates how to use MG_EV_READ events
-// to send large payload in smaller chunks. To test, use curl utility:
+// to get large payload in smaller chunks. To test, use curl utility:
 //
 // curl http://localhost:8000/upload?name=a.txt --data-binary @large_file.txt
 
@@ -12,21 +12,33 @@
 //   /upload - Saves the next file chunk
 //   all other URI - serves web_root/ directory
 static void cb(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
-  struct mg_http_message *hm = (struct mg_http_message *) ev_data;
+  if (ev == MG_EV_READ) {
-  if (ev == MG_EV_HTTP_CHUNK && mg_http_match_uri(hm, "/upload")) {
+    // Parse the incoming data ourselves. If we can parse the request,
-    MG_INFO(("Got chunk len %lu", (unsigned long) hm->chunk.len));
+    // store two size_t variables in the c->data: expected len and recv len.
-    MG_INFO(("Query string: [%.*s]", (int) hm->query.len, hm->query.ptr));
+    size_t *data = (size_t *) c->data;
-    // MG_INFO(("Chunk data:\n%.*s", (int) hm->chunk.len, hm->chunk.ptr));
+    if (data[0]) {  // Already parsed, simply print received data
-    mg_http_delete_chunk(c, hm);
+      data[1] += c->recv.len;
-    if (hm->chunk.len == 0) {
+      MG_INFO(("Got chunk len %lu, %lu total", c->recv.len, data[1]));
-      MG_INFO(("Last chunk received, sending response"));
+      c->recv.len = 0;  // And cleanup the receive buffer. Streaming!
-      mg_http_reply(c, 200, "", "ok (chunked)\n");
+      if (data[1] >= data[0]) mg_http_reply(c, 200, "", "ok\n");
+    } else {
+      struct mg_http_message hm;
+      int n = mg_http_parse((char *) c->recv.buf, c->recv.len, &hm);
+      if (n < 0) mg_error(c, "Bad response");
+      if (n > 0) {
+        if (mg_http_match_uri(&hm, "/upload")) {
+          MG_INFO(("Got chunk len %lu", c->recv.len - n));
+          data[0] = hm.body.len;
+          data[1] = c->recv.len - n;
+          if (data[1] >= data[0]) mg_http_reply(c, 200, "", "ok\n");
+        } else {
+          struct mg_http_serve_opts opts = {.root_dir = "web_root"};
+          mg_http_serve_dir(c, &hm, &opts);
+        }
+      }
     }
-  } else if (ev == MG_EV_HTTP_MSG) {
-    struct mg_http_serve_opts opts = {.root_dir = "web_root"};
-    mg_http_serve_dir(c, hm, &opts);
   }
-  (void) fn_data;
+  (void) fn_data, (void) ev_data;
 }
 
 int main(void) {

examples/file-upload-single-post/web_root/index.html

@@ -19,12 +19,10 @@
         of RAM to buffer everything.
         <br><br>
         In order to upload large files to a memory-constrained system, use
-        <code>MG_EV_HTTP_CHUNK</code> on a server side. It fires when
+        <code>MG_EV_READ</code> on a server side. It fires when
-        a partial HTTP message has been received (or a chunk-encoded chunk).
+        a partial HTTP message has been received. We discourage this method,
-        Use <code>mg_http_delete_chunk()</code> to release chunk memory.
+        since the client can send chunked-encoded data. Instead, please
-        When 0-sized chunk is received, that's the end of the message.
+        split the upload into smaller pieces and send sequentially.
-        Use <code>MG_MAX_RECV_SIZE</code> build constant to limit
-        maximum chunk size on a server side.
         <br><br>
         In this example, JavaScript code uses "fetch()" browser API.
         Uploaded file is not saved, but rather printed by server side.

examples/http-streaming-client/main.c

@@ -1,14 +1,12 @@
 // Copyright (c) 2021 Cesanta Software Limited
 // All rights reserved
 //
-// Example HTTP client.
+// Example streaming HTTP client.
-// Instead of buffering the whole HTTP response in memory and wait for the
+// The default HTTP handler waits until the whole HTTP message is buffered in
-// MG_EV_HTTP_MSG event, this client catches MG_EV_HTTP_CHUNK events and
+// c->recv IO buffer. It can be large. This example shows how to receive
-// prints them. This is useful to download large content without spooling
+// a potentially large response in chunks by handling the MG_EV_READ events.
-// everything to memory, or catch chunked content.
 //
 // You can change `s_url` from the command line by executing: ./example YOUR_URL
-//
 // To enable SSL/TLS, see https://mongoose.ws/tutorials/tls/#how-to-build
 
 #include "mongoose.h"
@@ -21,13 +19,11 @@ static void fn(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
   if (ev == MG_EV_CONNECT) {
     // Connected to server. Extract host name from URL
     struct mg_str host = mg_url_host(s_url);
-
     if (mg_url_is_ssl(s_url)) {
       struct mg_tls_opts opts = {.ca = mg_unpacked("/certs/ca.pem"),
                                  .name = host};
       mg_tls_init(c, &opts);
     }
-
     // Send request
     mg_printf(c,
               "GET %s HTTP/1.1\r\n"
@@ -36,21 +32,33 @@ static void fn(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
               "Host: %.*s\r\n"
               "\r\n",
               mg_url_uri(s_url), (int) host.len, host.ptr);
-  } else if (ev == MG_EV_HTTP_CHUNK) {
+  } else if (ev == MG_EV_READ) {
-    struct mg_http_message *hm = (struct mg_http_message *) ev_data;
+    // Parse the incoming data ourselves. If we can parse the request,
-    fwrite(hm->chunk.ptr, 1, hm->chunk.len, stdout);
+    // store two size_t variables in the c->data: expected len and recv len.
-    // fprintf(stderr, "c %u\n", (unsigned) hm->chunk.len);
+    size_t *data = (size_t *) c->data;
-    mg_http_delete_chunk(c, hm);
+    if (data[0]) {  // Already parsed, simply print received data
-    if (hm->chunk.len == 0) *(bool *) fn_data = true;  // Last chunk
+      data[1] += c->recv.len;
-  } else if (ev == MG_EV_HTTP_MSG) {
+      fwrite(c->recv.buf, 1, c->recv.len, stdout);
-    // Response is received. Print it
+      c->recv.len = 0;  // And cleanup the receive buffer. Streming!
-    struct mg_http_message *hm = (struct mg_http_message *) ev_data;
+      if (data[1] >= data[0]) * (bool *) fn_data = true;
-    fwrite(hm->body.ptr, 1, hm->body.len, stdout);
+    } else {
-    c->is_draining = 1;        // Tell mongoose to close this connection
+      struct mg_http_message hm;
-    *(bool *) fn_data = true;  // Tell event loop to stop
+      int n = mg_http_parse((char *) c->recv.buf, c->recv.len, &hm);
+      if (n < 0) mg_error(c, "Bad response");
+      if (n > 0) {
+        fwrite(c->recv.buf + n, 1, c->recv.len - n, stdout);  // Print body
+        data[0] = n + hm.body.len;
+        data[1] = c->recv.len;
+        c->recv.len = 0;  // Cleanup receive buffer
+        if (data[1] >= data[0]) * (bool *) fn_data = true;
+      }
+    }
+  } else if (ev == MG_EV_CLOSE) {
+    *(bool *) fn_data = true;  // tell event loop to stop
   } else if (ev == MG_EV_ERROR) {
     *(bool *) fn_data = true;  // Error, tell event loop to stop
   }
+  (void) ev_data;
 }
 
 int main(int argc, char *argv[]) {
@@ -59,7 +67,7 @@ int main(int argc, char *argv[]) {
   const char *log_level = getenv("V");  // Log level
 
   mg_mgr_init(&mgr);                       // Initialise event manager
-  if (log_level == NULL) log_level = "3";  // If not set, set to DEBUG
+  if (log_level == NULL) log_level = "2";  // If not set, set to DEBUG
   mg_log_set(atoi(log_level));             // Set to 0 to disable debug log
   if (argc > 1) s_url = argv[1];           // Use URL from command line
 

mongoose.c

@@ -1984,8 +1984,7 @@ int mg_http_parse(const char *s, size_t len, struct mg_http_message *hm) {
   hm->message.ptr = hm->head.ptr = s;
   hm->body.ptr = end;
   hm->head.len = (size_t) req_len;
-  hm->chunk.ptr = end;
+  hm->message.len = hm->body.len = (size_t) -1;  // Set body length to infinite
-  hm->message.len = hm->body.len = (size_t) ~0;  // Set body length to infinite
 
   // Parse request line
   hm->method.ptr = s;
@@ -2690,131 +2689,75 @@ static bool is_hex_digit(int c) {
          (c >= 'A' && c <= 'F');
 }
 
-// If a server sends data to the client using chunked encoding, Mongoose strips
+static int skip_chunk(const char *buf, int len, int *pl, int *dl) {
-// off the chunking prefix (hex length and \r\n) and suffix (\r\n), appends the
+  int i = 0, n = 0;
-// stripped data to the body, and fires the MG_EV_HTTP_CHUNK event.  When zero
+  if (len < 3) return 0;
-// chunk is received, we fire MG_EV_HTTP_MSG, and the body already has all
+  while (i < len && is_hex_digit(buf[i])) i++;
-// chunking prefixes/suffixes stripped.
+  if (len < i + 1 || buf[i] != '\r' || buf[i + 1] != '\n') return -1;  // Error
-//
+  n = (int) mg_unhexn(buf, (size_t) i);  // Decode hex length
-// If a server sends data without chunked encoding, we also fire a series of
+  if (n < 0) return -1;                  // Error
-// MG_EV_HTTP_CHUNK events for every received piece of data, and then we fire
+  if (len < i + n + 4) return 0;         // Chunk not yet fully buffered
-// MG_EV_HTTP_MSG event in the end.
+  if (buf[i + n + 2] != '\r' || buf[i + n + 3] != '\n') return -1;  // Error
-//
+  *pl = i + 2, *dl = n;
-// We track total processed length in the c->pfn_data, which is a void *
+  return i + 2 + n + 2;
-// pointer: we store a size_t value there.
-static bool getchunk(struct mg_str s, size_t *prefixlen, size_t *datalen) {
-  size_t i = 0, n;
-  while (i < s.len && is_hex_digit(s.ptr[i])) i++;
-  n = mg_unhexn(s.ptr, i);
-  // MG_INFO(("%d %d", (int) (i + n + 4), (int) s.len));
-  if (s.len < i + n + 4) return false;  // Chunk not yet fully buffered
-  if (s.ptr[i] != '\r' || s.ptr[i + 1] != '\n') return false;
-  if (s.ptr[i + n + 2] != '\r' || s.ptr[i + n + 3] != '\n') return false;
-  *prefixlen = i + 2;
-  *datalen = n;
-  return true;
 }
 
-static bool mg_is_chunked(struct mg_http_message *hm) {
+static bool is_chunked(struct mg_http_message *hm) {
   const char *needle = "chunked";
   struct mg_str *te = mg_http_get_header(hm, "Transfer-Encoding");
   return te != NULL && mg_vcasecmp(te, needle) == 0;
 }
 
-void mg_http_delete_chunk(struct mg_connection *c, struct mg_http_message *hm) {
-  size_t ofs = (size_t) (hm->chunk.ptr - (char *) c->recv.buf);
-  mg_iobuf_del(&c->recv, ofs, hm->chunk.len);
-  c->pfn_data = (void *) ((size_t) c->pfn_data | MG_DMARK);
-}
-
-static void deliver_chunked_chunks(struct mg_connection *c, size_t hlen,
-                                   struct mg_http_message *hm, bool *next) {
-  //  |  ... headers ... | HEXNUM\r\n ..data.. \r\n | ......
-  //  +------------------+--------------------------+----
-  //  |      hlen        |           chunk1         | ......
-  char *buf = (char *) &c->recv.buf[hlen], *p = buf;
-  size_t len = c->recv.len - hlen;
-  size_t processed = ((size_t) c->pfn_data) & ~MG_DMARK;
-  size_t mark, pl, dl, del = 0, ofs = 0;
-  bool last = false;
-  if (processed <= len) len -= processed, buf += processed;
-  while (!last && getchunk(mg_str_n(buf + ofs, len - ofs), &pl, &dl)) {
-    size_t saved = c->recv.len;
-    memmove(p + processed, buf + ofs + pl, dl);
-    // MG_INFO(("P2 [%.*s]", (int) (processed + dl), p));
-    hm->chunk = mg_str_n(p + processed, dl);
-    mg_call(c, MG_EV_HTTP_CHUNK, hm);
-    ofs += pl + dl + 2, del += pl + 2;  // 2 is for \r\n suffix
-    processed += dl;
-    if (c->recv.len != saved) processed -= dl, buf -= dl;
-    // mg_hexdump(c->recv.buf, hlen + processed);
-    last = (dl == 0);
-  }
-  mg_iobuf_del(&c->recv, hlen + processed, del);
-  mark = ((size_t) c->pfn_data) & MG_DMARK;
-  c->pfn_data = (void *) (processed | mark);
-  if (last) {
-    hm->body.len = processed;
-    hm->message.len = hlen + processed;
-    c->pfn_data = NULL;
-    if (mark) mg_iobuf_del(&c->recv, 0, hlen), *next = true;
-    // MG_INFO(("LAST, mark: %lx", mark));
-    // mg_hexdump(c->recv.buf, c->recv.len);
-  }
-}
-
-static void deliver_normal_chunks(struct mg_connection *c, size_t hlen,
-                                  struct mg_http_message *hm, bool *next) {
-  size_t left, processed = ((size_t) c->pfn_data) & ~MG_DMARK;
-  size_t deleted = ((size_t) c->pfn_data) & MG_DMARK;
-  hm->chunk = mg_str_n((char *) &c->recv.buf[hlen], c->recv.len - hlen);
-  if (processed <= hm->chunk.len && !deleted) {
-    hm->chunk.len -= processed;
-    hm->chunk.ptr += processed;
-  }
-  left = hm->body.len < processed ? 0 : hm->body.len - processed;
-  if (hm->chunk.len > left) hm->chunk.len = left;
-  if (hm->chunk.len > 0) mg_call(c, MG_EV_HTTP_CHUNK, hm);
-  processed += hm->chunk.len;
-  deleted = ((size_t) c->pfn_data) & MG_DMARK;  // Re-evaluate after user call
-  if (processed >= hm->body.len) {              // Last, 0-len chunk
-    hm->chunk.len = 0;                          // Reset length
-    mg_call(c, MG_EV_HTTP_CHUNK, hm);           // Call user handler
-    c->pfn_data = NULL;                         // Reset processed counter
-    if (processed && deleted) mg_iobuf_del(&c->recv, 0, hlen), *next = true;
-  } else {
-    c->pfn_data = (void *) (processed | deleted);  // if it is set
-  }
-}
-
 static void http_cb(struct mg_connection *c, int ev, void *evd, void *fnd) {
   if (ev == MG_EV_READ || ev == MG_EV_CLOSE) {
     struct mg_http_message hm;
-    while (c->recv.buf != NULL && c->recv.len > 0) {
+    size_t ofs = 0;  // Parsing offset
-      bool next = false;
+
-      int hlen = mg_http_parse((char *) c->recv.buf, c->recv.len, &hm);
+    while (c->is_resp == 0 && ofs < c->recv.len) {
-      if (hlen < 0) {
+      const char *buf = (char *) c->recv.buf + ofs;
-        mg_error(c, "HTTP parse:\n%.*s", (int) c->recv.len, c->recv.buf);
+      int n = mg_http_parse(buf, c->recv.len - ofs, &hm);
-        break;
+      if (n < 0) {
+        mg_error(c, "HTTP parse");
+        return;
       }
-      if (c->is_resp) break;           // Response is still generated
+      if (n == 0) break;        // Request is not buffered yet
-      if (hlen == 0) break;            // Request is not buffered yet
+      if (ev == MG_EV_CLOSE) {  // If client did not set Content-Length
-      if (ev == MG_EV_CLOSE) {         // If client did not set Content-Length
+        hm.message.len = c->recv.len - ofs;  // and closes now, deliver MSG
-        hm.message.len = c->recv.len;  // and closes now, deliver a MSG
         hm.body.len = hm.message.len - (size_t) (hm.body.ptr - hm.message.ptr);
       }
-      if (mg_is_chunked(&hm)) {
+
-        deliver_chunked_chunks(c, (size_t) hlen, &hm, &next);
+      if (is_chunked(&hm)) {
-      } else {
+        // For chunked data, strip off prefixes and suffixes from chunks
-        deliver_normal_chunks(c, (size_t) hlen, &hm, &next);
+        // and relocate them right after the headers, then report a message
+        char *s = (char *) c->recv.buf + ofs + n;
+        int o = 0, pl, dl, cl, len = (int) (c->recv.len - ofs - (size_t) n);
+
+        // Find zero-length chunk (the end of the body)
+        while ((cl = skip_chunk(s + o, len - o, &pl, &dl)) > 0 && dl) o += cl;
+        if (cl == 0) break;  // No zero-len chunk, buffer more data
+        if (cl < 0) {
+          mg_error(c, "Invalid chunk");
+          break;
+        }
+
+        // Zero chunk found. Second pass: strip + relocate
+        o = 0, hm.body.len = 0, hm.message.len = (size_t) n;
+        while ((cl = skip_chunk(s + o, len - o, &pl, &dl)) > 0) {
+          memmove(s + hm.body.len, s + o + pl, (size_t) dl);
+          o += cl, hm.body.len += (size_t) dl, hm.message.len += (size_t) dl;
+          if (dl == 0) break;
+        }
+        ofs += (size_t) (n + o);
+      } else { // Normal, non-chunked data
+        size_t len = c->recv.len - ofs - (size_t) n;
+        if (hm.body.len > len) break;  // Buffer more data
+        ofs += (size_t) n + hm.body.len;
       }
-      if (next) continue;  // Chunks & request were deleted
+
-      //  Chunk events are delivered. If we have full body, deliver MSG
-      if (c->recv.len < hm.message.len) break;
       if (c->is_accepted) c->is_resp = 1;  // Start generating response
       mg_call(c, MG_EV_HTTP_MSG, &hm);     // User handler can clear is_resp
-      mg_iobuf_del(&c->recv, 0, hm.message.len);
     }
+    if (ofs > 0) mg_iobuf_del(&c->recv, 0, ofs);  // Delete processed data
   }
   (void) evd, (void) fnd;
 }

mongoose.h

@@ -1145,26 +1145,25 @@ void mg_call(struct mg_connection *c, int ev, void *ev_data);
 void mg_error(struct mg_connection *c, const char *fmt, ...);
 
 enum {
-  MG_EV_ERROR,       // Error                        char *error_message
+  MG_EV_ERROR,      // Error                        char *error_message
-  MG_EV_OPEN,        // Connection created           NULL
+  MG_EV_OPEN,       // Connection created           NULL
-  MG_EV_POLL,        // mg_mgr_poll iteration        uint64_t *uptime_millis
+  MG_EV_POLL,       // mg_mgr_poll iteration        uint64_t *uptime_millis
-  MG_EV_RESOLVE,     // Host name is resolved        NULL
+  MG_EV_RESOLVE,    // Host name is resolved        NULL
-  MG_EV_CONNECT,     // Connection established       NULL
+  MG_EV_CONNECT,    // Connection established       NULL
-  MG_EV_ACCEPT,      // Connection accepted          NULL
+  MG_EV_ACCEPT,     // Connection accepted          NULL
-  MG_EV_TLS_HS,      // TLS handshake succeeded      NULL
+  MG_EV_TLS_HS,     // TLS handshake succeeded      NULL
-  MG_EV_READ,        // Data received from socket    long *bytes_read
+  MG_EV_READ,       // Data received from socket    long *bytes_read
-  MG_EV_WRITE,       // Data written to socket       long *bytes_written
+  MG_EV_WRITE,      // Data written to socket       long *bytes_written
-  MG_EV_CLOSE,       // Connection closed            NULL
+  MG_EV_CLOSE,      // Connection closed            NULL
-  MG_EV_HTTP_MSG,    // HTTP request/response        struct mg_http_message *
+  MG_EV_HTTP_MSG,   // HTTP request/response        struct mg_http_message *
-  MG_EV_HTTP_CHUNK,  // HTTP chunk (partial msg)     struct mg_http_message *
+  MG_EV_WS_OPEN,    // Websocket handshake done     struct mg_http_message *
-  MG_EV_WS_OPEN,     // Websocket handshake done     struct mg_http_message *
+  MG_EV_WS_MSG,     // Websocket msg, text or bin   struct mg_ws_message *
-  MG_EV_WS_MSG,      // Websocket msg, text or bin   struct mg_ws_message *
+  MG_EV_WS_CTL,     // Websocket control msg        struct mg_ws_message *
-  MG_EV_WS_CTL,      // Websocket control msg        struct mg_ws_message *
+  MG_EV_MQTT_CMD,   // MQTT low-level command       struct mg_mqtt_message *
-  MG_EV_MQTT_CMD,    // MQTT low-level command       struct mg_mqtt_message *
+  MG_EV_MQTT_MSG,   // MQTT PUBLISH received        struct mg_mqtt_message *
-  MG_EV_MQTT_MSG,    // MQTT PUBLISH received        struct mg_mqtt_message *
+  MG_EV_MQTT_OPEN,  // MQTT CONNACK received        int *connack_status_code
-  MG_EV_MQTT_OPEN,   // MQTT CONNACK received        int *connack_status_code
+  MG_EV_SNTP_TIME,  // SNTP time received           uint64_t *epoch_millis
-  MG_EV_SNTP_TIME,   // SNTP time received           uint64_t *epoch_millis
+  MG_EV_USER        // Starting ID for user events
-  MG_EV_USER         // Starting ID for user events
 };
 
 
@@ -1290,7 +1289,6 @@ struct mg_http_message {
   struct mg_http_header headers[MG_MAX_HTTP_HEADERS];  // Headers
   struct mg_str body;                                  // Body
   struct mg_str head;                                  // Request + headers
-  struct mg_str chunk;    // Chunk for chunked encoding,  or partial body
   struct mg_str message;  // Request + headers + body
 };
 

src/event.h

@@ -7,24 +7,23 @@ void mg_call(struct mg_connection *c, int ev, void *ev_data);
 void mg_error(struct mg_connection *c, const char *fmt, ...);
 
 enum {
-  MG_EV_ERROR,       // Error                        char *error_message
+  MG_EV_ERROR,      // Error                        char *error_message
-  MG_EV_OPEN,        // Connection created           NULL
+  MG_EV_OPEN,       // Connection created           NULL
-  MG_EV_POLL,        // mg_mgr_poll iteration        uint64_t *uptime_millis
+  MG_EV_POLL,       // mg_mgr_poll iteration        uint64_t *uptime_millis
-  MG_EV_RESOLVE,     // Host name is resolved        NULL
+  MG_EV_RESOLVE,    // Host name is resolved        NULL
-  MG_EV_CONNECT,     // Connection established       NULL
+  MG_EV_CONNECT,    // Connection established       NULL
-  MG_EV_ACCEPT,      // Connection accepted          NULL
+  MG_EV_ACCEPT,     // Connection accepted          NULL
-  MG_EV_TLS_HS,      // TLS handshake succeeded      NULL
+  MG_EV_TLS_HS,     // TLS handshake succeeded      NULL
-  MG_EV_READ,        // Data received from socket    long *bytes_read
+  MG_EV_READ,       // Data received from socket    long *bytes_read
-  MG_EV_WRITE,       // Data written to socket       long *bytes_written
+  MG_EV_WRITE,      // Data written to socket       long *bytes_written
-  MG_EV_CLOSE,       // Connection closed            NULL
+  MG_EV_CLOSE,      // Connection closed            NULL
-  MG_EV_HTTP_MSG,    // HTTP request/response        struct mg_http_message *
+  MG_EV_HTTP_MSG,   // HTTP request/response        struct mg_http_message *
-  MG_EV_HTTP_CHUNK,  // HTTP chunk (partial msg)     struct mg_http_message *
+  MG_EV_WS_OPEN,    // Websocket handshake done     struct mg_http_message *
-  MG_EV_WS_OPEN,     // Websocket handshake done     struct mg_http_message *
+  MG_EV_WS_MSG,     // Websocket msg, text or bin   struct mg_ws_message *
-  MG_EV_WS_MSG,      // Websocket msg, text or bin   struct mg_ws_message *
+  MG_EV_WS_CTL,     // Websocket control msg        struct mg_ws_message *
-  MG_EV_WS_CTL,      // Websocket control msg        struct mg_ws_message *
+  MG_EV_MQTT_CMD,   // MQTT low-level command       struct mg_mqtt_message *
-  MG_EV_MQTT_CMD,    // MQTT low-level command       struct mg_mqtt_message *
+  MG_EV_MQTT_MSG,   // MQTT PUBLISH received        struct mg_mqtt_message *
-  MG_EV_MQTT_MSG,    // MQTT PUBLISH received        struct mg_mqtt_message *
+  MG_EV_MQTT_OPEN,  // MQTT CONNACK received        int *connack_status_code
-  MG_EV_MQTT_OPEN,   // MQTT CONNACK received        int *connack_status_code
+  MG_EV_SNTP_TIME,  // SNTP time received           uint64_t *epoch_millis
-  MG_EV_SNTP_TIME,   // SNTP time received           uint64_t *epoch_millis
+  MG_EV_USER        // Starting ID for user events
-  MG_EV_USER         // Starting ID for user events
 };

src/http.c

@@ -245,8 +245,7 @@ int mg_http_parse(const char *s, size_t len, struct mg_http_message *hm) {
   hm->message.ptr = hm->head.ptr = s;
   hm->body.ptr = end;
   hm->head.len = (size_t) req_len;
-  hm->chunk.ptr = end;
+  hm->message.len = hm->body.len = (size_t) -1;  // Set body length to infinite
-  hm->message.len = hm->body.len = (size_t) ~0;  // Set body length to infinite
 
   // Parse request line
   hm->method.ptr = s;
@@ -951,131 +950,75 @@ static bool is_hex_digit(int c) {
          (c >= 'A' && c <= 'F');
 }
 
-// If a server sends data to the client using chunked encoding, Mongoose strips
+static int skip_chunk(const char *buf, int len, int *pl, int *dl) {
-// off the chunking prefix (hex length and \r\n) and suffix (\r\n), appends the
+  int i = 0, n = 0;
-// stripped data to the body, and fires the MG_EV_HTTP_CHUNK event.  When zero
+  if (len < 3) return 0;
-// chunk is received, we fire MG_EV_HTTP_MSG, and the body already has all
+  while (i < len && is_hex_digit(buf[i])) i++;
-// chunking prefixes/suffixes stripped.
+  if (len < i + 1 || buf[i] != '\r' || buf[i + 1] != '\n') return -1;  // Error
-//
+  n = (int) mg_unhexn(buf, (size_t) i);  // Decode hex length
-// If a server sends data without chunked encoding, we also fire a series of
+  if (n < 0) return -1;                  // Error
-// MG_EV_HTTP_CHUNK events for every received piece of data, and then we fire
+  if (len < i + n + 4) return 0;         // Chunk not yet fully buffered
-// MG_EV_HTTP_MSG event in the end.
+  if (buf[i + n + 2] != '\r' || buf[i + n + 3] != '\n') return -1;  // Error
-//
+  *pl = i + 2, *dl = n;
-// We track total processed length in the c->pfn_data, which is a void *
+  return i + 2 + n + 2;
-// pointer: we store a size_t value there.
-static bool getchunk(struct mg_str s, size_t *prefixlen, size_t *datalen) {
-  size_t i = 0, n;
-  while (i < s.len && is_hex_digit(s.ptr[i])) i++;
-  n = mg_unhexn(s.ptr, i);
-  // MG_INFO(("%d %d", (int) (i + n + 4), (int) s.len));
-  if (s.len < i + n + 4) return false;  // Chunk not yet fully buffered
-  if (s.ptr[i] != '\r' || s.ptr[i + 1] != '\n') return false;
-  if (s.ptr[i + n + 2] != '\r' || s.ptr[i + n + 3] != '\n') return false;
-  *prefixlen = i + 2;
-  *datalen = n;
-  return true;
 }
 
-static bool mg_is_chunked(struct mg_http_message *hm) {
+static bool is_chunked(struct mg_http_message *hm) {
   const char *needle = "chunked";
   struct mg_str *te = mg_http_get_header(hm, "Transfer-Encoding");
   return te != NULL && mg_vcasecmp(te, needle) == 0;
 }
 
-void mg_http_delete_chunk(struct mg_connection *c, struct mg_http_message *hm) {
-  size_t ofs = (size_t) (hm->chunk.ptr - (char *) c->recv.buf);
-  mg_iobuf_del(&c->recv, ofs, hm->chunk.len);
-  c->pfn_data = (void *) ((size_t) c->pfn_data | MG_DMARK);
-}
-
-static void deliver_chunked_chunks(struct mg_connection *c, size_t hlen,
-                                   struct mg_http_message *hm, bool *next) {
-  //  |  ... headers ... | HEXNUM\r\n ..data.. \r\n | ......
-  //  +------------------+--------------------------+----
-  //  |      hlen        |           chunk1         | ......
-  char *buf = (char *) &c->recv.buf[hlen], *p = buf;
-  size_t len = c->recv.len - hlen;
-  size_t processed = ((size_t) c->pfn_data) & ~MG_DMARK;
-  size_t mark, pl, dl, del = 0, ofs = 0;
-  bool last = false;
-  if (processed <= len) len -= processed, buf += processed;
-  while (!last && getchunk(mg_str_n(buf + ofs, len - ofs), &pl, &dl)) {
-    size_t saved = c->recv.len;
-    memmove(p + processed, buf + ofs + pl, dl);
-    // MG_INFO(("P2 [%.*s]", (int) (processed + dl), p));
-    hm->chunk = mg_str_n(p + processed, dl);
-    mg_call(c, MG_EV_HTTP_CHUNK, hm);
-    ofs += pl + dl + 2, del += pl + 2;  // 2 is for \r\n suffix
-    processed += dl;
-    if (c->recv.len != saved) processed -= dl, buf -= dl;
-    // mg_hexdump(c->recv.buf, hlen + processed);
-    last = (dl == 0);
-  }
-  mg_iobuf_del(&c->recv, hlen + processed, del);
-  mark = ((size_t) c->pfn_data) & MG_DMARK;
-  c->pfn_data = (void *) (processed | mark);
-  if (last) {
-    hm->body.len = processed;
-    hm->message.len = hlen + processed;
-    c->pfn_data = NULL;
-    if (mark) mg_iobuf_del(&c->recv, 0, hlen), *next = true;
-    // MG_INFO(("LAST, mark: %lx", mark));
-    // mg_hexdump(c->recv.buf, c->recv.len);
-  }
-}
-
-static void deliver_normal_chunks(struct mg_connection *c, size_t hlen,
-                                  struct mg_http_message *hm, bool *next) {
-  size_t left, processed = ((size_t) c->pfn_data) & ~MG_DMARK;
-  size_t deleted = ((size_t) c->pfn_data) & MG_DMARK;
-  hm->chunk = mg_str_n((char *) &c->recv.buf[hlen], c->recv.len - hlen);
-  if (processed <= hm->chunk.len && !deleted) {
-    hm->chunk.len -= processed;
-    hm->chunk.ptr += processed;
-  }
-  left = hm->body.len < processed ? 0 : hm->body.len - processed;
-  if (hm->chunk.len > left) hm->chunk.len = left;
-  if (hm->chunk.len > 0) mg_call(c, MG_EV_HTTP_CHUNK, hm);
-  processed += hm->chunk.len;
-  deleted = ((size_t) c->pfn_data) & MG_DMARK;  // Re-evaluate after user call
-  if (processed >= hm->body.len) {              // Last, 0-len chunk
-    hm->chunk.len = 0;                          // Reset length
-    mg_call(c, MG_EV_HTTP_CHUNK, hm);           // Call user handler
-    c->pfn_data = NULL;                         // Reset processed counter
-    if (processed && deleted) mg_iobuf_del(&c->recv, 0, hlen), *next = true;
-  } else {
-    c->pfn_data = (void *) (processed | deleted);  // if it is set
-  }
-}
-
 static void http_cb(struct mg_connection *c, int ev, void *evd, void *fnd) {
   if (ev == MG_EV_READ || ev == MG_EV_CLOSE) {
     struct mg_http_message hm;
-    while (c->recv.buf != NULL && c->recv.len > 0) {
+    size_t ofs = 0;  // Parsing offset
-      bool next = false;
+
-      int hlen = mg_http_parse((char *) c->recv.buf, c->recv.len, &hm);
+    while (c->is_resp == 0 && ofs < c->recv.len) {
-      if (hlen < 0) {
+      const char *buf = (char *) c->recv.buf + ofs;
-        mg_error(c, "HTTP parse:\n%.*s", (int) c->recv.len, c->recv.buf);
+      int n = mg_http_parse(buf, c->recv.len - ofs, &hm);
-        break;
+      if (n < 0) {
+        mg_error(c, "HTTP parse");
+        return;
       }
-      if (c->is_resp) break;           // Response is still generated
+      if (n == 0) break;        // Request is not buffered yet
-      if (hlen == 0) break;            // Request is not buffered yet
+      if (ev == MG_EV_CLOSE) {  // If client did not set Content-Length
-      if (ev == MG_EV_CLOSE) {         // If client did not set Content-Length
+        hm.message.len = c->recv.len - ofs;  // and closes now, deliver MSG
-        hm.message.len = c->recv.len;  // and closes now, deliver a MSG
         hm.body.len = hm.message.len - (size_t) (hm.body.ptr - hm.message.ptr);
       }
-      if (mg_is_chunked(&hm)) {
+
-        deliver_chunked_chunks(c, (size_t) hlen, &hm, &next);
+      if (is_chunked(&hm)) {
-      } else {
+        // For chunked data, strip off prefixes and suffixes from chunks
-        deliver_normal_chunks(c, (size_t) hlen, &hm, &next);
+        // and relocate them right after the headers, then report a message
+        char *s = (char *) c->recv.buf + ofs + n;
+        int o = 0, pl, dl, cl, len = (int) (c->recv.len - ofs - (size_t) n);
+
+        // Find zero-length chunk (the end of the body)
+        while ((cl = skip_chunk(s + o, len - o, &pl, &dl)) > 0 && dl) o += cl;
+        if (cl == 0) break;  // No zero-len chunk, buffer more data
+        if (cl < 0) {
+          mg_error(c, "Invalid chunk");
+          break;
+        }
+
+        // Zero chunk found. Second pass: strip + relocate
+        o = 0, hm.body.len = 0, hm.message.len = (size_t) n;
+        while ((cl = skip_chunk(s + o, len - o, &pl, &dl)) > 0) {
+          memmove(s + hm.body.len, s + o + pl, (size_t) dl);
+          o += cl, hm.body.len += (size_t) dl, hm.message.len += (size_t) dl;
+          if (dl == 0) break;
+        }
+        ofs += (size_t) (n + o);
+      } else { // Normal, non-chunked data
+        size_t len = c->recv.len - ofs - (size_t) n;
+        if (hm.body.len > len) break;  // Buffer more data
+        ofs += (size_t) n + hm.body.len;
       }
-      if (next) continue;  // Chunks & request were deleted
+
-      //  Chunk events are delivered. If we have full body, deliver MSG
-      if (c->recv.len < hm.message.len) break;
       if (c->is_accepted) c->is_resp = 1;  // Start generating response
       mg_call(c, MG_EV_HTTP_MSG, &hm);     // User handler can clear is_resp
-      mg_iobuf_del(&c->recv, 0, hm.message.len);
     }
+    if (ofs > 0) mg_iobuf_del(&c->recv, 0, ofs);  // Delete processed data
   }
   (void) evd, (void) fnd;
 }

src/http.h

@@ -16,7 +16,6 @@ struct mg_http_message {
   struct mg_http_header headers[MG_MAX_HTTP_HEADERS];  // Headers
   struct mg_str body;                                  // Body
   struct mg_str head;                                  // Request + headers
-  struct mg_str chunk;    // Chunk for chunked encoding,  or partial body
   struct mg_str message;  // Request + headers + body
 };
 

test/fuzz.c

@@ -2,7 +2,7 @@
 #define MG_ENABLE_LOG 0
 #define MG_ENABLE_LINES 1
 #define MG_ENABLE_TCPIP 1
-#define MG_IO_SIZE (4 * 1024 * 1024)  // 4M IO size, unless resize is too slow
+#define MG_IO_SIZE (32 * 1024 * 1024)  // Big IO size for fast resizes
 
 #include "mongoose.c"
 

test/unit_test.c

@@ -1342,8 +1342,6 @@ static void f4(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
               "abcdef");
     strcat((char *) fn_data, "m");
     c->is_draining = 1;
-  } else if (ev == MG_EV_HTTP_CHUNK) {
-    strcat((char *) fn_data, "f");
   } else if (ev == MG_EV_CLOSE) {
     strcat((char *) fn_data, "c");
   }
@@ -1356,10 +1354,6 @@ static void f4c(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
     struct mg_http_message *hm = (struct mg_http_message *) ev_data;
     ASSERT(mg_strcmp(hm->body, mg_str("/foo/bar/abcdef")) == 0);
     strcat((char *) fn_data, "m");
-  } else if (ev == MG_EV_HTTP_CHUNK) {
-    struct mg_http_message *hm = (struct mg_http_message *) ev_data;
-    MG_INFO(("FS [%.*s]", (int) hm->chunk.len, hm->chunk.ptr));
-    strcat((char *) fn_data, "f");
   } else if (ev == MG_EV_CLOSE) {
     strcat((char *) fn_data, "c");
   }
@@ -1375,8 +1369,8 @@ static void test_http_no_content_length(void) {
   mg_http_connect(&mgr, url, f4c, (void *) buf2);
   for (i = 0; i < 1000 && strchr(buf2, 'c') == NULL; i++) mg_mgr_poll(&mgr, 10);
   MG_INFO(("[%s] [%s]", buf1, buf2));
-  ASSERT(strcmp(buf1, "fmc") == 0);
+  ASSERT(strcmp(buf1, "mc") == 0);
-  ASSERT(strcmp(buf2, "fcfm") == 0);  // See #1475
+  ASSERT(strcmp(buf2, "cm") == 0);  // See #1475
   mg_mgr_free(&mgr);
   ASSERT(mgr.conns == NULL);
 }
@@ -2290,20 +2284,8 @@ static void test_crc32(void) {
 }
 
 static void us(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
-  int del = *(int *) fn_data;
   struct mg_http_message *hm = (struct mg_http_message *) ev_data;
-  if (ev == MG_EV_HTTP_CHUNK && mg_http_match_uri(hm, "/upload")) {
+  if (ev == MG_EV_HTTP_MSG && mg_http_match_uri(hm, "/upload")) {
-    MG_DEBUG(("Got chunk len %lu", (unsigned long) hm->chunk.len));
-    MG_DEBUG(("Query string: [%.*s]", (int) hm->query.len, hm->query.ptr));
-    // MG_DEBUG(("Chunk data:\n%.*s", (int) hm->chunk.len, hm->chunk.ptr));
-    if (del) {
-      mg_http_delete_chunk(c, hm);
-      if (hm->chunk.len == 0) {
-        MG_DEBUG(("Last chunk received, sending response"));
-        mg_http_reply(c, 200, "", "ok (chunked)\n");
-      }
-    }
-  } else if (ev == MG_EV_HTTP_MSG && mg_http_match_uri(hm, "/upload")) {
     MG_DEBUG(("Got all %lu bytes!", (unsigned long) hm->body.len));
     MG_DEBUG(("Query string: [%.*s]", (int) hm->query.len, hm->query.ptr));
     // MG_DEBUG(("Body:\n%.*s", (int) hm->body.len, hm->body.ptr));
@@ -2318,7 +2300,7 @@ static void us(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
 static void uc(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
   const char **s = (const char **) fn_data;
   if (ev == MG_EV_OPEN) {
-    c->is_hexdumping = 1;
+    // c->is_hexdumping = 1;
   } else if (ev == MG_EV_CONNECT) {
     mg_printf(c,
               "POST /upload HTTP/1.0\r\n"
@@ -2328,6 +2310,7 @@ static void uc(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
     mg_http_printf_chunk(c, "");
   } else if (ev == MG_EV_HTTP_MSG) {
     struct mg_http_message *hm = (struct mg_http_message *) ev_data;
+    // MG_INFO(("---> [%s] [%.*s]", *s, hm->body.len, hm->body.ptr));
     ASSERT(mg_strcmp(hm->body, mg_str(*s)) == 0);
     *s = NULL;
   }
@@ -2337,22 +2320,14 @@ static void uc(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
 static void test_http_upload(void) {
   struct mg_mgr mgr;
   const char *url = "http://127.0.0.1:12352";
-  int i, del = 1;
+  int i;
-  const char *s1 = "ok (chunked)\n";
+  const char *s = "ok (8 foo\nbar\n)\n";
-  const char *s2 = "ok (8 foo\nbar\n)\n";
 
   mg_mgr_init(&mgr);
-  mg_http_listen(&mgr, url, us, (void *) &del);
+  mg_http_listen(&mgr, url, us, NULL);
-
+  mg_http_connect(&mgr, url, uc, (void *) &s);
-  mg_http_connect(&mgr, url, uc, (void *) &s1);
   for (i = 0; i < 20; i++) mg_mgr_poll(&mgr, 5);
-  ASSERT(s1 == NULL);
+  ASSERT(s == NULL);
-
-  del = 0;
-  mg_http_connect(&mgr, url, uc, (void *) &s2);
-  for (i = 0; i < 20; i++) mg_mgr_poll(&mgr, 5);
-  ASSERT(s2 == NULL);
-
   mg_mgr_free(&mgr);
   ASSERT(mgr.conns == NULL);
 }
@@ -2363,7 +2338,7 @@ static void eX(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
   if (ev == MG_EV_HTTP_MSG) {
     mg_printf(c, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n");
     c->data[0] = 1;
-    c->is_hexdumping = 1;
+    // c->is_hexdumping = 1;
   } else if (ev == MG_EV_POLL && c->data[0] != 0) {
     c->data[0]++;
     if (c->data[0] == 10) mg_http_printf_chunk(c, "a");
@@ -2403,32 +2378,13 @@ static void eZ(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
 // Do not delete chunks as they arrive
 static void eh4(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
   uint32_t *crc = (uint32_t *) fn_data;
-  if (ev == MG_EV_HTTP_CHUNK) {
+  if (ev == MG_EV_HTTP_MSG) {
-    struct mg_http_message *hm = (struct mg_http_message *) ev_data;
-    *crc = mg_crc32(*crc, hm->chunk.ptr, hm->chunk.len);
-    *crc = mg_crc32(*crc, "x", 1);
-    MG_INFO(("%lu C [%.*s]", c->id, (int) hm->chunk.len, hm->chunk.ptr));
-  } else if (ev == MG_EV_HTTP_MSG) {
     struct mg_http_message *hm = (struct mg_http_message *) ev_data;
     *crc = mg_crc32(*crc, hm->body.ptr, hm->body.len);
     MG_INFO(("%lu M [%.*s]", c->id, (int) hm->body.len, hm->body.ptr));
   }
 }
 
-// Streaming client event handler. Delete chunks as they arrive
-static void eh5(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
-  uint32_t *crc = (uint32_t *) fn_data;
-  if (ev == MG_EV_HTTP_CHUNK) {
-    struct mg_http_message *hm = (struct mg_http_message *) ev_data;
-    *crc = mg_crc32(*crc, hm->chunk.ptr, hm->chunk.len);
-    *crc = mg_crc32(*crc, "x", 1);
-    MG_INFO(("%lu DELC [%.*s]", c->id, (int) hm->chunk.len, hm->chunk.ptr));
-    mg_http_delete_chunk(c, hm);
-  } else if (ev == MG_EV_HTTP_MSG) {
-    ASSERT(0);  // Must not be here, MSG must not be fired: chunks deleted!
-  }
-}
-
 static void test_http_chunked_case(mg_event_handler_t s, mg_event_handler_t c,
                                    int req_count, const char *expected) {
   char url[100];
@@ -2446,6 +2402,7 @@ static void test_http_chunked_case(mg_event_handler_t s, mg_event_handler_t c,
   for (i = 0; i < 100 && crc != expected_crc; i++) {
     mg_mgr_poll(&mgr, 1);
   }
+  // MG_INFO(("-------- %d [%s]", i, expected));
   ASSERT(i < 100);
   ASSERT(crc == expected_crc);
   mg_mgr_free(&mgr);
@@ -2454,20 +2411,14 @@ static void test_http_chunked_case(mg_event_handler_t s, mg_event_handler_t c,
 
 static void test_http_chunked(void) {
   // Non-chunked encoding
-  test_http_chunked_case(eY, eh4, 1, "axbcxdxxabcd");  // Chunks not deleted
+  test_http_chunked_case(eY, eh4, 1, "abcd");  // Chunks not deleted
-  test_http_chunked_case(eY, eh4, 2, "axbcxdxxabcdaxbcxdxxabcd");
+  test_http_chunked_case(eY, eh4, 2, "abcdabcd");
-  test_http_chunked_case(eY, eh5, 1, "axbcxdxx");  // Chunks deleted
+  test_http_chunked_case(eZ, eh4, 1, "abcd");  // Not deleted
-  test_http_chunked_case(eY, eh5, 2, "axbcxdxxaxbcxdxx");
+  test_http_chunked_case(eZ, eh4, 2, "abcdabcd");
-  test_http_chunked_case(eZ, eh4, 1, "abcdxxabcd");  // Not deleted
-  test_http_chunked_case(eZ, eh4, 2, "abcdxxabcdabcdxxabcd");
-  test_http_chunked_case(eZ, eh5, 1, "abcdxx");  // Deleted
-  test_http_chunked_case(eZ, eh5, 2, "abcdxxabcdxx");
 
   // Chunked encoding
-  test_http_chunked_case(eX, eh4, 1, "axbxcxdxxabcd");  // Chunks not deleted
+  test_http_chunked_case(eX, eh4, 1, "abcd");  // Chunks not deleted
-  test_http_chunked_case(eX, eh5, 1, "axbxcxdxx");      // Chunks deleted
+  test_http_chunked_case(eX, eh4, 2, "abcdabcd");
-  test_http_chunked_case(eX, eh4, 2, "axbxcxdxxabcdaxbxcxdxxabcd");
-  test_http_chunked_case(eX, eh5, 2, "axbxcxdxxaxbxcxdxx");
 }
 
 static void test_invalid_listen_addr(void) {