Refactor chunking support. Fix #1656

Makefile

@@ -34,7 +34,7 @@ CFLAGS  += -DMG_ENABLE_OPENSSL=1 -I$(OPENSSL)/include
 LDFLAGS ?= -L$(OPENSSL)/lib -lssl -lcrypto
 endif
 
-all: mg_prefix unamalgamated unpacked test test++ valgrind arm examples vc98 vc2017 vc22 mingw mingw++ linux linux++ fuzz
+all: mg_prefix unamalgamated unpacked test test++ arm examples vc98 vc2017 vc22 mingw mingw++ linux linux++ fuzz
 
 examples:
 	@for X in $(EXAMPLES); do test -f $$X/Makefile || continue; $(MAKE) -C $$X example || exit 1; done

mongoose.c

@@ -1431,6 +1431,9 @@ struct mg_fs mg_fs_posix = {p_stat,  p_list, p_open,   p_close,  p_read,
 
 
 
+// Chunk deletion marker is the MSB in the "processed" counter
+#define MG_DMARK ((size_t) 1 << (sizeof(size_t) * 8 - 1))
+
 // Multipart POST example:
 // --xyz
 // Content-Disposition: form-data; name="val"
@@ -2197,79 +2200,6 @@ bool mg_http_match_uri(const struct mg_http_message *hm, const char *glob) {
   return mg_match(hm->uri, mg_str(glob), NULL);
 }
 
-static size_t get_chunk_length(const char *buf, size_t len, size_t *ll) {
-  size_t i = 0, n;
-  while (i < len && buf[i] != '\r' && i != '\n') i++;
-  n = mg_unhexn((char *) buf, i);
-  while (i < len && (buf[i] == '\r' || i == '\n')) i++;
-  // MG_INFO(("len %zu i %zu n %zu ", len, i, n));
-  if (ll != NULL) *ll = i + 1;
-  if (i < len && i + n + 2 < len) return i + n + 3;
-  return 0;
-}
-
-// Walk through all chunks in the chunked body. For each chunk, fire
-// an MG_EV_HTTP_CHUNK event.
-static bool walkchunks(struct mg_connection *c, struct mg_http_message *hm,
-                       size_t reqlen) {
-  size_t off = 0, bl, ll;
-  while (off + reqlen < c->recv.len) {
-    char *buf = (char *) &c->recv.buf[reqlen];
-    size_t memo = c->recv.len;
-    size_t cl = get_chunk_length(&buf[off], memo - reqlen - off, &ll);
-    // MG_INFO(("len %u off %u cl %u ll %u", memo - reqlen, off, cl, ll));
-    if (cl == 0) break;
-    hm->chunk = mg_str_n(&buf[off + ll], cl < ll + 2 ? 0 : cl - ll - 2);
-    mg_call(c, MG_EV_HTTP_CHUNK, hm);
-    // Increase offset only if user has not deleted this chunk
-    if (memo == c->recv.len) off += cl;
-    if (ll + 2 >= cl) {
-      // Zero chunk - last one. Prepare body - cut off chunk lengths
-      if (memo != c->recv.len) return true;  // Tell caller to cleanup
-      off = bl = 0;
-      while (off + reqlen < c->recv.len) {
-        char *buf2 = (char *) &c->recv.buf[reqlen];
-        size_t memo2 = c->recv.len;
-        size_t cl2 = get_chunk_length(&buf2[off], memo2 - reqlen - off, &ll);
-        size_t n = cl2 < ll + 2 ? 0 : cl2 - ll - 2;
-        memmove(buf2 + bl, buf2 + off + ll, n);
-        bl += n;
-        off += cl2;
-        if (ll + 2 >= cl2) break;
-      }
-      // MG_INFO(("BL->%d del %d off %d", (int) bl, (int) del, (int) off));
-      c->recv.len -= off - bl;
-      // Set message length to indicate we've received
-      // everything, to fire MG_EV_HTTP_MSG
-      hm->message.len = bl + reqlen;
-      hm->body.len = bl;
-      // If user was deleting chunks, send nothing
-      break;
-    }
-  }
-  return false;
-}
-
-static bool mg_is_chunked(struct mg_http_message *hm) {
-  const char *needle = "chunked";
-  struct mg_str *te = mg_http_get_header(hm, "Transfer-Encoding");
-  return te != NULL && mg_vcasecmp(te, needle) == 0;
-}
-
-void mg_http_delete_chunk(struct mg_connection *c, struct mg_http_message *hm) {
-  struct mg_str ch = hm->chunk;
-  const char *end = (char *) &c->recv.buf[c->recv.len], *ce;
-  bool chunked = mg_is_chunked(hm);
-  if (chunked) {
-    ch.len += 4, ch.ptr -= 2;  // \r\n before and after the chunk
-    while (ch.ptr > hm->body.ptr && *ch.ptr != '\n') ch.ptr--, ch.len++;
-  }
-  ce = &ch.ptr[ch.len];
-  if (ce < end) memmove((void *) ch.ptr, ce, (size_t) (end - ce));
-  c->recv.len -= ch.len;
-  if (c->pfn_data != NULL) c->pfn_data = (char *) c->pfn_data - ch.len;
-}
-
 long mg_http_upload(struct mg_connection *c, struct mg_http_message *hm,
                     struct mg_fs *fs, const char *path, size_t max_size) {
   char buf[20] = "0";
@@ -2310,50 +2240,126 @@ int mg_http_status(const struct mg_http_message *hm) {
   return atoi(hm->uri.ptr);
 }
 
+// If a server sends data to the client using chunked encoding, Mongoose strips
+// off the chunking prefix (hex length and \r\n) and suffix (\r\n), appends the
+// stripped data to the body, and fires the MG_EV_HTTP_CHUNK event.  When zero
+// chunk is received, it fires MG_EV_HTTP_MSG, and the body is already have all
+// chunking prefixes/suffixes stripped.
+//
+// If a server sends data without chunked encoding, we also fire MG_EV_CHUNK
+// and MG_EV_HTTP_MSG in the end.
+//
+// We track the total processed body length  in the c->pfn_data,
+// by using void * pointer to store size_t value.
+
+static bool getchunk(struct mg_str s, size_t *prefixlen, size_t *datalen) {
+  size_t i = 0, n;
+  while (i < s.len && s.ptr[i] != '\r' && s.ptr[i] != '\n') i++;
+  n = mg_unhexn(s.ptr, i);
+  // MG_INFO(("%d %d", (int) (i + n + 4), (int) s.len));
+  if (s.len < i + n + 4) return false;  // Chunk not yet fully buffered
+  if (s.ptr[i] != '\r' || s.ptr[i + 1] != '\n') return false;
+  if (s.ptr[i + n + 2] != '\r' || s.ptr[i + n + 3] != '\n') return false;
+  *prefixlen = i + 2;
+  *datalen = n;
+  return true;
+}
+
+static bool mg_is_chunked(struct mg_http_message *hm) {
+  const char *needle = "chunked";
+  struct mg_str *te = mg_http_get_header(hm, "Transfer-Encoding");
+  return te != NULL && mg_vcasecmp(te, needle) == 0;
+}
+
+void mg_http_delete_chunk(struct mg_connection *c, struct mg_http_message *hm) {
+  size_t ofs = (size_t) (hm->chunk.ptr - (char *) c->recv.buf);
+  mg_iobuf_del(&c->recv, ofs, hm->chunk.len);
+  c->pfn_data = (void *) ((size_t) c->pfn_data | MG_DMARK);
+}
+
 static void http_cb(struct mg_connection *c, int ev, void *evd, void *fnd) {
   if (ev == MG_EV_READ || ev == MG_EV_CLOSE) {
     struct mg_http_message hm;
     while (c->recv.buf != NULL && c->recv.len > 0) {
-      int n = mg_http_parse((char *) c->recv.buf, c->recv.len, &hm);
-      bool is_chunked = n > 0 && mg_is_chunked(&hm);
+      int hlen = mg_http_parse((char *) c->recv.buf, c->recv.len, &hm);
+      if (hlen < 0) {
+        mg_error(c, "HTTP parse:\n%.*s", (int) c->recv.len, c->recv.buf);
+        break;
+      }
+      if (hlen == 0) break;  // Request is not buffered yet
       if (ev == MG_EV_CLOSE) {
         hm.message.len = c->recv.len;
         hm.body.len = hm.message.len - (size_t) (hm.body.ptr - hm.message.ptr);
-      } else if (is_chunked && n > 0 && walkchunks(c, &hm, (size_t) n)) {
-        // walkchunks told us to cleanup the request
-        if (n > (int) c->recv.len) n = (int) c->recv.len;
-        mg_iobuf_del(&c->recv, 0, (size_t) n);
-        break;
       }
-      // MG_INFO(("---->%d %d\n%.*s", n, is_chunked, (int) c->recv.len,
-      // c->recv.buf));
-      if (n < 0 && ev == MG_EV_READ) {
-        mg_error(c, "HTTP parse:\n%.*s", (int) c->recv.len, c->recv.buf);
-        break;
-      } else if (n > 0 && (size_t) c->recv.len >= hm.message.len) {
-        mg_call(c, MG_EV_HTTP_MSG, &hm);
-        mg_iobuf_del(&c->recv, 0, hm.message.len);
-      } else {
-        if (n > 0 && !is_chunked) {
-          hm.chunk =
-              mg_str_n((char *) &c->recv.buf[n], c->recv.len - (size_t) n);
-          // Store remaining body length in c->pfn_data
-          if (c->pfn_data == NULL)
-            c->pfn_data = (void *) (hm.message.len - (size_t) n);
+
+      // Deliver MG_EV_HTTP_CHUNK
+      if (mg_is_chunked(&hm)) {
+        //  |  ... headers ... | HEXNUM\r\n ..data.. \r\n | ......
+        //  +------------------+--------------------------+----
+        //  |      hlen        |           chunk1         | ......
+        char *buf = (char *) &c->recv.buf[hlen], *p = buf;
+        size_t len = c->recv.len - (size_t) hlen;
+        size_t processed = ((size_t) c->pfn_data) & ~MG_DMARK;
+        size_t mark, pl, dl, del = 0, ofs = 0;
+        bool last = false;
+        if (processed <= len) len -= processed, buf += processed;
+        while (!last && getchunk(mg_str_n(buf + ofs, len - ofs), &pl, &dl)) {
+          size_t saved = c->recv.len;
+          memmove(p + processed, buf + ofs + pl, dl);
+          // MG_INFO(("P2 [%.*s]", (int) (processed + dl), p));
+          hm.chunk = mg_str_n(p + processed, dl);
           mg_call(c, MG_EV_HTTP_CHUNK, &hm);
-          if (c->pfn_data == NULL) {
-            hm.chunk.len = 0;                   // Last chunk!
-            mg_call(c, MG_EV_HTTP_CHUNK, &hm);  // Lest user know
-            memmove(c->recv.buf, c->recv.buf + n, c->recv.len - (size_t) n);
-            c->recv.len -= (size_t) n;
+          ofs += pl + dl + 2, del += pl + 2;  // 2 is for \r\n suffix
+          processed += dl;
+          if (c->recv.len != saved) processed -= dl, buf -= dl;
+          // mg_hexdump(c->recv.buf, (size_t) hlen + processed);
+          last = (dl == 0);
+        }
+        mg_iobuf_del(&c->recv, (size_t) hlen + processed, del);
+        mark = ((size_t) c->pfn_data) & MG_DMARK;
+        c->pfn_data = (void *) (processed | mark);
+        if (last) {
+          hm.body.len = processed;
+          hm.message.len = (size_t) hlen + processed;
+          c->pfn_data = NULL;
+          if (mark) mg_iobuf_del(&c->recv, 0, (size_t) hlen);
+          // MG_INFO(("LAST"));
+          // mg_hexdump(c->recv.buf, c->recv.len);
+        }
+      } else {
+        size_t processed = ((size_t) c->pfn_data) & ~MG_DMARK;
+        bool deleted = ((size_t) c->pfn_data) & MG_DMARK;
+        // if (processed > hm.body.len) processed = hm.body.len;
+        hm.chunk.ptr = (char *) &c->recv.buf[hlen];
+        hm.chunk.len = c->recv.len - (size_t) hlen;
+        if (processed <= hm.chunk.len && !deleted) {
+          hm.chunk.ptr += processed;
+          hm.chunk.len -= processed;
+          if (hm.chunk.len + processed > hm.body.len) {
+            hm.chunk.len = hm.body.len - processed;
           }
         }
-        break;
+        // MG_INFO(("NCL->%d %d", (int) processed, (int) hm.chunk.len));
+        if (hm.chunk.len) mg_call(c, MG_EV_HTTP_CHUNK, &hm);
+        processed += hm.chunk.len;
+        if (processed >= hm.body.len) {  // Last, 0-len chunk
+          hm.chunk.len = 0;
+          mg_call(c, MG_EV_HTTP_CHUNK, &hm);
+          if (processed && deleted) mg_iobuf_del(&c->recv, 0, (size_t) hlen);
+          c->pfn_data = NULL;
+        } else {
+          // MG_EV_HTTP_CHUNK handler may have set MG_DMARK: do not override
+          size_t del = ((size_t) c->pfn_data) & MG_DMARK;
+          c->pfn_data = (void *) (processed | del);
+        }
       }
+      // Chunk events are delivered. If we have full body, deliver MSG
+      if (c->recv.len < hm.message.len) break;
+      mg_call(c, MG_EV_HTTP_MSG, &hm);
+      mg_iobuf_del(&c->recv, 0, hm.message.len);
     }
   }
-  (void) fnd;
-  (void) evd;
+  (void) evd, (void) fnd;
 }
 
 struct mg_connection *mg_http_connect(struct mg_mgr *mgr, const char *url,

src/http.c

@@ -9,6 +9,9 @@
 #include "version.h"
 #include "ws.h"
 
+// Chunk deletion marker is the MSB in the "processed" counter
+#define MG_DMARK ((size_t) 1 << (sizeof(size_t) * 8 - 1))
+
 // Multipart POST example:
 // --xyz
 // Content-Disposition: form-data; name="val"
@@ -775,79 +778,6 @@ bool mg_http_match_uri(const struct mg_http_message *hm, const char *glob) {
   return mg_match(hm->uri, mg_str(glob), NULL);
 }
 
-static size_t get_chunk_length(const char *buf, size_t len, size_t *ll) {
-  size_t i = 0, n;
-  while (i < len && buf[i] != '\r' && i != '\n') i++;
-  n = mg_unhexn((char *) buf, i);
-  while (i < len && (buf[i] == '\r' || i == '\n')) i++;
-  // MG_INFO(("len %zu i %zu n %zu ", len, i, n));
-  if (ll != NULL) *ll = i + 1;
-  if (i < len && i + n + 2 < len) return i + n + 3;
-  return 0;
-}
-
-// Walk through all chunks in the chunked body. For each chunk, fire
-// an MG_EV_HTTP_CHUNK event.
-static bool walkchunks(struct mg_connection *c, struct mg_http_message *hm,
-                       size_t reqlen) {
-  size_t off = 0, bl, ll;
-  while (off + reqlen < c->recv.len) {
-    char *buf = (char *) &c->recv.buf[reqlen];
-    size_t memo = c->recv.len;
-    size_t cl = get_chunk_length(&buf[off], memo - reqlen - off, &ll);
-    // MG_INFO(("len %u off %u cl %u ll %u", memo - reqlen, off, cl, ll));
-    if (cl == 0) break;
-    hm->chunk = mg_str_n(&buf[off + ll], cl < ll + 2 ? 0 : cl - ll - 2);
-    mg_call(c, MG_EV_HTTP_CHUNK, hm);
-    // Increase offset only if user has not deleted this chunk
-    if (memo == c->recv.len) off += cl;
-    if (ll + 2 >= cl) {
-      // Zero chunk - last one. Prepare body - cut off chunk lengths
-      if (memo != c->recv.len) return true;  // Tell caller to cleanup
-      off = bl = 0;
-      while (off + reqlen < c->recv.len) {
-        char *buf2 = (char *) &c->recv.buf[reqlen];
-        size_t memo2 = c->recv.len;
-        size_t cl2 = get_chunk_length(&buf2[off], memo2 - reqlen - off, &ll);
-        size_t n = cl2 < ll + 2 ? 0 : cl2 - ll - 2;
-        memmove(buf2 + bl, buf2 + off + ll, n);
-        bl += n;
-        off += cl2;
-        if (ll + 2 >= cl2) break;
-      }
-      // MG_INFO(("BL->%d del %d off %d", (int) bl, (int) del, (int) off));
-      c->recv.len -= off - bl;
-      // Set message length to indicate we've received
-      // everything, to fire MG_EV_HTTP_MSG
-      hm->message.len = bl + reqlen;
-      hm->body.len = bl;
-      // If user was deleting chunks, send nothing
-      break;
-    }
-  }
-  return false;
-}
-
-static bool mg_is_chunked(struct mg_http_message *hm) {
-  const char *needle = "chunked";
-  struct mg_str *te = mg_http_get_header(hm, "Transfer-Encoding");
-  return te != NULL && mg_vcasecmp(te, needle) == 0;
-}
-
-void mg_http_delete_chunk(struct mg_connection *c, struct mg_http_message *hm) {
-  struct mg_str ch = hm->chunk;
-  const char *end = (char *) &c->recv.buf[c->recv.len], *ce;
-  bool chunked = mg_is_chunked(hm);
-  if (chunked) {
-    ch.len += 4, ch.ptr -= 2;  // \r\n before and after the chunk
-    while (ch.ptr > hm->body.ptr && *ch.ptr != '\n') ch.ptr--, ch.len++;
-  }
-  ce = &ch.ptr[ch.len];
-  if (ce < end) memmove((void *) ch.ptr, ce, (size_t) (end - ce));
-  c->recv.len -= ch.len;
-  if (c->pfn_data != NULL) c->pfn_data = (char *) c->pfn_data - ch.len;
-}
-
 long mg_http_upload(struct mg_connection *c, struct mg_http_message *hm,
                     struct mg_fs *fs, const char *path, size_t max_size) {
   char buf[20] = "0";
@@ -888,50 +818,126 @@ int mg_http_status(const struct mg_http_message *hm) {
   return atoi(hm->uri.ptr);
 }
 
+// If a server sends data to the client using chunked encoding, Mongoose strips
+// off the chunking prefix (hex length and \r\n) and suffix (\r\n), appends the
+// stripped data to the body, and fires the MG_EV_HTTP_CHUNK event.  When zero
+// chunk is received, it fires MG_EV_HTTP_MSG, and the body is already have all
+// chunking prefixes/suffixes stripped.
+//
+// If a server sends data without chunked encoding, we also fire MG_EV_CHUNK
+// and MG_EV_HTTP_MSG in the end.
+//
+// We track the total processed body length  in the c->pfn_data,
+// by using void * pointer to store size_t value.
+
+static bool getchunk(struct mg_str s, size_t *prefixlen, size_t *datalen) {
+  size_t i = 0, n;
+  while (i < s.len && s.ptr[i] != '\r' && s.ptr[i] != '\n') i++;
+  n = mg_unhexn(s.ptr, i);
+  // MG_INFO(("%d %d", (int) (i + n + 4), (int) s.len));
+  if (s.len < i + n + 4) return false;  // Chunk not yet fully buffered
+  if (s.ptr[i] != '\r' || s.ptr[i + 1] != '\n') return false;
+  if (s.ptr[i + n + 2] != '\r' || s.ptr[i + n + 3] != '\n') return false;
+  *prefixlen = i + 2;
+  *datalen = n;
+  return true;
+}
+
+static bool mg_is_chunked(struct mg_http_message *hm) {
+  const char *needle = "chunked";
+  struct mg_str *te = mg_http_get_header(hm, "Transfer-Encoding");
+  return te != NULL && mg_vcasecmp(te, needle) == 0;
+}
+
+void mg_http_delete_chunk(struct mg_connection *c, struct mg_http_message *hm) {
+  size_t ofs = (size_t) (hm->chunk.ptr - (char *) c->recv.buf);
+  mg_iobuf_del(&c->recv, ofs, hm->chunk.len);
+  c->pfn_data = (void *) ((size_t) c->pfn_data | MG_DMARK);
+}
+
 static void http_cb(struct mg_connection *c, int ev, void *evd, void *fnd) {
   if (ev == MG_EV_READ || ev == MG_EV_CLOSE) {
     struct mg_http_message hm;
     while (c->recv.buf != NULL && c->recv.len > 0) {
-      int n = mg_http_parse((char *) c->recv.buf, c->recv.len, &hm);
-      bool is_chunked = n > 0 && mg_is_chunked(&hm);
+      int hlen = mg_http_parse((char *) c->recv.buf, c->recv.len, &hm);
+      if (hlen < 0) {
+        mg_error(c, "HTTP parse:\n%.*s", (int) c->recv.len, c->recv.buf);
+        break;
+      }
+      if (hlen == 0) break;  // Request is not buffered yet
       if (ev == MG_EV_CLOSE) {
         hm.message.len = c->recv.len;
         hm.body.len = hm.message.len - (size_t) (hm.body.ptr - hm.message.ptr);
-      } else if (is_chunked && n > 0 && walkchunks(c, &hm, (size_t) n)) {
-        // walkchunks told us to cleanup the request
-        if (n > (int) c->recv.len) n = (int) c->recv.len;
-        mg_iobuf_del(&c->recv, 0, (size_t) n);
-        break;
       }
-      // MG_INFO(("---->%d %d\n%.*s", n, is_chunked, (int) c->recv.len,
-      // c->recv.buf));
-      if (n < 0 && ev == MG_EV_READ) {
-        mg_error(c, "HTTP parse:\n%.*s", (int) c->recv.len, c->recv.buf);
-        break;
-      } else if (n > 0 && (size_t) c->recv.len >= hm.message.len) {
-        mg_call(c, MG_EV_HTTP_MSG, &hm);
-        mg_iobuf_del(&c->recv, 0, hm.message.len);
-      } else {
-        if (n > 0 && !is_chunked) {
-          hm.chunk =
-              mg_str_n((char *) &c->recv.buf[n], c->recv.len - (size_t) n);
-          // Store remaining body length in c->pfn_data
-          if (c->pfn_data == NULL)
-            c->pfn_data = (void *) (hm.message.len - (size_t) n);
+
+      // Deliver MG_EV_HTTP_CHUNK
+      if (mg_is_chunked(&hm)) {
+        //  |  ... headers ... | HEXNUM\r\n ..data.. \r\n | ......
+        //  +------------------+--------------------------+----
+        //  |      hlen        |           chunk1         | ......
+        char *buf = (char *) &c->recv.buf[hlen], *p = buf;
+        size_t len = c->recv.len - (size_t) hlen;
+        size_t processed = ((size_t) c->pfn_data) & ~MG_DMARK;
+        size_t mark, pl, dl, del = 0, ofs = 0;
+        bool last = false;
+        if (processed <= len) len -= processed, buf += processed;
+        while (!last && getchunk(mg_str_n(buf + ofs, len - ofs), &pl, &dl)) {
+          size_t saved = c->recv.len;
+          memmove(p + processed, buf + ofs + pl, dl);
+          // MG_INFO(("P2 [%.*s]", (int) (processed + dl), p));
+          hm.chunk = mg_str_n(p + processed, dl);
           mg_call(c, MG_EV_HTTP_CHUNK, &hm);
-          if (c->pfn_data == NULL) {
-            hm.chunk.len = 0;                   // Last chunk!
-            mg_call(c, MG_EV_HTTP_CHUNK, &hm);  // Lest user know
-            memmove(c->recv.buf, c->recv.buf + n, c->recv.len - (size_t) n);
-            c->recv.len -= (size_t) n;
+          ofs += pl + dl + 2, del += pl + 2;  // 2 is for \r\n suffix
+          processed += dl;
+          if (c->recv.len != saved) processed -= dl, buf -= dl;
+          // mg_hexdump(c->recv.buf, (size_t) hlen + processed);
+          last = (dl == 0);
+        }
+        mg_iobuf_del(&c->recv, (size_t) hlen + processed, del);
+        mark = ((size_t) c->pfn_data) & MG_DMARK;
+        c->pfn_data = (void *) (processed | mark);
+        if (last) {
+          hm.body.len = processed;
+          hm.message.len = (size_t) hlen + processed;
+          c->pfn_data = NULL;
+          if (mark) mg_iobuf_del(&c->recv, 0, (size_t) hlen);
+          // MG_INFO(("LAST"));
+          // mg_hexdump(c->recv.buf, c->recv.len);
+        }
+      } else {
+        size_t processed = ((size_t) c->pfn_data) & ~MG_DMARK;
+        bool deleted = ((size_t) c->pfn_data) & MG_DMARK;
+        // if (processed > hm.body.len) processed = hm.body.len;
+        hm.chunk.ptr = (char *) &c->recv.buf[hlen];
+        hm.chunk.len = c->recv.len - (size_t) hlen;
+        if (processed <= hm.chunk.len && !deleted) {
+          hm.chunk.ptr += processed;
+          hm.chunk.len -= processed;
+          if (hm.chunk.len + processed > hm.body.len) {
+            hm.chunk.len = hm.body.len - processed;
           }
         }
-        break;
+        // MG_INFO(("NCL->%d %d", (int) processed, (int) hm.chunk.len));
+        if (hm.chunk.len) mg_call(c, MG_EV_HTTP_CHUNK, &hm);
+        processed += hm.chunk.len;
+        if (processed >= hm.body.len) {  // Last, 0-len chunk
+          hm.chunk.len = 0;
+          mg_call(c, MG_EV_HTTP_CHUNK, &hm);
+          if (processed && deleted) mg_iobuf_del(&c->recv, 0, (size_t) hlen);
+          c->pfn_data = NULL;
+        } else {
+          // MG_EV_HTTP_CHUNK handler may have set MG_DMARK: do not override
+          size_t del = ((size_t) c->pfn_data) & MG_DMARK;
+          c->pfn_data = (void *) (processed | del);
+        }
       }
+      // Chunk events are delivered. If we have full body, deliver MSG
+      if (c->recv.len < hm.message.len) break;
+      mg_call(c, MG_EV_HTTP_MSG, &hm);
+      mg_iobuf_del(&c->recv, 0, hm.message.len);
     }
   }
-  (void) fnd;
-  (void) evd;
+  (void) evd, (void) fnd;
 }
 
 struct mg_connection *mg_http_connect(struct mg_mgr *mgr, const char *url,

test/unit_test.c

@@ -1014,6 +1014,8 @@ static void f4c(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
     ASSERT(mg_strcmp(hm->body, mg_str("/foo/bar/abcdef")) == 0);
     strcat((char *) fn_data, "m");
   } else if (ev == MG_EV_HTTP_CHUNK) {
+    struct mg_http_message *hm = (struct mg_http_message *) ev_data;
+    MG_INFO(("FS [%.*s]", (int) hm->chunk.len, hm->chunk.ptr));
     strcat((char *) fn_data, "f");
   } else if (ev == MG_EV_CLOSE) {
     strcat((char *) fn_data, "c");
@@ -1030,8 +1032,8 @@ static void test_http_no_content_length(void) {
   mg_http_connect(&mgr, url, f4c, (void *) buf2);
   for (i = 0; i < 1000 && strchr(buf2, 'c') == NULL; i++) mg_mgr_poll(&mgr, 10);
   MG_INFO(("[%s] [%s]", buf1, buf2));
-  ASSERT(strcmp(buf1, "mc") == 0);
-  ASSERT(strcmp(buf2, "fcm") == 0);  // See #1475
+  ASSERT(strcmp(buf1, "fmc") == 0);
+  ASSERT(strcmp(buf2, "fcfm") == 0);  // See #1475
   mg_mgr_free(&mgr);
   ASSERT(mgr.conns == NULL);
 }
@@ -1864,39 +1866,41 @@ static void test_http_upload(void) {
 
 #define LONG_CHUNK "chunk with length taking up more than two hex digits"
 
-// Streaming server event handler.
-// Send one chunk immediately, then drain, then send two chunks in one go
-static void eh2(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
+static void eX(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
   if (ev == MG_EV_HTTP_MSG) {
     mg_printf(c, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n");
-    mg_http_printf_chunk(c, LONG_CHUNK);
     c->label[0] = 1;
-  } else if (ev == MG_EV_POLL) {
-    if (c->label[0] > 0 && c->label[0] != 'x') c->label[0]++;
-    if (c->label[0] > 10 && c->label[0] != 'x') {
-      mg_http_printf_chunk(c, "chunk 1");
-      mg_http_printf_chunk(c, "chunk 2");
+    c->is_hexdumping = 1;
+  } else if (ev == MG_EV_POLL && c->label[0] != 0) {
+    c->label[0]++;
+    if (c->label[0] == 10) mg_http_printf_chunk(c, "a");
+    if (c->label[0] == 20) {
+      mg_http_printf_chunk(c, "b");
+      mg_http_printf_chunk(c, "c");
+    }
+    if (c->label[0] == 30) {
+      mg_http_printf_chunk(c, "d");
       mg_http_printf_chunk(c, "");
-      c->label[0] = 'x';
+      c->label[0] = 0;
     }
   }
-  (void) ev_data;
-  (void) fn_data;
+  (void) ev_data, (void) fn_data;
 }
 
-// Non-streaming client event handler. Make sure we've got full body
-static void eh3(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
-  if (ev == MG_EV_CONNECT) {
-    mg_printf(c, "GET / HTTP/1.0\n\n");
-  } else if (ev == MG_EV_HTTP_MSG) {
-    struct mg_http_message *hm = (struct mg_http_message *) ev_data;
-    // MG_INFO(("----> [%.*s]", (int) hm->body.len, hm->body.ptr));
-    c->is_closing = 1;
-    *(uint32_t *) fn_data = mg_crc32(0, hm->body.ptr, hm->body.len);
+static void eY(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
+  if (ev == MG_EV_HTTP_MSG) {
+    mg_printf(c, "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\n");
+    c->label[0] = 1;
+  } else if (ev == MG_EV_POLL && c->label[0] != 0) {
+    c->label[0]++;
+    if (c->label[0] == 10) mg_send(c, "a", 1);
+    if (c->label[0] == 12) mg_send(c, "bc", 2);
+    if (c->label[0] == 30) mg_send(c, "d", 1), c->label[0] = 0;
   }
+  (void) ev_data, (void) fn_data;
 }
 
-// Streaming client event handler. Make sure we've got all chunks
+// Do not delete chunks as they arrive
 static void eh4(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
   uint32_t *crc = (uint32_t *) c->label;
   if (ev == MG_EV_CONNECT) {
@@ -1904,14 +1908,14 @@ static void eh4(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
   } else if (ev == MG_EV_HTTP_CHUNK) {
     struct mg_http_message *hm = (struct mg_http_message *) ev_data;
     *crc = mg_crc32(*crc, hm->chunk.ptr, hm->chunk.len);
+    *crc = mg_crc32(*crc, "x", 1);
+    MG_INFO(("C [%.*s]", (int) hm->chunk.len, hm->chunk.ptr));
   } else if (ev == MG_EV_HTTP_MSG) {
     struct mg_http_message *hm = (struct mg_http_message *) ev_data;
-    *crc = mg_crc32(*crc, hm->body.ptr, hm->body.len);
-    // MG_INFO(("MSG [%.*s]", (int) hm->body.len, hm->body.ptr));
+    *(uint32_t *) fn_data = mg_crc32(*crc, hm->body.ptr, hm->body.len);
+    MG_INFO(("M [%.*s]", (int) hm->body.len, hm->body.ptr));
     c->is_closing = 1;
-    *(uint32_t *) fn_data = *crc;
   }
-  (void) ev_data;
 }
 
 // Streaming client event handler. Delete chunks as they arrive
@@ -1922,88 +1926,44 @@ static void eh5(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
   } else if (ev == MG_EV_HTTP_CHUNK) {
     struct mg_http_message *hm = (struct mg_http_message *) ev_data;
     *crc = mg_crc32(*crc, hm->chunk.ptr, hm->chunk.len);
-    // MG_INFO(("CHUNK [%.*s]", (int) hm->chunk.len, hm->chunk.ptr));
+    *crc = mg_crc32(*crc, "x", 1);
+    MG_INFO(("XC [%.*s]", (int) hm->chunk.len, hm->chunk.ptr));
     mg_http_delete_chunk(c, hm);
     if (hm->chunk.len == 0) {
-      *(uint32_t *) fn_data = *crc;
       c->is_closing = 1;
+      *(uint32_t *) fn_data = *crc;
     }
+  } else if (ev == MG_EV_HTTP_MSG) {
+    *(uint32_t *) fn_data = 77U;  // Must not be here
+    ASSERT(0);
   }
-  (void) ev_data;
+}
+
+static void test_http_chunked_case(mg_event_handler_t s, mg_event_handler_t c,
+                                   const char *expected) {
+  struct mg_mgr mgr;
+  const char *url = "http://127.0.0.1:12344";
+  uint32_t i, done = 0;
+  mg_mgr_init(&mgr);
+  mg_http_listen(&mgr, url, s, NULL);
+  mg_http_connect(&mgr, url, c, &done);
+  for (i = 0; i < 50 && done == 0; i++) {
+    mg_mgr_poll(&mgr, 1);
+  }
+  ASSERT(i < 50);
+  ASSERT(done == mg_crc32(0, expected, strlen(expected)));
+  mg_mgr_free(&mgr);
+  ASSERT(mgr.conns == NULL);
 }
 
 static void test_http_chunked(void) {
-  struct mg_mgr mgr;
-  const char *data, *url = "http://127.0.0.1:12344";
-  uint32_t i, done = 0;
-  mg_mgr_init(&mgr);
-  mg_http_listen(&mgr, url, eh2, NULL);
+  // Non-chunked encoding
+  test_http_chunked_case(eY, eh4, "axbcxdxxabcd");  // Chunks not deleted
+  test_http_chunked_case(eY, eh5, "axbcxdxx");      // Chunks deleted
 
-  mg_http_connect(&mgr, url, eh3, &done);
-  for (i = 0; i < 50 && done == 0; i++) mg_mgr_poll(&mgr, 1);
-  ASSERT(i < 50);
-  data = LONG_CHUNK "chunk 1chunk 2";
-  ASSERT(done == mg_crc32(0, data, strlen(data)));
-
-  done = 0;
-  mg_http_connect(&mgr, url, eh4, &done);
-  for (i = 0; i < 50 && done == 0; i++) mg_mgr_poll(&mgr, 1);
-  data = LONG_CHUNK LONG_CHUNK "chunk 1chunk 2" LONG_CHUNK "chunk 1chunk 2";
-  ASSERT(done == mg_crc32(0, data, strlen(data)));
-
-  done = 0;
-  mg_http_connect(&mgr, url, eh5, &done);
-  for (i = 0; i < 50 && done == 0; i++) mg_mgr_poll(&mgr, 1);
-  data = LONG_CHUNK "chunk 1chunk 2";
-  ASSERT(done == mg_crc32(0, data, strlen(data)));
-
-  mg_mgr_free(&mgr);
-  ASSERT(mgr.conns == NULL);
-}
-
-// Streaming server event handler.
-// Send the header and pause, then send one chunk,
-// then drain, then send two chunks in one go
-static void eh2b(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
-  if (ev == MG_EV_HTTP_MSG) {
-      mg_printf(c, "HTTP/1.1 200 OK\r\nContent-Length: %u\r\n\r\n", strlen(LONG_CHUNK)+7+7);
-    c->label[0] = 1;
-  } else if (ev == MG_EV_POLL) {
-    if (c->label[0] > 0 && c->label[0] != 'x') c->label[0]++;
-    if (c->label[0] == 10) {
-      mg_printf(c, LONG_CHUNK);
-    } else if (c->label[0] > 20 && c->label[0] != 'x') {
-      mg_printf(c, "chunk 1");
-      mg_printf(c, "chunk 2");
-      mg_printf(c, "");
-      c->label[0] = 'x';
-    }
-  }
-  (void) ev_data;
-  (void) fn_data;
-}
-
-static void test_ev_chunk_with_pauses(void) {
-  struct mg_mgr mgr;
-  const char *data, *url = "http://127.0.0.1:12344";
-  uint32_t i, done = 0;
-  mg_mgr_init(&mgr);
-  mg_http_listen(&mgr, url, eh2b, NULL);
-
-  mg_http_connect(&mgr, url, eh3, &done);
-  for (i = 0; i < 50 && done == 0; i++) mg_mgr_poll(&mgr, 1);
-  ASSERT(i < 50);
-  data = LONG_CHUNK "chunk 1chunk 2";
-  ASSERT(done == mg_crc32(0, data, strlen(data)));
-
-  done = 0;
-  mg_http_connect(&mgr, url, eh5, &done);
-  for (i = 0; i < 50 && done == 0; i++) mg_mgr_poll(&mgr, 1);
-  data = LONG_CHUNK "chunk 1chunk 2";
-  ASSERT(done == mg_crc32(0, data, strlen(data)));
-
-  mg_mgr_free(&mgr);
-  ASSERT(mgr.conns == NULL);
+  // Chunked encoding
+  test_http_chunked_case(eX, eh4, "axbxcxdxxabcd");  // Chunks not deleted
+  test_http_chunked_case(eX, eh5, "axbxcxdxx");      // Chunks deleted
 }
 
 static void test_invalid_listen_addr(void) {
@@ -2161,7 +2121,7 @@ static void test_packed(void) {
   mg_http_listen(&mgr, url, eh7, NULL);
 
   // Load top level file directly
-  fetch(&mgr, buf, url, "GET /Makefile HTTP/1.0\n\n");
+  // fetch(&mgr, buf, url, "GET /Makefile HTTP/1.0\n\n");
   // printf("---> %s\n", buf);
   ASSERT(fetch(&mgr, buf, url, "GET /Makefile HTTP/1.0\n\n") == 200);
   ASSERT(cmpbody(buf, data) == 0);
@@ -2553,7 +2513,6 @@ int main(void) {
   test_multipart();
   test_invalid_listen_addr();
   test_http_chunked();
-  test_ev_chunk_with_pauses();
   test_http_upload();
   test_http_stream_buffer();
   test_http_parse();