From c438bb0bc5c4cac8870d1bd32356a1de98fb694e Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Tue, 9 Oct 2018 12:44:32 +0100 Subject: [PATCH 01/25] Make mg_rpc more easily portable CL: none PUBLISHED_FROM=2e77d72cdf82454349a7fe29cc77bd0b90ac4369 --- mongoose.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoose.c b/mongoose.c index 3174523a..fd390088 100644 --- a/mongoose.c +++ b/mongoose.c @@ -1645,6 +1645,7 @@ void mbuf_remove(struct mbuf *mb, size_t n) { /* Amalgamated: #include "common/mg_str.h" */ /* Amalgamated: #include "common/platform.h" */ +#include #include #include From 220231e64763a75f46abc764a90c5314a8a9c7d8 Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Mon, 15 Oct 2018 16:16:14 +0100 Subject: [PATCH 02/25] Fix digest auth result comparison Should use length of the exected response, not the response sent by the user. CL: mg: Fix digest auth result comparison PUBLISHED_FROM=23375133b8babac0b276b51395917ad975e1bd3c --- mongoose.c | 11 +++++------ src/mg_http.c | 11 +++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/mongoose.c b/mongoose.c index fd390088..8b58d2e9 100644 --- a/mongoose.c +++ b/mongoose.c @@ -7668,7 +7668,7 @@ int mg_check_digest_auth(struct mg_str method, struct mg_str uri, struct mg_str nc, struct mg_str nonce, struct mg_str auth_domain, FILE *fp) { char buf[128], f_user[sizeof(buf)], f_ha1[sizeof(buf)], f_domain[sizeof(buf)]; - char expected_response[33]; + char exp_resp[33]; /* * Read passwords file line by line. If should have htdigest format, @@ -7682,11 +7682,10 @@ int mg_check_digest_auth(struct mg_str method, struct mg_str uri, /* Username and domain matched, check the password */ mg_mkmd5resp(method.p, method.len, uri.p, uri.len, f_ha1, strlen(f_ha1), nonce.p, nonce.len, nc.p, nc.len, cnonce.p, cnonce.len, - qop.p, qop.len, expected_response); - LOG(LL_DEBUG, - ("%.*s %s %.*s %s", (int) username.len, username.p, f_domain, - (int) response.len, response.p, expected_response)); - return mg_ncasecmp(response.p, expected_response, response.len) == 0; + qop.p, qop.len, exp_resp); + LOG(LL_DEBUG, ("%.*s %s %.*s %s", (int) username.len, username.p, + f_domain, (int) response.len, response.p, exp_resp)); + return mg_ncasecmp(response.p, exp_resp, strlen(exp_resp)) == 0; } } diff --git a/src/mg_http.c b/src/mg_http.c index f8161a4b..3fc497b0 100644 --- a/src/mg_http.c +++ b/src/mg_http.c @@ -1923,7 +1923,7 @@ int mg_check_digest_auth(struct mg_str method, struct mg_str uri, struct mg_str nc, struct mg_str nonce, struct mg_str auth_domain, FILE *fp) { char buf[128], f_user[sizeof(buf)], f_ha1[sizeof(buf)], f_domain[sizeof(buf)]; - char expected_response[33]; + char exp_resp[33]; /* * Read passwords file line by line. If should have htdigest format, @@ -1937,11 +1937,10 @@ int mg_check_digest_auth(struct mg_str method, struct mg_str uri, /* Username and domain matched, check the password */ mg_mkmd5resp(method.p, method.len, uri.p, uri.len, f_ha1, strlen(f_ha1), nonce.p, nonce.len, nc.p, nc.len, cnonce.p, cnonce.len, - qop.p, qop.len, expected_response); - LOG(LL_DEBUG, - ("%.*s %s %.*s %s", (int) username.len, username.p, f_domain, - (int) response.len, response.p, expected_response)); - return mg_ncasecmp(response.p, expected_response, response.len) == 0; + qop.p, qop.len, exp_resp); + LOG(LL_DEBUG, ("%.*s %s %.*s %s", (int) username.len, username.p, + f_domain, (int) response.len, response.p, exp_resp)); + return mg_ncasecmp(response.p, exp_resp, strlen(exp_resp)) == 0; } } From 91b57aa9a869bbc2a9247735a2306538ce2ffdbe Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Mon, 29 Oct 2018 14:42:32 +0000 Subject: [PATCH 03/25] Add MG_MK_STR_N() and mg_str_starts_with() CL: Add MG_MK_STR_N() and mg_str_starts_with() PUBLISHED_FROM=f01003efc6aa6ff45fa99c7ad6e5319f5002f22d --- mongoose.c | 7 +++++++ mongoose.h | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/mongoose.c b/mongoose.c index 8b58d2e9..7c4e691d 100644 --- a/mongoose.c +++ b/mongoose.c @@ -1770,6 +1770,13 @@ struct mg_str mg_strstrip(struct mg_str s) { } return s; } + +int mg_str_starts_with(struct mg_str s, struct mg_str prefix) WEAK; +int mg_str_starts_with(struct mg_str s, struct mg_str prefix) { + const struct mg_str sp = MG_MK_STR_N(s.p, prefix.len); + if (s.len < prefix.len) return 0; + return (mg_strcmp(sp, prefix) == 0); +} #ifdef MG_MODULE_LINES #line 1 "common/str_util.c" #endif diff --git a/mongoose.h b/mongoose.h index 8fe13cd6..4b5b0db6 100644 --- a/mongoose.h +++ b/mongoose.h @@ -2239,6 +2239,8 @@ struct mg_str mg_mk_str_n(const char *s, size_t len); /* Macro for initializing mg_str. */ #define MG_MK_STR(str_literal) \ { str_literal, sizeof(str_literal) - 1 } +#define MG_MK_STR_N(str_literal, len) \ + { str_literal, len } #define MG_NULL_STR \ { NULL, 0 } @@ -2286,6 +2288,9 @@ const char *mg_strstr(const struct mg_str haystack, const struct mg_str needle); /* Strip whitespace at the start and the end of s */ struct mg_str mg_strstrip(struct mg_str s); +/* Returns 1 if s starts with the given prefix. */ +int mg_str_starts_with(struct mg_str s, struct mg_str prefix); + #ifdef __cplusplus } #endif From 93286836f944cf71c0f344eab8c1b0a29666f82a Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Fri, 2 Nov 2018 14:32:29 +0000 Subject: [PATCH 04/25] Read all available data during recv poll CL: mg: Read all available data during recv poll PUBLISHED_FROM=b081025b9fb392c988924c7097b5333c38053f6f --- mongoose.c | 26 ++++++++++++++------------ src/mg_net.c | 26 ++++++++++++++------------ 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/mongoose.c b/mongoose.c index 7c4e691d..75332c61 100644 --- a/mongoose.c +++ b/mongoose.c @@ -2882,18 +2882,20 @@ static int mg_do_recv(struct mg_connection *nc) { ((nc->flags & MG_F_LISTENING) && !(nc->flags & MG_F_UDP))) { return -1; } - len = recv_avail_size(nc, len); - if (len == 0) return -2; - if (nc->recv_mbuf.size < nc->recv_mbuf.len + len) { - mbuf_resize(&nc->recv_mbuf, nc->recv_mbuf.len + len); - } - buf = nc->recv_mbuf.buf + nc->recv_mbuf.len; - len = nc->recv_mbuf.size - nc->recv_mbuf.len; - if (nc->flags & MG_F_UDP) { - res = mg_recv_udp(nc, buf, len); - } else { - res = mg_recv_tcp(nc, buf, len); - } + do { + len = recv_avail_size(nc, len); + if (len == 0) return -2; + if (nc->recv_mbuf.size < nc->recv_mbuf.len + len) { + mbuf_resize(&nc->recv_mbuf, nc->recv_mbuf.len + len); + } + buf = nc->recv_mbuf.buf + nc->recv_mbuf.len; + len = nc->recv_mbuf.size - nc->recv_mbuf.len; + if (nc->flags & MG_F_UDP) { + res = mg_recv_udp(nc, buf, len); + } else { + res = mg_recv_tcp(nc, buf, len); + } + } while (res > 0); return res; } diff --git a/src/mg_net.c b/src/mg_net.c index 97afe9ab..809895c6 100644 --- a/src/mg_net.c +++ b/src/mg_net.c @@ -565,18 +565,20 @@ static int mg_do_recv(struct mg_connection *nc) { ((nc->flags & MG_F_LISTENING) && !(nc->flags & MG_F_UDP))) { return -1; } - len = recv_avail_size(nc, len); - if (len == 0) return -2; - if (nc->recv_mbuf.size < nc->recv_mbuf.len + len) { - mbuf_resize(&nc->recv_mbuf, nc->recv_mbuf.len + len); - } - buf = nc->recv_mbuf.buf + nc->recv_mbuf.len; - len = nc->recv_mbuf.size - nc->recv_mbuf.len; - if (nc->flags & MG_F_UDP) { - res = mg_recv_udp(nc, buf, len); - } else { - res = mg_recv_tcp(nc, buf, len); - } + do { + len = recv_avail_size(nc, len); + if (len == 0) return -2; + if (nc->recv_mbuf.size < nc->recv_mbuf.len + len) { + mbuf_resize(&nc->recv_mbuf, nc->recv_mbuf.len + len); + } + buf = nc->recv_mbuf.buf + nc->recv_mbuf.len; + len = nc->recv_mbuf.size - nc->recv_mbuf.len; + if (nc->flags & MG_F_UDP) { + res = mg_recv_udp(nc, buf, len); + } else { + res = mg_recv_tcp(nc, buf, len); + } + } while (res > 0); return res; } From 3130e592443dfad81ceac63b4af9976b9804ca21 Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Fri, 2 Nov 2018 17:31:58 +0000 Subject: [PATCH 05/25] Stop reading if connection is closing CL: none PUBLISHED_FROM=3b74ca02e2960bbbf130f6d95aef679f2917e824 --- mongoose.c | 6 ++++-- src/mg_net.c | 6 ++++-- test/unit_test.c | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/mongoose.c b/mongoose.c index 75332c61..76f35fe3 100644 --- a/mongoose.c +++ b/mongoose.c @@ -2895,7 +2895,7 @@ static int mg_do_recv(struct mg_connection *nc) { } else { res = mg_recv_tcp(nc, buf, len); } - } while (res > 0); + } while (res > 0 && !(nc->flags & (MG_F_CLOSE_IMMEDIATELY | MG_F_UDP))); return res; } @@ -3012,7 +3012,9 @@ static int mg_recv_udp(struct mg_connection *nc, char *buf, size_t len) { mg_hexdump_connection(nc, nc->mgr->hexdump_file, buf, n, MG_EV_RECV); } #endif - mg_call(nc, NULL, nc->user_data, MG_EV_RECV, &n); + if (n != 0) { + mg_call(nc, NULL, nc->user_data, MG_EV_RECV, &n); + } } out: diff --git a/src/mg_net.c b/src/mg_net.c index 809895c6..ebf0513f 100644 --- a/src/mg_net.c +++ b/src/mg_net.c @@ -578,7 +578,7 @@ static int mg_do_recv(struct mg_connection *nc) { } else { res = mg_recv_tcp(nc, buf, len); } - } while (res > 0); + } while (res > 0 && !(nc->flags & (MG_F_CLOSE_IMMEDIATELY | MG_F_UDP))); return res; } @@ -695,7 +695,9 @@ static int mg_recv_udp(struct mg_connection *nc, char *buf, size_t len) { mg_hexdump_connection(nc, nc->mgr->hexdump_file, buf, n, MG_EV_RECV); } #endif - mg_call(nc, NULL, nc->user_data, MG_EV_RECV, &n); + if (n != 0) { + mg_call(nc, NULL, nc->user_data, MG_EV_RECV, &n); + } } out: diff --git a/test/unit_test.c b/test/unit_test.c index cde8df8a..114d20f3 100644 --- a/test/unit_test.c +++ b/test/unit_test.c @@ -5665,7 +5665,7 @@ static const char *test_socks(void) { mbuf_resize(&c->recv_mbuf, 10000000); /* Run event loop. Use more cycles to let file download complete. */ - poll_until(&mgr, 10, c_str_ne, status, (void *) ""); + poll_until(&mgr, 15, c_str_ne, status, (void *) ""); ASSERT_STREQ(status, "success"); mg_mgr_free(&mgr); From 15b361fbac9b6a640116b25faad721085ac3adbe Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Fri, 23 Nov 2018 14:54:44 +0000 Subject: [PATCH 06/25] Add mbuf_clear() PUBLISHED_FROM=754b98b3000b24ad3b3d9cc0b4c480ca005a1c3f --- docs/c-api/mbuf.h/intro.md | 1 + docs/c-api/mbuf.h/mbuf_clear.md | 10 ++++++++++ mongoose.c | 5 +++++ mongoose.h | 3 +++ 4 files changed, 19 insertions(+) create mode 100644 docs/c-api/mbuf.h/mbuf_clear.md diff --git a/docs/c-api/mbuf.h/intro.md b/docs/c-api/mbuf.h/intro.md index 056369a2..5779ab9c 100644 --- a/docs/c-api/mbuf.h/intro.md +++ b/docs/c-api/mbuf.h/intro.md @@ -4,6 +4,7 @@ symbol_kind: "intro" decl_name: "mbuf.h" items: - { name: mbuf_append.md } + - { name: mbuf_clear.md } - { name: mbuf_free.md } - { name: mbuf_init.md } - { name: mbuf_insert.md } diff --git a/docs/c-api/mbuf.h/mbuf_clear.md b/docs/c-api/mbuf.h/mbuf_clear.md new file mode 100644 index 00000000..afefc21a --- /dev/null +++ b/docs/c-api/mbuf.h/mbuf_clear.md @@ -0,0 +1,10 @@ +--- +title: "mbuf_clear()" +decl_name: "mbuf_clear" +symbol_kind: "func" +signature: | + void mbuf_clear(struct mbuf *); +--- + +Removes all the data from mbuf (if any). + diff --git a/mongoose.c b/mongoose.c index 76f35fe3..d3cd2bd5 100644 --- a/mongoose.c +++ b/mongoose.c @@ -1620,6 +1620,11 @@ void mbuf_remove(struct mbuf *mb, size_t n) { } } +void mbuf_clear(struct mbuf *mb) WEAK; +void mbuf_clear(struct mbuf *mb) { + mb->len = 0; +} + #endif /* EXCLUDE_COMMON */ #ifdef MG_MODULE_LINES #line 1 "common/mg_str.c" diff --git a/mongoose.h b/mongoose.h index 4b5b0db6..07cf98a1 100644 --- a/mongoose.h +++ b/mongoose.h @@ -2388,6 +2388,9 @@ void mbuf_remove(struct mbuf *, size_t data_size); */ void mbuf_resize(struct mbuf *, size_t new_size); +/* Removes all the data from mbuf (if any). */ +void mbuf_clear(struct mbuf *); + /* Shrinks an Mbuf by resizing its `size` to `len`. */ void mbuf_trim(struct mbuf *); From 139c2061a8707d7e042134afb8b2fd6f5130b709 Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Wed, 28 Nov 2018 19:44:11 +0000 Subject: [PATCH 07/25] Add create-fw-bundle command Replacement for fw_meta.py commands and also supports unbundling .hex files. General syntax is: ``` $ mos create-fw-bundle -o fw.zip part1:prop1=val1,prop2=val2,... part2:... ``` Example of building a mos fw bundle for ESP8266: ``` $ mos create-fw-bundle -o fw.zip --platform esp8266 --build-info=build_info.json \ boot:addr=0x0,src=rboot.bin,update=false \ boot_cfg:addr=0x7000,size=0x1000,fill=0xff \ fw:addr=0x100000,src=demo-c.bin \ fs:src=fs.bin,type=fs,addr=0x8000,fs_size=262144,fs_block_size=4096,fs_page_size=256,fs_erase_size=4096 ``` Example of converting a HEX file to a bundle: ``` $ mos create-fw-bundle -o fw.zip host:type=host,src=hostfw.hex ``` CL: mos: Add create-fw-bundle command PUBLISHED_FROM=35f52ea53e4b5900463415257323a4c08cf45deb --- src/common/platforms/stm32/flasher.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/platforms/stm32/flasher.go b/src/common/platforms/stm32/flasher.go index 8ddde035..36cbad11 100644 --- a/src/common/platforms/stm32/flasher.go +++ b/src/common/platforms/stm32/flasher.go @@ -11,7 +11,7 @@ import ( "time" "unsafe" - "cesanta.com/mos/flash/common" + "cesanta.com/common/go/fwbundle" "github.com/cesanta/errors" ) @@ -20,7 +20,7 @@ type FlashOpts struct { Timeout time.Duration } -func Flash(fw *common.FirmwareBundle, opts *FlashOpts) error { +func Flash(fw *fwbundle.FirmwareBundle, opts *FlashOpts) error { data, err := fw.GetPartData("boot") if err != nil { return errors.Annotatef(err, "invalid manifest") From 53e1c469a2ef5632e31ec5a8dabc2c154f214d11 Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Wed, 5 Dec 2018 15:06:28 +0000 Subject: [PATCH 08/25] Add ability for multipart data handler to provide pushback It can specify how much data was actually processed and the rest will be re-delivered on next poll. CL: mg: Add ability for multipart data handler to provide pushback PUBLISHED_FROM=e0168c5064c3a32921c9209bc09f4da2079cd616 --- .../struct_mg_http_multipart_part.md | 8 +++ mongoose.c | 59 +++++++++++++------ mongoose.h | 8 +++ src/mg_http.c | 59 +++++++++++++------ src/mg_http.h | 8 +++ 5 files changed, 104 insertions(+), 38 deletions(-) diff --git a/docs/c-api/mg_http.h/struct_mg_http_multipart_part.md b/docs/c-api/mg_http.h/struct_mg_http_multipart_part.md index cd2d0c2e..ec65f9d9 100644 --- a/docs/c-api/mg_http.h/struct_mg_http_multipart_part.md +++ b/docs/c-api/mg_http.h/struct_mg_http_multipart_part.md @@ -9,6 +9,14 @@ signature: | struct mg_str data; int status; /* <0 on error */ void *user_data; + /* + * User handler can indicate how much of the data was consumed + * by setting this variable. By default, it is assumed that all + * data has been consumed by the handler. + * If not all data was consumed, user's handler will be invoked again later + * with the remainder. + */ + size_t num_data_consumed; }; --- diff --git a/mongoose.c b/mongoose.c index d3cd2bd5..3b0faf95 100644 --- a/mongoose.c +++ b/mongoose.c @@ -5901,6 +5901,7 @@ struct mg_http_multipart_stream { void *user_data; enum mg_http_multipart_stream_state state; int processing_part; + int data_avail; }; struct mg_reverse_proxy_data { @@ -6529,6 +6530,9 @@ void mg_http_handler(struct mg_connection *nc, int ev, mg_http_call_endpoint_handler(nc, ev2, hm); } pd->rcvd = 0; + if (pd->endpoint_handler != NULL && pd->endpoint_handler != nc->handler) { + mg_call(nc, pd->endpoint_handler, nc->user_data, ev, NULL); + } } #if MG_ENABLE_FILESYSTEM @@ -6539,17 +6543,24 @@ void mg_http_handler(struct mg_connection *nc, int ev, mg_call(nc, nc->handler, nc->user_data, ev, ev_data); +#if MG_ENABLE_HTTP_STREAMING_MULTIPART + if (pd->mp_stream.boundary != NULL && + (ev == MG_EV_RECV || ev == MG_EV_POLL)) { + if (ev == MG_EV_RECV) { + pd->rcvd += *(int *) ev_data; + mg_http_multipart_continue(nc); + } else if (pd->mp_stream.data_avail) { + /* Try re-delivering the data. */ + mg_http_multipart_continue(nc); + } + return; + } +#endif /* MG_ENABLE_HTTP_STREAMING_MULTIPART */ + if (ev == MG_EV_RECV) { struct mg_str *s; pd->rcvd += *(int *) ev_data; -#if MG_ENABLE_HTTP_STREAMING_MULTIPART - if (pd->mp_stream.boundary != NULL) { - mg_http_multipart_continue(nc); - return; - } -#endif /* MG_ENABLE_HTTP_STREAMING_MULTIPART */ - again: req_len = mg_parse_http(io->buf, io->len, hm, is_req); @@ -6744,8 +6755,9 @@ exit_mp: #define CONTENT_DISPOSITION "Content-Disposition: " -static void mg_http_multipart_call_handler(struct mg_connection *c, int ev, - const char *data, size_t data_len) { +static size_t mg_http_multipart_call_handler(struct mg_connection *c, int ev, + const char *data, + size_t data_len) { struct mg_http_multipart_part mp; struct mg_http_proto_data *pd = mg_http_get_proto_data(c); memset(&mp, 0, sizeof(mp)); @@ -6755,8 +6767,11 @@ static void mg_http_multipart_call_handler(struct mg_connection *c, int ev, mp.user_data = pd->mp_stream.user_data; mp.data.p = data; mp.data.len = data_len; + mp.num_data_consumed = data_len; mg_call(c, pd->endpoint_handler, c->user_data, ev, &mp); pd->mp_stream.user_data = mp.user_data; + pd->mp_stream.data_avail = (mp.num_data_consumed != data_len); + return mp.num_data_consumed; } static int mg_http_multipart_finalize(struct mg_connection *c) { @@ -6894,19 +6909,25 @@ static int mg_http_multipart_continue_wait_for_chunk(struct mg_connection *c) { boundary = c_strnstr(io->buf, pd->mp_stream.boundary, io->len); if (boundary == NULL) { - int data_size = (io->len - (pd->mp_stream.boundary_len + 6)); - if (data_size > 0) { - mg_http_multipart_call_handler(c, MG_EV_HTTP_PART_DATA, io->buf, - data_size); - mbuf_remove(io, data_size); + int data_len = (io->len - (pd->mp_stream.boundary_len + 6)); + if (data_len > 0) { + size_t consumed = mg_http_multipart_call_handler( + c, MG_EV_HTTP_PART_DATA, io->buf, (size_t) data_len); + mbuf_remove(io, consumed); } return 0; } else if (boundary != NULL) { - int data_size = (boundary - io->buf - 4); - mg_http_multipart_call_handler(c, MG_EV_HTTP_PART_DATA, io->buf, data_size); - mbuf_remove(io, (boundary - io->buf)); - pd->mp_stream.state = MPS_WAITING_FOR_BOUNDARY; - return 1; + size_t data_len = ((size_t)(boundary - io->buf) - 4); + size_t consumed = mg_http_multipart_call_handler(c, MG_EV_HTTP_PART_DATA, + io->buf, data_len); + mbuf_remove(io, consumed); + if (consumed == data_len) { + mbuf_remove(io, 4); + pd->mp_stream.state = MPS_WAITING_FOR_BOUNDARY; + return 1; + } else { + return 0; + } } else { return 0; } diff --git a/mongoose.h b/mongoose.h index 07cf98a1..85d62e3d 100644 --- a/mongoose.h +++ b/mongoose.h @@ -4761,6 +4761,14 @@ struct mg_http_multipart_part { struct mg_str data; int status; /* <0 on error */ void *user_data; + /* + * User handler can indicate how much of the data was consumed + * by setting this variable. By default, it is assumed that all + * data has been consumed by the handler. + * If not all data was consumed, user's handler will be invoked again later + * with the remainder. + */ + size_t num_data_consumed; }; /* SSI call context */ diff --git a/src/mg_http.c b/src/mg_http.c index 3fc497b0..460277cd 100644 --- a/src/mg_http.c +++ b/src/mg_http.c @@ -140,6 +140,7 @@ struct mg_http_multipart_stream { void *user_data; enum mg_http_multipart_stream_state state; int processing_part; + int data_avail; }; struct mg_reverse_proxy_data { @@ -768,6 +769,9 @@ void mg_http_handler(struct mg_connection *nc, int ev, mg_http_call_endpoint_handler(nc, ev2, hm); } pd->rcvd = 0; + if (pd->endpoint_handler != NULL && pd->endpoint_handler != nc->handler) { + mg_call(nc, pd->endpoint_handler, nc->user_data, ev, NULL); + } } #if MG_ENABLE_FILESYSTEM @@ -778,17 +782,24 @@ void mg_http_handler(struct mg_connection *nc, int ev, mg_call(nc, nc->handler, nc->user_data, ev, ev_data); +#if MG_ENABLE_HTTP_STREAMING_MULTIPART + if (pd->mp_stream.boundary != NULL && + (ev == MG_EV_RECV || ev == MG_EV_POLL)) { + if (ev == MG_EV_RECV) { + pd->rcvd += *(int *) ev_data; + mg_http_multipart_continue(nc); + } else if (pd->mp_stream.data_avail) { + /* Try re-delivering the data. */ + mg_http_multipart_continue(nc); + } + return; + } +#endif /* MG_ENABLE_HTTP_STREAMING_MULTIPART */ + if (ev == MG_EV_RECV) { struct mg_str *s; pd->rcvd += *(int *) ev_data; -#if MG_ENABLE_HTTP_STREAMING_MULTIPART - if (pd->mp_stream.boundary != NULL) { - mg_http_multipart_continue(nc); - return; - } -#endif /* MG_ENABLE_HTTP_STREAMING_MULTIPART */ - again: req_len = mg_parse_http(io->buf, io->len, hm, is_req); @@ -983,8 +994,9 @@ exit_mp: #define CONTENT_DISPOSITION "Content-Disposition: " -static void mg_http_multipart_call_handler(struct mg_connection *c, int ev, - const char *data, size_t data_len) { +static size_t mg_http_multipart_call_handler(struct mg_connection *c, int ev, + const char *data, + size_t data_len) { struct mg_http_multipart_part mp; struct mg_http_proto_data *pd = mg_http_get_proto_data(c); memset(&mp, 0, sizeof(mp)); @@ -994,8 +1006,11 @@ static void mg_http_multipart_call_handler(struct mg_connection *c, int ev, mp.user_data = pd->mp_stream.user_data; mp.data.p = data; mp.data.len = data_len; + mp.num_data_consumed = data_len; mg_call(c, pd->endpoint_handler, c->user_data, ev, &mp); pd->mp_stream.user_data = mp.user_data; + pd->mp_stream.data_avail = (mp.num_data_consumed != data_len); + return mp.num_data_consumed; } static int mg_http_multipart_finalize(struct mg_connection *c) { @@ -1133,19 +1148,25 @@ static int mg_http_multipart_continue_wait_for_chunk(struct mg_connection *c) { boundary = c_strnstr(io->buf, pd->mp_stream.boundary, io->len); if (boundary == NULL) { - int data_size = (io->len - (pd->mp_stream.boundary_len + 6)); - if (data_size > 0) { - mg_http_multipart_call_handler(c, MG_EV_HTTP_PART_DATA, io->buf, - data_size); - mbuf_remove(io, data_size); + int data_len = (io->len - (pd->mp_stream.boundary_len + 6)); + if (data_len > 0) { + size_t consumed = mg_http_multipart_call_handler( + c, MG_EV_HTTP_PART_DATA, io->buf, (size_t) data_len); + mbuf_remove(io, consumed); } return 0; } else if (boundary != NULL) { - int data_size = (boundary - io->buf - 4); - mg_http_multipart_call_handler(c, MG_EV_HTTP_PART_DATA, io->buf, data_size); - mbuf_remove(io, (boundary - io->buf)); - pd->mp_stream.state = MPS_WAITING_FOR_BOUNDARY; - return 1; + size_t data_len = ((size_t)(boundary - io->buf) - 4); + size_t consumed = mg_http_multipart_call_handler(c, MG_EV_HTTP_PART_DATA, + io->buf, data_len); + mbuf_remove(io, consumed); + if (consumed == data_len) { + mbuf_remove(io, 4); + pd->mp_stream.state = MPS_WAITING_FOR_BOUNDARY; + return 1; + } else { + return 0; + } } else { return 0; } diff --git a/src/mg_http.h b/src/mg_http.h index c89d38f3..32849058 100644 --- a/src/mg_http.h +++ b/src/mg_http.h @@ -80,6 +80,14 @@ struct mg_http_multipart_part { struct mg_str data; int status; /* <0 on error */ void *user_data; + /* + * User handler can indicate how much of the data was consumed + * by setting this variable. By default, it is assumed that all + * data has been consumed by the handler. + * If not all data was consumed, user's handler will be invoked again later + * with the remainder. + */ + size_t num_data_consumed; }; /* SSI call context */ From 9a70abef2428d1b2e35cfbd09522ec7bb78d0c44 Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Wed, 5 Dec 2018 20:46:00 +0000 Subject: [PATCH 09/25] Add mbuf_append_and_free() and mbuf_move() CL: Add mbuf_append_and_free() and mbuf_move() PUBLISHED_FROM=bdf73906dd93b2ebeace160596508e15480b54b4 --- docs/c-api/mbuf.h/intro.md | 2 ++ docs/c-api/mbuf.h/mbuf_append_and_free.md | 13 +++++++++++++ docs/c-api/mbuf.h/mbuf_move.md | 10 ++++++++++ mongoose.c | 22 ++++++++++++++++++++++ mongoose.h | 11 +++++++++++ 5 files changed, 58 insertions(+) create mode 100644 docs/c-api/mbuf.h/mbuf_append_and_free.md create mode 100644 docs/c-api/mbuf.h/mbuf_move.md diff --git a/docs/c-api/mbuf.h/intro.md b/docs/c-api/mbuf.h/intro.md index 5779ab9c..600a1e09 100644 --- a/docs/c-api/mbuf.h/intro.md +++ b/docs/c-api/mbuf.h/intro.md @@ -4,10 +4,12 @@ symbol_kind: "intro" decl_name: "mbuf.h" items: - { name: mbuf_append.md } + - { name: mbuf_append_and_free.md } - { name: mbuf_clear.md } - { name: mbuf_free.md } - { name: mbuf_init.md } - { name: mbuf_insert.md } + - { name: mbuf_move.md } - { name: mbuf_remove.md } - { name: mbuf_resize.md } - { name: mbuf_trim.md } diff --git a/docs/c-api/mbuf.h/mbuf_append_and_free.md b/docs/c-api/mbuf.h/mbuf_append_and_free.md new file mode 100644 index 00000000..e7be33ac --- /dev/null +++ b/docs/c-api/mbuf.h/mbuf_append_and_free.md @@ -0,0 +1,13 @@ +--- +title: "mbuf_append_and_free()" +decl_name: "mbuf_append_and_free" +symbol_kind: "func" +signature: | + size_t mbuf_append_and_free(struct mbuf *, void *data, size_t data_size); +--- + +Appends data to the Mbuf and frees it (data must be heap-allocated). + +Returns the number of bytes appended or 0 if out of memory. +data is freed irrespective of return value. + diff --git a/docs/c-api/mbuf.h/mbuf_move.md b/docs/c-api/mbuf.h/mbuf_move.md new file mode 100644 index 00000000..61e20853 --- /dev/null +++ b/docs/c-api/mbuf.h/mbuf_move.md @@ -0,0 +1,10 @@ +--- +title: "mbuf_move()" +decl_name: "mbuf_move" +symbol_kind: "func" +signature: | + void mbuf_move(struct mbuf *from, struct mbuf *to); +--- + +Moves the state from one mbuf to the other. + diff --git a/mongoose.c b/mongoose.c index 3b0faf95..ffa97c9b 100644 --- a/mongoose.c +++ b/mongoose.c @@ -1612,6 +1612,22 @@ size_t mbuf_append(struct mbuf *a, const void *buf, size_t len) { return mbuf_insert(a, a->len, buf, len); } +size_t mbuf_append_and_free(struct mbuf *a, void *buf, size_t len) WEAK; +size_t mbuf_append_and_free(struct mbuf *a, void *data, size_t len) { + size_t ret; + /* Optimization: if the buffer is currently empty, + * take over the user-provided buffer. */ + if (a->len == 0) { + if (a->buf != NULL) free(a->buf); + a->buf = (char *) data; + a->len = a->size = len; + return len; + } + ret = mbuf_insert(a, a->len, data, len); + free(data); + return ret; +} + void mbuf_remove(struct mbuf *mb, size_t n) WEAK; void mbuf_remove(struct mbuf *mb, size_t n) { if (n > 0 && n <= mb->len) { @@ -1625,6 +1641,12 @@ void mbuf_clear(struct mbuf *mb) { mb->len = 0; } +void mbuf_move(struct mbuf *from, struct mbuf *to) WEAK; +void mbuf_move(struct mbuf *from, struct mbuf *to) { + memcpy(to, from, sizeof(*to)); + memset(from, 0, sizeof(*from)); +} + #endif /* EXCLUDE_COMMON */ #ifdef MG_MODULE_LINES #line 1 "common/mg_str.c" diff --git a/mongoose.h b/mongoose.h index 85d62e3d..8d9e8f29 100644 --- a/mongoose.h +++ b/mongoose.h @@ -2368,6 +2368,14 @@ void mbuf_free(struct mbuf *); */ size_t mbuf_append(struct mbuf *, const void *data, size_t data_size); +/* + * Appends data to the Mbuf and frees it (data must be heap-allocated). + * + * Returns the number of bytes appended or 0 if out of memory. + * data is freed irrespective of return value. + */ +size_t mbuf_append_and_free(struct mbuf *, void *data, size_t data_size); + /* * Inserts data at a specified offset in the Mbuf. * @@ -2388,6 +2396,9 @@ void mbuf_remove(struct mbuf *, size_t data_size); */ void mbuf_resize(struct mbuf *, size_t new_size); +/* Moves the state from one mbuf to the other. */ +void mbuf_move(struct mbuf *from, struct mbuf *to); + /* Removes all the data from mbuf (if any). */ void mbuf_clear(struct mbuf *); From c198d2e5f145799fcd0d3027de62f01c29777c88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D0=BE=D0=B1=D0=B1=D0=B8?= Date: Fri, 7 Dec 2018 09:18:29 +0000 Subject: [PATCH 10/25] Make MG_CTL_MSG_MESSAGE_SIZE overridable CL: Make MG_CTL_MSG_MESSAGE_SIZE overridable Fix https://github.com/cesanta/mongoose/pull/987 PUBLISHED_FROM=b0a7c67297b98f616c9eadcd3c34b8747939c918 --- docs/c-api/mg_net.h/mg_broadcast.md | 2 +- mongoose.c | 2 ++ mongoose.h | 2 +- src/mg_internal.h | 2 ++ src/mg_net.h | 2 +- 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/c-api/mg_net.h/mg_broadcast.md b/docs/c-api/mg_net.h/mg_broadcast.md index cef0b379..6e966afe 100644 --- a/docs/c-api/mg_net.h/mg_broadcast.md +++ b/docs/c-api/mg_net.h/mg_broadcast.md @@ -16,5 +16,5 @@ that can be, and must be, called from a different (non-IO) thread. `func` callback function will be called by the IO thread for each connection. When called, the event will be `MG_EV_POLL`, and a message will be passed as the `ev_data` pointer. Maximum message size is capped -by `MG_CTL_MSG_MESSAGE_SIZE` which is set to 8192 bytes. +by `MG_CTL_MSG_MESSAGE_SIZE` which is set to 8192 bytes by default. diff --git a/mongoose.c b/mongoose.c index ffa97c9b..3196c065 100644 --- a/mongoose.c +++ b/mongoose.c @@ -38,7 +38,9 @@ /* Amalgamated: #include "mg_http.h" */ /* Amalgamated: #include "mg_net.h" */ +#ifndef MG_CTL_MSG_MESSAGE_SIZE #define MG_CTL_MSG_MESSAGE_SIZE 8192 +#endif /* internals that need to be accessible in unit tests */ MG_INTERNAL struct mg_connection *mg_do_connect(struct mg_connection *nc, diff --git a/mongoose.h b/mongoose.h index 8d9e8f29..2b7c0830 100644 --- a/mongoose.h +++ b/mongoose.h @@ -4034,7 +4034,7 @@ int mg_mgr_poll(struct mg_mgr *mgr, int milli); * `func` callback function will be called by the IO thread for each * connection. When called, the event will be `MG_EV_POLL`, and a message will * be passed as the `ev_data` pointer. Maximum message size is capped - * by `MG_CTL_MSG_MESSAGE_SIZE` which is set to 8192 bytes. + * by `MG_CTL_MSG_MESSAGE_SIZE` which is set to 8192 bytes by default. */ void mg_broadcast(struct mg_mgr *mgr, mg_event_handler_t cb, void *data, size_t len); diff --git a/src/mg_internal.h b/src/mg_internal.h index 20614234..0e62448e 100644 --- a/src/mg_internal.h +++ b/src/mg_internal.h @@ -34,7 +34,9 @@ #include "mg_http.h" #include "mg_net.h" +#ifndef MG_CTL_MSG_MESSAGE_SIZE #define MG_CTL_MSG_MESSAGE_SIZE 8192 +#endif /* internals that need to be accessible in unit tests */ MG_INTERNAL struct mg_connection *mg_do_connect(struct mg_connection *nc, diff --git a/src/mg_net.h b/src/mg_net.h index d854e770..aa8ed171 100644 --- a/src/mg_net.h +++ b/src/mg_net.h @@ -225,7 +225,7 @@ int mg_mgr_poll(struct mg_mgr *mgr, int milli); * `func` callback function will be called by the IO thread for each * connection. When called, the event will be `MG_EV_POLL`, and a message will * be passed as the `ev_data` pointer. Maximum message size is capped - * by `MG_CTL_MSG_MESSAGE_SIZE` which is set to 8192 bytes. + * by `MG_CTL_MSG_MESSAGE_SIZE` which is set to 8192 bytes by default. */ void mg_broadcast(struct mg_mgr *mgr, mg_event_handler_t cb, void *data, size_t len); From e2dfac946dd4e4fa2312fde0aae90b18e738ad12 Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Mon, 10 Dec 2018 19:08:12 +0000 Subject: [PATCH 11/25] Ensure that user sees all the data before connection is closed If user throttles receive by setting recv_mbuf_limit, after the net interface reports connection as closed we must wait for data to trickle through before disposing of it. There can still b data in the buffers (e.g. SSL). CL: mg: Ensure that user sees all the data before connection is closed PUBLISHED_FROM=22be0fa368950a9fdb03cfb00febc7c0a1674b01 --- docs/c-api/mg_net.h/struct_mg_connection.md | 1 + mongoose.c | 26 +++++++++++++++++---- mongoose.h | 1 + src/common/platforms/lwip/mg_lwip_ev_mgr.c | 1 - src/mg_net.c | 25 +++++++++++++++++--- src/mg_net.h | 1 + 6 files changed, 47 insertions(+), 8 deletions(-) diff --git a/docs/c-api/mg_net.h/struct_mg_connection.md b/docs/c-api/mg_net.h/struct_mg_connection.md index be69d16f..cf6bd7ff 100644 --- a/docs/c-api/mg_net.h/struct_mg_connection.md +++ b/docs/c-api/mg_net.h/struct_mg_connection.md @@ -46,6 +46,7 @@ signature: | #define MG_F_WANT_READ (1 << 6) /* SSL specific */ #define MG_F_WANT_WRITE (1 << 7) /* SSL specific */ #define MG_F_IS_WEBSOCKET (1 << 8) /* Websocket specific */ + #define MG_F_RECV_AND_CLOSE (1 << 9) /* Drain rx and close the connection. */ /* Flags that are settable by user */ #define MG_F_SEND_AND_CLOSE (1 << 10) /* Push remaining data and close */ diff --git a/mongoose.c b/mongoose.c index 3196c065..457b23b9 100644 --- a/mongoose.c +++ b/mongoose.c @@ -2474,8 +2474,16 @@ MG_INTERNAL size_t recv_avail_size(struct mg_connection *conn, size_t max) { static int mg_do_recv(struct mg_connection *nc); int mg_if_poll(struct mg_connection *nc, double now) { - if ((nc->flags & MG_F_CLOSE_IMMEDIATELY) || - (nc->send_mbuf.len == 0 && (nc->flags & MG_F_SEND_AND_CLOSE))) { + if (nc->flags & MG_F_CLOSE_IMMEDIATELY) { + mg_close_conn(nc); + return 0; + } else if (nc->flags & MG_F_SEND_AND_CLOSE) { + if (nc->send_mbuf.len == 0) { + nc->flags |= MG_F_CLOSE_IMMEDIATELY; + mg_close_conn(nc); + return 0; + } + } else if (nc->flags & MG_F_RECV_AND_CLOSE) { mg_close_conn(nc); return 0; } @@ -2518,6 +2526,13 @@ void mg_destroy_conn(struct mg_connection *conn, int destroy_if) { } void mg_close_conn(struct mg_connection *conn) { + /* See if there's any remaining data to deliver. Skip if user completely + * throttled the connection there will be no progress anyway. */ + if (conn->sock != INVALID_SOCKET && mg_do_recv(conn) == -2) { + /* Receive is throttled, wait. */ + conn->flags |= MG_F_RECV_AND_CLOSE; + return; + } #if MG_ENABLE_SSL if (conn->flags & MG_F_SSL_HANDSHAKE_DONE) { mg_ssl_if_conn_close_notify(conn); @@ -2608,6 +2623,7 @@ void mg_mgr_free(struct mg_mgr *m) { for (conn = m->active_connections; conn != NULL; conn = tmp_conn) { tmp_conn = conn->next; + conn->flags |= MG_F_CLOSE_IMMEDIATELY; mg_close_conn(conn); } @@ -2913,7 +2929,10 @@ static int mg_do_recv(struct mg_connection *nc) { } do { len = recv_avail_size(nc, len); - if (len == 0) return -2; + if (len == 0) { + res = -2; + break; + } if (nc->recv_mbuf.size < nc->recv_mbuf.len + len) { mbuf_resize(&nc->recv_mbuf, nc->recv_mbuf.len + len); } @@ -15866,7 +15885,6 @@ void mg_ev_mgr_lwip_process_signals(struct mg_mgr *mgr) { break; } case MG_SIG_CLOSE_CONN: { - nc->flags |= MG_F_SEND_AND_CLOSE; mg_close_conn(nc); break; } diff --git a/mongoose.h b/mongoose.h index 2b7c0830..f9dd87d4 100644 --- a/mongoose.h +++ b/mongoose.h @@ -3949,6 +3949,7 @@ struct mg_connection { #define MG_F_WANT_READ (1 << 6) /* SSL specific */ #define MG_F_WANT_WRITE (1 << 7) /* SSL specific */ #define MG_F_IS_WEBSOCKET (1 << 8) /* Websocket specific */ +#define MG_F_RECV_AND_CLOSE (1 << 9) /* Drain rx and close the connection. */ /* Flags that are settable by user */ #define MG_F_SEND_AND_CLOSE (1 << 10) /* Push remaining data and close */ diff --git a/src/common/platforms/lwip/mg_lwip_ev_mgr.c b/src/common/platforms/lwip/mg_lwip_ev_mgr.c index b495464e..ab8f19f7 100644 --- a/src/common/platforms/lwip/mg_lwip_ev_mgr.c +++ b/src/common/platforms/lwip/mg_lwip_ev_mgr.c @@ -67,7 +67,6 @@ void mg_ev_mgr_lwip_process_signals(struct mg_mgr *mgr) { break; } case MG_SIG_CLOSE_CONN: { - nc->flags |= MG_F_SEND_AND_CLOSE; mg_close_conn(nc); break; } diff --git a/src/mg_net.c b/src/mg_net.c index ebf0513f..3b1b7038 100644 --- a/src/mg_net.c +++ b/src/mg_net.c @@ -128,8 +128,16 @@ MG_INTERNAL size_t recv_avail_size(struct mg_connection *conn, size_t max) { static int mg_do_recv(struct mg_connection *nc); int mg_if_poll(struct mg_connection *nc, double now) { - if ((nc->flags & MG_F_CLOSE_IMMEDIATELY) || - (nc->send_mbuf.len == 0 && (nc->flags & MG_F_SEND_AND_CLOSE))) { + if (nc->flags & MG_F_CLOSE_IMMEDIATELY) { + mg_close_conn(nc); + return 0; + } else if (nc->flags & MG_F_SEND_AND_CLOSE) { + if (nc->send_mbuf.len == 0) { + nc->flags |= MG_F_CLOSE_IMMEDIATELY; + mg_close_conn(nc); + return 0; + } + } else if (nc->flags & MG_F_RECV_AND_CLOSE) { mg_close_conn(nc); return 0; } @@ -172,6 +180,13 @@ void mg_destroy_conn(struct mg_connection *conn, int destroy_if) { } void mg_close_conn(struct mg_connection *conn) { + /* See if there's any remaining data to deliver. Skip if user completely + * throttled the connection there will be no progress anyway. */ + if (conn->sock != INVALID_SOCKET && mg_do_recv(conn) == -2) { + /* Receive is throttled, wait. */ + conn->flags |= MG_F_RECV_AND_CLOSE; + return; + } #if MG_ENABLE_SSL if (conn->flags & MG_F_SSL_HANDSHAKE_DONE) { mg_ssl_if_conn_close_notify(conn); @@ -262,6 +277,7 @@ void mg_mgr_free(struct mg_mgr *m) { for (conn = m->active_connections; conn != NULL; conn = tmp_conn) { tmp_conn = conn->next; + conn->flags |= MG_F_CLOSE_IMMEDIATELY; mg_close_conn(conn); } @@ -567,7 +583,10 @@ static int mg_do_recv(struct mg_connection *nc) { } do { len = recv_avail_size(nc, len); - if (len == 0) return -2; + if (len == 0) { + res = -2; + break; + } if (nc->recv_mbuf.size < nc->recv_mbuf.len + len) { mbuf_resize(&nc->recv_mbuf, nc->recv_mbuf.len + len); } diff --git a/src/mg_net.h b/src/mg_net.h index aa8ed171..cbe75b4c 100644 --- a/src/mg_net.h +++ b/src/mg_net.h @@ -140,6 +140,7 @@ struct mg_connection { #define MG_F_WANT_READ (1 << 6) /* SSL specific */ #define MG_F_WANT_WRITE (1 << 7) /* SSL specific */ #define MG_F_IS_WEBSOCKET (1 << 8) /* Websocket specific */ +#define MG_F_RECV_AND_CLOSE (1 << 9) /* Drain rx and close the connection. */ /* Flags that are settable by user */ #define MG_F_SEND_AND_CLOSE (1 << 10) /* Push remaining data and close */ From 455cec67a50177c42e335c046fb2ae4339e7afd0 Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Tue, 18 Dec 2018 16:17:03 +0000 Subject: [PATCH 12/25] CC3220: Enterprise auth support CL: wifi: CC3220: Enterprise auth support PUBLISHED_FROM=ce445eb5b740073c9a3f9811f1acd404ce134f2e --- examples/CC3200/main.c | 4 +- mongoose.c | 102 +++++++++++++----- mongoose.h | 2 +- .../platforms/simplelink/cs_simplelink.h | 2 +- src/common/platforms/simplelink/sl_fs_slfs.c | 94 +++++++++++----- src/common/platforms/simplelink/sl_fs_slfs.h | 4 +- src/common/platforms/simplelink/sl_ssl_if.c | 4 +- 7 files changed, 152 insertions(+), 60 deletions(-) diff --git a/examples/CC3200/main.c b/examples/CC3200/main.c index 700419aa..c37c0e1f 100644 --- a/examples/CC3200/main.c +++ b/examples/CC3200/main.c @@ -50,7 +50,7 @@ #define BM222_ADDR 0x18 #define TMP006_ADDR 0x41 -void fs_slfs_set_new_file_size(const char *name, size_t size); +void fs_slfs_set_file_size(const char *name, size_t size); static const char *upload_form = "\ @@ -70,7 +70,7 @@ static struct mg_str upload_fname(struct mg_connection *nc, if (nc->user_data != NULL) { intptr_t cl = (intptr_t) nc->user_data; if (cl >= 0) { - fs_slfs_set_new_file_size(fn + 3, cl); + fs_slfs_set_file_size(fn + 3, cl); } } lfn.len = fname.len + 4; diff --git a/mongoose.c b/mongoose.c index 457b23b9..710ded3d 100644 --- a/mongoose.c +++ b/mongoose.c @@ -13485,7 +13485,9 @@ off_t fs_slfs_lseek(int fd, off_t offset, int whence); int fs_slfs_unlink(const char *filename); int fs_slfs_rename(const char *from, const char *to); -void fs_slfs_set_new_file_size(const char *name, size_t size); +void fs_slfs_set_file_size(const char *name, size_t size); +void fs_slfs_set_file_flags(const char *name, uint32_t flags, uint32_t *token); +void fs_slfs_unset_file_flags(const char *name); #endif /* defined(MG_FS_SLFS) */ @@ -13526,14 +13528,14 @@ void fs_slfs_set_new_file_size(const char *name, size_t size); /* Amalgamated: #include "common/mg_mem.h" */ #if SL_MAJOR_VERSION_NUM < 2 -int slfs_open(const unsigned char *fname, uint32_t flags) { +int slfs_open(const unsigned char *fname, uint32_t flags, uint32_t *token) { _i32 fh; - _i32 r = sl_FsOpen(fname, flags, NULL /* token */, &fh); + _i32 r = sl_FsOpen(fname, flags, (unsigned long *) token, &fh); return (r < 0 ? r : fh); } #else /* SL_MAJOR_VERSION_NUM >= 2 */ -int slfs_open(const unsigned char *fname, uint32_t flags) { - return sl_FsOpen(fname, flags, NULL /* token */); +int slfs_open(const unsigned char *fname, uint32_t flags, uint32_t *token) { + return sl_FsOpen(fname, flags, (unsigned long *) token); } #endif @@ -13549,9 +13551,11 @@ const char *drop_dir(const char *fname, bool *is_slfs); #define FS_SLFS_MAX_FILE_SIZE (64 * 1024) #endif -struct sl_file_size_hint { +struct sl_file_open_info { char *name; size_t size; + uint32_t flags; + uint32_t *token; }; struct sl_fd_info { @@ -13561,7 +13565,10 @@ struct sl_fd_info { }; static struct sl_fd_info s_sl_fds[MAX_OPEN_SLFS_FILES]; -static struct sl_file_size_hint s_sl_file_size_hints[MAX_OPEN_SLFS_FILES]; +static struct sl_file_open_info s_sl_file_open_infos[MAX_OPEN_SLFS_FILES]; + +static struct sl_file_open_info *fs_slfs_find_foi(const char *name, + bool create); static int sl_fs_to_errno(_i32 r) { DBG(("SL error: %d", (int) r)); @@ -13602,7 +13609,13 @@ int fs_slfs_open(const char *pathname, int flags, mode_t mode) { _u32 am = 0; fi->size = (size_t) -1; int rw = (flags & 3); - size_t new_size = FS_SLFS_MAX_FILE_SIZE; + size_t new_size = 0; + struct sl_file_open_info *foi = + fs_slfs_find_foi(pathname, false /* create */); + if (foi != NULL) { + LOG(LL_DEBUG, ("FOI for %s: %d 0x%x %p", pathname, (int) foi->size, + (unsigned int) foi->flags, foi->token)); + } if (rw == O_RDONLY) { SlFsFileInfo_t sl_fi; _i32 r = sl_FsGetInfo((const _u8 *) pathname, 0, &sl_fi); @@ -13617,24 +13630,27 @@ int fs_slfs_open(const char *pathname, int flags, mode_t mode) { return set_errno(ENOTSUP); } if (flags & O_CREAT) { - size_t i; - for (i = 0; i < MAX_OPEN_SLFS_FILES; i++) { - if (s_sl_file_size_hints[i].name != NULL && - strcmp(s_sl_file_size_hints[i].name, pathname) == 0) { - new_size = s_sl_file_size_hints[i].size; - MG_FREE(s_sl_file_size_hints[i].name); - s_sl_file_size_hints[i].name = NULL; - break; - } + if (foi->size > 0) { + new_size = foi->size; + } else { + new_size = FS_SLFS_MAX_FILE_SIZE; } am = FS_MODE_OPEN_CREATE(new_size, 0); } else { am = SL_FS_WRITE; } +#if SL_MAJOR_VERSION_NUM >= 2 + am |= SL_FS_OVERWRITE; +#endif } - fi->fh = slfs_open((_u8 *) pathname, am); - LOG(LL_DEBUG, ("sl_FsOpen(%s, 0x%x) sz %u = %d", pathname, (int) am, - (unsigned int) new_size, (int) fi->fh)); + uint32_t *token = NULL; + if (foi != NULL) { + am |= foi->flags; + token = foi->token; + } + fi->fh = slfs_open((_u8 *) pathname, am, token); + LOG(LL_DEBUG, ("sl_FsOpen(%s, 0x%x, %p) sz %u = %d", pathname, (int) am, + token, (unsigned int) new_size, (int) fi->fh)); int r; if (fi->fh >= 0) { fi->pos = 0; @@ -13738,16 +13754,46 @@ int fs_slfs_rename(const char *from, const char *to) { return set_errno(ENOTSUP); } -void fs_slfs_set_new_file_size(const char *name, size_t size) { - int i; +static struct sl_file_open_info *fs_slfs_find_foi(const char *name, + bool create) { + int i = 0; for (i = 0; i < MAX_OPEN_SLFS_FILES; i++) { - if (s_sl_file_size_hints[i].name == NULL) { - DBG(("File size hint: %s %d", name, (int) size)); - s_sl_file_size_hints[i].name = strdup(name); - s_sl_file_size_hints[i].size = size; + if (s_sl_file_open_infos[i].name != NULL && + strcmp(drop_dir(s_sl_file_open_infos[i].name, NULL), name) == 0) { break; } } + if (i != MAX_OPEN_SLFS_FILES) return &s_sl_file_open_infos[i]; + if (!create) return NULL; + for (i = 0; i < MAX_OPEN_SLFS_FILES; i++) { + if (s_sl_file_open_infos[i].name == NULL) break; + } + if (i == MAX_OPEN_SLFS_FILES) { + i = 0; /* Evict a random slot. */ + } + if (s_sl_file_open_infos[i].name != NULL) { + free(s_sl_file_open_infos[i].name); + } + s_sl_file_open_infos[i].name = strdup(name); + return &s_sl_file_open_infos[i]; +} + +void fs_slfs_set_file_size(const char *name, size_t size) { + struct sl_file_open_info *foi = fs_slfs_find_foi(name, true /* create */); + foi->size = size; +} + +void fs_slfs_set_file_flags(const char *name, uint32_t flags, uint32_t *token) { + struct sl_file_open_info *foi = fs_slfs_find_foi(name, true /* create */); + foi->flags = flags; + foi->token = token; +} + +void fs_slfs_unset_file_flags(const char *name) { + struct sl_file_open_info *foi = fs_slfs_find_foi(name, false /* create */); + if (foi == NULL) return; + free(foi->name); + memset(foi, 0, sizeof(*foi)); } #endif /* defined(MG_FS_SLFS) || defined(CC3200_FS_SLFS) */ @@ -14890,9 +14936,9 @@ bool pem_to_der(const char *pem_file, const char *der_file) { pf = fopen(pem_file, "r"); if (pf == NULL) goto clean; remove(der_file); - fs_slfs_set_new_file_size(der_file + MG_SSL_IF_SIMPLELINK_SLFS_PREFIX_LEN, - 2048); + fs_slfs_set_file_size(der_file + MG_SSL_IF_SIMPLELINK_SLFS_PREFIX_LEN, 2048); df = fopen(der_file, "w"); + fs_slfs_unset_file_flags(der_file + MG_SSL_IF_SIMPLELINK_SLFS_PREFIX_LEN); if (df == NULL) goto clean; while (1) { char pem_buf[70]; diff --git a/mongoose.h b/mongoose.h index f9dd87d4..98c6730e 100644 --- a/mongoose.h +++ b/mongoose.h @@ -1531,7 +1531,7 @@ int sl_set_ssl_opts(int sock, struct mg_connection *nc); #endif /* SL_MAJOR_VERSION_NUM < 2 */ -int slfs_open(const unsigned char *fname, uint32_t flags); +int slfs_open(const unsigned char *fname, uint32_t flags, uint32_t *token); #endif /* MG_NET_IF == MG_NET_IF_SIMPLELINK */ diff --git a/src/common/platforms/simplelink/cs_simplelink.h b/src/common/platforms/simplelink/cs_simplelink.h index 2b415410..71af2dcb 100644 --- a/src/common/platforms/simplelink/cs_simplelink.h +++ b/src/common/platforms/simplelink/cs_simplelink.h @@ -169,7 +169,7 @@ int sl_set_ssl_opts(int sock, struct mg_connection *nc); #endif /* SL_MAJOR_VERSION_NUM < 2 */ -int slfs_open(const unsigned char *fname, uint32_t flags); +int slfs_open(const unsigned char *fname, uint32_t flags, uint32_t *token); #endif /* MG_NET_IF == MG_NET_IF_SIMPLELINK */ diff --git a/src/common/platforms/simplelink/sl_fs_slfs.c b/src/common/platforms/simplelink/sl_fs_slfs.c index 8170310b..7dcff308 100644 --- a/src/common/platforms/simplelink/sl_fs_slfs.c +++ b/src/common/platforms/simplelink/sl_fs_slfs.c @@ -31,14 +31,14 @@ #include "common/mg_mem.h" #if SL_MAJOR_VERSION_NUM < 2 -int slfs_open(const unsigned char *fname, uint32_t flags) { +int slfs_open(const unsigned char *fname, uint32_t flags, uint32_t *token) { _i32 fh; - _i32 r = sl_FsOpen(fname, flags, NULL /* token */, &fh); + _i32 r = sl_FsOpen(fname, flags, (unsigned long *) token, &fh); return (r < 0 ? r : fh); } #else /* SL_MAJOR_VERSION_NUM >= 2 */ -int slfs_open(const unsigned char *fname, uint32_t flags) { - return sl_FsOpen(fname, flags, NULL /* token */); +int slfs_open(const unsigned char *fname, uint32_t flags, uint32_t *token) { + return sl_FsOpen(fname, flags, (unsigned long *) token); } #endif @@ -54,9 +54,11 @@ const char *drop_dir(const char *fname, bool *is_slfs); #define FS_SLFS_MAX_FILE_SIZE (64 * 1024) #endif -struct sl_file_size_hint { +struct sl_file_open_info { char *name; size_t size; + uint32_t flags; + uint32_t *token; }; struct sl_fd_info { @@ -66,7 +68,10 @@ struct sl_fd_info { }; static struct sl_fd_info s_sl_fds[MAX_OPEN_SLFS_FILES]; -static struct sl_file_size_hint s_sl_file_size_hints[MAX_OPEN_SLFS_FILES]; +static struct sl_file_open_info s_sl_file_open_infos[MAX_OPEN_SLFS_FILES]; + +static struct sl_file_open_info *fs_slfs_find_foi(const char *name, + bool create); static int sl_fs_to_errno(_i32 r) { DBG(("SL error: %d", (int) r)); @@ -107,7 +112,13 @@ int fs_slfs_open(const char *pathname, int flags, mode_t mode) { _u32 am = 0; fi->size = (size_t) -1; int rw = (flags & 3); - size_t new_size = FS_SLFS_MAX_FILE_SIZE; + size_t new_size = 0; + struct sl_file_open_info *foi = + fs_slfs_find_foi(pathname, false /* create */); + if (foi != NULL) { + LOG(LL_DEBUG, ("FOI for %s: %d 0x%x %p", pathname, (int) foi->size, + (unsigned int) foi->flags, foi->token)); + } if (rw == O_RDONLY) { SlFsFileInfo_t sl_fi; _i32 r = sl_FsGetInfo((const _u8 *) pathname, 0, &sl_fi); @@ -122,24 +133,27 @@ int fs_slfs_open(const char *pathname, int flags, mode_t mode) { return set_errno(ENOTSUP); } if (flags & O_CREAT) { - size_t i; - for (i = 0; i < MAX_OPEN_SLFS_FILES; i++) { - if (s_sl_file_size_hints[i].name != NULL && - strcmp(s_sl_file_size_hints[i].name, pathname) == 0) { - new_size = s_sl_file_size_hints[i].size; - MG_FREE(s_sl_file_size_hints[i].name); - s_sl_file_size_hints[i].name = NULL; - break; - } + if (foi->size > 0) { + new_size = foi->size; + } else { + new_size = FS_SLFS_MAX_FILE_SIZE; } am = FS_MODE_OPEN_CREATE(new_size, 0); } else { am = SL_FS_WRITE; } +#if SL_MAJOR_VERSION_NUM >= 2 + am |= SL_FS_OVERWRITE; +#endif } - fi->fh = slfs_open((_u8 *) pathname, am); - LOG(LL_DEBUG, ("sl_FsOpen(%s, 0x%x) sz %u = %d", pathname, (int) am, - (unsigned int) new_size, (int) fi->fh)); + uint32_t *token = NULL; + if (foi != NULL) { + am |= foi->flags; + token = foi->token; + } + fi->fh = slfs_open((_u8 *) pathname, am, token); + LOG(LL_DEBUG, ("sl_FsOpen(%s, 0x%x, %p) sz %u = %d", pathname, (int) am, + token, (unsigned int) new_size, (int) fi->fh)); int r; if (fi->fh >= 0) { fi->pos = 0; @@ -243,16 +257,46 @@ int fs_slfs_rename(const char *from, const char *to) { return set_errno(ENOTSUP); } -void fs_slfs_set_new_file_size(const char *name, size_t size) { - int i; +static struct sl_file_open_info *fs_slfs_find_foi(const char *name, + bool create) { + int i = 0; for (i = 0; i < MAX_OPEN_SLFS_FILES; i++) { - if (s_sl_file_size_hints[i].name == NULL) { - DBG(("File size hint: %s %d", name, (int) size)); - s_sl_file_size_hints[i].name = strdup(name); - s_sl_file_size_hints[i].size = size; + if (s_sl_file_open_infos[i].name != NULL && + strcmp(drop_dir(s_sl_file_open_infos[i].name, NULL), name) == 0) { break; } } + if (i != MAX_OPEN_SLFS_FILES) return &s_sl_file_open_infos[i]; + if (!create) return NULL; + for (i = 0; i < MAX_OPEN_SLFS_FILES; i++) { + if (s_sl_file_open_infos[i].name == NULL) break; + } + if (i == MAX_OPEN_SLFS_FILES) { + i = 0; /* Evict a random slot. */ + } + if (s_sl_file_open_infos[i].name != NULL) { + free(s_sl_file_open_infos[i].name); + } + s_sl_file_open_infos[i].name = strdup(name); + return &s_sl_file_open_infos[i]; +} + +void fs_slfs_set_file_size(const char *name, size_t size) { + struct sl_file_open_info *foi = fs_slfs_find_foi(name, true /* create */); + foi->size = size; +} + +void fs_slfs_set_file_flags(const char *name, uint32_t flags, uint32_t *token) { + struct sl_file_open_info *foi = fs_slfs_find_foi(name, true /* create */); + foi->flags = flags; + foi->token = token; +} + +void fs_slfs_unset_file_flags(const char *name) { + struct sl_file_open_info *foi = fs_slfs_find_foi(name, false /* create */); + if (foi == NULL) return; + free(foi->name); + memset(foi, 0, sizeof(*foi)); } #endif /* defined(MG_FS_SLFS) || defined(CC3200_FS_SLFS) */ diff --git a/src/common/platforms/simplelink/sl_fs_slfs.h b/src/common/platforms/simplelink/sl_fs_slfs.h index eb909c1b..4e4cb57b 100644 --- a/src/common/platforms/simplelink/sl_fs_slfs.h +++ b/src/common/platforms/simplelink/sl_fs_slfs.h @@ -39,7 +39,9 @@ off_t fs_slfs_lseek(int fd, off_t offset, int whence); int fs_slfs_unlink(const char *filename); int fs_slfs_rename(const char *from, const char *to); -void fs_slfs_set_new_file_size(const char *name, size_t size); +void fs_slfs_set_file_size(const char *name, size_t size); +void fs_slfs_set_file_flags(const char *name, uint32_t flags, uint32_t *token); +void fs_slfs_unset_file_flags(const char *name); #endif /* defined(MG_FS_SLFS) */ diff --git a/src/common/platforms/simplelink/sl_ssl_if.c b/src/common/platforms/simplelink/sl_ssl_if.c index 6e4daa6e..92453e3d 100644 --- a/src/common/platforms/simplelink/sl_ssl_if.c +++ b/src/common/platforms/simplelink/sl_ssl_if.c @@ -115,9 +115,9 @@ bool pem_to_der(const char *pem_file, const char *der_file) { pf = fopen(pem_file, "r"); if (pf == NULL) goto clean; remove(der_file); - fs_slfs_set_new_file_size(der_file + MG_SSL_IF_SIMPLELINK_SLFS_PREFIX_LEN, - 2048); + fs_slfs_set_file_size(der_file + MG_SSL_IF_SIMPLELINK_SLFS_PREFIX_LEN, 2048); df = fopen(der_file, "w"); + fs_slfs_unset_file_flags(der_file + MG_SSL_IF_SIMPLELINK_SLFS_PREFIX_LEN); if (df == NULL) goto clean; while (1) { char pem_buf[70]; From 22e602debbc9df0503562d1eb7902f7ef9d6886f Mon Sep 17 00:00:00 2001 From: Sergey Lyubka Date: Fri, 21 Dec 2018 12:50:52 +0000 Subject: [PATCH 13/25] Check for NULL in tcp_recved_tcpip() CL: Check for NULL in tcp_recved_tcpip() PUBLISHED_FROM=3f5bbc2e35b79d363eac4b99a1f0d17c9b81d4ec --- mongoose.c | 2 +- src/common/platforms/lwip/mg_lwip_net_if.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoose.c b/mongoose.c index 710ded3d..325bbb35 100644 --- a/mongoose.c +++ b/mongoose.c @@ -15724,7 +15724,7 @@ struct tcp_recved_ctx { void tcp_recved_tcpip(void *arg) { struct tcp_recved_ctx *ctx = (struct tcp_recved_ctx *) arg; - tcp_recved(ctx->tpcb, ctx->len); + if (ctx->tpcb != NULL) tcp_recved(ctx->tpcb, ctx->len); } static int mg_lwip_if_tcp_recv(struct mg_connection *nc, void *buf, diff --git a/src/common/platforms/lwip/mg_lwip_net_if.c b/src/common/platforms/lwip/mg_lwip_net_if.c index d0777029..e78d93ce 100644 --- a/src/common/platforms/lwip/mg_lwip_net_if.c +++ b/src/common/platforms/lwip/mg_lwip_net_if.c @@ -581,7 +581,7 @@ struct tcp_recved_ctx { void tcp_recved_tcpip(void *arg) { struct tcp_recved_ctx *ctx = (struct tcp_recved_ctx *) arg; - tcp_recved(ctx->tpcb, ctx->len); + if (ctx->tpcb != NULL) tcp_recved(ctx->tpcb, ctx->len); } static int mg_lwip_if_tcp_recv(struct mg_connection *nc, void *buf, From e9a8e5468d9e6e37fd2813decfd49f0d5312e66e Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Mon, 24 Dec 2018 20:15:42 +0000 Subject: [PATCH 14/25] Convert Python scripts to Python 3, delete unused ones CL: none PUBLISHED_FROM=4bbc2175634e20a726dd4b2dadff57f945b261d0 --- .../esp8266/tools/malloc_analyzer.py | 23 ------------------- 1 file changed, 23 deletions(-) delete mode 100644 src/common/platforms/esp8266/tools/malloc_analyzer.py diff --git a/src/common/platforms/esp8266/tools/malloc_analyzer.py b/src/common/platforms/esp8266/tools/malloc_analyzer.py deleted file mode 100644 index 91d4d0c4..00000000 --- a/src/common/platforms/esp8266/tools/malloc_analyzer.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/python - -# Analyzers output of firmware compiler with -DESP_ENABLE_MALLOC_TRACES - -import re -import sys - -allocs = {} - -for l in sys.stdin: - m = re.match(r'(ca|ma|re|fr) (\S+)\s*(\S*)\s*(\S*)', l) - if not m: continue - op = m.group(1) - if op in ('ca', 'ma'): - allocs[m.group(2)] = long(m.group(3)) - else: - if m.group(2) in allocs: - del allocs[m.group(2)] - if op == 're': - allocs[m.group(3)] = long(m.group(4)) - -for k, v in sorted(allocs.iteritems()): - print k, v From c2a10601b9486f28b75307f003f5d9078acdce10 Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Tue, 8 Jan 2019 19:56:19 +0000 Subject: [PATCH 15/25] Fix nsleep100 on STM32 CL: Fix nsleep100 on STM32 PUBLISHED_FROM=cabb98a0541bac9bb66bfa702d54bf8ff20063e7 --- src/common/platforms/arm/arm_nsleep100_m4.S | 8 ++++++++ src/common/platforms/arm/arm_nsleep100_m7.S | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/src/common/platforms/arm/arm_nsleep100_m4.S b/src/common/platforms/arm/arm_nsleep100_m4.S index d40820f6..afb596fa 100644 --- a/src/common/platforms/arm/arm_nsleep100_m4.S +++ b/src/common/platforms/arm/arm_nsleep100_m4.S @@ -26,7 +26,11 @@ .global mgos_nsleep100_impl .global mgos_nsleep100_loop_count +#ifdef STM32 +.section .RamFunc.mgos_nsleep100_impl +#else .section .iram.mgos_nsleep100_impl +#endif .type mgos_nsleep100_impl, %function .align 4 @@ -34,7 +38,11 @@ mgos_nsleep100_impl: ldr r3, =mgos_nsleep100_loop_count ldr r3, [r3] mul r0, r3 +#ifdef STM32L4 + mov r1, #3 +#else mov r1, #6 +#endif udiv r0, r0, r1 cbz r0, xxx lxx: diff --git a/src/common/platforms/arm/arm_nsleep100_m7.S b/src/common/platforms/arm/arm_nsleep100_m7.S index c0e8f490..36c878fd 100644 --- a/src/common/platforms/arm/arm_nsleep100_m7.S +++ b/src/common/platforms/arm/arm_nsleep100_m7.S @@ -26,7 +26,11 @@ .global mgos_nsleep100_impl .global mgos_nsleep100_loop_count +#ifdef STM32 +.section .RamFunc.mgos_nsleep100_impl +#else .section .iram.mgos_nsleep100_impl +#endif .type mgos_nsleep100_impl, %function .align 4 From c431904b56252672c0b0e9214a96ec840358f7f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D0=BE=D0=B1=D0=B1=D0=B8?= Date: Wed, 16 Jan 2019 09:55:52 +0000 Subject: [PATCH 16/25] Send extra options with OPTIONS CL: Send extra options with OPTIONS. Integrate https://github.com/cesanta/mongoose/pull/657 PUBLISHED_FROM=6ee609a92febd8d908f7713f94fb73657443a09a --- mongoose.c | 14 ++++++++------ src/mg_http.c | 14 ++++++++------ test/unit_test.c | 9 +++++---- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/mongoose.c b/mongoose.c index 325bbb35..2a3eb07d 100644 --- a/mongoose.c +++ b/mongoose.c @@ -6563,9 +6563,9 @@ void mg_http_handler(struct mg_connection *nc, int ev, if (io->len > 0 && (req_len = mg_parse_http(io->buf, io->len, hm, is_req)) > 0) { /* - * For HTTP messages without Content-Length, always send HTTP message - * before MG_EV_CLOSE message. - */ + * For HTTP messages without Content-Length, always send HTTP message + * before MG_EV_CLOSE message. + */ int ev2 = is_req ? MG_EV_HTTP_REQUEST : MG_EV_HTTP_REPLY; hm->message.len = io->len; hm->body.len = io->buf + io->len - hm->body.p; @@ -8360,9 +8360,11 @@ void mg_http_send_digest_auth_request(struct mg_connection *c, domain, (unsigned long) mg_time()); } -static void mg_http_send_options(struct mg_connection *nc) { +static void mg_http_send_options(struct mg_connection *nc, + struct mg_serve_http_opts *opts) { + mg_send_response_line(nc, 200, opts->extra_headers); mg_printf(nc, "%s", - "HTTP/1.1 200 OK\r\nAllow: GET, POST, HEAD, CONNECT, OPTIONS" + "Allow: GET, POST, HEAD, CONNECT, OPTIONS" #if MG_ENABLE_HTTP_WEBDAV ", MKCOL, PUT, DELETE, PROPFIND, MOVE\r\nDAV: 1,2" #endif @@ -8469,7 +8471,7 @@ MG_INTERNAL void mg_send_http_file(struct mg_connection *nc, char *path, #endif #endif /* MG_ENABLE_HTTP_WEBDAV */ } else if (!mg_vcmp(&hm->method, "OPTIONS")) { - mg_http_send_options(nc); + mg_http_send_options(nc, opts); } else if (is_directory && index_file == NULL) { #if MG_ENABLE_DIRECTORY_LISTING if (strcmp(opts->enable_directory_listing, "yes") == 0) { diff --git a/src/mg_http.c b/src/mg_http.c index 460277cd..46643773 100644 --- a/src/mg_http.c +++ b/src/mg_http.c @@ -759,9 +759,9 @@ void mg_http_handler(struct mg_connection *nc, int ev, if (io->len > 0 && (req_len = mg_parse_http(io->buf, io->len, hm, is_req)) > 0) { /* - * For HTTP messages without Content-Length, always send HTTP message - * before MG_EV_CLOSE message. - */ + * For HTTP messages without Content-Length, always send HTTP message + * before MG_EV_CLOSE message. + */ int ev2 = is_req ? MG_EV_HTTP_REQUEST : MG_EV_HTTP_REPLY; hm->message.len = io->len; hm->body.len = io->buf + io->len - hm->body.p; @@ -2556,9 +2556,11 @@ void mg_http_send_digest_auth_request(struct mg_connection *c, domain, (unsigned long) mg_time()); } -static void mg_http_send_options(struct mg_connection *nc) { +static void mg_http_send_options(struct mg_connection *nc, + struct mg_serve_http_opts *opts) { + mg_send_response_line(nc, 200, opts->extra_headers); mg_printf(nc, "%s", - "HTTP/1.1 200 OK\r\nAllow: GET, POST, HEAD, CONNECT, OPTIONS" + "Allow: GET, POST, HEAD, CONNECT, OPTIONS" #if MG_ENABLE_HTTP_WEBDAV ", MKCOL, PUT, DELETE, PROPFIND, MOVE\r\nDAV: 1,2" #endif @@ -2665,7 +2667,7 @@ MG_INTERNAL void mg_send_http_file(struct mg_connection *nc, char *path, #endif #endif /* MG_ENABLE_HTTP_WEBDAV */ } else if (!mg_vcmp(&hm->method, "OPTIONS")) { - mg_http_send_options(nc); + mg_http_send_options(nc, opts); } else if (is_directory && index_file == NULL) { #if MG_ENABLE_DIRECTORY_LISTING if (strcmp(opts->enable_directory_listing, "yes") == 0) { diff --git a/test/unit_test.c b/test/unit_test.c index 114d20f3..84f9f8bc 100644 --- a/test/unit_test.c +++ b/test/unit_test.c @@ -15,12 +15,12 @@ * license, as set out in . */ -#include "mongoose.h" -#include "src/mg_internal.h" #include "unit_test.h" +#include "common/cs_md5.h" #include "common/test_main.h" #include "common/test_util.h" -#include "common/cs_md5.h" +#include "mongoose.h" +#include "src/mg_internal.h" #if defined(__STDC_VERSION__) && __STDC_VERSION__ < 199901L && !defined(WIN32) #define __func__ "" @@ -2011,7 +2011,8 @@ static const char *test_http(void) { char buf[50] = "", status[100] = "", mime1[20] = "", mime2[100] = ""; char opt_buf[1024] = ""; const char *opt_answer = - "HTTP/1.1 200 OK\r\nAllow: GET, POST, HEAD, CONNECT, OPTIONS, MKCOL, " + "HTTP/1.1 200 OK\r\nServer: Mongoose/" MG_VERSION + "\r\nAllow: GET, POST, HEAD, CONNECT, OPTIONS, MKCOL, " "PUT, DELETE, PROPFIND, MOVE"; char url[1000]; From 69e2b2985da29e8755fb7837b06dbe7232c0447d Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Wed, 16 Jan 2019 19:44:59 +0000 Subject: [PATCH 17/25] Add mg_strfree() CL: Add mg_strfree() PUBLISHED_FROM=b80566218986e0607a9d2758bf776a1031b27212 --- mongoose.c | 8 ++++++++ mongoose.h | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/mongoose.c b/mongoose.c index 2a3eb07d..a7851188 100644 --- a/mongoose.c +++ b/mongoose.c @@ -1774,6 +1774,14 @@ int mg_strncmp(const struct mg_str str1, const struct mg_str str2, size_t n) { return mg_strcmp(s1, s2); } +void mg_strfree(struct mg_str *s) WEAK; +void mg_strfree(struct mg_str *s) { + char *sp = (char *) s->p; + s->p = NULL; + s->len = 0; + if (sp != NULL) free(sp); +} + const char *mg_strstr(const struct mg_str haystack, const struct mg_str needle) WEAK; const char *mg_strstr(const struct mg_str haystack, diff --git a/mongoose.h b/mongoose.h index 98c6730e..6f6d513f 100644 --- a/mongoose.h +++ b/mongoose.h @@ -2280,6 +2280,11 @@ int mg_strcmp(const struct mg_str str1, const struct mg_str str2); */ int mg_strncmp(const struct mg_str str1, const struct mg_str str2, size_t n); +/* + * Free the string (assuming it was heap allocated). + */ +void mg_strfree(struct mg_str *s); + /* * Finds the first occurrence of a substring `needle` in the `haystack`. */ From 86ffa3a3ea1fb742116766680a95a22e7753d79a Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Mon, 21 Jan 2019 17:36:42 +0000 Subject: [PATCH 18/25] Properly support MQTT wildcards when matching topics CL: mg: Properly support MQTT wildcards when matching topics PUBLISHED_FROM=aa60a71d488f4b56dff0951886c8a9a4066fd4f2 --- mongoose.c | 50 ++++++++++++++++++++++++++++++++---------------- src/mg_mqtt.c | 50 ++++++++++++++++++++++++++++++++---------------- test/unit_test.c | 37 +++++++++++++++++++++++++++-------- 3 files changed, 97 insertions(+), 40 deletions(-) diff --git a/mongoose.c b/mongoose.c index a7851188..2b6cd18d 100644 --- a/mongoose.c +++ b/mongoose.c @@ -10977,26 +10977,44 @@ static void mg_mqtt_proto_data_destructor(void *proto_data) { MG_FREE(proto_data); } +static struct mg_str mg_mqtt_next_topic_component(struct mg_str *topic) { + struct mg_str res = *topic; + const char *c = mg_strchr(*topic, '/'); + if (c != NULL) { + res.len = (c - topic->p); + topic->len -= (res.len + 1); + topic->p += (res.len + 1); + } else { + topic->len = 0; + } + return res; +} + +/* Refernce: https://mosquitto.org/man/mqtt-7.html */ int mg_mqtt_match_topic_expression(struct mg_str exp, struct mg_str topic) { - /* TODO(mkm): implement real matching */ - if (memchr(exp.p, '#', exp.len)) { - /* exp `foo/#` will become `foo/` */ - exp.len -= 1; - /* - * topic should be longer than the expression: e.g. topic `foo/bar` does - * match `foo/#`, but neither `foo` nor `foo/` do. - */ - if (topic.len <= exp.len) { + struct mg_str ec, tc; + if (exp.len == 0) return 0; + while (1) { + ec = mg_mqtt_next_topic_component(&exp); + tc = mg_mqtt_next_topic_component(&topic); + if (ec.len == 0) { + if (tc.len != 0) return 0; + if (exp.len == 0) break; + continue; + } + if (mg_vcmp(&ec, "+") == 0) { + if (tc.len == 0 && topic.len == 0) return 0; + continue; + } + if (mg_vcmp(&ec, "#") == 0) { + /* Must be the last component in the expression or it's invalid. */ + return (exp.len == 0); + } + if (mg_strcmp(ec, tc) != 0) { return 0; } - - /* Truncate topic so that it'll pass the next length check */ - topic.len = exp.len; } - if (topic.len != exp.len) { - return 0; - } - return strncmp(topic.p, exp.p, exp.len) == 0; + return (tc.len == 0 && topic.len == 0); } int mg_mqtt_vmatch_topic_expression(const char *exp, struct mg_str topic) { diff --git a/src/mg_mqtt.c b/src/mg_mqtt.c index 5c0b194e..68069e8d 100644 --- a/src/mg_mqtt.c +++ b/src/mg_mqtt.c @@ -196,26 +196,44 @@ static void mg_mqtt_proto_data_destructor(void *proto_data) { MG_FREE(proto_data); } +static struct mg_str mg_mqtt_next_topic_component(struct mg_str *topic) { + struct mg_str res = *topic; + const char *c = mg_strchr(*topic, '/'); + if (c != NULL) { + res.len = (c - topic->p); + topic->len -= (res.len + 1); + topic->p += (res.len + 1); + } else { + topic->len = 0; + } + return res; +} + +/* Refernce: https://mosquitto.org/man/mqtt-7.html */ int mg_mqtt_match_topic_expression(struct mg_str exp, struct mg_str topic) { - /* TODO(mkm): implement real matching */ - if (memchr(exp.p, '#', exp.len)) { - /* exp `foo/#` will become `foo/` */ - exp.len -= 1; - /* - * topic should be longer than the expression: e.g. topic `foo/bar` does - * match `foo/#`, but neither `foo` nor `foo/` do. - */ - if (topic.len <= exp.len) { + struct mg_str ec, tc; + if (exp.len == 0) return 0; + while (1) { + ec = mg_mqtt_next_topic_component(&exp); + tc = mg_mqtt_next_topic_component(&topic); + if (ec.len == 0) { + if (tc.len != 0) return 0; + if (exp.len == 0) break; + continue; + } + if (mg_vcmp(&ec, "+") == 0) { + if (tc.len == 0 && topic.len == 0) return 0; + continue; + } + if (mg_vcmp(&ec, "#") == 0) { + /* Must be the last component in the expression or it's invalid. */ + return (exp.len == 0); + } + if (mg_strcmp(ec, tc) != 0) { return 0; } - - /* Truncate topic so that it'll pass the next length check */ - topic.len = exp.len; } - if (topic.len != exp.len) { - return 0; - } - return strncmp(topic.p, exp.p, exp.len) == 0; + return (tc.len == 0 && topic.len == 0); } int mg_mqtt_vmatch_topic_expression(const char *exp, struct mg_str topic) { diff --git a/test/unit_test.c b/test/unit_test.c index 84f9f8bc..96c9e459 100644 --- a/test/unit_test.c +++ b/test/unit_test.c @@ -3380,15 +3380,36 @@ static const char *test_mqtt_parse_mqtt_qos1(void) { } static const char *test_mqtt_match_topic_expression(void) { - ASSERT_EQ(mg_mqtt_vmatch_topic_expression("foo", mg_mk_str("foo")), 1); - ASSERT_EQ(mg_mqtt_vmatch_topic_expression("foo", mg_mk_str("foo/")), 0); - ASSERT_EQ(mg_mqtt_vmatch_topic_expression("foo", mg_mk_str("foo/bar")), 0); + ASSERT(mg_mqtt_vmatch_topic_expression("foo", mg_mk_str("foo"))); + ASSERT(mg_mqtt_vmatch_topic_expression("/foo", mg_mk_str("/foo"))); + ASSERT(mg_mqtt_vmatch_topic_expression("+/foo", mg_mk_str("/foo"))); + ASSERT(!mg_mqtt_vmatch_topic_expression("foo", mg_mk_str("foobar"))); + ASSERT(mg_mqtt_vmatch_topic_expression("foo", mg_mk_str("foo/"))); + ASSERT(mg_mqtt_vmatch_topic_expression("foo", mg_mk_str("foo//"))); + ASSERT(!mg_mqtt_vmatch_topic_expression("foo", mg_mk_str("foo/bar"))); + ASSERT(!mg_mqtt_vmatch_topic_expression("foo", mg_mk_str("foo/+"))); + ASSERT(mg_mqtt_vmatch_topic_expression("foo/bar", mg_mk_str("foo/bar"))); + ASSERT(mg_mqtt_vmatch_topic_expression("foo/+", mg_mk_str("foo/bar"))); + ASSERT(mg_mqtt_vmatch_topic_expression("+/bar", mg_mk_str("foo/bar"))); + ASSERT(mg_mqtt_vmatch_topic_expression("+/+", mg_mk_str("foo/bar"))); + ASSERT(mg_mqtt_vmatch_topic_expression("foo/+/bar", mg_mk_str("foo//bar"))); + ASSERT(!mg_mqtt_vmatch_topic_expression("foo/+/+", mg_mk_str("foo/bar"))); + ASSERT(mg_mqtt_vmatch_topic_expression("foo/+/#", mg_mk_str("foo/bar"))); + ASSERT(mg_mqtt_vmatch_topic_expression("+/foo/bar", mg_mk_str("/foo/bar"))); - ASSERT_EQ(mg_mqtt_vmatch_topic_expression("foo/#", mg_mk_str("foo")), 0); - ASSERT_EQ(mg_mqtt_vmatch_topic_expression("foo/#", mg_mk_str("foo/")), 0); - ASSERT_EQ(mg_mqtt_vmatch_topic_expression("foo/#", mg_mk_str("foo/bar")), 1); - ASSERT_EQ(mg_mqtt_vmatch_topic_expression("foo/#", mg_mk_str("foo/bar/baz")), - 1); + ASSERT(!mg_mqtt_vmatch_topic_expression("", mg_mk_str(""))); + ASSERT(mg_mqtt_vmatch_topic_expression("/", mg_mk_str(""))); + ASSERT(mg_mqtt_vmatch_topic_expression("/", mg_mk_str("/"))); + + ASSERT(mg_mqtt_vmatch_topic_expression("#", mg_mk_str(""))); + ASSERT(mg_mqtt_vmatch_topic_expression("#", mg_mk_str("foo"))); + ASSERT(mg_mqtt_vmatch_topic_expression("#", mg_mk_str("foo/bar"))); + ASSERT(mg_mqtt_vmatch_topic_expression("foo/#", mg_mk_str("foo"))); + ASSERT(mg_mqtt_vmatch_topic_expression("foo/#", mg_mk_str("foo/"))); + ASSERT(mg_mqtt_vmatch_topic_expression("foo/#", mg_mk_str("foo/bar"))); + ASSERT(mg_mqtt_vmatch_topic_expression("foo/#", mg_mk_str("foo/bar/baz"))); + ASSERT(!mg_mqtt_vmatch_topic_expression("#/foo", mg_mk_str("foo"))); + ASSERT(!mg_mqtt_vmatch_topic_expression("#/foo", mg_mk_str("bar/foo"))); return NULL; } From 95a36b17597276a74629b21828c0e9b1346b8739 Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Sat, 26 Jan 2019 04:37:17 +0000 Subject: [PATCH 19/25] Log file:line instead of function; add cs_log_set_file_level() cs_log_set_filter() is removed in favor of cs_log_set_file_level() which allows setting log verbosity of individual files and/or lines. E.g.: `mg_=1,mjs=1,=4` - level for for everything except mjs and mongoose. We use the fact that we are usually pretty careful with our file names and use prefixes consistently. In mos, `debug.file_level` sets the option on boot and `Sys.SetDebug` is updated to parse `file_level` option to change it at runtime. CL: Log file:line instead of function; add cs_log_set_file_level() PUBLISHED_FROM=49f18d4d0d3dd224865129a4f8fe6cccd1d9bc12 --- examples/CC3200/bm222.c | 3 +- examples/CC3200/cs_dbg.h | 98 +++++++++++++++++++++++---- examples/CC3200/wifi.c | 4 +- mongoose.c | 140 +++++++++++++++++++++------------------ mongoose.h | 2 + src/common/cs_dbg.c | 79 ++++++++++++++-------- src/common/cs_dbg.h | 61 ++++++----------- 7 files changed, 240 insertions(+), 147 deletions(-) diff --git a/examples/CC3200/bm222.c b/examples/CC3200/bm222.c index 416c410f..6fb2517a 100644 --- a/examples/CC3200/bm222.c +++ b/examples/CC3200/bm222.c @@ -98,7 +98,8 @@ bool bm222_get_data(struct bm222_ctx *ctx) { s->y = v[3]; s->z = v[5]; if (d > max_d) max_d = d; - LOG(LL_VERBOSE_DEBUG, ("dx %d dy %d dz %d d %d", dx, dy, dz, d)); + LOG(LL_VERBOSE_DEBUG, + ("dx %d dy %d dz %d d %d", (int) dx, (int) dy, (int) dz, (int) d)); } } return (overflow ? bm222_fifo_init(ctx) : true); /* Clear the ovf flag. */ diff --git a/examples/CC3200/cs_dbg.h b/examples/CC3200/cs_dbg.h index e987c427..8422ca3b 100644 --- a/examples/CC3200/cs_dbg.h +++ b/examples/CC3200/cs_dbg.h @@ -1,11 +1,25 @@ /* - * Copyright (c) 2014-2016 Cesanta Software Limited + * Copyright (c) 2014-2018 Cesanta Software Limited * All rights reserved + * + * Licensed under the Apache License, Version 2.0 (the ""License""); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an ""AS IS"" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #ifndef CS_COMMON_CS_DBG_H_ #define CS_COMMON_CS_DBG_H_ +#include "common/platform.h" + #if CS_ENABLE_STDIO #include #endif @@ -14,6 +28,10 @@ #define CS_ENABLE_DEBUG 0 #endif +#ifndef CS_LOG_PREFIX_LEN +#define CS_LOG_PREFIX_LEN 24 +#endif + #ifndef CS_LOG_ENABLE_TS_DIFF #define CS_LOG_ENABLE_TS_DIFF 0 #endif @@ -22,6 +40,9 @@ extern "C" { #endif /* __cplusplus */ +/* + * Log level; `LL_INFO` is the default. Use `cs_log_set_level()` to change it. + */ enum cs_log_level { LL_NONE = -1, LL_ERROR = 0, @@ -34,29 +55,78 @@ enum cs_log_level { _LL_MAX = 5, }; +/* + * Set max log level to print; messages with the level above the given one will + * not be printed. + */ void cs_log_set_level(enum cs_log_level level); +/* + * A comma-separated set of prefix=level. + * prefix is matched against the log prefix exactly as printed, including line + * number, but partial match is ok. Check stops on first matching entry. + * If nothing matches, default level is used. + * + * Examples: + * main.c:=4 - everything from main C at verbose debug level. + * mongoose.c=1,mjs.c=1,=4 - everything at verbose debug except mg_* and mjs_* + * + */ +void cs_log_set_file_level(const char *file_level); + +/* + * Helper function which prints message prefix with the given `level`. + * If message should be printed (according to the current log level + * and filter), prints the prefix and returns 1, otherwise returns 0. + * + * Clients should typically just use `LOG()` macro. + */ +int cs_log_print_prefix(enum cs_log_level level, const char *fname, int line); + +extern enum cs_log_level cs_log_level; + #if CS_ENABLE_STDIO +/* + * Set file to write logs into. If `NULL`, logs go to `stderr`. + */ void cs_log_set_file(FILE *file); -extern enum cs_log_level cs_log_threshold; -void cs_log_print_prefix(const char *func); -void cs_log_printf(const char *fmt, ...); +/* + * Prints log to the current log file, appends "\n" in the end and flushes the + * stream. + */ +void cs_log_printf(const char *fmt, ...) PRINTF_LIKE(1, 2); -#define LOG(l, x) \ - if (cs_log_threshold >= l) { \ - cs_log_print_prefix(__func__); \ - cs_log_printf x; \ - } +#if CS_ENABLE_STDIO + +/* + * Format and print message `x` with the given level `l`. Example: + * + * ```c + * LOG(LL_INFO, ("my info message: %d", 123)); + * LOG(LL_DEBUG, ("my debug message: %d", 123)); + * ``` + */ +#define LOG(l, x) \ + do { \ + if (cs_log_print_prefix(l, __FILE__, __LINE__)) { \ + cs_log_printf x; \ + } \ + } while (0) + +#else + +#define LOG(l, x) ((void) l) + +#endif #ifndef CS_NDEBUG -#define DBG(x) \ - if (cs_log_threshold >= LL_VERBOSE_DEBUG) { \ - cs_log_print_prefix(__func__); \ - cs_log_printf x; \ - } +/* + * Shortcut for `LOG(LL_VERBOSE_DEBUG, (...))` + */ +#define DBG(x) LOG(LL_VERBOSE_DEBUG, x) #else /* NDEBUG */ diff --git a/examples/CC3200/wifi.c b/examples/CC3200/wifi.c index 035772f6..9f5b9644 100644 --- a/examples/CC3200/wifi.c +++ b/examples/CC3200/wifi.c @@ -25,7 +25,7 @@ void SimpleLinkWlanEventHandler(SlWlanEvent_t *e) { LOG(LL_INFO, ("WiFi: station disconnected")); break; default: - LOG(LL_INFO, ("WiFi: event %d", e->Event)); + LOG(LL_INFO, ("WiFi: event %d", (int) e->Event)); } } @@ -39,7 +39,7 @@ void SimpleLinkNetAppEventHandler(SlNetAppEvent_t *e) { } else if (e->Event == SL_NETAPP_IP_LEASED_EVENT) { LOG(LL_INFO, ("IP leased")); } else { - LOG(LL_INFO, ("NetApp event %d", e->Event)); + LOG(LL_INFO, ("NetApp event %d", (int) e->Event)); } } diff --git a/mongoose.c b/mongoose.c index 2b6cd18d..645cf6a7 100644 --- a/mongoose.c +++ b/mongoose.c @@ -462,6 +462,10 @@ int cs_base64_decode(const unsigned char *s, int len, char *dst, int *dec_len) { #define CS_ENABLE_DEBUG 0 #endif +#ifndef CS_LOG_PREFIX_LEN +#define CS_LOG_PREFIX_LEN 24 +#endif + #ifndef CS_LOG_ENABLE_TS_DIFF #define CS_LOG_ENABLE_TS_DIFF 0 #endif @@ -492,51 +496,28 @@ enum cs_log_level { void cs_log_set_level(enum cs_log_level level); /* - * Set log filter. NULL (a default) logs everything. - * Otherwise, function name and file name will be tested against the given - * pattern, and only matching messages will be printed. + * A comma-separated set of prefix=level. + * prefix is matched against the log prefix exactly as printed, including line + * number, but partial match is ok. Check stops on first matching entry. + * If nothing matches, default level is used. * - * For the pattern syntax, refer to `mg_match_prefix()` in `str_util.h`. + * Examples: + * main.c:=4 - everything from main C at verbose debug level. + * mongoose.c=1,mjs.c=1,=4 - everything at verbose debug except mg_* and mjs_* * - * Example: - * ```c - * void foo(void) { - * LOG(LL_INFO, ("hello from foo")); - * } - * - * void bar(void) { - * LOG(LL_INFO, ("hello from bar")); - * } - * - * void test(void) { - * cs_log_set_filter(NULL); - * foo(); - * bar(); - * - * cs_log_set_filter("f*"); - * foo(); - * bar(); // Will NOT print anything - * - * cs_log_set_filter("bar"); - * foo(); // Will NOT print anything - * bar(); - * } - * ``` */ -void cs_log_set_filter(const char *pattern); +void cs_log_set_file_level(const char *file_level); /* - * Helper function which prints message prefix with the given `level`, function - * name `func` and `filename`. If message should be printed (accordingly to the - * current log level and filter), prints the prefix and returns 1, otherwise - * returns 0. + * Helper function which prints message prefix with the given `level`. + * If message should be printed (according to the current log level + * and filter), prints the prefix and returns 1, otherwise returns 0. * * Clients should typically just use `LOG()` macro. */ -int cs_log_print_prefix(enum cs_log_level level, const char *func, - const char *filename); +int cs_log_print_prefix(enum cs_log_level level, const char *fname, int line); -extern enum cs_log_level cs_log_threshold; +extern enum cs_log_level cs_log_level; #if CS_ENABLE_STDIO @@ -561,9 +542,11 @@ void cs_log_printf(const char *fmt, ...) PRINTF_LIKE(1, 2); * LOG(LL_DEBUG, ("my debug message: %d", 123)); * ``` */ -#define LOG(l, x) \ - do { \ - if (cs_log_print_prefix(l, __func__, __FILE__)) cs_log_printf x; \ +#define LOG(l, x) \ + do { \ + if (cs_log_print_prefix(l, __FILE__, __LINE__)) { \ + cs_log_printf x; \ + } \ } while (0) #else @@ -626,7 +609,7 @@ void cs_log_printf(const char *fmt, ...) PRINTF_LIKE(1, 2); /* Amalgamated: #include "common/cs_time.h" */ /* Amalgamated: #include "common/str_util.h" */ -enum cs_log_level cs_log_threshold WEAK = +enum cs_log_level cs_log_level WEAK = #if CS_ENABLE_DEBUG LL_VERBOSE_DEBUG; #else @@ -634,10 +617,9 @@ enum cs_log_level cs_log_threshold WEAK = #endif #if CS_ENABLE_STDIO -static char *s_filter_pattern = NULL; -static size_t s_filter_pattern_len; +static char *s_file_level = NULL; -void cs_log_set_filter(const char *pattern) WEAK; +void cs_log_set_file_level(const char *file_level) WEAK; FILE *cs_log_file WEAK = NULL; @@ -647,34 +629,62 @@ double cs_log_ts WEAK; enum cs_log_level cs_log_cur_msg_level WEAK = LL_NONE; -void cs_log_set_filter(const char *pattern) { - free(s_filter_pattern); - if (pattern != NULL) { - s_filter_pattern = strdup(pattern); - s_filter_pattern_len = strlen(pattern); +void cs_log_set_file_level(const char *file_level) { + char *fl = s_file_level; + if (file_level != NULL) { + s_file_level = strdup(file_level); } else { - s_filter_pattern = NULL; - s_filter_pattern_len = 0; + s_file_level = NULL; } + free(fl); } -int cs_log_print_prefix(enum cs_log_level, const char *, const char *) WEAK; -int cs_log_print_prefix(enum cs_log_level level, const char *func, - const char *filename) { - char prefix[21]; +int cs_log_print_prefix(enum cs_log_level level, const char *file, int ln) WEAK; +int cs_log_print_prefix(enum cs_log_level level, const char *file, int ln) { + char prefix[CS_LOG_PREFIX_LEN], *q; + const char *p; + size_t fl = 0, ll = 0, pl = 0; - if (level > cs_log_threshold) return 0; - if (s_filter_pattern != NULL && - mg_match_prefix(s_filter_pattern, s_filter_pattern_len, func) == 0 && - mg_match_prefix(s_filter_pattern, s_filter_pattern_len, filename) == 0) { - return 0; + if (level > cs_log_level && s_file_level == NULL) return 0; + + p = file + strlen(file); + + while (p != file) { + const char c = *(p - 1); + if (c == '/' || c == '\\') break; + p--; + fl++; + } + + ll = (ln < 10000 ? (ln < 1000 ? (ln < 100 ? (ln < 10 ? 1 : 2) : 3) : 4) : 5); + if (fl > (sizeof(prefix) - ll - 2)) fl = (sizeof(prefix) - ll - 2); + + pl = fl + 1 + ll; + memcpy(prefix, p, fl); + q = prefix + pl; + memset(q, ' ', sizeof(prefix) - pl); + do { + *(--q) = '0' + (ln % 10); + ln /= 10; + } while (ln > 0); + *(--q) = ':'; + + if (s_file_level != NULL) { + enum cs_log_level pll = cs_log_level; + struct mg_str fl = mg_mk_str(s_file_level), ps = MG_MK_STR_N(prefix, pl); + struct mg_str k, v; + while ((fl = mg_next_comma_list_entry_n(fl, &k, &v)).p != NULL) { + bool yes = !(!mg_str_starts_with(ps, k) || v.len == 0); + if (!yes) continue; + pll = (enum cs_log_level)(*v.p - '0'); + break; + } + if (level > pll) return 0; } - strncpy(prefix, func, 20); - prefix[20] = '\0'; if (cs_log_file == NULL) cs_log_file = stderr; cs_log_cur_msg_level = level; - fprintf(cs_log_file, "%-20s ", prefix); + fwrite(prefix, 1, sizeof(prefix), cs_log_file); #if CS_LOG_ENABLE_TS_DIFF { double now = cs_time(); @@ -703,15 +713,15 @@ void cs_log_set_file(FILE *file) { #else -void cs_log_set_filter(const char *pattern) { - (void) pattern; +void cs_log_set_file_level(const char *file_level) { + (void) file_level; } #endif /* CS_ENABLE_STDIO */ void cs_log_set_level(enum cs_log_level level) WEAK; void cs_log_set_level(enum cs_log_level level) { - cs_log_threshold = level; + cs_log_level = level; #if CS_LOG_ENABLE_TS_DIFF && CS_ENABLE_STDIO cs_log_ts = cs_time(); #endif diff --git a/mongoose.h b/mongoose.h index 6f6d513f..fc36705c 100644 --- a/mongoose.h +++ b/mongoose.h @@ -2647,6 +2647,8 @@ const char *mg_next_comma_list_entry(const char *list, struct mg_str *val, /* * Like `mg_next_comma_list_entry()`, but takes `list` as `struct mg_str`. + * NB: Test return value's .p, not .len. On last itreation that yields result + * .len will be 0 but .p will not. When finished, .p will be NULL. */ struct mg_str mg_next_comma_list_entry_n(struct mg_str list, struct mg_str *val, struct mg_str *eq_val); diff --git a/src/common/cs_dbg.c b/src/common/cs_dbg.c index 594aa153..35c457e3 100644 --- a/src/common/cs_dbg.c +++ b/src/common/cs_dbg.c @@ -24,7 +24,7 @@ #include "common/cs_time.h" #include "common/str_util.h" -enum cs_log_level cs_log_threshold WEAK = +enum cs_log_level cs_log_level WEAK = #if CS_ENABLE_DEBUG LL_VERBOSE_DEBUG; #else @@ -32,10 +32,9 @@ enum cs_log_level cs_log_threshold WEAK = #endif #if CS_ENABLE_STDIO -static char *s_filter_pattern = NULL; -static size_t s_filter_pattern_len; +static char *s_file_level = NULL; -void cs_log_set_filter(const char *pattern) WEAK; +void cs_log_set_file_level(const char *file_level) WEAK; FILE *cs_log_file WEAK = NULL; @@ -45,34 +44,62 @@ double cs_log_ts WEAK; enum cs_log_level cs_log_cur_msg_level WEAK = LL_NONE; -void cs_log_set_filter(const char *pattern) { - free(s_filter_pattern); - if (pattern != NULL) { - s_filter_pattern = strdup(pattern); - s_filter_pattern_len = strlen(pattern); +void cs_log_set_file_level(const char *file_level) { + char *fl = s_file_level; + if (file_level != NULL) { + s_file_level = strdup(file_level); } else { - s_filter_pattern = NULL; - s_filter_pattern_len = 0; + s_file_level = NULL; } + free(fl); } -int cs_log_print_prefix(enum cs_log_level, const char *, const char *) WEAK; -int cs_log_print_prefix(enum cs_log_level level, const char *func, - const char *filename) { - char prefix[21]; +int cs_log_print_prefix(enum cs_log_level level, const char *file, int ln) WEAK; +int cs_log_print_prefix(enum cs_log_level level, const char *file, int ln) { + char prefix[CS_LOG_PREFIX_LEN], *q; + const char *p; + size_t fl = 0, ll = 0, pl = 0; - if (level > cs_log_threshold) return 0; - if (s_filter_pattern != NULL && - mg_match_prefix(s_filter_pattern, s_filter_pattern_len, func) == 0 && - mg_match_prefix(s_filter_pattern, s_filter_pattern_len, filename) == 0) { - return 0; + if (level > cs_log_level && s_file_level == NULL) return 0; + + p = file + strlen(file); + + while (p != file) { + const char c = *(p - 1); + if (c == '/' || c == '\\') break; + p--; + fl++; + } + + ll = (ln < 10000 ? (ln < 1000 ? (ln < 100 ? (ln < 10 ? 1 : 2) : 3) : 4) : 5); + if (fl > (sizeof(prefix) - ll - 2)) fl = (sizeof(prefix) - ll - 2); + + pl = fl + 1 + ll; + memcpy(prefix, p, fl); + q = prefix + pl; + memset(q, ' ', sizeof(prefix) - pl); + do { + *(--q) = '0' + (ln % 10); + ln /= 10; + } while (ln > 0); + *(--q) = ':'; + + if (s_file_level != NULL) { + enum cs_log_level pll = cs_log_level; + struct mg_str fl = mg_mk_str(s_file_level), ps = MG_MK_STR_N(prefix, pl); + struct mg_str k, v; + while ((fl = mg_next_comma_list_entry_n(fl, &k, &v)).p != NULL) { + bool yes = !(!mg_str_starts_with(ps, k) || v.len == 0); + if (!yes) continue; + pll = (enum cs_log_level)(*v.p - '0'); + break; + } + if (level > pll) return 0; } - strncpy(prefix, func, 20); - prefix[20] = '\0'; if (cs_log_file == NULL) cs_log_file = stderr; cs_log_cur_msg_level = level; - fprintf(cs_log_file, "%-20s ", prefix); + fwrite(prefix, 1, sizeof(prefix), cs_log_file); #if CS_LOG_ENABLE_TS_DIFF { double now = cs_time(); @@ -101,15 +128,15 @@ void cs_log_set_file(FILE *file) { #else -void cs_log_set_filter(const char *pattern) { - (void) pattern; +void cs_log_set_file_level(const char *file_level) { + (void) file_level; } #endif /* CS_ENABLE_STDIO */ void cs_log_set_level(enum cs_log_level level) WEAK; void cs_log_set_level(enum cs_log_level level) { - cs_log_threshold = level; + cs_log_level = level; #if CS_LOG_ENABLE_TS_DIFF && CS_ENABLE_STDIO cs_log_ts = cs_time(); #endif diff --git a/src/common/cs_dbg.h b/src/common/cs_dbg.h index 5b9777c1..8422ca3b 100644 --- a/src/common/cs_dbg.h +++ b/src/common/cs_dbg.h @@ -28,6 +28,10 @@ #define CS_ENABLE_DEBUG 0 #endif +#ifndef CS_LOG_PREFIX_LEN +#define CS_LOG_PREFIX_LEN 24 +#endif + #ifndef CS_LOG_ENABLE_TS_DIFF #define CS_LOG_ENABLE_TS_DIFF 0 #endif @@ -58,51 +62,28 @@ enum cs_log_level { void cs_log_set_level(enum cs_log_level level); /* - * Set log filter. NULL (a default) logs everything. - * Otherwise, function name and file name will be tested against the given - * pattern, and only matching messages will be printed. + * A comma-separated set of prefix=level. + * prefix is matched against the log prefix exactly as printed, including line + * number, but partial match is ok. Check stops on first matching entry. + * If nothing matches, default level is used. * - * For the pattern syntax, refer to `mg_match_prefix()` in `str_util.h`. + * Examples: + * main.c:=4 - everything from main C at verbose debug level. + * mongoose.c=1,mjs.c=1,=4 - everything at verbose debug except mg_* and mjs_* * - * Example: - * ```c - * void foo(void) { - * LOG(LL_INFO, ("hello from foo")); - * } - * - * void bar(void) { - * LOG(LL_INFO, ("hello from bar")); - * } - * - * void test(void) { - * cs_log_set_filter(NULL); - * foo(); - * bar(); - * - * cs_log_set_filter("f*"); - * foo(); - * bar(); // Will NOT print anything - * - * cs_log_set_filter("bar"); - * foo(); // Will NOT print anything - * bar(); - * } - * ``` */ -void cs_log_set_filter(const char *pattern); +void cs_log_set_file_level(const char *file_level); /* - * Helper function which prints message prefix with the given `level`, function - * name `func` and `filename`. If message should be printed (accordingly to the - * current log level and filter), prints the prefix and returns 1, otherwise - * returns 0. + * Helper function which prints message prefix with the given `level`. + * If message should be printed (according to the current log level + * and filter), prints the prefix and returns 1, otherwise returns 0. * * Clients should typically just use `LOG()` macro. */ -int cs_log_print_prefix(enum cs_log_level level, const char *func, - const char *filename); +int cs_log_print_prefix(enum cs_log_level level, const char *fname, int line); -extern enum cs_log_level cs_log_threshold; +extern enum cs_log_level cs_log_level; #if CS_ENABLE_STDIO @@ -127,9 +108,11 @@ void cs_log_printf(const char *fmt, ...) PRINTF_LIKE(1, 2); * LOG(LL_DEBUG, ("my debug message: %d", 123)); * ``` */ -#define LOG(l, x) \ - do { \ - if (cs_log_print_prefix(l, __func__, __FILE__)) cs_log_printf x; \ +#define LOG(l, x) \ + do { \ + if (cs_log_print_prefix(l, __FILE__, __LINE__)) { \ + cs_log_printf x; \ + } \ } while (0) #else From 0b2e474d52180dcfac80772e6a62999444333353 Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Fri, 8 Feb 2019 16:20:51 +0000 Subject: [PATCH 20/25] Improve ESP flasher baud rate setting Use old baud rate and autodetected divider to calculate new divider. This works better with higher baud rates and drifting clocks. CL: mos: Improve ESP flasher baud rate setting PUBLISHED_FROM=7761e756c3ae7cc76e3b9a4fe3e40d06a779b2e8 --- src/common/platforms/esp/stub_flasher.c | 17 +++++++++-------- src/common/platforms/esp32/stubs/stub.ld | 5 +++++ src/common/platforms/esp32/stubs/uart.c | 14 +++++++++----- src/common/platforms/esp32/stubs/uart.h | 3 ++- src/common/platforms/esp8266/stubs/Makefile | 6 +++--- src/common/platforms/esp8266/stubs/stub.ld | 1 + src/common/platforms/esp8266/stubs/uart.c | 14 ++++++++++---- src/common/platforms/esp8266/stubs/uart.h | 6 ++++-- 8 files changed, 43 insertions(+), 23 deletions(-) diff --git a/src/common/platforms/esp/stub_flasher.c b/src/common/platforms/esp/stub_flasher.c index d733d359..09afcfe3 100644 --- a/src/common/platforms/esp/stub_flasher.c +++ b/src/common/platforms/esp/stub_flasher.c @@ -25,7 +25,7 @@ #if defined(ESP8266) #include "eagle_soc.h" #include "ets_sys.h" -#include "../../../miniz.c" +#include "miniz.c" #elif defined(ESP32) #include "rom/efuse.h" #include "rom/miniz.h" @@ -38,7 +38,7 @@ #include "uart.h" /* Param: baud rate. */ -uint32_t params[1] __attribute__((section(".params"))); +uint32_t params[2] __attribute__((section(".params"))); #define FLASH_BLOCK_SIZE 65536 #define FLASH_SECTOR_SIZE 4096 @@ -477,8 +477,7 @@ uint8_t cmd_loop(void) { } void stub_main1(void) { - uint32_t baud_rate = params[0]; - uint32_t greeting = 0x4941484f; /* OHAI */ + uint32_t old_baud_rate = params[0], new_baud_rate = params[1]; uint8_t last_cmd; /* This points at us right now, reset for next boot. */ @@ -487,6 +486,7 @@ void stub_main1(void) { /* Selects SPI functions for flash pins. */ #if defined(ESP8266) SelectSpiFunction(); + spi_flash_attach(); SET_PERI_REG_MASK(0x3FF00014, 1); /* Switch to 160 MHz */ #elif defined(ESP32) esp_rom_spiflash_attach(ets_efuse_get_spiconfig(), 0 /* legacy */); @@ -496,9 +496,10 @@ void stub_main1(void) { 0 /* deviceId */, 16 * 1024 * 1024 /* chip_size */, FLASH_BLOCK_SIZE, FLASH_SECTOR_SIZE, FLASH_PAGE_SIZE, 0xffff /* status_mask */); - if (baud_rate > 0) { + uint32_t old_div = 0; + if (new_baud_rate > 0) { ets_delay_us(10000); - set_baud_rate(0, baud_rate); + old_div = set_baud_rate(0, old_baud_rate, new_baud_rate); } /* Give host time to get ready too. */ @@ -509,7 +510,7 @@ void stub_main1(void) { WRITE_PERI_REG(UART_FIFO_REG(0), 0x55); } #else - SLIP_send(&greeting, 4); + SLIP_send(&old_div, 4); #endif last_cmd = cmd_loop(); @@ -525,7 +526,7 @@ void stub_main1(void) { * then jumps to 0x4000108a, then checks strapping bits again (which will * not have changed), and then proceeds to 0x400010a8. */ - volatile uint32_t *sp = &baud_rate; + volatile uint32_t *sp = &old_baud_rate; while (*sp != (uint32_t) 0x40001100) sp++; *sp = 0x400010a8; /* diff --git a/src/common/platforms/esp32/stubs/stub.ld b/src/common/platforms/esp32/stubs/stub.ld index 6c2af65a..84085bb3 100644 --- a/src/common/platforms/esp32/stubs/stub.ld +++ b/src/common/platforms/esp32/stubs/stub.ld @@ -42,5 +42,10 @@ SECTIONS { INCLUDE "components/esp32/ld/esp32.rom.ld" INCLUDE "components/esp32/ld/esp32.rom.spiram_incompatible_fns.ld" +PROVIDE(ets_isr_mask = 0x400067fc); +PROVIDE(ets_isr_unmask = 0x40006808); +PROVIDE(MD5Init = 0x4005da7c); +PROVIDE(MD5Update = 0x4005da9c); +PROVIDE(MD5Final = 0x4005db1c); PROVIDE(esp_rom_spiflash_attach = 0x40062a6c); PROVIDE(esp_rom_spiflash_config_clk = 0x40062bc8); diff --git a/src/common/platforms/esp32/stubs/uart.c b/src/common/platforms/esp32/stubs/uart.c index 8d2bca74..d6d1faad 100644 --- a/src/common/platforms/esp32/stubs/uart.c +++ b/src/common/platforms/esp32/stubs/uart.c @@ -19,9 +19,13 @@ #include "rom_functions.h" -void set_baud_rate(uint32_t uart_no, uint32_t baud_rate) { - uint32_t master_freq = ets_get_detected_xtal_freq() << 4; - master_freq += (baud_rate / 2); - uint32_t div = master_freq / baud_rate; - uart_div_modify(uart_no, div); +uint32_t set_baud_rate(uint32_t uart_no, uint32_t old_baud_rate, + uint32_t new_baud_rate) { + uint32_t uart_reg = REG_READ(UART_CLKDIV_REG(uart_no)); + uint32_t uart_div = uart_reg & UART_CLKDIV_M; + uint32_t fraction = (uart_reg >> UART_CLKDIV_FRAG_S) & UART_CLKDIV_FRAG_V; + uart_div = (uart_div << 4) + fraction; + uint32_t master_freq = uart_div * old_baud_rate; + uart_div_modify(uart_no, master_freq / new_baud_rate); + return uart_div; } diff --git a/src/common/platforms/esp32/stubs/uart.h b/src/common/platforms/esp32/stubs/uart.h index b28edf88..99e351ac 100644 --- a/src/common/platforms/esp32/stubs/uart.h +++ b/src/common/platforms/esp32/stubs/uart.h @@ -20,6 +20,7 @@ #include -void set_baud_rate(uint32_t uart_no, uint32_t baud_rate); +uint32_t set_baud_rate(uint32_t uart_no, uint32_t old_baud_rate, + uint32_t new_baud_rate); #endif /* CS_COMMON_PLATFORMS_ESP32_STUBS_UART_H_ */ diff --git a/src/common/platforms/esp8266/stubs/Makefile b/src/common/platforms/esp8266/stubs/Makefile index 7aee5613..f7a76122 100644 --- a/src/common/platforms/esp8266/stubs/Makefile +++ b/src/common/platforms/esp8266/stubs/Makefile @@ -23,13 +23,13 @@ all: $(STUB_ELF) $(STUB_ELF): $(STUB) $(LIBS) @echo " CC $^ -> $@" @[ -d $(BUILD_DIR) ] || mkdir $(BUILD_DIR) - @docker run --rm -i -v $(CURDIR)/../../../..:/src $(SDK) //bin/bash -c \ + docker run --rm -i -v $(CURDIR)/../../../..:/src $(SDK) //bin/bash -c \ "cd /src/common/platforms/esp8266/stubs && \ $(XT_CC) -std=c99 -Wall -Werror -Os -DESP8266 \ -mtext-section-literals -mlongcalls -nostdlib -fno-builtin \ - -I. -I/src/common/platforms/esp \ + -I. -I/src/common/platforms/esp -I/src/common/platforms/esp8266 \ -I/opt/Espressif/ESP8266_SDK \ - -Wl,-static -ffunction-sections -Wl,--gc-sections \ + -Wl,-static -ffunction-sections -lgcc -Wl,--gc-sections \ -Tstub.ld $(CFLAGS) -o $@ $^" wrap: $(STUB_JSON) diff --git a/src/common/platforms/esp8266/stubs/stub.ld b/src/common/platforms/esp8266/stubs/stub.ld index f54a4f84..08c416c4 100644 --- a/src/common/platforms/esp8266/stubs/stub.ld +++ b/src/common/platforms/esp8266/stubs/stub.ld @@ -41,3 +41,4 @@ INCLUDE "eagle.rom.addr.v6.ld" PROVIDE(SPIFlashModeConfig = 0x40004568); PROVIDE(SPI_erase_sector = 0x400040c0); PROVIDE(SPI_erase_block = 0x40004120); +PROVIDE(uart_div_modify = 0x400039d8); diff --git a/src/common/platforms/esp8266/stubs/uart.c b/src/common/platforms/esp8266/stubs/uart.c index cb99b2d9..39e28117 100644 --- a/src/common/platforms/esp8266/stubs/uart.c +++ b/src/common/platforms/esp8266/stubs/uart.c @@ -18,9 +18,15 @@ #include "uart.h" #include "ets_sys.h" -#define UART_CLKDIV_26MHZ(B) (52000000 + B / 2) / B +#define UART_CLKDIV_M (UART_CLKDIV_CNT << UART_CLKDIV_S) -void set_baud_rate(uint32_t uart_no, uint32_t baud_rate) { - uint32_t div = UART_CLKDIV_26MHZ(baud_rate); - WRITE_PERI_REG(UART_CLKDIV_REG(uart_no), div & 0xfffff); +void uart_div_modify(uint8_t uart_no, uint32_t div); + +uint32_t set_baud_rate(uint32_t uart_no, uint32_t old_baud_rate, + uint32_t new_baud_rate) { + uint32_t uart_reg = READ_PERI_REG(UART_CLKDIV_REG(uart_no)); + uint32_t uart_div = uart_reg & UART_CLKDIV_M; + uint32_t master_freq = uart_div * old_baud_rate; + uart_div_modify(uart_no, master_freq / new_baud_rate); + return uart_div; } diff --git a/src/common/platforms/esp8266/stubs/uart.h b/src/common/platforms/esp8266/stubs/uart.h index ad3ef1dc..5afd9e38 100644 --- a/src/common/platforms/esp8266/stubs/uart.h +++ b/src/common/platforms/esp8266/stubs/uart.h @@ -20,9 +20,11 @@ #include -void set_baud_rate(uint32_t uart_no, uint32_t baud_rate); +#include "uart_register.h" + +uint32_t set_baud_rate(uint32_t uart_no, uint32_t old_baud_rate, + uint32_t new_baud_rate); -#define REG_UART_BASE(i) (0x60000000 + (i) *0xf00) #define UART_FIFO_REG(i) (REG_UART_BASE(i) + 0x0) #define UART_CONF1_REG(i) (REG_UART_BASE(i) + 0x24) #define UART_RX_TOUT_EN (BIT(31)) From c3057df3f507e15a04267ac83fd23e22e4851574 Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Fri, 8 Feb 2019 17:33:27 +0000 Subject: [PATCH 21/25] Fix mongoose CI test CL: none PUBLISHED_FROM=9510e460251c250cb6a67a40edc0af515044f9fe --- test/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/Makefile b/test/Makefile index 89223ab8..97f41c9a 100644 --- a/test/Makefile +++ b/test/Makefile @@ -9,7 +9,9 @@ SRC_DIR = ../src COMMON_DIR_PUB = ../src/common COMMON_DIR_DEV = ../../common -ifneq ("$(wildcard ../../fw/defs)","") +# Our dev repo and public mongoose repo have different layouts, so here +# we make it work on both +ifneq ("$(wildcard ../../fw)","") COMMON_PARENT = ../.. else COMMON_PARENT = $(SRC_DIR) @@ -19,8 +21,6 @@ TEST_SOURCES = unit_test.c test_util.c test_main.c AMALGAMATED_SOURCES = ../mongoose.c KRYPTON_PATH = $(REPO_ROOT)/krypton -# Our dev repo and public mongoose repo have different layouts, so here -# we make it work on both VPATH = $(MONGOOSE_DIR) $(SRC_DIR) $(COMMON_DIR_PUB) $(COMMON_DIR_DEV) $(COMMON_PARENT) # or Krypton, or mbedTLS From 148d3f05625afa98d64db3906985fd5d25590b05 Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Tue, 12 Feb 2019 12:13:50 +0000 Subject: [PATCH 22/25] STM32F2 and Particle Electron board support CL: STM32F2 and Particle Electron board support PUBLISHED_FROM=20dede6546e9f99d2829508177924790d87e985a --- src/common/platforms/arm/arm_nsleep100_m3.S | 50 +++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 src/common/platforms/arm/arm_nsleep100_m3.S diff --git a/src/common/platforms/arm/arm_nsleep100_m3.S b/src/common/platforms/arm/arm_nsleep100_m3.S new file mode 100644 index 00000000..cae920e3 --- /dev/null +++ b/src/common/platforms/arm/arm_nsleep100_m3.S @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2014-2019 Cesanta Software Limited + * All rights reserved + * + * Licensed under the Apache License, Version 2.0 (the ""License""); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an ""AS IS"" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.arch armv7e-m +.syntax unified +.thumb + +/* These are required to satisfy TI linker. */ +.eabi_attribute Tag_ABI_align_needed, 1 +.eabi_attribute Tag_ABI_align_preserved, 1 + +.global mgos_nsleep100_impl +.global mgos_nsleep100_loop_count + +#ifdef STM32 +.section .RamFunc.mgos_nsleep100_impl +#else +.section .iram.mgos_nsleep100_impl +#endif +.type mgos_nsleep100_impl, %function +.align 4 + +mgos_nsleep100_impl: + ldr r3, =mgos_nsleep100_loop_count + ldr r3, [r3] + mul r0, r3 + mov r1, #6 + udiv r0, r0, r1 + cbz r0, xxx +lxx: + subs r0, #1 + bne lxx +xxx: + bx lr +.align 4 +.size mgos_nsleep100_impl, . - mgos_nsleep100_impl From 1e9fabe1ca6e02f5426086075ec50ed5ea7ead1d Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Wed, 13 Feb 2019 16:10:44 +0000 Subject: [PATCH 23/25] Fix handling of WS handshake error response Check response code, make sure it's 101. Pass http_message to the client to keep it appraised. This represents a slight change in the API - in case of an error MG_EV_WEBSOCKET_HANDSHAKE_DONE will now be delivered where previosuly connection would just hang. Clients that do not examine the argument may for a moment think handshake has succeeded but in fact connection will be closed immediately. CL: mg: Fix handling of WS handshake error response PUBLISHED_FROM=645a43d9e5bee216e54411f85827c9b974e9a7d1 --- .../mg_set_protocol_http_websocket.md | 4 +- .../websocket_chat_client.c | 12 ++- mongoose.c | 29 ++++--- mongoose.h | 6 +- src/mg_http.c | 27 ++++--- src/mg_http.h | 6 +- src/mg_http_websocket.c | 2 + test/unit_test.c | 80 ++++++++++++------- 8 files changed, 108 insertions(+), 58 deletions(-) diff --git a/docs/c-api/mg_http.h/mg_set_protocol_http_websocket.md b/docs/c-api/mg_http.h/mg_set_protocol_http_websocket.md index a78ff358..7320ae76 100644 --- a/docs/c-api/mg_http.h/mg_set_protocol_http_websocket.md +++ b/docs/c-api/mg_http.h/mg_set_protocol_http_websocket.md @@ -30,7 +30,9 @@ The user-defined event handler will receive following extra events: - MG_EV_WEBSOCKET_HANDSHAKE_REQUEST: server has received the WebSocket handshake request. `ev_data` contains parsed HTTP request. - MG_EV_WEBSOCKET_HANDSHAKE_DONE: server has completed the WebSocket - handshake. `ev_data` is `NULL`. + handshake. `ev_data` is a `struct http_message` containing the + client's request (server mode) or server's response (client). + In client mode handler can examine `resp_code`, which should be 101. - MG_EV_WEBSOCKET_FRAME: new WebSocket frame has arrived. `ev_data` is `struct websocket_message *` diff --git a/examples/websocket_chat_client/websocket_chat_client.c b/examples/websocket_chat_client/websocket_chat_client.c index 8191b4bf..c74ef4bb 100644 --- a/examples/websocket_chat_client/websocket_chat_client.c +++ b/examples/websocket_chat_client/websocket_chat_client.c @@ -14,7 +14,6 @@ static int s_done = 0; static int s_is_connected = 0; static void ev_handler(struct mg_connection *nc, int ev, void *ev_data) { - struct websocket_message *wm = (struct websocket_message *) ev_data; (void) nc; switch (ev) { @@ -26,8 +25,14 @@ static void ev_handler(struct mg_connection *nc, int ev, void *ev_data) { break; } case MG_EV_WEBSOCKET_HANDSHAKE_DONE: { - printf("-- Connected\n"); - s_is_connected = 1; + struct http_message *hm = (struct http_message *) ev_data; + if (hm->resp_code == 101) { + printf("-- Connected\n"); + s_is_connected = 1; + } else { + printf("-- Connection failed! HTTP code %d\n", hm->resp_code); + /* Connection will be closed after this. */ + } break; } case MG_EV_POLL: { @@ -66,6 +71,7 @@ static void ev_handler(struct mg_connection *nc, int ev, void *ev_data) { break; } case MG_EV_WEBSOCKET_FRAME: { + struct websocket_message *wm = (struct websocket_message *) ev_data; printf("%.*s\n", (int) wm->size, wm->data); break; } diff --git a/mongoose.c b/mongoose.c index 645cf6a7..9026762e 100644 --- a/mongoose.c +++ b/mongoose.c @@ -6649,16 +6649,23 @@ void mg_http_handler(struct mg_connection *nc, int ev, /* Do nothing, request is not yet fully buffered */ } #if MG_ENABLE_HTTP_WEBSOCKET - else if (nc->listener == NULL && - mg_get_http_header(hm, "Sec-WebSocket-Accept")) { + else if (nc->listener == NULL && (nc->flags & MG_F_IS_WEBSOCKET)) { /* We're websocket client, got handshake response from server. */ - /* TODO(lsm): check the validity of accept Sec-WebSocket-Accept */ - mbuf_remove(io, req_len); - nc->proto_handler = mg_ws_handler; - nc->flags |= MG_F_IS_WEBSOCKET; - mg_call(nc, nc->handler, nc->user_data, MG_EV_WEBSOCKET_HANDSHAKE_DONE, - NULL); - mg_ws_handler(nc, MG_EV_RECV, ev_data MG_UD_ARG(user_data)); + DBG(("%p WebSocket upgrade code %d", nc, hm->resp_code)); + if (hm->resp_code == 101 && + mg_get_http_header(hm, "Sec-WebSocket-Accept")) { + /* TODO(lsm): check the validity of accept Sec-WebSocket-Accept */ + mg_call(nc, nc->handler, nc->user_data, MG_EV_WEBSOCKET_HANDSHAKE_DONE, + hm); + mbuf_remove(io, req_len); + nc->proto_handler = mg_ws_handler; + mg_ws_handler(nc, MG_EV_RECV, ev_data MG_UD_ARG(user_data)); + } else { + mg_call(nc, nc->handler, nc->user_data, MG_EV_WEBSOCKET_HANDSHAKE_DONE, + hm); + nc->flags |= MG_F_CLOSE_IMMEDIATELY; + mbuf_remove(io, req_len); + } } else if (nc->listener != NULL && (vec = mg_get_http_header(hm, "Sec-WebSocket-Key")) != NULL) { struct mg_http_endpoint *ep; @@ -6689,7 +6696,7 @@ void mg_http_handler(struct mg_connection *nc, int ev, mg_ws_handshake(nc, vec, hm); } mg_call(nc, nc->handler, nc->user_data, MG_EV_WEBSOCKET_HANDSHAKE_DONE, - NULL); + hm); mg_ws_handler(nc, MG_EV_RECV, ev_data MG_UD_ARG(user_data)); } } @@ -10402,6 +10409,8 @@ void mg_send_websocket_handshake3v(struct mg_connection *nc, } mg_printf(nc, "\r\n"); + nc->flags |= MG_F_IS_WEBSOCKET; + mbuf_free(&auth); } diff --git a/mongoose.h b/mongoose.h index fc36705c..88a552aa 100644 --- a/mongoose.h +++ b/mongoose.h @@ -4806,7 +4806,7 @@ struct mg_ssi_call_ctx { #if MG_ENABLE_HTTP_WEBSOCKET #define MG_EV_WEBSOCKET_HANDSHAKE_REQUEST 111 /* struct http_message * */ -#define MG_EV_WEBSOCKET_HANDSHAKE_DONE 112 /* NULL */ +#define MG_EV_WEBSOCKET_HANDSHAKE_DONE 112 /* struct http_message * */ #define MG_EV_WEBSOCKET_FRAME 113 /* struct websocket_message * */ #define MG_EV_WEBSOCKET_CONTROL_FRAME 114 /* struct websocket_message * */ #endif @@ -4845,7 +4845,9 @@ struct mg_ssi_call_ctx { * - MG_EV_WEBSOCKET_HANDSHAKE_REQUEST: server has received the WebSocket * handshake request. `ev_data` contains parsed HTTP request. * - MG_EV_WEBSOCKET_HANDSHAKE_DONE: server has completed the WebSocket - * handshake. `ev_data` is `NULL`. + * handshake. `ev_data` is a `struct http_message` containing the + * client's request (server mode) or server's response (client). + * In client mode handler can examine `resp_code`, which should be 101. * - MG_EV_WEBSOCKET_FRAME: new WebSocket frame has arrived. `ev_data` is * `struct websocket_message *` * diff --git a/src/mg_http.c b/src/mg_http.c index 46643773..5cd6f623 100644 --- a/src/mg_http.c +++ b/src/mg_http.c @@ -827,16 +827,23 @@ void mg_http_handler(struct mg_connection *nc, int ev, /* Do nothing, request is not yet fully buffered */ } #if MG_ENABLE_HTTP_WEBSOCKET - else if (nc->listener == NULL && - mg_get_http_header(hm, "Sec-WebSocket-Accept")) { + else if (nc->listener == NULL && (nc->flags & MG_F_IS_WEBSOCKET)) { /* We're websocket client, got handshake response from server. */ - /* TODO(lsm): check the validity of accept Sec-WebSocket-Accept */ - mbuf_remove(io, req_len); - nc->proto_handler = mg_ws_handler; - nc->flags |= MG_F_IS_WEBSOCKET; - mg_call(nc, nc->handler, nc->user_data, MG_EV_WEBSOCKET_HANDSHAKE_DONE, - NULL); - mg_ws_handler(nc, MG_EV_RECV, ev_data MG_UD_ARG(user_data)); + DBG(("%p WebSocket upgrade code %d", nc, hm->resp_code)); + if (hm->resp_code == 101 && + mg_get_http_header(hm, "Sec-WebSocket-Accept")) { + /* TODO(lsm): check the validity of accept Sec-WebSocket-Accept */ + mg_call(nc, nc->handler, nc->user_data, MG_EV_WEBSOCKET_HANDSHAKE_DONE, + hm); + mbuf_remove(io, req_len); + nc->proto_handler = mg_ws_handler; + mg_ws_handler(nc, MG_EV_RECV, ev_data MG_UD_ARG(user_data)); + } else { + mg_call(nc, nc->handler, nc->user_data, MG_EV_WEBSOCKET_HANDSHAKE_DONE, + hm); + nc->flags |= MG_F_CLOSE_IMMEDIATELY; + mbuf_remove(io, req_len); + } } else if (nc->listener != NULL && (vec = mg_get_http_header(hm, "Sec-WebSocket-Key")) != NULL) { struct mg_http_endpoint *ep; @@ -867,7 +874,7 @@ void mg_http_handler(struct mg_connection *nc, int ev, mg_ws_handshake(nc, vec, hm); } mg_call(nc, nc->handler, nc->user_data, MG_EV_WEBSOCKET_HANDSHAKE_DONE, - NULL); + hm); mg_ws_handler(nc, MG_EV_RECV, ev_data MG_UD_ARG(user_data)); } } diff --git a/src/mg_http.h b/src/mg_http.h index 32849058..3be7656f 100644 --- a/src/mg_http.h +++ b/src/mg_http.h @@ -106,7 +106,7 @@ struct mg_ssi_call_ctx { #if MG_ENABLE_HTTP_WEBSOCKET #define MG_EV_WEBSOCKET_HANDSHAKE_REQUEST 111 /* struct http_message * */ -#define MG_EV_WEBSOCKET_HANDSHAKE_DONE 112 /* NULL */ +#define MG_EV_WEBSOCKET_HANDSHAKE_DONE 112 /* struct http_message * */ #define MG_EV_WEBSOCKET_FRAME 113 /* struct websocket_message * */ #define MG_EV_WEBSOCKET_CONTROL_FRAME 114 /* struct websocket_message * */ #endif @@ -145,7 +145,9 @@ struct mg_ssi_call_ctx { * - MG_EV_WEBSOCKET_HANDSHAKE_REQUEST: server has received the WebSocket * handshake request. `ev_data` contains parsed HTTP request. * - MG_EV_WEBSOCKET_HANDSHAKE_DONE: server has completed the WebSocket - * handshake. `ev_data` is `NULL`. + * handshake. `ev_data` is a `struct http_message` containing the + * client's request (server mode) or server's response (client). + * In client mode handler can examine `resp_code`, which should be 101. * - MG_EV_WEBSOCKET_FRAME: new WebSocket frame has arrived. `ev_data` is * `struct websocket_message *` * diff --git a/src/mg_http_websocket.c b/src/mg_http_websocket.c index e69e4b73..317b4997 100644 --- a/src/mg_http_websocket.c +++ b/src/mg_http_websocket.c @@ -478,6 +478,8 @@ void mg_send_websocket_handshake3v(struct mg_connection *nc, } mg_printf(nc, "\r\n"); + nc->flags |= MG_F_IS_WEBSOCKET; + mbuf_free(&auth); } diff --git a/test/unit_test.c b/test/unit_test.c index 96c9e459..5901c21c 100644 --- a/test/unit_test.c +++ b/test/unit_test.c @@ -2540,32 +2540,6 @@ static const char *test_http_range(void) { return NULL; } -static void cb3(struct mg_connection *nc, int ev, void *ev_data) { - struct websocket_message *wm = (struct websocket_message *) ev_data; - - if (ev == MG_EV_WEBSOCKET_FRAME) { - const char *reply = wm->size == 2 && !memcmp(wm->data, "hi", 2) ? "A" : "B"; - mg_printf_websocket_frame(nc, WEBSOCKET_OP_TEXT, "%s", reply); - } -} - -static void cb4(struct mg_connection *nc, int ev, void *ev_data) { - struct websocket_message *wm = (struct websocket_message *) ev_data; - - if (ev == MG_EV_WEBSOCKET_FRAME) { - memcpy(nc->user_data, wm->data, wm->size); - mg_send_websocket_frame(nc, WEBSOCKET_OP_CLOSE, NULL, 0); - } else if (ev == MG_EV_WEBSOCKET_HANDSHAKE_DONE) { - /* Send "hi" to server. server must reply "A". */ - struct mg_str h[2]; - h[0].p = "h"; - h[0].len = 1; - h[1].p = "i"; - h[1].len = 1; - mg_send_websocket_framev(nc, WEBSOCKET_OP_TEXT, h, 2); - } -} - static void cb_ws_server(struct mg_connection *nc, int ev, void *ev_data) { struct websocket_message *wm = (struct websocket_message *) ev_data; @@ -2780,6 +2754,38 @@ static const char *test_websocket(void) { return NULL; } +static void cb3(struct mg_connection *nc, int ev, void *ev_data) { + struct websocket_message *wm = (struct websocket_message *) ev_data; + if (ev != MG_EV_WEBSOCKET_FRAME) return; + DBG(("server data '%.*s'", (int) wm->size, wm->data)); + const char *reply = wm->size == 2 && !memcmp(wm->data, "hi", 2) ? "A" : "B"; + mg_printf_websocket_frame(nc, WEBSOCKET_OP_TEXT, "%s", reply); +} + +static void cb4(struct mg_connection *nc, int ev, void *ev_data) { + char *buf = (char *) nc->user_data; + if (ev == MG_EV_WEBSOCKET_FRAME) { + struct websocket_message *wm = (struct websocket_message *) ev_data; + DBG(("client data '%.*s'", (int) wm->size, wm->data)); + memcpy(buf, wm->data, wm->size); + mg_send_websocket_frame(nc, WEBSOCKET_OP_CLOSE, NULL, 0); + } else if (ev == MG_EV_WEBSOCKET_HANDSHAKE_DONE) { + struct http_message *hm = (struct http_message *) ev_data; + DBG(("code %d", hm->resp_code)); + if (hm->resp_code == 101) { + /* Send "hi" to server. server must reply "A". */ + struct mg_str h[2]; + h[0].p = "h"; + h[0].len = 1; + h[1].p = "i"; + h[1].len = 1; + mg_send_websocket_framev(nc, WEBSOCKET_OP_TEXT, h, 2); + } else { + snprintf(buf, 20, "code %d", hm->resp_code); + } + } +} + static void cbwep(struct mg_connection *c, int ev, void *ev_data) { struct websocket_message *wm = (struct websocket_message *) ev_data; char *buf = (char *) c->user_data; @@ -2787,6 +2793,9 @@ static void cbwep(struct mg_connection *c, int ev, void *ev_data) { switch (ev) { case MG_EV_WEBSOCKET_HANDSHAKE_REQUEST: strcat(buf, "R"); + if (buf[0] != '0') { + mg_http_send_error(c, 403, "I don't like you"); + } break; case MG_EV_WEBSOCKET_HANDSHAKE_DONE: strcat(buf, "D"); @@ -2803,7 +2812,7 @@ static const char *test_websocket_endpoint(void) { struct mg_mgr mgr; struct mg_connection *nc; const char *local_addr = "127.0.0.1:7798"; - char buf[20] = "", buf2[20] = ""; + char buf[20] = "", buf2[20] = "0"; mg_mgr_init(&mgr, NULL); /* mgr.hexdump_file = "-"; */ @@ -2818,10 +2827,21 @@ static const char *test_websocket_endpoint(void) { nc->user_data = buf; mg_send_websocket_handshake(nc, "/boo", NULL); poll_until(&mgr, 1, c_str_ne, buf, (void *) ""); - mg_mgr_free(&mgr); - /* Check that test buffer has been filled by the callback properly. */ - ASSERT_STREQ(buf, "RDF|hi"); + ASSERT_STREQ(buf, "0RDF|hi"); + + /* Test handshake failure */ + ASSERT((nc = mg_connect(&mgr, local_addr, cb4)) != NULL); + mg_set_protocol_http_websocket(nc); + buf[0] = '\0'; + buf2[0] = '1'; + buf2[1] = '\0'; + nc->user_data = buf; + mg_send_websocket_handshake(nc, "/boo", NULL); + poll_until(&mgr, 1, c_str_ne, buf, (void *) ""); + ASSERT_STREQ(buf, "code 403"); + + mg_mgr_free(&mgr); return NULL; } From d0213ee90a70c624c4d62118e15702afc4825ea3 Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Mon, 4 Mar 2019 00:12:25 +0000 Subject: [PATCH 24/25] ESP32: Add support for the 3/4 key encoding scheme * Add calculated redundancy values when writing eFuse blocks 1-3 * Perform 192 -> 256 bit key extension when encrypting images for flashing Fixes https://github.com/cesanta/mos-tool/issues/37 CL: mos: ESP32: Add support for the 3/4 key encoding scheme PUBLISHED_FROM=16639fb25d5200550ad2097929239d896e188b5b --- src/common/platforms/esp/stub_flasher.c | 21 +++++++++++++++++++++ src/common/platforms/esp/stub_flasher.h | 18 ++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/common/platforms/esp/stub_flasher.c b/src/common/platforms/esp/stub_flasher.c index 09afcfe3..51f1ffa7 100644 --- a/src/common/platforms/esp/stub_flasher.c +++ b/src/common/platforms/esp/stub_flasher.c @@ -470,6 +470,27 @@ uint8_t cmd_loop(void) { resp = 0; break; } + case CMD_READ_REG: { + len = SLIP_recv(args, sizeof(args)); + if (len == 4) { + uint32_t value = READ_PERI_REG((uint32_t *) args[0]); + SLIP_send(&value, sizeof(value)); + resp = 0; + } else { + resp = 0x91; + } + break; + } + case CMD_WRITE_REG: { + len = SLIP_recv(args, sizeof(args)); + if (len == 8) { + WRITE_PERI_REG((uint32_t *) args[0], args[1]); + resp = 0; + } else { + resp = 0xa1; + } + break; + } } SLIP_send(&resp, 1); } while (cmd != CMD_BOOT_FW && cmd != CMD_REBOOT); diff --git a/src/common/platforms/esp/stub_flasher.h b/src/common/platforms/esp/stub_flasher.h index 4bae4c6a..dd1f948b 100644 --- a/src/common/platforms/esp/stub_flasher.h +++ b/src/common/platforms/esp/stub_flasher.h @@ -91,6 +91,24 @@ enum stub_cmd { * Output: arguments. */ CMD_ECHO = 8, + + /* + * Read register value. + * + * Args: register address. + * Input: None. + * Output: register value. + */ + CMD_READ_REG = 9, + + /* + * Write register value. + * + * Args: register address, value. + * Input: None. + * Output: None. + */ + CMD_WRITE_REG = 10, }; #endif /* CS_COMMON_PLATFORMS_ESP8266_STUBS_STUB_FLASHER_H_ */ From 8f365970489afe6ee41aa91668359811f1b833c5 Mon Sep 17 00:00:00 2001 From: Deomid Ryabkov Date: Mon, 4 Mar 2019 15:10:24 +0000 Subject: [PATCH 25/25] Mongoose 6.14 PUBLISHED_FROM=3e6e17d1f1cfb05efc37d76279dcbedb7f1add5d --- mongoose.h | 2 +- src/mg_common.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoose.h b/mongoose.h index 88a552aa..0f85b18b 100644 --- a/mongoose.h +++ b/mongoose.h @@ -23,7 +23,7 @@ #ifndef CS_MONGOOSE_SRC_COMMON_H_ #define CS_MONGOOSE_SRC_COMMON_H_ -#define MG_VERSION "6.13" +#define MG_VERSION "6.14" /* Local tweaks, applied before any of Mongoose's own headers. */ #ifdef MG_LOCALS diff --git a/src/mg_common.h b/src/mg_common.h index 69900255..94a77232 100644 --- a/src/mg_common.h +++ b/src/mg_common.h @@ -20,7 +20,7 @@ #ifndef CS_MONGOOSE_SRC_COMMON_H_ #define CS_MONGOOSE_SRC_COMMON_H_ -#define MG_VERSION "6.13" +#define MG_VERSION "6.14" /* Local tweaks, applied before any of Mongoose's own headers. */ #ifdef MG_LOCALS