// Copyright (c) 2004-2013 Sergey Lyubka
// Copyright (c) 2013-2025 Cesanta Software Limited
// All rights reserved
//
// This software is dual-licensed: you can redistribute it and/or modify
// it under the terms of the GNU General Public License version 2 as
// published by the Free Software Foundation. For the terms of this
// license, see http://www.gnu.org/licenses/
//
// You are free to use this software under the terms of the GNU General
// Public License, but WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU General Public License for more details.
//
// Alternatively, you can license this software under a commercial
// license, as set out in https://www.mongoose.ws/licensing/
//
// SPDX-License-Identifier: GPL-2.0-only or commercial

#include "mongoose.h"

#ifdef MG_ENABLE_LINES
#line 1 "src/base64.c"
#endif


static int mg_base64_encode_single(int c) {
  if (c < 26) {
    return c + 'A';
  } else if (c < 52) {
    return c - 26 + 'a';
  } else if (c < 62) {
    return c - 52 + '0';
  } else {
    return c == 62 ? '+' : '/';
  }
}

static int base64_decode_single(int c, char plus, char slash) {
  if (c >= 'A' && c <= 'Z') {
    return c - 'A';
  } else if (c >= 'a' && c <= 'z') {
    return c + 26 - 'a';
  } else if (c >= '0' && c <= '9') {
    return c + 52 - '0';
  } else if (c == plus) {
    return 62;
  } else if (c == slash) {
    return 63;
  } else if (c == '=') {
    return 64;
  } else {
    return -1;
  }
}

static int mg_base64_decode_single(int c) {
  return base64_decode_single(c, '+', '/');
}

static int mg_base64url_decode_single(int c) {
  return base64_decode_single(c, '-', '_');
}

size_t mg_base64_update(unsigned char ch, char *to, size_t n) {
  unsigned long rem = (n & 3) % 3;
  if (rem == 0) {
    to[n] = (char) mg_base64_encode_single(ch >> 2);
    to[++n] = (char) ((ch & 3) << 4);
  } else if (rem == 1) {
    to[n] = (char) mg_base64_encode_single(to[n] | (ch >> 4));
    to[++n] = (char) ((ch & 15) << 2);
  } else {
    to[n] = (char) mg_base64_encode_single(to[n] | (ch >> 6));
    to[++n] = (char) mg_base64_encode_single(ch & 63);
    n++;
  }
  return n;
}

size_t mg_base64_final(char *to, size_t n) {
  size_t saved = n;
  // printf("---[%.*s]\n", n, to);
  if (n & 3) n = mg_base64_update(0, to, n);
  if ((saved & 3) == 2) n--;
  // printf("    %d[%.*s]\n", n, n, to);
  while (n & 3) to[n++] = '=';
  to[n] = '\0';
  return n;
}

size_t mg_base64_encode(const unsigned char *p, size_t n, char *to, size_t dl) {
  size_t i, len = 0;
  if (dl > 0) to[0] = '\0';
  if (dl < ((n / 3) + (n % 3 ? 1 : 0)) * 4 + 1) return 0;
  for (i = 0; i < n; i++) len = mg_base64_update(p[i], to, len);
  len = mg_base64_final(to, len);
  return len;
}

size_t mg_base64_decode(const char *src, size_t n, char *dst, size_t dl) {
  const char *end = src == NULL ? NULL : src + n;  // Cannot add to NULL
  size_t len = 0;
  if (dl < n / 4 * 3 + 1) goto fail;
  while (src != NULL && src + 3 < end) {
    int a = mg_base64_decode_single(src[0]),
        b = mg_base64_decode_single(src[1]),
        c = mg_base64_decode_single(src[2]),
        d = mg_base64_decode_single(src[3]);
    if (a == 64 || a < 0 || b == 64 || b < 0 || c < 0 || d < 0) {
      goto fail;
    }
    dst[len++] = (char) ((a << 2) | (b >> 4));
    if (src[2] != '=') {
      dst[len++] = (char) ((b << 4) | (c >> 2));
      if (src[3] != '=') dst[len++] = (char) ((c << 6) | d);
    }
    src += 4;
  }
  dst[len] = '\0';
  return len;
fail:
  if (dl > 0) dst[0] = '\0';
  return 0;
}

size_t mg_base64url_encode(const unsigned char *p, size_t n, char *to,
                           size_t dl) {
  size_t i, len = mg_base64_encode(p, n, to, dl);
  if (len == 0) return 0;
  for (i = 0; i < len; i++) {
    if (to[i] == '+') {
      to[i] = '-';
    } else if (to[i] == '/') {
      to[i] = '_';
    }
  }
  while (len > 0 && to[len - 1] == '=') to[--len] = '\0';
  return len;
}

size_t mg_base64url_decode(const char *src, size_t n, char *dst, size_t dl) {
  size_t i, len = 0;
  unsigned int bits = 0, v = 0;
  if (dl == 0 || (n & 3) == 1) goto fail;
  for (i = 0; src != NULL && i < n; i++) {
    int c = mg_base64url_decode_single(src[i]);
    if (c == 64) break;
    if (c < 0) goto fail;
    v = (v << 6) | (unsigned int) c;
    bits += 6;
    if (bits >= 8) {
      bits -= 8;
      if (len + 1 >= dl) goto fail;
      dst[len++] = (char) ((v >> bits) & 255U);
    }
  }
  dst[len] = '\0';
  return len;
fail:
  if (dl > 0) dst[0] = '\0';
  return 0;
}

#ifdef MG_ENABLE_LINES
#line 1 "src/bsd.c"
#endif


#include <stdarg.h>

#if MG_ENABLE_BSD_SOCKETS && MG_ENABLE_FREERTOS
#include <queue.h>

static QueueHandle_t s_cmd_q;
static struct mg_bsd_cmd *s_waiters;

static bool bsd_qsend(QueueHandle_t q, const void *item, TickType_t ticks) {
  if (xQueueSend(q, item, ticks) == pdTRUE) return true;
  MG_ERROR(("queue send failed %p", q));
  return false;
}
#endif

#if MG_ENABLE_BSD_SOCKETS
static struct mg_mgr *s_mgr;

#ifndef MG_BSD_FD_BASE
#define MG_BSD_FD_BASE 11
#endif
#ifndef MG_BSD_MAX_SOCKETS
#define MG_BSD_MAX_SOCKETS 64
#endif
#ifndef MG_BSD_ACCEPT_MS
#define MG_BSD_ACCEPT_MS 3000
#endif

// clang-format off
enum bsd_op {
  BSD_OP_NONE, BSD_OP_SOCKET, BSD_OP_BIND, BSD_OP_LISTEN, BSD_OP_FCNTL,
  BSD_OP_ACCEPT, BSD_OP_CONNECT, BSD_OP_RECV, BSD_OP_SEND, BSD_OP_SENDTO,
  BSD_OP_RECVFROM, BSD_OP_CLOSE, BSD_OP_POLL, BSD_OP_SELECT,
  BSD_OP_GETHOSTBYNAME,
};

// A BSD API request from a user task. Gets sent via the RTOS queue, and
// executed by the network task (mg_bsd_poll). Baremetal code calls directly
struct mg_bsd_cmd {
  int result;                // return value
  int err;                   // errno
  enum bsd_op op;
  bool started;              // operation initiated (connect/resolve)
  void *caller;              // task to notify when done
  struct mg_bsd_cmd *next;   // waiter list link
  union {                    // op-specific arguments
    struct { int domain, type, proto; } op_socket;
    struct { int fd, backlog; } op_listen;
    struct { int fd, cmd, arg; } op_fcntl;
    struct { int fd; } op_close;
    struct { int fd; const struct sockaddr *addr; socklen_t len; } op_bind;
    struct { int fd; const struct sockaddr *addr; socklen_t len; } op_connect;
    struct { int fd; struct sockaddr *addr; socklen_t *addrlen; } op_accept;
    struct { int fd; void *buf; size_t len; int flags; } op_recv;
    struct { int fd; const void *buf; size_t len; int flags; } op_send;
    struct { int fd; const void *buf; size_t len; int flags; const struct sockaddr *addr; socklen_t addrlen; } op_sendto;
    struct { int fd; void *buf; size_t len; int flags; struct sockaddr *src; socklen_t *addrlen; } op_recvfrom;
    struct { struct pollfd *fds; nfds_t nfds; uint64_t deadline; } op_poll;
    struct { int nfds; fd_set *r, *w, *e; uint64_t deadline; } op_select;
    struct { const char *name; struct hostent *result; } op_gethostbyname;
  };
};
#define BSD_CMD_EMPTY {0, 0, BSD_OP_NONE, 0, 0, 0, {{0, 0, 0}}}
// clang-format on

// The BSD socket outlives its Mongoose connection until the caller closes fd.
struct mg_bsd_sock {
  int fd, err, backlog;
  uint64_t accept_expire;
  struct sockaddr_in addr;
  struct mg_connection *c;
  struct mg_bsd_sock *listener;
  bool used, bound, closed, nonblock, is_udp, orphan;  // closed: c was closed
};

static struct mg_bsd_sock s_socks[MG_BSD_MAX_SOCKETS > 0 ?
                                  MG_BSD_MAX_SOCKETS : 1];

static void bsd_ev(struct mg_connection *, int, void *);

// BSD-owned fd registry
static struct mg_bsd_sock *bsd_get(int fd) {
  int offset;
  struct mg_bsd_sock *s;

  if (fd < MG_BSD_FD_BASE) return NULL;
  offset = fd - MG_BSD_FD_BASE;
  if (offset >= MG_BSD_MAX_SOCKETS) return NULL;
  s = &s_socks[offset];
  if (!s->used || s->orphan) return NULL;
  return s;
}

static void bsd_attach(struct mg_bsd_sock *s, struct mg_connection *c) {
  s->c = c;
  c->fn = bsd_ev;
  c->fn_data = s;
  c->is_bsd = 1;
  c->is_udp = s->is_udp;
}

static struct mg_connection *bsd_new_conn(struct mg_bsd_sock *s) {
  struct mg_connection *c = mg_alloc_conn(s_mgr);
  if (c != NULL) {
    bsd_attach(s, c);
    LIST_ADD_HEAD(struct mg_connection, &s_mgr->conns, c);
  }
  return c;
}

#if MG_ENABLE_FREERTOS
// Complete operations blocked on c before Mongoose destroys it.
static void bsd_finish_waiters(struct mg_bsd_sock *s, bool error) {
  struct mg_bsd_cmd **p = &s_waiters;

  if (s->orphan) return;  // No BSD socket owner yet

  while (*p != NULL) {
    struct mg_bsd_cmd *cmd = *p;
    int fd = -1;
    bool read = cmd->op == BSD_OP_RECV || cmd->op == BSD_OP_RECVFROM;
    bool send = cmd->op == BSD_OP_SEND || cmd->op == BSD_OP_SENDTO;

    switch (cmd->op) {  // clang-format off
      case BSD_OP_ACCEPT: fd = cmd->op_accept.fd; break;
      case BSD_OP_CONNECT: fd = cmd->op_connect.fd; break;
      case BSD_OP_RECV: fd = cmd->op_recv.fd; break;
      case BSD_OP_SEND: fd = cmd->op_send.fd; break;
      case BSD_OP_SENDTO: fd = cmd->op_sendto.fd; break;
      case BSD_OP_RECVFROM: fd = cmd->op_recvfrom.fd; break;
      default: break;
    }  // clang-format on
    if (fd != s->fd) {
      p = &cmd->next;
      continue;
    }

    *p = cmd->next;
    if (!error && s->c != NULL && s->c->is_draining && read) {
      cmd->result = 0;  // Orderly EOF
    } else {
      cmd->result = -1;
      cmd->err = send ? EPIPE : EIO;  // Mongoose error / forced close
    }
    xTaskNotifyGive((TaskHandle_t) cmd->caller);
  }
}
#endif

static void bsd_ev(struct mg_connection *c, int ev, void *ev_data) {
  struct mg_bsd_sock *s = (struct mg_bsd_sock *) c->fn_data;
  if (s == NULL) return;

  if (ev == MG_EV_ACCEPT) {
    struct mg_bsd_sock *child = NULL, *t;
    int count = 0;
    for (t = s_socks; t < s_socks + MG_BSD_MAX_SOCKETS; t++) {
      if (!t->used && child == NULL) child = t;  // Find a free BSD socket slot,
      if (t->orphan && t->listener == s) count++;  // and count pending accepts
    }
    if (count >= s->backlog) {
      c->fn_data = NULL;
      mg_error(c, "accept backlog full");
    } else if (child == NULL) {
      c->fn_data = NULL;
      mg_error(c, "accept rejected");
    } else {
      memset(child, 0, sizeof(*child));  // Claim slot
      child->used = true;
      child->c = c;
      child->listener = s;
      child->accept_expire = mg_millis() + MG_BSD_ACCEPT_MS;
      child->is_udp = s->is_udp;
      child->orphan = true;
      c->fn_data = child;
      c->is_bsd = 1;
      c->is_udp = child->is_udp;
    }
  } else if (ev == MG_EV_ERROR) {
    if (s->err == 0) s->err = EIO;
#if MG_ENABLE_FREERTOS
    bsd_finish_waiters(s, true);
#endif
  } else if (ev == MG_EV_CLOSE) {
#if MG_ENABLE_FREERTOS
    bsd_finish_waiters(s, false);
#endif
    s->c = NULL;
    s->closed = true;
    c->fn_data = NULL;
    if (s->orphan) memset(s, 0, sizeof(*s)); // release
  } else if (ev == MG_EV_POLL && s->orphan &&
             mg_millis() > s->accept_expire) {
    mg_error(c, "accept timeout");
  }
  (void) ev_data;
}

#if MG_ENABLE_FREERTOS
#ifndef MG_WAKEUP_QUEUE_DEPTH
#define MG_WAKEUP_QUEUE_DEPTH 4
#endif

struct wumsg {
  unsigned long id;
  size_t len;
  uint8_t data[];
};

static void wufn(struct mg_connection *c, int ev, void *ev_data) {
  QueueHandle_t q = (QueueHandle_t) c->mgr->pipe.q;
  if (ev == MG_EV_POLL) {
    struct wumsg *m;
    if (xQueueReceive(q, &m, 0) == pdTRUE) {
      struct mg_connection *t;
      for (t = c->mgr->conns; t != NULL; t = t->next) {
        if (t->id == m->id) {
          struct mg_str data = mg_str_n((char *) m->data, m->len);
          mg_call(t, MG_EV_WAKEUP, &data);
          break;
        }
      }
      free(m);
    }
  } else if (ev == MG_EV_CLOSE) {
    struct wumsg *m;
    while (xQueueReceive(q, &m, 0) == pdTRUE) free(m);
    vQueueDelete(q);
    c->mgr->pipe.q = NULL;
  }
  (void) ev_data;
}

bool mg_wakeup_init(struct mg_mgr *mgr) {
  struct mg_connection *c;
  if (mgr->pipe.q != NULL) return true;
  mgr->pipe.q = xQueueCreate(MG_WAKEUP_QUEUE_DEPTH, sizeof(void *));
  if (mgr->pipe.q == NULL) {
    MG_ERROR(("Cannot create queue"));
    return false;
  }
  c = mg_alloc_conn(mgr);
  if (c == NULL) {
    vQueueDelete((QueueHandle_t) mgr->pipe.q);
    mgr->pipe.q = NULL;
    return false;
  }
  c->fd = (void *) (size_t) MG_INVALID_SOCKET;
  c->fn = wufn;
  LIST_ADD_HEAD(struct mg_connection, &mgr->conns, c);
  MG_DEBUG(("%lu queue %p", c->id, mgr->pipe.q));
  mg_call(c, MG_EV_OPEN, NULL);
  return true;
}

bool mg_wakeup(struct mg_mgr *mgr, unsigned long conn_id, const void *buf,
               size_t len) {
  struct wumsg *m;
  if (mgr->pipe.q == NULL || conn_id == 0) return false;
  m = (struct wumsg *) calloc(1, sizeof(*m) + len);
  if (m == NULL) {
    MG_ERROR(("OOM"));
    return false;
  }
  m->id = conn_id;
  m->len = len;
  memcpy(m->data, buf, len);
  if (!bsd_qsend((QueueHandle_t) mgr->pipe.q, &m, 0)) {
    free(m);
    return false;
  }
  return true;
}
#endif

enum { BSD_READ = 1, BSD_WRITE = 2, BSD_ERROR = 4 };

static int bsd_ready(struct mg_bsd_sock *s) {
  struct mg_bsd_sock *t;
  int ready = 0;

  if (s->err != 0) return BSD_READ | BSD_ERROR;
  if (s->c == NULL) return s->closed ? BSD_READ | BSD_ERROR : 0;
  if (s->c->is_listening) {
    for (t = s_socks; t < s_socks + MG_BSD_MAX_SOCKETS; t++) {
      if (t->orphan && t->listener == s) return BSD_READ;
    }
    return 0;
  }
  if (s->c->recv.len > 0 || s->c->is_draining) ready |= BSD_READ;
  if (!s->is_udp && !s->c->is_connecting && !s->c->is_closing &&
      !s->c->is_draining)
    ready |= BSD_WRITE;  // Established TCP socket
  return ready;
}

static void bsd_close(struct mg_bsd_sock *s) {
  struct mg_bsd_sock *t;

#if MG_ENABLE_FREERTOS
  bsd_finish_waiters(s, true);
#endif
  for (t = s_socks; t < s_socks + MG_BSD_MAX_SOCKETS; t++) {
    if (t->orphan && t->listener == s) {
      if (t->c != NULL) {
        t->c->fn_data = NULL;
        mg_close_conn(t->c);
        t->c = NULL;
      }
      memset(t, 0, sizeof(*t));  // Release orphan
    }
  }
  if (s->c != NULL) {
    s->c->fn_data = NULL;
    mg_close_conn(s->c);
    s->c = NULL;
  }
  memset(s, 0, sizeof(*s));  // Release socket
}

static char s_hostent_name[64];
static uint32_t s_hostent_addr;
static char *s_hostent_addr_list[2] = {(char *) &s_hostent_addr, NULL};
static struct hostent s_hostent = {s_hostent_name, NULL, AF_INET, 4,
                                   s_hostent_addr_list};
static volatile int s_resolve_done;

static void bsd_resolve_fn(struct mg_connection *c, int ev, void *ev_data) {
  (void) ev_data;
  if (ev == MG_EV_RESOLVE) {
    s_hostent_addr = c->rem.addr.ip4;
    s_resolve_done = 1;
    c->is_closing = 1;
  } else if (ev == MG_EV_ERROR) {
    s_resolve_done = -1;
  }
}

// Execute a command. Runs in the network task (mg_bsd_poll) on FreeRTOS, or
// in the caller otherwise. Returns true when done, false when the operation
// must wait for the network task and be re-checked later.
static bool bsd_run(struct mg_bsd_cmd *cmd) {
  struct mg_connection *c;
  struct mg_bsd_sock *s;
  switch (cmd->op) {
    case BSD_OP_NONE:
      cmd->result = -1;
      return true;
    case BSD_OP_SOCKET:
      if (s_mgr == NULL || cmd->op_socket.domain != AF_INET ||
          (cmd->op_socket.type != SOCK_STREAM &&
           cmd->op_socket.type != SOCK_DGRAM)) {
        cmd->result = -1, cmd->err = EPERM;
      } else {
        for (s = s_socks; s < s_socks + MG_BSD_MAX_SOCKETS; s++) {
          if (!s->used) break;  // Find a free BSD socket registry slot
        }
        if (s == s_socks + MG_BSD_MAX_SOCKETS) {
          cmd->result = -1, cmd->err = EMFILE;
        } else {
          memset(s, 0, sizeof(*s));  // Claim slot
          s->used = true;
          s->is_udp = cmd->op_socket.type == SOCK_DGRAM;
          s->fd = MG_BSD_FD_BASE + (int) (s - s_socks);
          cmd->result = s->fd;
        }
      }
      return true;
    case BSD_OP_BIND:
      if ((s = bsd_get(cmd->op_bind.fd)) == NULL) {
        cmd->result = -1, cmd->err = EBADF;
      } else {
        const struct sockaddr_in *sa =
            (const struct sockaddr_in *) cmd->op_bind.addr;
        memcpy(&s->addr, sa, sizeof(s->addr));
        s->bound = true;
        if (!s->is_udp) {
          cmd->result = 0;
        } else if (s->c == NULL && (c = bsd_new_conn(s)) == NULL) {
          cmd->result = -1, cmd->err = ENOMEM;
        } else {
          c = s->c;
          c->loc.port = s->addr.sin_port;
          c->loc.addr.ip4 = s->addr.sin_addr.s_addr;
          c->loc.is_ip6 = false;
          cmd->result = 0;
        }
      }
      return true;
    case BSD_OP_LISTEN: {
      char url[32];

      s = bsd_get(cmd->op_listen.fd);
      if (s == NULL) {
        cmd->result = -1, cmd->err = EBADF;
      } else if (!s->bound || s->is_udp || cmd->op_listen.backlog < 1) {
        cmd->result = -1, cmd->err = EINVAL;
      } else if (s->c != NULL && !s->c->is_listening) {
        cmd->result = -1, cmd->err = EINVAL;
      } else if (s->c == NULL) {
        mg_snprintf(url, sizeof(url), "tcp://%M:%hu", mg_print_ip4,
                    &s->addr.sin_addr.s_addr, mg_ntohs(s->addr.sin_port));
        if ((c = mg_listen(s_mgr, url, bsd_ev, s)) == NULL) {
          cmd->result = -1, cmd->err = ENOMEM;
        } else {
          bsd_attach(s, c);
          s->backlog = cmd->op_listen.backlog;
          cmd->result = 0;
        }
      } else {
        s->backlog = cmd->op_listen.backlog;
        cmd->result = 0;
      }
      return true;
    }
    case BSD_OP_FCNTL:
      if ((s = bsd_get(cmd->op_fcntl.fd)) == NULL) {
        cmd->result = -1, cmd->err = EBADF;
      } else if (cmd->op_fcntl.cmd == F_GETFL) {
        cmd->result = s->nonblock ? O_NONBLOCK : 0;
      } else if (cmd->op_fcntl.cmd == F_SETFL) {
        s->nonblock = (cmd->op_fcntl.arg & O_NONBLOCK) != 0;
        cmd->result = 0;
      } else {
        cmd->result = -1, cmd->err = EINVAL;
      }
      return true;
    case BSD_OP_ACCEPT: {
      int i;
      struct mg_bsd_sock *child;

      s = bsd_get(cmd->op_accept.fd);
      if (s == NULL || s->c == NULL || !s->c->is_listening) {
        cmd->result = -1, cmd->err = EINVAL;
        return true;
      }
      for (i = 0, child = s_socks; i < MG_BSD_MAX_SOCKETS; i++, child++) {
        if (child->orphan && child->listener == s) {
          c = child->c;
          child->orphan = false;
          child->listener = NULL;
          child->fd = MG_BSD_FD_BASE + i;
          cmd->result = child->fd;
          if (cmd->op_accept.addr != NULL && cmd->op_accept.addrlen != NULL) {
            struct sockaddr_in *sa = (struct sockaddr_in *) cmd->op_accept.addr;
            sa->sin_family = AF_INET;
            sa->sin_port = c->rem.port;
            sa->sin_addr.s_addr = c->rem.addr.ip4;
            *cmd->op_accept.addrlen = sizeof(*sa);
          }
          return true;
        }
      }
      if (s->nonblock) {
        cmd->result = -1, cmd->err = EAGAIN;
        return true;
      }
#if MG_ENABLE_FREERTOS
      return false;  // No connection yet, wait
#else
      cmd->result = -1, cmd->err = EAGAIN;
      return true;
#endif
    }
    case BSD_OP_CONNECT:
      if ((s = bsd_get(cmd->op_connect.fd)) == NULL) {
        cmd->result = -1, cmd->err = EBADF;
        return true;
      }
      if (!cmd->started && s->c == NULL) {
        if (s->closed || (c = bsd_new_conn(s)) == NULL) {
          cmd->result = -1, cmd->err = s->closed ? EIO : ENOMEM;
          return true;
        }
      }
      if ((c = s->c) == NULL) {
        cmd->result = -1, cmd->err = s->err ? s->err : EIO;
        return true;
      }
      if (!cmd->started) {
        const struct sockaddr_in *sa =
            (const struct sockaddr_in *) cmd->op_connect.addr;
        c->rem.port = sa->sin_port;
        c->rem.addr.ip4 = sa->sin_addr.s_addr;
        c->rem.is_ip6 = false;
        c->is_client = 1;
        mg_connect_resolved(c);
        cmd->started = true;
      }
      if (c->is_closing) {
        cmd->result = -1, cmd->err = ECONNREFUSED;
        return true;
      }
      if (!c->is_connecting) {
        cmd->result = 0;
        return true;
      }
      if (s->nonblock) {
        cmd->result = -1, cmd->err = EINPROGRESS;
        return true;
      }
#if MG_ENABLE_FREERTOS
      return false;  // Still connecting, wait
#else
      cmd->result = -1, cmd->err = EINPROGRESS;
      return true;
#endif
    case BSD_OP_RECV:
      if ((s = bsd_get(cmd->op_recv.fd)) == NULL) {
        cmd->result = -1, cmd->err = EBADF;
        return true;
      }
      if ((c = s->c) == NULL) {
        cmd->result = s->closed && s->err == 0 ? 0 : -1;
        cmd->err = s->closed ? s->err : ENOTCONN;
        return true;
      }
      if (c->recv.len > 0) {
        size_t n = cmd->op_recv.len;
        if (c->recv.len < n) n = c->recv.len;
        memcpy(cmd->op_recv.buf, c->recv.buf, n);
        if (!(cmd->op_recv.flags & MSG_PEEK)) mg_iobuf_del(&c->recv, 0, n);
        cmd->result = (int) n;
        return true;
      }
      if (s->err != 0) {
        cmd->result = -1, cmd->err = s->err;
        return true;
      }
      if (c->is_draining) {  // Orderly shutdown by peer
        cmd->result = 0;
        return true;
      }
      if (!s->nonblock && !(cmd->op_recv.flags & MSG_DONTWAIT)) {
#if MG_ENABLE_FREERTOS
        return false;  // No data yet, wait
#endif
      }
      cmd->result = -1, cmd->err = EAGAIN;
      return true;
    case BSD_OP_SEND:
      if ((s = bsd_get(cmd->op_send.fd)) == NULL) {
        cmd->result = -1, cmd->err = EBADF;
      } else if ((c = s->c) == NULL || s->err != 0 || c->is_closing ||
                 c->is_draining) {
        cmd->result = -1, cmd->err = EPIPE;
      } else if (mg_send(c, cmd->op_send.buf, cmd->op_send.len)) {
        cmd->result = (int) cmd->op_send.len;
      } else if (s->nonblock || (cmd->op_send.flags & MSG_DONTWAIT)) {
        cmd->result = -1, cmd->err = EAGAIN;
      } else {
#if MG_ENABLE_FREERTOS
        return false;  // No send buffer space yet, wait
#else
        cmd->result = -1, cmd->err = EAGAIN;
#endif
      }
      return true;
    case BSD_OP_SENDTO:
      if ((s = bsd_get(cmd->op_sendto.fd)) == NULL) {
        cmd->result = -1, cmd->err = EBADF;
      } else if ((c = s->c) == NULL || s->err != 0 || c->is_closing ||
                 c->is_draining) {
        cmd->result = -1, cmd->err = EPIPE;
      } else {
        const struct sockaddr_in *sa =
            (const struct sockaddr_in *) cmd->op_sendto.addr;
        c->rem.port = sa->sin_port;
        c->rem.addr.ip4 = sa->sin_addr.s_addr;
        c->rem.is_ip6 = false;
        if (mg_send(c, cmd->op_sendto.buf, cmd->op_sendto.len)) {
          cmd->result = (int) cmd->op_sendto.len;
        } else if (s->nonblock || (cmd->op_sendto.flags & MSG_DONTWAIT)) {
          cmd->result = -1, cmd->err = EAGAIN;
        } else {
#if MG_ENABLE_FREERTOS
          return false;  // No send buffer space yet, wait
#else
          cmd->result = -1, cmd->err = EAGAIN;
#endif
        }
      }
      return true;
    case BSD_OP_RECVFROM:
      if ((s = bsd_get(cmd->op_recvfrom.fd)) == NULL) {
        cmd->result = -1, cmd->err = EBADF;
        return true;
      }
      if ((c = s->c) == NULL) {
        cmd->result = s->closed && s->err == 0 ? 0 : -1;
        cmd->err = s->closed ? s->err : ENOTCONN;
        return true;
      }
      if (c->recv.len > 0) {
        size_t n = cmd->op_recvfrom.len;
        if (c->recv.len < n) n = c->recv.len;
        memcpy(cmd->op_recvfrom.buf, c->recv.buf, n);
        if (!(cmd->op_recvfrom.flags & MSG_PEEK)) mg_iobuf_del(&c->recv, 0, n);
        if (cmd->op_recvfrom.src != NULL && cmd->op_recvfrom.addrlen != NULL) {
          struct sockaddr_in *sa = (struct sockaddr_in *) cmd->op_recvfrom.src;
          sa->sin_family = AF_INET;
          sa->sin_port = c->rem.port;
          sa->sin_addr.s_addr = c->rem.addr.ip4;
          *cmd->op_recvfrom.addrlen = sizeof(*sa);
        }
        cmd->result = (int) n;
        return true;
      }
      if (s->err != 0) {
        cmd->result = -1, cmd->err = s->err;
        return true;
      }
      if (!s->nonblock && !(cmd->op_recvfrom.flags & MSG_DONTWAIT)) {
#if MG_ENABLE_FREERTOS
        return false;  // No datagram yet, wait
#endif
      }
      cmd->result = -1, cmd->err = EAGAIN;
      return true;
    case BSD_OP_CLOSE:
      if ((s = bsd_get(cmd->op_close.fd)) == NULL) {
        cmd->result = -1, cmd->err = EBADF;
      } else {
        bsd_close(s);
        cmd->result = 0;
      }
      return true;
    case BSD_OP_POLL: {
      struct pollfd *p;
      int i, n = 0, ready;

      for (i = 0, p = cmd->op_poll.fds; i < (int) cmd->op_poll.nfds;
           i++, p++) {
        p->revents = 0;
        if (p->fd < 0) continue;
        if ((s = bsd_get(p->fd)) == NULL) {
          p->revents = POLLNVAL;
        } else {
          ready = bsd_ready(s);
          if ((p->events & POLLIN) && (ready & BSD_READ))
            p->revents |= POLLIN;
          if ((p->events & POLLOUT) && (ready & BSD_WRITE))
            p->revents |= POLLOUT;
          if (s->err != 0) p->revents |= POLLERR;
          if (s->closed) p->revents |= POLLHUP;
        }
        if (p->revents != 0) n++;
      }
#if MG_ENABLE_FREERTOS
      if (n == 0 && mg_millis() < cmd->op_poll.deadline)
        return false;  // Keep caller pollfd[] intact while waiting
#endif
      cmd->result = n;
      return true;
    }
    case BSD_OP_SELECT: {
      fd_set rr, ww, ee;
      fd_set *r = cmd->op_select.r;
      fd_set *w = cmd->op_select.w;
      fd_set *e = cmd->op_select.e;
      int fd, n = 0, want_read, want_write, want_error, ready;
#if defined(MG_BSD_SELECT_H)
      int i;
      __fd_mask bits;
      int bitsize = (int) (sizeof(bits) * 8);
#endif

      if (cmd->op_select.nfds < 0 || cmd->op_select.nfds > FD_SETSIZE) {
        cmd->result = -1, cmd->err = EINVAL;
        return true;
      }
      FD_ZERO(&rr);
      FD_ZERO(&ww);
      FD_ZERO(&ee);
#if defined(MG_BSD_SELECT_H)
      for (i = 0; i < (cmd->op_select.nfds + bitsize - 1) / bitsize; i++) {
        bits = 0;
        if (r != NULL) bits |= r->__fds_bits[i];
        if (w != NULL) bits |= w->__fds_bits[i];
        if (e != NULL) bits |= e->__fds_bits[i];
        while (bits != 0) {
          fd = i * bitsize + (int) __builtin_ctzl(bits);
          bits &= bits - 1;
          if (fd >= cmd->op_select.nfds) continue;
          want_read = r != NULL && FD_ISSET(fd, r);
          want_write = w != NULL && FD_ISSET(fd, w);
          want_error = e != NULL && FD_ISSET(fd, e);
          if ((s = bsd_get(fd)) == NULL) goto err;
          ready = bsd_ready(s);
          if (want_read && (ready & BSD_READ)) FD_SET(fd, &rr);
          if (want_write && (ready & BSD_WRITE)) FD_SET(fd, &ww);
          if (want_error && (ready & BSD_ERROR)) FD_SET(fd, &ee);
          if ((want_read && (ready & BSD_READ)) ||
              (want_write && (ready & BSD_WRITE)) ||
              (want_error && (ready & BSD_ERROR)))
            n++;
        }
      }
#else
      for (fd = 0; fd < cmd->op_select.nfds; fd++) {
        want_read = r != NULL && FD_ISSET(fd, r);
        want_write = w != NULL && FD_ISSET(fd, w);
        want_error = e != NULL && FD_ISSET(fd, e);
        if (!want_read && !want_write && !want_error) continue;
        if ((s = bsd_get(fd)) == NULL) goto err;
        ready = bsd_ready(s);
        if (want_read && (ready & BSD_READ)) FD_SET(fd, &rr);
        if (want_write && (ready & BSD_WRITE)) FD_SET(fd, &ww);
        if (want_error && (ready & BSD_ERROR)) FD_SET(fd, &ee);
        if ((want_read && (ready & BSD_READ)) ||
            (want_write && (ready & BSD_WRITE)) ||
            (want_error && (ready & BSD_ERROR)))
          n++;
      }
#endif
#if MG_ENABLE_FREERTOS
      if (n == 0 && mg_millis() < cmd->op_select.deadline)
        return false;  // Keep fd sets while waiting
#endif
      if (r != NULL) *r = rr;
      if (w != NULL) *w = ww;
      if (e != NULL) *e = ee;
      cmd->result = n;
      return true;

err:
      cmd->result = -1, cmd->err = EBADF;
      return true;
    }
    case BSD_OP_GETHOSTBYNAME:
      if (!cmd->started) {
        struct mg_connection *r = mg_alloc_conn(s_mgr);
        if (r == NULL) {
          cmd->op_gethostbyname.result = NULL;
          return true;
        }
        r->fn = bsd_resolve_fn;
        LIST_ADD_HEAD(struct mg_connection, &s_mgr->conns, r);
        mg_call(r, MG_EV_OPEN, NULL);
        s_resolve_done = 0;
        mg_resolve(r, cmd->op_gethostbyname.name);
        cmd->started = true;
#if MG_ENABLE_FREERTOS
        return false;  // Wait for DNS response
#endif
      }
      if (s_resolve_done != 0) {
        if (s_resolve_done < 0) {
          cmd->op_gethostbyname.result = NULL;
        } else {
          mg_snprintf(s_hostent_name, sizeof(s_hostent_name), "%s",
                      cmd->op_gethostbyname.name);
          cmd->op_gethostbyname.result = &s_hostent;
        }
        return true;
      }
#if MG_ENABLE_FREERTOS
      return false;  // Still resolving
#else
      cmd->op_gethostbyname.result = NULL;
      return true;
#endif
  }
  return true;
}

static void bsd_dispatch(struct mg_bsd_cmd *cmd) {
#if MG_ENABLE_FREERTOS
  if (s_cmd_q == NULL) {
    cmd->result = -1, cmd->err = EPERM;
  } else {
    cmd->caller = xTaskGetCurrentTaskHandle(); 
    if (!bsd_qsend(s_cmd_q, &cmd, portMAX_DELAY)) {
      cmd->result = -1, cmd->err = EIO;
    } else {
      ulTaskNotifyTake(pdTRUE, portMAX_DELAY);
    }
  }
#else
  bsd_run(cmd);
#endif
}

int socket(int domain, int type, int proto) {
  struct mg_bsd_cmd cmd = BSD_CMD_EMPTY;
  cmd.op = BSD_OP_SOCKET, cmd.op_socket.domain = domain,
  cmd.op_socket.type = type, cmd.op_socket.proto = proto;
  bsd_dispatch(&cmd);
  errno = cmd.err;
  return cmd.result;
}

int bind(int fd, const struct sockaddr *addr, socklen_t len) {
  struct mg_bsd_cmd cmd = BSD_CMD_EMPTY;
  cmd.op = BSD_OP_BIND, cmd.op_bind.fd = fd, cmd.op_bind.addr = addr,
  cmd.op_bind.len = len;
  bsd_dispatch(&cmd);
  errno = cmd.err;
  return cmd.result;
}

int listen(int fd, int backlog) {
  struct mg_bsd_cmd cmd = BSD_CMD_EMPTY;
  cmd.op = BSD_OP_LISTEN, cmd.op_listen.fd = fd,
  cmd.op_listen.backlog = backlog;
  bsd_dispatch(&cmd);
  errno = cmd.err;
  return cmd.result;
}

int fcntl(int fd, int fcmd, ...) {
  struct mg_bsd_cmd cmd = BSD_CMD_EMPTY;
  int arg = 0;

  if (fcmd == F_SETFL) {
    va_list ap;

    va_start(ap, fcmd);
    arg = va_arg(ap, int);
    va_end(ap);
  }
  cmd.op = BSD_OP_FCNTL, cmd.op_fcntl.fd = fd, cmd.op_fcntl.cmd = fcmd,
  cmd.op_fcntl.arg = arg;
  bsd_dispatch(&cmd);
  errno = cmd.err;
  return cmd.result;
}

int accept(int fd, struct sockaddr *addr, socklen_t *addrlen) {
  struct mg_bsd_cmd cmd = BSD_CMD_EMPTY;
  cmd.op = BSD_OP_ACCEPT, cmd.op_accept.fd = fd, cmd.op_accept.addr = addr,
  cmd.op_accept.addrlen = addrlen;
  bsd_dispatch(&cmd);
  errno = cmd.err;
  return cmd.result;
}

int connect(int fd, const struct sockaddr *addr, socklen_t len) {
  struct mg_bsd_cmd cmd = BSD_CMD_EMPTY;
  cmd.op = BSD_OP_CONNECT, cmd.op_connect.fd = fd, cmd.op_connect.addr = addr,
  cmd.op_connect.len = len;
  bsd_dispatch(&cmd);
  errno = cmd.err;
  return cmd.result;
}

ssize_t recv(int fd, void *buf, size_t len, int flags) {
  struct mg_bsd_cmd cmd = BSD_CMD_EMPTY;
  cmd.op = BSD_OP_RECV, cmd.op_recv.fd = fd, cmd.op_recv.buf = buf,
  cmd.op_recv.len = len, cmd.op_recv.flags = flags;
  bsd_dispatch(&cmd);
  errno = cmd.err;
  return cmd.result;
}

ssize_t send(int fd, const void *buf, size_t len, int flags) {
  struct mg_bsd_cmd cmd = BSD_CMD_EMPTY;
  cmd.op = BSD_OP_SEND, cmd.op_send.fd = fd, cmd.op_send.buf = (void *) buf,
  cmd.op_send.len = len, cmd.op_send.flags = flags;
  bsd_dispatch(&cmd);
  errno = cmd.err;
  return cmd.result;
}

ssize_t sendto(int fd, const void *buf, size_t len, int flags,
               const struct sockaddr *dest, socklen_t addrlen) {
  struct mg_bsd_cmd cmd = BSD_CMD_EMPTY;
  cmd.op = BSD_OP_SENDTO, cmd.op_sendto.fd = fd,
  cmd.op_sendto.buf = (void *) buf, cmd.op_sendto.len = len,
  cmd.op_sendto.flags = flags, cmd.op_sendto.addr = dest,
  cmd.op_sendto.addrlen = addrlen;
  bsd_dispatch(&cmd);
  errno = cmd.err;
  return cmd.result;
}

ssize_t recvfrom(int fd, void *buf, size_t len, int flags, struct sockaddr *src,
                 socklen_t *addrlen) {
  struct mg_bsd_cmd cmd = BSD_CMD_EMPTY;
  cmd.op = BSD_OP_RECVFROM, cmd.op_recvfrom.fd = fd, cmd.op_recvfrom.buf = buf,
  cmd.op_recvfrom.len = len, cmd.op_recvfrom.flags = flags,
  cmd.op_recvfrom.src = src, cmd.op_recvfrom.addrlen = addrlen;
  bsd_dispatch(&cmd);
  errno = cmd.err;
  return cmd.result;
}

int close(int fd) {
  struct mg_bsd_cmd cmd = BSD_CMD_EMPTY;
  cmd.op = BSD_OP_CLOSE, cmd.op_close.fd = fd;
  bsd_dispatch(&cmd);
  errno = cmd.err;
  return cmd.result;
}

int poll(struct pollfd *fds, nfds_t nfds, int timeout) {
  struct mg_bsd_cmd cmd = BSD_CMD_EMPTY;
  uint64_t deadline = timeout < 0 ? (uint64_t) ~0
                                  : mg_millis() + (uint64_t) timeout;

  cmd.op = BSD_OP_POLL, cmd.op_poll.fds = fds, cmd.op_poll.nfds = nfds,
  cmd.op_poll.deadline = deadline;
  bsd_dispatch(&cmd);
  errno = cmd.err;
  return cmd.result;
}

int select(int nfds, fd_set *r, fd_set *w, fd_set *e, struct timeval *tv) {
  struct mg_bsd_cmd cmd = BSD_CMD_EMPTY;
  uint64_t deadline = tv == NULL ? (uint64_t) ~0
                                 : mg_millis() + (uint64_t) tv->tv_sec * 1000 +
                                       tv->tv_usec / 1000;
  cmd.op = BSD_OP_SELECT, cmd.op_select.nfds = nfds, cmd.op_select.r = r,
  cmd.op_select.w = w, cmd.op_select.e = e, cmd.op_select.deadline = deadline;
  bsd_dispatch(&cmd);
  errno = cmd.err;
  return cmd.result;
}

struct hostent *gethostbyname(const char *name) {
  struct mg_bsd_cmd cmd = BSD_CMD_EMPTY;
  cmd.op = BSD_OP_GETHOSTBYNAME, cmd.op_gethostbyname.name = name;
  bsd_dispatch(&cmd);
  return cmd.op_gethostbyname.result;
}

#endif  // MG_ENABLE_BSD_SOCKETS

#if MG_ENABLE_BSD_SOCKETS
void mg_bsd_init(struct mg_mgr *mgr) {
  s_mgr = mgr;
#if MG_ENABLE_BSD_SOCKETS && MG_ENABLE_FREERTOS
  s_cmd_q = xQueueCreate(16, sizeof(struct mg_bsd_cmd *));
#endif
}

void mg_bsd_poll(struct mg_mgr *mgr) {
  (void) mgr;
#if MG_ENABLE_BSD_SOCKETS && MG_ENABLE_FREERTOS
  struct mg_bsd_cmd **p, *cmd;
  while (xQueueReceive(s_cmd_q, &cmd, 0) == pdTRUE) {
    if (bsd_run(cmd)) {
      xTaskNotifyGive((TaskHandle_t) cmd->caller);
    } else {
      cmd->next = s_waiters;
      s_waiters = cmd;
    }
  }
  p = &s_waiters;
  while (*p != NULL) {
    struct mg_bsd_cmd *w = *p;
    if (bsd_run(w)) {
      *p = w->next;
      xTaskNotifyGive((TaskHandle_t) w->caller);
    } else {
      p = &w->next;
    }
  }
  taskYIELD();
#endif
}
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/dash.c"
#endif





#define MG_NO_CACHE_HEADERS "Cache-Control: no-cache\r\n"
#define MG_JSON_HEADERS "Content-Type: application/json\r\n" MG_NO_CACHE_HEADERS

#define CONN_HANDLED 'Z'

struct mg_dash_cdata {
  char marker;
  struct mg_dash_user *u;
  struct mg_dash *dash;
};

static struct mg_dash_user s_guest;
static struct mg_dash_user *s_users;  // List of authenticated users

static struct mg_str trimq(struct mg_str s) {  // Trim double quotes
  if (s.len > 1 && s.buf[0] == '"') s.len -= 2, s.buf++;
  return s;
}

static struct mg_field_set *mg_dash_find_field_set(struct mg_dash *dash,
                                                   struct mg_str name) {
  struct mg_field_set *fs;
  for (fs = dash->sets; fs != NULL; fs = fs->next) {
    if (mg_strcmp(name, mg_str(fs->name)) == 0) return fs;
  }
  return NULL;
}

// static struct mg_field *mg_dash_find_field(struct mg_field *fields,
//                                            struct mg_str name) {
//   size_t i;
//   for (i = 0; fields != NULL && fields[i].name != NULL; i++) {
//     if (mg_strcmp(name, mg_str(fields[i].name)) == 0) return &fields[i];
//   }
//   return NULL;
// }

static size_t mg_print_field(mg_pfn_t fn, void *arg, va_list *ap) {
  struct mg_field *f = va_arg(*ap, struct mg_field *);
  size_t n = 0;
  n += mg_xprintf(fn, arg, "%m:", MG_ESC(f->name));
  if (f->type == MG_VAL_BOOL) {
    n += mg_xprintf(fn, arg, "%s", *(bool *) f->value ? "true" : "false");
  } else if (f->type == MG_VAL_INT) {
    n += mg_xprintf(fn, arg, "%d", *(int *) f->value);
  } else if (f->type == MG_VAL_UINT64) {
    n += mg_xprintf(fn, arg, "%llu", (uint64_t) *(uint64_t *) f->value);
  } else if (f->type == MG_VAL_DBL) {
    n += mg_xprintf(fn, arg, "%.2f", *(double *) f->value);
  } else if (f->type == MG_VAL_STR) {
    n += mg_xprintf(fn, arg, "%m", MG_ESC(f->value));
  } else if (f->type == MG_VAL_RAW) {
    n += mg_xprintf(fn, arg, "%s", f->value);
  } else {
    n += mg_xprintf(fn, arg, "null");
  }
  return n;
}

static size_t mg_print_field_set(mg_pfn_t fn, void *arg, va_list *ap) {
  struct mg_field_set *set = va_arg(*ap, struct mg_field_set *);
  size_t i, n = 0;
  n += mg_xprintf(fn, arg, "{");
  for (i = 0; set != NULL && set->fields[i].name != NULL; i++) {
    if (i > 0) n += mg_xprintf(fn, arg, ",");
    n += mg_xprintf(fn, arg, "%M", mg_print_field, &set->fields[i]);
  }
  n += mg_xprintf(fn, arg, "}");
  return n;
}

static int mg_dash_array_size(struct mg_field_set *set,
                              struct mg_dash_user *u) {
  int saved = *set->index, sz = -1;
  *set->index = -1;
  if (set->fn && set->fn(MG_DASH_READ, u)) sz = *set->index;
  *set->index = saved;
  return sz;
}

static size_t mg_dash_print_array(mg_pfn_t fn, void *arg, va_list *ap) {
  struct mg_field_set *set = va_arg(*ap, struct mg_field_set *);
  int from = va_arg(*ap, int);
  int to = va_arg(*ap, int);
  struct mg_dash_user *u = va_arg(*ap, struct mg_dash_user *);
  bool started = false;
  int saved = *set->index;
  size_t n = 0;
  *set->index = from;
  n += mg_xprintf(fn, arg, "[");
  for (;;) {
    bool done = to >= 0 && *set->index > to;
    if (!done) {
      if (set->fn) set->fn(MG_DASH_READ, u);
      done = *set->index < 0;
    }
    if (done) break;
    n += mg_xprintf(fn, arg, "%s%M", started ? "," : "", mg_print_field_set,
                    set);
    started = true;
    (*set->index)++;
  }
  n += mg_xprintf(fn, arg, "]");
  *set->index = saved;
  return n;
}

static size_t mg_dash_print_endpoint(mg_pfn_t fn, void *arg, va_list *ap) {
  struct mg_dash *dash = va_arg(*ap, struct mg_dash *);
  struct mg_dash_user *u = va_arg(*ap, struct mg_dash_user *);
  struct mg_str *name = va_arg(*ap, struct mg_str *);
  struct mg_str *from_str = va_arg(*ap, struct mg_str *);
  struct mg_str *to_str = va_arg(*ap, struct mg_str *);
  struct mg_field_set *set = mg_dash_find_field_set(dash, *name);
  size_t n = 0;
  if (name->len == 0) {
    struct mg_field_set *fs;
    const char *comma = "";
    n += mg_xprintf(fn, arg, "{");
    for (fs = dash->sets; fs != NULL; fs = fs->next) {
      if (fs->index != NULL) {
        int sz = mg_dash_array_size(fs, u);
        if (sz < 0) continue;
        n += mg_xprintf(fn, arg, comma);
        n += mg_xprintf(fn, arg, "%m:%d", MG_ESC(fs->name), sz);
      } else {
        if (fs->fn && !fs->fn(MG_DASH_READ, u)) continue;
        n += mg_xprintf(fn, arg, comma);
        n += mg_xprintf(fn, arg, "%m:%M", MG_ESC(fs->name), mg_print_field_set,
                        fs);
      }
      comma = ",";
    }
    n += mg_xprintf(fn, arg, "}");
  } else if (set != NULL && (set->fn == NULL || set->fn(MG_DASH_READ, u))) {
    if (set->index != NULL && from_str != NULL && from_str->len > 0) {
      int from = 0, to = 0;
      mg_str_to_num(*from_str, 10, &from, sizeof(from));
      to = from;
      if (to_str != NULL && to_str->len > 0) {
        mg_str_to_num(*to_str, 10, &to, sizeof(to));
      }
      n += mg_xprintf(fn, arg, "%M", mg_dash_print_array, set, from, to, u);
    } else if (set->index != NULL) {
      n += mg_xprintf(fn, arg, "%d", mg_dash_array_size(set, u));
    } else {
      n += mg_xprintf(fn, arg, "%M", mg_print_field_set, set);
    }
  } else {
    n += mg_xprintf(fn, arg, "null");
  }
  return n;
}

void mg_dash_send_change(struct mg_mgr *mgr, struct mg_field_set *set) {
  struct mg_connection *c;
  for (c = mgr->conns; c != NULL; c = c->next) {
    struct mg_dash_cdata *d;
    struct mg_dash_user *u;
    if (!c->is_websocket) continue;
    if (c->send.len > MG_DASH_MAX_SEND_BUF_SIZE) {  // Skip slow consumers
      MG_ERROR(("skipping, send buf %lu > %d", (unsigned long) c->send.len,
                MG_DASH_MAX_SEND_BUF_SIZE));
      continue;
    }
    d = (struct mg_dash_cdata *) c->data;
    u = d->u;
    if (u == NULL) continue;
    if (set->index != NULL && *set->index < 0) {
      int sz = mg_dash_array_size(set, u);
      if (sz < 0) continue;
      mg_ws_printf(c, WEBSOCKET_OP_TEXT, "{%m:%m,%m:{%m:%d}}", MG_ESC("method"),
                   MG_ESC("change"), MG_ESC("params"), MG_ESC(set->name), sz);
    } else {
      int saved_idx = set->index != NULL ? *set->index : 0;
      bool ok = set->fn ? set->fn(MG_DASH_READ, u) : true;
      if (!ok) {
        if (set->index != NULL) *set->index = saved_idx;
        continue;
      }
      if (set->index != NULL) {
        char key[64];
        mg_snprintf(key, sizeof(key), "%s/%d", set->name, *set->index);
        mg_ws_printf(c, WEBSOCKET_OP_TEXT, "{%m:%m,%m:{%m:%M}}",
                     MG_ESC("method"), MG_ESC("change"), MG_ESC("params"),
                     MG_ESC(key), mg_print_field_set, set);
        *set->index = saved_idx;
      } else {
        mg_ws_printf(c, WEBSOCKET_OP_TEXT, "{%m:%m,%m:{%m:%M}}",
                     MG_ESC("method"), MG_ESC("change"), MG_ESC("params"),
                     MG_ESC(set->name), mg_print_field_set, set);
      }
    }
  }
}

static int mg_dash_parse_field(struct mg_str json, struct mg_field *f) {
  char json_path[128];
  bool ok = false;
  mg_snprintf(json_path, sizeof(json_path), "$.%s", f->name);
  if (f->type == MG_VAL_BOOL) {
    ok = f->value_size == sizeof(bool) &&
         mg_json_get_bool(json, json_path, (bool *) f->value);
  } else if (f->type == MG_VAL_INT) {
    double d;
    if (f->value_size == sizeof(int) && mg_json_get_num(json, json_path, &d) &&
        d == (int) d) {
      *(int *) f->value = (int) d;
      ok = true;
    }
  } else if (f->type == MG_VAL_UINT64) {
    double d;
    if (f->value_size == sizeof(uint64_t) &&
        mg_json_get_num(json, json_path, &d) && d == (double) (int64_t) d) {
      *(uint64_t *) f->value = (uint64_t) d;
      ok = true;
    }
  } else if (f->type == MG_VAL_DBL) {
    ok = f->value_size == sizeof(double) &&
         mg_json_get_num(json, json_path, (double *) f->value);
  } else if (f->type == MG_VAL_STR && f->value_size > 0) {
    struct mg_str tok = mg_json_get_tok(json, json_path);
    ok = tok.len >= 2 && tok.buf[0] == '"' && tok.buf[tok.len - 1] == '"';
    if (ok) mg_json_unescape(json, json_path, (char *) f->value, f->value_size);
  } else if (f->type == MG_VAL_RAW && f->value_size > 0) {
    ok = mg_snprintf((char *) f->value, f->value_size, "%.*s", json.len,
                     json.buf) == json.len;
  }
  return ok;
}

static int mg_dash_apply(struct mg_connection *c, struct mg_dash *dash,
                         struct mg_str json, struct mg_dash_user *u) {
  struct mg_str key, val;
  size_t ofs = 0;
  int total_count = 0;
  while ((ofs = mg_json_next(json, ofs, &key, &val)) > 0) {
    struct mg_field_set *set = mg_dash_find_field_set(dash, trimq(key));
    int count = 0;
    if (set == NULL) {
      MG_ERROR(("UNKNOWN SET: [%.*s]", key.len, key.buf));
      continue;
    }
    if (set->fn != NULL && !set->fn(MG_DASH_WRITE, u)) continue;  // auth check
    {
      size_t i;
      for (i = 0; set->fields[i].name != NULL; i++) {
        if (mg_dash_parse_field(val, &set->fields[i])) count++;
      }
    }
    if (count) {
      if (set->fn) set->fn(MG_DASH_WRITE, u);  // apply side effects
      mg_dash_send_change(c->mgr, set);
      total_count += count;
    }
  }
  return total_count;
}

static inline void mg_log_http_req(struct mg_connection *c,
                                   struct mg_http_message *hm) {
  int len = 0;
  size_t n, spaces = 0, body_n = hm->body.len;
  struct mg_http_message tmp;
  memset(&tmp, 0, sizeof(tmp));
  len = mg_http_parse((char *) c->send.buf, c->send.len, &tmp);
  n = (len < 0 || (size_t) len > c->send.len) ? c->send.len : (size_t) len;
  while ((n + spaces) < c->send.len && spaces < c->send.len &&
         (c->send.buf[c->send.len - spaces - 1] == '\r' ||
          c->send.buf[c->send.len - spaces - 1] == '\n'))
    spaces++;
  // hm->body.len comes from Content-Length and can be larger than the bytes
  // actually buffered so far (e.g. mid-stream uploads); cap the preview to
  // what's actually present in c->recv, or we'd read past its end
  {
    char *recv_end = (char *) c->recv.buf + c->recv.len;
    if (hm->body.buf >= (char *) c->recv.buf && hm->body.buf <= recv_end) {
      size_t avail = (size_t) (recv_end - hm->body.buf);
      if (body_n > avail) body_n = avail;
    } else {
      body_n = 0;
    }
  }
  MG_DEBUG(("%lu %.*s %.*s%s%.*s %.*s: %lu %.*s -> %lu %.*s", c->id,
            hm->method.len, hm->method.buf, hm->uri.len, hm->uri.buf,
            hm->query.len > 0 ? "?" : "", hm->query.len, hm->query.buf,
            c->send.len > 15 ? 3 : 0, &c->send.buf[9], hm->body.len, body_n,
            hm->body.buf, c->send.len - n, c->send.len - n - spaces,
            c->send.buf + n));
}

static uint64_t mg_dash_make_expiration_time(struct mg_dash *dash) {
  unsigned t = (unsigned) dash->session_auto_expiration_seconds;
  if (t == 0) t = 3600;  // Default expiration time in seconds
  return mg_millis() + t * 1000;
}

static struct mg_dash_user *mg_dash_add_user(struct mg_dash_user **users,
                                             struct mg_dash *dash,
                                             const char *name,
                                             const char *token, int level) {
  struct mg_dash_user *u = (struct mg_dash_user *) mg_calloc(1, sizeof(*u));
  if (u != NULL) {
    mg_snprintf(u->name, sizeof(u->name), "%s", name);
    if (token == NULL) {
      mg_random_str(u->token, sizeof(u->token) - 1);
    } else {
      mg_snprintf(u->token, sizeof(u->token), "%s", token);
    }
    u->level = level;
    u->expire = mg_dash_make_expiration_time(dash);
    u->dash = dash;
    u->next = *users;
    *users = u;
  }
  return u;
}

static struct mg_dash_user *mg_dash_find_user(struct mg_dash_user *users,
                                              struct mg_dash *dash,
                                              const char *name) {
  struct mg_dash_user *u;
  for (u = users; u != NULL; u = u->next) {
    if (u->dash == dash && strcmp(u->name, name) == 0) return u;
  }
  return NULL;
}

static struct mg_dash_user *mg_dash_find_token(struct mg_dash_user *users,
                                               struct mg_dash *dash,
                                               const char *token) {
  struct mg_dash_user *u;
  for (u = users; u != NULL; u = u->next) {
    if (u->dash == dash && strcmp(u->token, token) == 0) return u;
  }
  return NULL;
}

static void mg_dash_refresh_user(struct mg_dash *dash,
                                 struct mg_dash_user *user) {
  user->expire = mg_dash_make_expiration_time(dash);
}

static void mg_dash_delete_user(struct mg_mgr *mgr, struct mg_dash_user *u) {
  struct mg_connection *conn;
  for (conn = mgr->conns; conn != NULL; conn = conn->next) {
    struct mg_dash_cdata *d = (struct mg_dash_cdata *) conn->data;
    if (conn->is_websocket && d->dash == u->dash && d->u == u) {
      d->u = NULL;
      conn->is_closing = 1;
    }
  }
  LIST_DELETE(struct mg_dash_user, &s_users, u);
  mg_free(u);
}

static void mg_dash_delete_users(struct mg_mgr *mgr, struct mg_dash *dash) {
  struct mg_dash_user *u, *tmp;
  for (u = s_users; u != NULL; u = tmp) {
    tmp = u->next;
    if (u->dash == dash) mg_dash_delete_user(mgr, u);
  }
}

// Parse HTTP requests, return authenticated user or NULL
static struct mg_dash_user *mg_dash_authenticate(struct mg_connection *c,
                                                 struct mg_http_message *hm,
                                                 struct mg_dash *dash) {
  char user[100], pass[100];
  struct mg_dash_user *u, *tmp;
  struct mg_str *ah;
  int level = 0, num_users = 0;

  if (dash->authenticate == NULL) {
    mg_snprintf(s_guest.name, sizeof(s_guest.name), "%s", "guest");
    s_guest.level = 9;
    s_guest.dash = dash;
    dash->guest = &s_guest;
    return dash->guest;
  }
  mg_http_creds(hm, user, sizeof(user), pass, sizeof(pass));
  ah = mg_http_get_header(hm, "Authorization");
  // MG_DEBUG(("user [%s], pass: [%s], h: %.*s", user, pass, hm->head.len,
  // hm->head.buf));

  // Remove expired users
  for (u = s_users; u != NULL; u = tmp) {
    tmp = u->next;
    if (u->expire < mg_millis()) {
      MG_DEBUG(("Deleting expired auth %s/%d %llu %u", u->name, u->level,
                u->expire, mg_millis() - u->expire));
      mg_dash_delete_user(c->mgr, u);
    }
  }

  if (pass[0] == '\0') return NULL;

  for (u = s_users; u != NULL; u = u->next) {
    if (u->dash == dash) num_users++;
  }
  if (ah == NULL) {
    u = mg_dash_find_token(s_users, dash, pass);
    if (u != NULL) {
      mg_dash_refresh_user(dash, u);
      return u;
    }
  }

  level = dash->authenticate(user, sizeof(user), pass);
  MG_DEBUG(("user %s, level: %d", user, level));
  if (level <= 0) return NULL;

  u = mg_dash_find_user(s_users, dash, user);
  if (u != NULL) {
    if (ah == NULL) mg_snprintf(u->token, sizeof(u->token), "%s", pass);
    mg_dash_refresh_user(dash, u);
    return u;
  }

  if (num_users < 10)
    return mg_dash_add_user(&s_users, dash, user, ah == NULL ? pass : NULL,
                            level);
  // MG_DEBUG(("[%s/%s] -> %s", user, pass, result ? "OK" : "FAIL"));
  return NULL;
}

static void mg_handle_login(struct mg_connection *c, struct mg_dash_user *u) {
  char cookie[256];
  mg_snprintf(cookie, sizeof(cookie),
              "Set-Cookie: access_token=%s; Path=/; "
              "%sHttpOnly; SameSite=Lax; Max-Age=%d\r\n%s",
              u->token, c->is_tls ? "Secure; " : "", 3600 * 24,
              MG_JSON_HEADERS);
  mg_http_reply(c, 200, cookie, "{%m:%m,%m:%d}\n",  //
                MG_ESC("user"), MG_ESC(u->name),    //
                MG_ESC("level"), u->level);
}

static void mg_handle_logout(struct mg_connection *c) {
  char cookie[256];
  mg_snprintf(cookie, sizeof(cookie),
              "Set-Cookie: access_token=; Path=/; "
              "Expires=Thu, 01 Jan 1970 00:00:00 UTC; "
              "%sHttpOnly; Max-Age=0; \r\n",
              c->is_tls ? "Secure; " : "");
  mg_http_reply(c, 401, cookie, "Unauthorized\n");
}

static void mg_dash_handle_del(struct mg_connection *c, struct mg_dash *dash,
                               struct mg_dash_user *u, struct mg_str *parts) {
  struct mg_field_set *set = mg_dash_find_field_set(dash, parts[0]);
  if (set == NULL || set->index == NULL) {
    mg_http_reply(c, 404, MG_JSON_HEADERS, "null\n");
  } else {
    int from = 0, to = 0, count = 0;
    if (parts[1].len) mg_str_to_num(parts[1], 10, &from, sizeof(from));
    to = parts[2].len ? 0 : from;
    if (parts[2].len) mg_str_to_num(parts[2], 10, &to, sizeof(to));
    for (*set->index = from; *set->index <= to; (*set->index)++) {
      if (set->fn && set->fn(MG_DASH_DELETE, u))
        count++;
      else
        break;
    }
    if (count) {
      *set->index = -1;
      mg_dash_send_change(c->mgr, set);
      mg_http_reply(c, 200, MG_JSON_HEADERS, "%d\n", count);
    } else {
      mg_http_reply(c, 403, MG_JSON_HEADERS, "false\n");
    }
  }
}

// Handle "POST /api/get/<set>/<index>": modify one array element. Loads the
// element at <index> first - that doubles as a read-access check and as a
// pre-fill, so that JSON keys absent from the body keep their old values -
// then overlays the values from the body and asks fn to persist them
static void mg_dash_handle_mod(struct mg_connection *c, struct mg_dash *dash,
                               struct mg_dash_user *u, struct mg_str *parts,
                               struct mg_str body) {
  struct mg_field_set *set = mg_dash_find_field_set(dash, parts[0]);
  int index = 0, count = 0;
  size_t i;
  if (set == NULL || set->index == NULL || set->fn == NULL) {
    mg_http_reply(c, 404, MG_JSON_HEADERS, "null\n");
    return;
  }
  mg_str_to_num(parts[1], 10, &index, sizeof(index));
  *set->index = index;
  if (!set->fn(MG_DASH_READ, u) || *set->index != index) {
    mg_http_reply(c, 404, MG_JSON_HEADERS, "null\n");
    return;
  }
  for (i = 0; set->fields[i].name != NULL; i++) {
    if (mg_dash_parse_field(body, &set->fields[i])) count++;
  }
  if (count && set->fn(MG_DASH_WRITE, u)) {
    mg_dash_send_change(c->mgr, set);
    mg_http_reply(c, 200, MG_JSON_HEADERS, "%d\n", count);
  } else {
    mg_http_reply(c, 403, MG_JSON_HEADERS, "false\n");
  }
}

// Handle "POST /api/add/<set>": append a new array element. Parses the body
// straight into the bound fields, then asks fn to accept and persist them as
// a new element - fn returns false to reject, e.g. when a cap is reached
static void mg_dash_handle_add(struct mg_connection *c, struct mg_dash *dash,
                               struct mg_dash_user *u, struct mg_str *parts,
                               struct mg_str body) {
  struct mg_field_set *set = mg_dash_find_field_set(dash, parts[0]);
  int count = 0;
  size_t i;
  if (set == NULL || set->index == NULL || set->fn == NULL) {
    mg_http_reply(c, 404, MG_JSON_HEADERS, "null\n");
    return;
  }
  for (i = 0; set->fields[i].name != NULL; i++) {
    if (mg_dash_parse_field(body, &set->fields[i])) count++;
  }
  if (count && set->fn(MG_DASH_ADD, u)) {
    *set->index = -1;  // Signal mg_dash_send_change() to send new size
    mg_dash_send_change(c->mgr, set);
    mg_http_reply(c, 200, MG_JSON_HEADERS, "true\n");
  } else {
    mg_http_reply(c, 403, MG_JSON_HEADERS, "false\n");
  }
}

// Firmware upload callback for mg_http_stream_body(). Uses *p as a flag:
// non-NULL means an OTA session is open and must be closed by mg_ota_end()
static bool otacb(struct mg_http_message *hm, struct mg_str *data, void **p) {
  if (hm != NULL) {  // Start
    if (!mg_ota_begin(hm->body.len)) return false;
    *p = (void *) (ptrdiff_t) 1;
    return true;
  } else if (data != NULL) {  // Next chunk
    return mg_ota_write(data->buf, data->len);
  }
  if (*p == NULL) return false;  // End. Begin failed, nothing to finish
  *p = NULL;
  return mg_ota_end();
}

static void mg_serve_fs(struct mg_connection *c, struct mg_http_message *hm,
                        struct mg_dash_user *u, bool stream) {
  struct mg_dash *dash = (struct mg_dash *) c->fn_data;
  char root[128] = "/fs/=";
  if (u == NULL) {
    mg_http_reply(c, 403, MG_JSON_HEADERS, "Not Authorised\n");
  } else if (dash->files_dir(u, root + 5, sizeof(root) - 5) == false) {
    mg_http_reply(c, 500, MG_JSON_HEADERS, "Upload dir error\n");
  } else {
    struct mg_http_serve_opts o;
    memset(&o, 0, sizeof(o));
    o.root_dir = root;
    o.fs = dash->upload_fs ? dash->upload_fs : &mg_fs_posix;
    o.enable_delete = o.enable_upload = o.enable_dir_listing = true;
    if (stream) mg_http_serve_upload(c, hm, &o);
    else mg_http_serve_dir(c, hm, &o);
  }
}

void mg_dash_ev_handler(struct mg_connection *c, int ev, void *ev_data) {
  struct mg_dash *dash = (struct mg_dash *) c->fn_data;
  struct mg_dash_cdata *d = (struct mg_dash_cdata *) c->data;

  if (ev == MG_EV_OPEN) {
    d->dash = dash;
    // c->is_hexdumping = 1;
  } else if (ev == MG_EV_CLOSE && c->is_listening) {
    mg_dash_delete_users(c->mgr, dash);
  } else if (ev == MG_EV_HTTP_HDRS && d->marker == 0) {
    // Received headers - check authentication and possibly start uploads/ota
    struct mg_http_message *hm = (struct mg_http_message *) ev_data;
    struct mg_dash_user *u = mg_dash_authenticate(c, hm, dash);
    if (mg_match(hm->uri, mg_str("/api/hi"), NULL) ||
        mg_match(hm->uri, mg_str("/api/logout"), NULL)) {
      // Do nothing, handle them MG_EV_HTTP_MSG. We bypass auth for those
    } else if (u == NULL && mg_match(hm->uri, mg_str("/api/#"), NULL)) {
      mg_http_reply(c, 403, MG_JSON_HEADERS, "Not Authorised\n");
      d->marker = CONN_HANDLED;
    } else if (mg_match(hm->uri, mg_str("/api/login"), NULL) && u != NULL) {
      mg_handle_login(c, u);
      d->marker = CONN_HANDLED;
    } else if (u != NULL && dash->files_dir != NULL &&
               mg_match(hm->uri, mg_str("/fs/#"), NULL)) {
      mg_serve_fs(c, hm, u, true);
      return;  // Important - the serve_fs() call can invalidate hm
    } else if (mg_match(hm->uri, mg_str("/api/ota"), NULL)) {
      // Authenticated above. If streaming started, the stream handler owns
      // the connection and c->data, so skip the marker check below
      if (mg_http_stream_body(c, hm, otacb, NULL)) return;
    }
    if (d->marker != '\0') mg_log_http_req(c, hm);
  } else if (ev == MG_EV_HTTP_MSG && d->marker != '\0') {
    // The response has been send in EV_HDRS path, so we're not reponding
    // anything but clearing the marker for the next request.
    d->marker = 0;
    c->pfn_data = NULL;  // Response done
  } else if (ev == MG_EV_HTTP_MSG && d->marker == '\0') {
    struct mg_http_message *hm = (struct mg_http_message *) ev_data;
    struct mg_dash_user *u = mg_dash_authenticate(c, hm, dash);
    struct mg_str parts[5];
    memset(parts, 0, sizeof(parts));

    if (mg_match(hm->uri, mg_str("/api/hi"), NULL)) {
      mg_http_reply(c, 200, MG_JSON_HEADERS, "hi\n");
    } else if (mg_match(hm->uri, mg_str("/api/logout"), NULL)) {
      mg_handle_logout(c);
      mg_ws_printf(c, WEBSOCKET_OP_TEXT, "{%m:%m}", MG_ESC("method"),
                   MG_ESC("logout"));
    } else if (mg_match(hm->uri, mg_str("/api/websocket"), NULL)) {
      d->u = u;
      mg_ws_upgrade(c, hm, NULL);
    } else if (dash->files_dir != NULL &&
               mg_match(hm->uri, mg_str("/fs/#"), NULL)) {
      mg_serve_fs(c, hm, u, false);
    } else if (mg_match(hm->uri, mg_str("/api/del/*/*/*"), parts) ||
               mg_match(hm->uri, mg_str("/api/del/*/*"), parts)) {
      mg_dash_handle_del(c, dash, u, parts);
    } else if (mg_match(hm->uri, mg_str("/api/add/*"), parts)) {
      mg_dash_handle_add(c, dash, u, parts, hm->body);
    } else if (mg_match(hm->uri, mg_str("/api/get/*/*"), parts) &&
               mg_strcasecmp(hm->method, mg_str("POST")) == 0) {
      mg_dash_handle_mod(c, dash, u, parts, hm->body);
    } else if (mg_match(hm->uri, mg_str("/api/get/*/*/*"), parts) ||
               mg_match(hm->uri, mg_str("/api/get/*/*"), parts) ||
               mg_match(hm->uri, mg_str("/api/get/*"), parts) ||
               mg_match(hm->uri, mg_str("/api/get"), NULL)) {
      mg_http_reply(c, 200, MG_JSON_HEADERS, "%M\n", mg_dash_print_endpoint,
                    dash, u, &parts[0], &parts[1], &parts[2]);
    } else if (mg_match(hm->uri, mg_str("/api/set"), NULL)) {
      int count = mg_dash_apply(c, dash, hm->body, u);
      mg_http_reply(c, 200, MG_JSON_HEADERS, "%d\n", count);
    } else if (mg_match(hm->uri, mg_str("/"), NULL)) {
      struct mg_http_serve_opts opts;
      memset(&opts, 0, sizeof(opts));
      opts.fs = &mg_fs_packed;
      mg_http_serve_file(c, hm, "/dashboard.html", &opts);
    } else {
      struct mg_dash_custom_handler *ch = dash->custom_handlers;
      for (ch = dash->custom_handlers; ch != NULL; ch = ch->next) {
        if (mg_match(hm->uri, ch->uri_pattern, NULL)) {
          ch->handler(c, ev, ev_data);
          break;
        }
      }
      if (ch == NULL) mg_http_reply(c, 404, MG_JSON_HEADERS, "Not Found\n");
      mg_log_http_req(c, hm);
    }
  }
}

#ifdef MG_ENABLE_LINES
#line 1 "src/dns.c"
#endif








struct dns_data {
  struct dns_data *next;
  struct mg_connection *c;
  uint64_t expire;
  uint16_t txnid;
};

static void sendnsreq(struct mg_connection *, struct mg_str *, int,
                      struct mg_dns *, bool);

struct mdns_data {
  struct mdns_data *next;
  struct mg_connection *c;
  uint64_t expire;
  struct mg_str name;
};

static void sendmdnsreq(struct mg_connection *, struct mg_str *, int,
                        struct mg_connection *, bool);

static void dns_free(struct dns_data **head, struct dns_data *d) {
  LIST_DELETE(struct dns_data, head, d);
  mg_free(d);
}

static void mdns_free(struct mdns_data **head, struct mdns_data *d) {
  LIST_DELETE(struct mdns_data, head, d);
  mg_free((void *) d->name.buf);
  mg_free(d);
}

void mg_resolve_cancel(struct mg_connection *c) {
  struct dns_data *tmp, *d;
  struct mdns_data *mtmp, *md;
  struct dns_data **head = (struct dns_data **) &c->mgr->active_dns_requests;
  struct mdns_data **mhead =
      (struct mdns_data **) &c->mgr->active_mdns_requests;
  for (d = *head; d != NULL; d = tmp) {
    tmp = d->next;
    if (d->c == c) dns_free(head, d);
  }
  for (md = *mhead; md != NULL; md = mtmp) {
    mtmp = md->next;
    if (md->c == c) mdns_free(mhead, md);
  }
}

static size_t mg_dns_parse_name_depth(const uint8_t *s, size_t len, size_t ofs,
                                      char *to, size_t tolen, size_t j,
                                      int depth) {
  size_t i = 0;
  if (tolen > 0 && depth == 0) to[0] = '\0';
  if (depth > 5) return 0;
  // MG_INFO(("ofs %lx %x %x", (unsigned long) ofs, s[ofs], s[ofs + 1]));
  while (ofs + i + 1 < len) {
    size_t n = s[ofs + i];
    if (n == 0) {
      i++;
      break;
    }
    if (n & 0xc0) {
      size_t ptr = (((n & 0x3f) << 8) | s[ofs + i + 1]);  // 12 is hdr len
      // MG_INFO(("PTR %lx", (unsigned long) ptr));
      if (ptr + 1 < len && (s[ptr] & 0xc0) == 0 &&
          mg_dns_parse_name_depth(s, len, ptr, to, tolen, j, depth + 1) == 0)
        return 0;
      i += 2;
      break;
    }
    if (ofs + i + n + 1 >= len) return 0;
    if (j > 0) {
      if (j < tolen) to[j] = '.';
      j++;
    }
    if (j + n < tolen) memcpy(&to[j], &s[ofs + i + 1], n);
    j += n;
    i += n + 1;
    if (j < tolen) to[j] = '\0';  // Zero-terminate this chunk
    // MG_INFO(("--> [%s]", to));
  }
  if (tolen > 0) to[tolen - 1] = '\0';  // Make sure it is nul-term
  return i;
}

static size_t mg_dns_parse_name(const uint8_t *s, size_t n, size_t ofs,
                                char *dst, size_t dstlen) {
  return mg_dns_parse_name_depth(s, n, ofs, dst, dstlen, 0, 0);
}

size_t mg_dns_parse_rr(const uint8_t *buf, size_t len, size_t ofs,
                       bool is_question, struct mg_dns_rr *rr) {
  const uint8_t *s = buf + ofs, *e = &buf[len];

  memset(rr, 0, sizeof(*rr));
  if (len < sizeof(struct mg_dns_header)) return 0;  // Too small
  if (len > 512) return 0;  //  Too large, we don't expect that
  if (s >= e) return 0;     //  Overflow

  if ((rr->nlen = (uint16_t) mg_dns_parse_name(buf, len, ofs, NULL, 0)) == 0)
    return 0;
  s += rr->nlen + 4;
  if (s > e) return 0;
  rr->atype = (uint16_t) (((uint16_t) s[-4] << 8) | s[-3]);
  rr->aclass = (uint16_t) (((uint16_t) s[-2] << 8) | s[-1]);
  if (is_question) return (size_t) (rr->nlen + 4);

  s += 6;
  if (s > e) return 0;
  rr->alen = (uint16_t) (((uint16_t) s[-2] << 8) | s[-1]);
  if (s + rr->alen > e) return 0;
  return (size_t) (rr->nlen + rr->alen + 10);
}

bool mg_dns_parse(const uint8_t *buf, size_t len, struct mg_dns_message *dm) {
  const struct mg_dns_header *h = (struct mg_dns_header *) buf;
  struct mg_dns_rr rr;
  size_t i, n, num_answers, ofs = sizeof(*h);
  bool is_response;
  memset(dm, 0, sizeof(*dm));

  if (len < sizeof(*h)) return 0;                // Too small, headers dont fit
  if (mg_ntohs(h->num_questions) > 1) return 0;  // Sanity
  num_answers = mg_ntohs(h->num_answers);
  if (num_answers > 10) {
    MG_DEBUG(("Got %u answers, ignoring beyond 10th one", num_answers));
    num_answers = 10;  // Sanity cap
  }
  dm->txnid = mg_ntohs(h->txnid);
  is_response = mg_ntohs(h->flags) & 0x8000;

  for (i = 0; i < mg_ntohs(h->num_questions); i++) {
    if ((n = mg_dns_parse_rr(buf, len, ofs, true, &rr)) == 0) return false;
    // MG_INFO(("Q %lu %lu %hu/%hu", ofs, n, rr.atype, rr.aclass));
    mg_dns_parse_name(buf, len, ofs, dm->name, sizeof(dm->name));
    ofs += n;
  }

  if (!is_response) {
    // For queries, there is no need to parse the answers. In this way,
    // we also ensure the domain name (dm->name) is parsed from
    // the question field.
    return true;
  }

  for (i = 0; i < num_answers; i++) {
    if ((n = mg_dns_parse_rr(buf, len, ofs, false, &rr)) == 0) return false;
    // MG_INFO(("A -- %lu %lu %hu/%hu %s", ofs, n, rr.atype, rr.aclass,
    // dm->name));
    mg_dns_parse_name(buf, len, ofs, dm->name, sizeof(dm->name));
    ofs += n;

    if (rr.alen == 4 && rr.atype == MG_DNS_RTYPE_A && rr.aclass == 1) {
      dm->addr.is_ip6 = false;
      memcpy(&dm->addr.addr.ip, &buf[ofs - 4], 4);
      dm->resolved = true;
      break;  // Return success
    } else if (rr.alen == 16 && rr.atype == MG_DNS_RTYPE_AAAA &&
               rr.aclass == 1) {
      dm->addr.is_ip6 = true;
      memcpy(&dm->addr.addr.ip, &buf[ofs - 16], 16);
      dm->resolved = true;
      break;  // Return success
    }
  }
  return true;
}

static void dns_cb(struct mg_connection *c, int ev, void *ev_data) {
  struct dns_data *d, *tmp;
  struct dns_data **head = (struct dns_data **) &c->mgr->active_dns_requests;
  if (ev == MG_EV_POLL) {
    uint64_t now = *(uint64_t *) ev_data;
    for (d = *head; d != NULL; d = tmp) {
      tmp = d->next;
      // MG_DEBUG(("%lu %lu dns poll", d->expire, now));
      if (now > d->expire) mg_error(d->c, "DNS timeout");  // will remove entry
    }
  } else if (ev == MG_EV_READ) {
    struct mg_dns_message dm;
    int resolved = 0;
    if (mg_dns_parse(c->recv.buf, c->recv.len, &dm) == false) {
      MG_ERROR(("Unexpected DNS response:"));
      mg_hexdump(c->recv.buf, c->recv.len);
    } else {
      // MG_VERBOSE(("%s %d", dm.name, dm.resolved));
      for (d = *head; d != NULL; d = tmp) {
        tmp = d->next;
        // MG_INFO(("d %p %hu %hu", d, d->txnid, dm.txnid));
        if (dm.txnid != d->txnid) continue;
        if (d->c->is_resolving) {
          if (dm.resolved) {
            dm.addr.port = d->c->rem.port;  // Save port
            d->c->rem = dm.addr;            // Copy resolved address
            MG_DEBUG(
                ("%lu %s is %M", d->c->id, dm.name, mg_print_ip, &d->c->rem));
            mg_connect_resolved(d->c);
#if MG_ENABLE_IPV6
          } else if (dm.addr.is_ip6 == false && dm.name[0] != '\0' &&
                     c->mgr->use_dns6 == false) {
            struct mg_str x = mg_str(dm.name);
            sendnsreq(d->c, &x, c->mgr->dnstimeout, &c->mgr->dns6, true);
#endif
          } else {
            mg_error(d->c, "%s DNS lookup failed", dm.name);
          }
        } else {
          MG_ERROR(("%lu already resolved", d->c->id));
        }
        dns_free(head, d);
        resolved = 1;
      }
    }
    if (!resolved) MG_ERROR(("stray DNS reply"));
    c->recv.len = 0;
  } else if (ev == MG_EV_CLOSE) {
    for (d = *head; d != NULL; d = tmp) {
      tmp = d->next;
      mg_error(d->c, "DNS error");  // will remove entry
    }
  }
}

static bool dns_send(struct mg_connection *c, const struct mg_str *name,
                     unsigned int rtype, uint16_t txnid, uint16_t flags) {
  struct { // RFC-1035 4.1.2
    struct mg_dns_header header;
    uint8_t data[256];
  } pkt;
  size_t i, n;
  memset(&pkt, 0, sizeof(pkt));
  pkt.header.txnid = mg_htons(txnid);
  pkt.header.flags = mg_htons(flags);
  pkt.header.num_questions = mg_htons(1);
  for (i = n = 0; i < sizeof(pkt.data) - 5; i++) {
    if (name->buf[i] == '.' || i >= name->len) {
      pkt.data[n] = (uint8_t) (i - n);
      memcpy(&pkt.data[n + 1], name->buf + n, i - n);
      n = i + 1;
    }
    if (i >= name->len) break;
  }
  memset(&pkt.data[n], 0, 5);         // nul, QTYPE, QCLASS
  pkt.data[n + 2] = (uint8_t) rtype;  // QTYPE = rtype, only 0-255 supported
  pkt.data[n + 4] = 1;                // QCLASS = IN
  n += 5;
  return mg_send(c, &pkt, sizeof(pkt.header) + n);
}

static bool mg_dns_send(struct mg_connection *c, const struct mg_str *name,
                        uint16_t txnid, bool ipv6) {
  return dns_send(c, name, ipv6 ? MG_DNS_RTYPE_AAAA : MG_DNS_RTYPE_A, txnid,
                  0x100); // RD
}

bool mg_dnsc_init(struct mg_mgr *mgr, struct mg_dns *dnsc);
bool mg_dnsc_init(struct mg_mgr *mgr, struct mg_dns *dnsc) {
  if (dnsc->url == NULL) {
    mg_error(0, "DNS server URL is NULL. Call mg_mgr_init()");
    return false;
  }
  if (dnsc->c == NULL) {
    dnsc->c = mg_connect(mgr, dnsc->url, NULL, NULL);
    if (dnsc->c == NULL) return false;
    dnsc->c->pfn = dns_cb;
  }
  return true;
}

static void sendnsreq(struct mg_connection *c, struct mg_str *name, int ms,
                      struct mg_dns *dnsc, bool ipv6) {
  struct dns_data *d = NULL;
  if (!mg_dnsc_init(c->mgr, dnsc)) {
    mg_error(c, "resolver");
  } else if ((d = (struct dns_data *) mg_calloc(1, sizeof(*d))) == NULL) {
    mg_error(c, "resolve OOM");
  } else {
    struct dns_data *reqs = (struct dns_data *) c->mgr->active_dns_requests;
    uint16_t id;
    mg_random(&id, sizeof(uint16_t));
    if (reqs != NULL) // no seq, no collision for upto 256 in-flight requests
      id = (uint16_t) (reqs->txnid + (id & 0xFF) + 1);
    d->txnid = id;
    d->next = reqs;
    c->mgr->active_dns_requests = d;
    d->expire = mg_millis() + (uint64_t) ms;
    d->c = c;
    c->is_resolving = 1;
    MG_VERBOSE(("%lu resolving %.*s @ %s, txnid %hu", c->id, (int) name->len,
                name->buf, dnsc->url, d->txnid));
    if (!mg_dns_send(dnsc->c, name, d->txnid, ipv6)) {
      mg_error(dnsc->c, "DNS send");
    }
  }
}

void mg_resolve(struct mg_connection *c, const char *url) {
  struct mg_str host = mg_url_host(url);
  c->rem.port = mg_htons(mg_url_port(url));
  if (mg_aton(host, &c->rem)) {
    // host is an IP address, do not fire name resolution
    mg_connect_resolved(c);
  } else if (host.len > 6 &&
             strncmp(".local", &host.buf[host.len - 6], 6) == 0) {
    // this is a request for a .local name (mDNS)
    sendmdnsreq(c, &host, 500, c->mgr->mdns, c->mgr->use_dns6);  // 500ms tmout
  } else {
    // host is not an IP nor a .local, send DNS resolution request
    struct mg_dns *dns = c->mgr->use_dns6 ? &c->mgr->dns6 : &c->mgr->dns4;
    sendnsreq(c, &host, c->mgr->dnstimeout, dns, c->mgr->use_dns6);
  }
}

// Response header length is 10 bytes
static const uint8_t mdns_answer[] = {
    0, 1,          // 2 bytes - record type, A
    0, 1,          // 2 bytes - address class, INET
    0, 0, 0, 120,  // 4 bytes - TTL
    0, 4           // 2 bytes - address length
};

// A name length is name->len + '.local' + 2 = name->len + 8
static uint8_t *build_name(struct mg_str *name, uint8_t *p) {
  *p++ = (uint8_t) name->len;  // label 1
  memcpy(p, name->buf, name->len), p += name->len;
  *p++ = 5;  // label 2
  memcpy(p, "local", 5), p += 5;
  *p++ = 0;  // no more labels
  return p;
}

void mg_getlocaddr(struct mg_connection *, struct mg_addr *, struct mg_addr *);

// An A record length is 10 + 4 = 14 bytes
static uint8_t *build_a_record(struct mg_connection *c, uint8_t *p,
                               struct mg_addr *addr) {
  memcpy(p, mdns_answer, sizeof(mdns_answer)), p += sizeof(mdns_answer);
  if (addr != NULL && !addr->is_ip6) {
    memcpy(p, &addr->addr.ip4, 4), p += 4;
  } else {
#if MG_ENABLE_TCPIP
    memcpy(p, &c->mgr->ifp->ip, 4), p += 4;
#else
    struct mg_addr loc, to;
    memset(&loc, 0, sizeof(loc));
    to.is_ip6 = false;
    to.port = mg_htons(5353);
    to.addr.ip4 = MG_IPV4(224, 0, 0, 51);
    mg_getlocaddr(c, &to, &loc);
    memcpy(p, &loc.addr.ip4, 4), p += 4;
#endif
  }
  return p;
}

// A srv name length is r->srvcproto.len + '.local' + 2 = r->srvcproto.len + 8
static uint8_t *build_srv_name(uint8_t *p, struct mg_dnssd_record *r) {
  *p++ = (uint8_t) r->srvcproto.len - 5;  // label 1, up to '._tcp'
  memcpy(p, r->srvcproto.buf, r->srvcproto.len), p += r->srvcproto.len;
  p[-5] = 4;  // label 2, '_tcp', overwrite '.'
  *p++ = 5;   // label 3
  memcpy(p, "local", 5), p += 5;
  *p++ = 0;  // no more labels
  return p;
}

#if 0
// TODO(): for listing
static uint8_t *build_mysrv_name(struct mg_str *name, uint8_t *p,
                                 struct mg_dnssd_record *r) {
  *p++ = name->len;  // label 1
  memcpy(p, name->buf, name->len), p += name->len;
  return build_srv_name(p, r);
}
#endif

// A PTR record length is 10 + name->len + 3 = name->len + 13
static uint8_t *build_ptr_record(struct mg_str *name, uint8_t *p, uint16_t o) {
  uint16_t offset = mg_htons(o);
  memcpy(p, mdns_answer, sizeof(mdns_answer));
  p[1] = MG_DNS_RTYPE_PTR;  // overwrite record type
  p += sizeof(mdns_answer);
  p[-1] = (uint8_t) name->len +
          3;  // overwrite response length, label length + label + offset
  *p++ = (uint8_t) name->len;                       // response: label 1
  memcpy(p, name->buf, name->len), p += name->len;  // copy label
  memcpy(p, &offset, 2);
  *p |= 0xC0, p += 2;
  return p;
}

// An SRV record length is 10 + name->len + 9 = name->len + 19
static uint8_t *build_srv_record(struct mg_str *name, uint8_t *p,
                                 struct mg_dnssd_record *r, uint16_t o) {
  uint16_t port = mg_htons(r->port);
  uint16_t offset = mg_htons(o);
  memcpy(p, mdns_answer, sizeof(mdns_answer));
  p[1] = MG_DNS_RTYPE_SRV;  // overwrite record type
  p += sizeof(mdns_answer);
  p[-1] = (uint8_t) name->len + 9;  // overwrite response length (4+2+1+2)
  *p++ = 0;                         // priority
  *p++ = 0;
  *p++ = 0;  // weight
  *p++ = 0;
  memcpy(p, &port, 2), p += 2;  // port
  *p++ = (uint8_t) name->len;   // label 1
  memcpy(p, name->buf, name->len), p += name->len;
  memcpy(p, &offset, 2);
  *p |= 0xC0, p += 2;
  return p;
}

// A TXT record length is r->txt.len (txt contents) + 10
static uint8_t *build_txt_record(uint8_t *p, struct mg_dnssd_record *r) {
  uint16_t len = mg_htons((uint16_t) r->txt.len);
  memcpy(p, mdns_answer, sizeof(mdns_answer));
  p[1] = MG_DNS_RTYPE_TXT;  // overwrite record type
  p += sizeof(mdns_answer);
  memcpy(p - 2, &len, 2);  // overwrite response length
  memcpy(p, r->txt.buf, r->txt.len), p += r->txt.len;  // copy record verbatim
  return p;
}

// Each additional record has a 2-byte field pointing to the name label

// RFC-6762 16: case-insensitivity --> RFC-1034, 1035

static void handle_mdns_query(struct mg_connection *c) {
  struct mg_dns_header *qh = (struct mg_dns_header *) c->recv.buf;
  struct mg_dns_rr rr;
  size_t n;
  // Parse first question, offset 12 is header size
  n = mg_dns_parse_rr(c->recv.buf, c->recv.len, 12, true, &rr);
  MG_VERBOSE(("mDNS request parsed, result=%d", (int) n));
  if (n > 0) {
    // RFC-6762 Appendix C, RFC2181 11: m(n + 1-63), max 255 + 0x0
    uint8_t buf[sizeof(struct mg_dns_header) + 256 + sizeof(mdns_answer) + 4];
    struct mg_dns_header *h = (struct mg_dns_header *) buf;
    uint8_t *p = &buf[sizeof(*h)];
    char name[256];
    uint8_t name_len;
    // uint16_t q = mg_ntohs(qh->num_questions);
    struct mg_str defname = mg_str((const char *) c->fn_data);
    struct mg_str *respname;
    struct mg_mdns_req req;
    memset(&req, 0, sizeof(req));
    req.is_unicast = (rr.aclass & MG_BIT(15)) != 0;  // QU
    rr.aclass &= (uint16_t) ~MG_BIT(15);  // remove "QU" (unicast response)
    qh->num_questions = mg_htons(1);      // parser sanity
    mg_dns_parse_name(c->recv.buf, c->recv.len, 12, name, sizeof(name));
    name_len = (uint8_t) strlen(name);  // verify it ends in .local
    if (name_len <= 6 || strcmp(".local", &name[name_len - 6]) != 0 ||
        (rr.aclass != 1 && rr.aclass != 0xff))
      return;
    name[name_len -= 6] = '\0';  // remove .local
    MG_VERBOSE(("RR %u %u %s", (unsigned int) rr.atype,
                (unsigned int) rr.aclass, name));
    if (rr.atype == MG_DNS_RTYPE_A) {
      // if we have a name to match, go; otherwise users will match and fill
      // req.r.name and set req.is_resp
      if (c->fn_data != NULL && mg_casecmp((char *) c->fn_data, name) != 0)
        return;
      req.is_resp = (c->fn_data != NULL);
      req.reqname = mg_str_n(name, name_len);
    } else  // users have to match the request to something in their db, then
            // fill req.r and set req.is_resp
      if (rr.atype == MG_DNS_RTYPE_PTR) {
        if (strcmp("_services._dns-sd._udp", name) == 0) req.is_listing = true;
        MG_DEBUG(
            ("PTR request for %s", req.is_listing ? "services listing" : name));
        req.reqname = mg_str_n(name, name_len);
      } else if (rr.atype == MG_DNS_RTYPE_SRV || rr.atype == MG_DNS_RTYPE_TXT) {
        MG_DEBUG(("%s request for %s",
                  rr.atype == MG_DNS_RTYPE_SRV ? "SRV" : "TXT", name));
        // if possible, check it starts with our name, users will check it ends
        // in a service name they handle
        if (c->fn_data != NULL) {
          if (mg_strcasecmp(defname, mg_str_n(name, defname.len)) != 0 ||
              name[defname.len] != '.')
            return;
          req.reqname =
              mg_str_n(name + defname.len + 1, name_len - defname.len - 1);
          MG_DEBUG(
              ("That's us, handing %.*s", req.reqname.len, req.reqname.buf));
        } else {
          req.reqname = mg_str_n(name, name_len);
        }
      } else {  // unhandled record
        return;
      }
    req.rr = &rr;
    mg_call(c, MG_EV_MDNS_REQ, &req);
    if (!req.is_resp) return;
    respname = req.respname.buf != NULL ? &req.respname : &defname;

    memset(h, 0, sizeof(*h));                   // clear header
    h->txnid = req.is_unicast ? qh->txnid : 0;  // RFC-6762 18.1
    h->num_answers = mg_htons(1);  // RFC-6762 6: 0 questions, 1 Answer
    h->flags = mg_htons(0x8400);   // Authoritative response
    if (req.is_listing) {
      // TODO(): RFC-6762 6: each responder SHOULD delay its response by a
      // random amount of time selected with uniform random distribution in the
      // range 20-120 ms.
      // TODO():
      return;
    } else if (rr.atype == MG_DNS_RTYPE_PTR) {  // serve PTR + SRV + TXT + A
      // TODO(): RFC-6762 6: each responder SHOULD delay its response by a
      // random amount of time selected with uniform random distribution in the
      // range 20-120 ms. Response to PTR is local_name._myservice._tcp.local
      uint8_t *o = p, *aux;
      uint16_t offset;
      if (respname->buf == NULL || respname->len == 0) return;
      if ((sizeof(*h) + req.r->srvcproto.len + 8 + respname->len + 13 + 2 +
           respname->len + 19 + 2 + req.r->txt.len + 10 + 2 + 14) >
          sizeof(buf))  // srv name + PTR + 2 + SRV + 2 + TXT + 2 + A
        return;
      h->num_other_prs = mg_htons(3);  // 3 additional records
      p = build_srv_name(p, req.r);
      aux = build_ptr_record(respname, p, (uint16_t) (o - buf));
      o = p + sizeof(mdns_answer);  // point to PTR response (full srvc name)
      offset = mg_htons((uint16_t) (o - buf));
      o = p - 7;  // point to '.local' label (\x05local\x00)
      p = aux;
      memcpy(p, &offset, 2);  // point to full srvc name, in record
      *p |= 0xC0, p += 2;
      aux = p;
      p = build_srv_record(respname, p, req.r, (uint16_t) (o - buf));
      o = aux + sizeof(mdns_answer) + 6;  // point to target in SRV
      memcpy(p, &offset, 2);              // point to full srvc name, in record
      *p |= 0xC0, p += 2;
      p = build_txt_record(p, req.r);
      offset = mg_htons((uint16_t) (o - buf));
      memcpy(p, &offset, 2);  // point to target name, in record
      *p |= 0xC0, p += 2;
      p = build_a_record(c, p, req.addr);
    } else if (rr.atype == MG_DNS_RTYPE_TXT) {
      if ((sizeof(*h) + req.r->srvcproto.len + 8 + req.r->txt.len + 10) >
          sizeof(buf))  // srv name + TXT
        return;
      p = build_srv_name(p, req.r);
      p = build_txt_record(p, req.r);
    } else if (rr.atype == MG_DNS_RTYPE_SRV) {  // serve SRV + A
      uint8_t *o, *aux;
      uint16_t offset;
      if (respname->buf == NULL || respname->len == 0) return;
      if ((sizeof(*h) + req.r->srvcproto.len + 8 + respname->len + 19 + 2 +
           14) > sizeof(buf))  // srv name + SRV + 2 + A
        return;
      h->num_other_prs = mg_htons(1);  // 1 additional record
      p = build_srv_name(p, req.r);
      o = p - 7;  // point to '.local' label (\x05local\x00)
      aux = p;
      p = build_srv_record(respname, p, req.r, (uint16_t) (o - buf));
      o = aux + sizeof(mdns_answer) + 6;  // point to target in SRV
      offset = mg_htons((uint16_t) (o - buf));
      memcpy(p, &offset, 2);  // point to target name, in record
      *p |= 0xC0, p += 2;
      p = build_a_record(c, p, req.addr);
    } else {  // A requested
      // RFC-6762 6: 0 Auth, 0 Additional RRs
      if (respname->buf == NULL || respname->len == 0) return;
      if ((sizeof(*h) + respname->len + 8 + 14) > sizeof(buf))  // name + A
        return;
      p = build_name(respname, p);
      p = build_a_record(c, p, req.addr);
    }
    if (!req.is_unicast) mg_multicast_restore(c, (uint8_t *) &c->loc);
    mg_send(c, buf, (size_t) (p - buf));  // And send it!
    MG_DEBUG(("%M > %M", mg_print_ip_port, &c->loc, mg_print_ip_port, &c->rem));
    MG_DEBUG(("mDNS %s response sent", req.is_unicast ? "unicast" : "mcast"));
  }
}


static size_t srvtoname(char *svc) {
  char *dot = strchr(svc, '.');
  if (dot == NULL) return 0;
  *dot = '\0';
  return (size_t) (dot - svc);
}

#define MG_MAX_MDNS_RECORDS 8
// Condense one useful mDNS/DNS-SD chain from a response datagram into a single
// MG_EV_MDNS_RESP event. A/AAAA as the first supported record are terminal
// address answers (AAAA currently ignored). For PTR/SRV/TXT, subsequent
// records are attached only when their label matches the service instance, or
// the SRV target for address records.
// Some devices send SRV/TXT as extra Answer RRs instead of Additional RRs, so
// both sections are parsed as one chain.
static void handle_mdns_response(struct mg_connection *c) {
  struct mg_dns_header *rh = (struct mg_dns_header *) c->recv.buf;
  struct mg_dns_rr rr, rr_[5]; // A, PTR, SRV, TXT, AAAA, in that order
  struct mg_mdns_resp resp;
  size_t n, i, num, answers, roff_[5], roff = 12;  // offset 12 is header size
  // RFC-6762 Appendix C, RFC2181 11: m(n + 1-63), max 255 + 0x0
  char name[256], srvcproto[256], instance[256], host[256];
  uint16_t atype = 0;
  answers = mg_ntohs(rh->num_answers);
  num = answers + mg_ntohs(rh->num_other_prs);
  if (answers == 0) return;
  MG_VERBOSE(("mDNS response: %u answer, %u additional", (unsigned) answers,
              (unsigned) (num - answers)));
  if (num > MG_MAX_MDNS_RECORDS) {
    num = MG_MAX_MDNS_RECORDS;
    MG_DEBUG(("ignoring > %u records", MG_MAX_MDNS_RECORDS));
  }
  memset(roff_, 0, sizeof(roff_));
  memset(&rr_, 0, sizeof(rr_));
  memset(&resp, 0, sizeof(resp));
  instance[0] = host[0] = '\0';
  for (i = 0; i < num; i++) { // First Answer RR is primary; the rest must match it
    n = mg_dns_parse_rr(c->recv.buf, c->recv.len, roff, false, &rr);
    MG_VERBOSE(("mDNS record parsed, result=%u", n));
    if (n == 0) return;
    if (mg_dns_parse_name(c->recv.buf, c->recv.len, roff, name, sizeof(name)) == 0) return;
    MG_VERBOSE(("RR %u %u %s", (unsigned int) rr.atype, (unsigned int) rr.aclass, name));
    if (rr.alen == 4 && rr.atype == MG_DNS_RTYPE_A && (rr.aclass & 0x7FFF) == 1 &&
        (i == 0 || (host[0] != '\0' && strcmp(name, host) == 0) ||
         (host[0] == '\0' && instance[0] != '\0' && strcmp(name, instance) == 0))) {
      if (rr_[0].atype != 0) break;
      if (i == 0) atype = rr.atype;
      resp.name = mg_str(name);
      resp.addr.is_ip6 = false;
      memcpy(resp.addr.addr.ip, c->recv.buf + roff + n - 4, 4);
      MG_VERBOSE(("A record"));
      roff_[0] = roff, rr_[0] = rr;
      if (atype == MG_DNS_RTYPE_A) break;
    } else if (rr.alen == 16 && rr.atype == MG_DNS_RTYPE_AAAA && (rr.aclass & 0x7FFF) == 1 &&
               (i == 0 || (host[0] != '\0' && strcmp(name, host) == 0) ||
                (host[0] == '\0' && instance[0] != '\0' && strcmp(name, instance) == 0))) {
# if 0
      if (rr_[4].atype != 0) break;
      resp.addr.is_ip6 = true;
      if (i == 0) atype = rr.atype;
      resp.name = mg_str(name);
      memcpy(resp.addr.addr.ip, c->recv.buf + roff + n - 16, 16);
      MG_VERBOSE(("AAAA record"));
      roff_[4] = roff, rr_[4] = rr;
      if (atype == MG_DNS_RTYPE_AAAA) break;
#else
      MG_VERBOSE(("ignored AAAA record"));
#endif
    } else if (rr.atype == MG_DNS_RTYPE_PTR && (rr.aclass & 0x7FFF) == 1 && i == 0) {
      if (rr_[1].atype != 0) break;
      atype = rr.atype;
      if (mg_dns_parse_name(c->recv.buf, c->recv.len, roff + rr.nlen + 10,
                            instance, sizeof(instance)) == 0) return;
      MG_VERBOSE(("PTR record"));
      roff_[1] = roff, rr_[1] = rr;
    } else if (rr.atype == MG_DNS_RTYPE_SRV && rr.alen >= 6 && (rr.aclass & 0x7FFF) == 1 &&
               (i == 0 || (instance[0] != '\0' && strcmp(name, instance) == 0))) {
      if (rr_[2].atype != 0) break;
      if (i == 0) atype = rr.atype;
      resp.sd.port = MG_LOAD_BE16(c->recv.buf + roff + rr.nlen + 14);
      resp.addr.port = mg_htons(resp.sd.port);
      if (instance[0] == '\0') mg_snprintf(instance, sizeof(instance), "%s", name);
      if (mg_dns_parse_name(c->recv.buf, c->recv.len, roff + rr.nlen + 16,
                            host, sizeof(host)) == 0) return;
      MG_VERBOSE(("SRV record"));
      roff_[2] = roff, rr_[2] = rr;
    } else if (rr.atype == MG_DNS_RTYPE_TXT && (rr.aclass & 0x7FFF) == 1 &&
               (i == 0 || (instance[0] != '\0' && strcmp(name, instance) == 0))) {
      if (rr_[3].atype != 0) break;
      if (i == 0) atype = rr.atype;
      if (instance[0] == '\0') mg_snprintf(instance, sizeof(instance), "%s", name);
      resp.sd.txt = mg_str_n((char *) c->recv.buf + roff + rr.nlen + 10, rr.alen);
      MG_VERBOSE(("TXT record"));
      roff_[3] = roff, rr_[3] = rr;
    }
    roff += n;
    if (i == 0 && atype == 0) return;
  }
  if (atype == MG_DNS_RTYPE_A) {
    resp.rr = &rr_[0];
    MG_DEBUG(("A response from %s = %M", name, mg_print_ip, &resp.addr));
#if 0
  } else if (atype == MG_DNS_RTYPE_AAAA) {
    resp.rr = &rr_[4];
    MG_DEBUG(("AAAA response from %s = %M", name, mg_print_ip6, resp.addr.addr.ip));
#endif
  } else if (atype == MG_DNS_RTYPE_PTR) {
    size_t len;
    resp.rr = &rr_[1];
    if (mg_dns_parse_name(c->recv.buf, c->recv.len, roff_[1], srvcproto,
                          sizeof(srvcproto)) == 0) return;
    len = strlen(srvcproto);
    if (len <= 6) return;
    resp.sd.srvcproto = mg_str_n(srvcproto, len - 6);  // remove .local
    if (host[0] != '\0') {
      resp.name = mg_str(host);
    } else if (rr_[0].atype == 0) {
      mg_snprintf(name, sizeof(name), "%s", instance);
      resp.name = mg_str_n(name, srvtoname(name));
    } else {
      if (mg_dns_parse_name(c->recv.buf, c->recv.len, roff_[0], name,
                            sizeof(name)) == 0) return;
      resp.name = mg_str(name);
    }
    MG_DEBUG(("PTR response for %s from %s", srvcproto, name));
  } else if (atype == MG_DNS_RTYPE_SRV || atype == MG_DNS_RTYPE_TXT) {
    size_t len, name_len;
    resp.rr = atype == MG_DNS_RTYPE_SRV ? &rr_[2] : &rr_[3];
    if (mg_dns_parse_name(c->recv.buf, c->recv.len,
                          atype == MG_DNS_RTYPE_SRV ? roff_[2] : roff_[3],
                          srvcproto, sizeof(srvcproto)) == 0)
      return;
    len = strlen(srvcproto);
    name_len = srvtoname(srvcproto);
    if (name_len == 0 || len <= name_len + 7) return;  // remove .local
    resp.sd.srvcproto = mg_str_n(srvcproto + name_len + 1, len - name_len - 7);
    resp.name = mg_str_n(srvcproto, name_len);
    MG_DEBUG(("%s response for %s from %s", atype == MG_DNS_RTYPE_SRV ? "SRV" : "TXT", srvcproto, name));
  } else {
    return;
  }
  mg_call(c, MG_EV_MDNS_RESP, &resp);
}

static void handle_mdns_record(struct mg_connection *c) {
  struct mg_dns_header *h = (struct mg_dns_header *) c->recv.buf;
  if (c->recv.len <= 12) return;
  if ((h->flags & mg_htons(0xF800)) == 0) {
    // flags -> !resp, opcode=0 => query; ignore other opcodes
    handle_mdns_query(c);
  } else if ((h->flags & mg_htons(0xF800)) == mg_htons(0x8000)) {
    // flags -> resp, opcode=0 => response; ignore other opcodes
    handle_mdns_response(c);
  }
}

static void mdns_cb(struct mg_connection *c, int ev, void *ev_data) {
  struct mdns_data *d, *tmp;
  struct mdns_data **head = (struct mdns_data **) &c->mgr->active_mdns_requests;
  // mDNS resolver
  if (ev == MG_EV_POLL) {
    uint64_t now = *(uint64_t *) ev_data;
    for (d = *head; d != NULL; d = tmp) {
      tmp = d->next;
      // MG_DEBUG(("%lu %lu mdns poll", d->expire, now));
      if (now > d->expire) mg_error(d->c, "mDNS timeout");  // will remove entry
    }
  } else if (ev == MG_EV_CLOSE) {
    for (d = *head; d != NULL; d = tmp) {
      tmp = d->next;
      mg_error(d->c, "mDNS listener error");  // this will remove entry
    }
  } else if (ev == MG_EV_MDNS_RESP) {
    struct mg_mdns_resp *resp = (struct mg_mdns_resp *) ev_data;
    if (resp->rr->atype == MG_DNS_RTYPE_A) {
      for (d = *head; d != NULL; d = tmp) {
        tmp = d->next;
        if (mg_strcasecmp(d->name, resp->name) != 0) continue;
        if (d->c->is_resolving) {
          resp->addr.port = d->c->rem.port;  // Save port
          d->c->rem = resp->addr;            // Copy resolved address
          MG_DEBUG(("%lu %.*s is %M", d->c->id, resp->name.len, resp->name.buf,
                    mg_print_ip, &d->c->rem));
          mg_connect_resolved(d->c);
        } else {
          // this should not happen, unless above does not clear c->is_resolving
          MG_ERROR(("%lu already resolved", d->c->id));
        }
        mdns_free(head, d);
      }
    }
  } else if (ev == MG_EV_READ) {
    // generic mDNS[-SD] handling
    handle_mdns_record(c);  // this will call us back with MG_EV_MDNS_RESP
    mg_iobuf_del(&c->recv, 0, c->recv.len);
  }

  (void) ev_data;
}

void mg_multicast_add(struct mg_connection *c, char *ip);
struct mg_connection *mg_mdns_listen(struct mg_mgr *mgr, mg_event_handler_t fn,
                                     void *fn_data) {
  struct mg_connection *c =
      mg_listen(mgr, "udp://224.0.0.251:5353", fn, fn_data);
  if (c == NULL) return NULL;
  c->mgr->mdns = c;  // Add mDNS entry to enable resolver to use it
  c->pfn = mdns_cb, c->pfn_data = fn_data;
  mg_multicast_add(c, (char *) "224.0.0.251");
  return c;
}

static bool mdns_query(struct mg_connection *c, struct mg_str *name,
                       unsigned int rtype) {
  mg_multicast_restore(c, (uint8_t *) &c->loc);
  return dns_send(c, name, rtype, 0, 0);  // RFC-6762 18.1 id = 0, 18.6 RD = 0
}

bool mg_mdns_query(struct mg_connection *c, const char *name,
                   unsigned int rtype) {
  struct mg_str name_;
  name_.buf = (char *) name, name_.len = strlen(name);
  return mdns_query(c, &name_, rtype);
}

static void sendmdnsreq(struct mg_connection *c, struct mg_str *name, int ms,
                        struct mg_connection *mdnsc, bool ipv6) {
  struct mdns_data *d = NULL;
  if (mdnsc == NULL) {
    mg_error(c, "no mDNS listener, see mg_mdns_listen()");
  } else if ((d = (struct mdns_data *) mg_calloc(1, sizeof(*d))) == NULL) {
    mg_error(c, "resolve OOM");
  } else {
    struct mdns_data *reqs = (struct mdns_data *) c->mgr->active_mdns_requests;
    d->next = reqs;
    c->mgr->active_mdns_requests = d;
    d->expire = mg_millis() + (uint64_t) ms;
    d->name = mg_strdup(*name);
    d->c = c;
    c->is_resolving = 1;
    MG_VERBOSE(
        ("%lu resolving %.*s via mDNS", c->id, (int) name->len, name->buf));
    if (!mdns_query(mdnsc, name, MG_DNS_RTYPE_A)) {
      mg_error(c, "mDNS send");  // will remove newly created entry
    }
  }
  (void) ipv6;
}

#ifdef MG_ENABLE_LINES
#line 1 "src/event.c"
#endif






void mg_call(struct mg_connection *c, int ev, void *ev_data) {
#if MG_ENABLE_PROFILE
  const char *names[] = {
      "EV_ERROR",    "EV_OPEN",      "EV_POLL",      "EV_RESOLVE",
      "EV_CONNECT",  "EV_ACCEPT",    "EV_TLS_HS",    "EV_READ",
      "EV_WRITE",    "EV_CLOSE",     "EV_HTTP_MSG",  "EV_HTTP_CHUNK",
      "EV_WS_OPEN",  "EV_WS_MSG",    "EV_WS_CTL",    "EV_MQTT_CMD",
      "EV_MQTT_MSG", "EV_MQTT_OPEN", "EV_SNTP_TIME", "EV_USER"};
  if (ev != MG_EV_POLL && ev < (int) (sizeof(names) / sizeof(names[0]))) {
    MG_PROF_ADD(c, names[ev]);
  }
#endif
  // Fire protocol handler first, user handler second. See #2559
  if (c->pfn != NULL) c->pfn(c, ev, ev_data);
  if (c->fn != NULL) c->fn(c, ev, ev_data);
}

void mg_error(struct mg_connection *c, const char *fmt, ...) {
  char buf[64];
  va_list ap;
  va_start(ap, fmt);
  mg_vsnprintf(buf, sizeof(buf), fmt, &ap);
  va_end(ap);
  MG_ERROR(("%lu %ld %s", c->id, c->fd, buf));
  c->is_closing = 1;             // Set is_closing before sending MG_EV_CALL
  mg_call(c, MG_EV_ERROR, buf);  // Let user handler override it
}

#ifdef MG_ENABLE_LINES
#line 1 "src/flash.c"
#endif




#ifdef MG_OTA_PUBLIC_KEY


#endif

#if MG_OTA != MG_OTA_NONE && MG_OTA != MG_OTA_CUSTOM

static char *s_addr;      // Current address to write to
static size_t s_size;     // Firmware size to flash. In-progress indicator
static uint32_t s_crc32;  // Firmware checksum

bool mg_ota_flash_begin(size_t new_firmware_size, struct mg_flash *flash) {
  bool ok = false;
  if (s_size) {
    MG_ERROR(("OTA already in progress. Call mg_ota_end()"));
  } else {
    size_t half = flash->size / 2;
    s_crc32 = 0;
    s_addr = (char *) flash->start + half;
    MG_DEBUG(("FW %lu bytes, max %lu", new_firmware_size, half));
    if (new_firmware_size < half) {
      ok = true;
      s_size = new_firmware_size;
      MG_INFO(("Starting OTA, firmware size %lu", s_size));
    } else {
      MG_ERROR(("Firmware %lu is too big to fit %lu", new_firmware_size, half - flash->align));
    }
  }
  return ok;
}

bool mg_ota_flash_write(const void *buf, size_t len, struct mg_flash *flash) {
  bool ok = false;
  if (s_size == 0) {
    MG_ERROR(("OTA is not started, call mg_ota_begin()"));
  } else if (s_addr + MG_ROUND_UP(len, flash->align) >
             (char *) flash->start + flash->size) {
    MG_ERROR(("Flash overflow: attempting to write past the flash boundary"));
  } else {
    size_t len_aligned_down = MG_ROUND_DOWN(len, flash->align);
    if (len_aligned_down) ok = flash->write_fn(s_addr, buf, len_aligned_down);
    if (len_aligned_down < len) {
      size_t left = len - len_aligned_down;
      char tmp[flash->align];
      memset(tmp, 0xff, sizeof(tmp));
      memcpy(tmp, (char *) buf + len_aligned_down, left);
      ok = flash->write_fn(s_addr + len_aligned_down, tmp, sizeof(tmp));
    }
    s_crc32 = mg_crc32(s_crc32, (char *) buf, len);  // Update CRC
    MG_DEBUG(("%#x %p %lu -> %d", s_addr - len, buf, len, ok));
    s_addr += len;
  }
  return ok;
}

bool mg_ota_flash_end(struct mg_flash *flash) {
  char *base = (char *) flash->start + flash->size / 2;
  bool ok = false;
  int prev_state = MG_OTA_STATE_GET();
  if (s_size) {
    size_t size = (size_t) (s_addr - base);
    uint32_t crc32 = mg_crc32(0, base, s_size);
    if (size == s_size && crc32 == s_crc32) ok = true;
    MG_DEBUG(("CRC: %x/%x, size: %lu/%lu, status: %s", s_crc32, crc32, s_size,
              size, ok ? "ok" : "fail"));
#ifdef MG_OTA_PUBLIC_KEY
    if (ok) {
      bool signed_fw = s_size > 68 &&
                       memcmp((uint8_t *) base + s_size - 4, "MGSG", 4) == 0;
      if (signed_fw) {
        static const uint8_t s_pubkey[] = MG_OTA_PUBLIC_KEY;
        uint8_t hash[32];
        size_t fw_size = s_size - 68;  // strip 64-byte sig + 4-byte magic
        mg_sha256(hash, (uint8_t *) base, fw_size);
        ok = mg_p256_verify(s_pubkey, hash, sizeof(hash),
                            (uint8_t *) base + fw_size) == 1;
        MG_INFO(("Signature: %s", ok ? "ok" : "fail"));
      } else {
        ok = false;
        MG_ERROR(("Unsigned firmware rejected"));
      }
    }
#endif
    s_size = 0;
    if (ok) MG_OTA_STATE_SET(MG_OTA_TESTING);
    if (ok) ok = flash->swap_fn();
    if (!ok) MG_OTA_STATE_SET(prev_state); // undo state in case of failure
  }
  MG_INFO(("Finishing OTA: %s", ok ? "ok" : "fail"));
  return ok;
}

#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/fmt.c"
#endif




static bool is_digit(int c) {
  return c >= '0' && c <= '9';
}

static int addexp(char *buf, int e, int sign) {
  int n = 0;
  buf[n++] = 'e';
  buf[n++] = (char) sign;
  if (e > 400) return 0;
  if (e < 10) buf[n++] = '0';
  if (e >= 100) buf[n++] = (char) (e / 100 + '0'), e -= 100 * (e / 100);
  if (e >= 10) buf[n++] = (char) (e / 10 + '0'), e -= 10 * (e / 10);
  buf[n++] = (char) (e + '0');
  return n;
}

static int xisinf(double x) {
  union {
    double f;
    uint64_t u;
  } ieee754;
  ieee754.f = x;
  return ((unsigned) (ieee754.u >> 32) & 0x7fffffff) == 0x7ff00000 &&
         ((unsigned) ieee754.u == 0);
}

static int xisnan(double x) {
  union {
    double f;
    uint64_t u;
  } ieee754;
  ieee754.f = x;
  return ((unsigned) (ieee754.u >> 32) & 0x7fffffff) +
             ((unsigned) ieee754.u != 0) >
         0x7ff00000;
}

static size_t mg_dtoa(char *dst, size_t dstlen, double d, int width, bool tz) {
  char buf[40];
  int i, s = 0, n = 0, e = 0;
  double t, mul, saved;
  if (d == 0.0) return mg_snprintf(dst, dstlen, "%s", "0");
  if (xisinf(d)) return mg_snprintf(dst, dstlen, "%s", d > 0 ? "inf" : "-inf");
  if (xisnan(d)) return mg_snprintf(dst, dstlen, "%s", "nan");
  if (d < 0.0) d = -d, buf[s++] = '-';

  // Round
  saved = d;
  if (tz) {
    mul = 1.0;
    while (d >= 10.0 && d / mul >= 10.0) mul *= 10.0;
  } else {
    mul = 0.1;
  }

  while (d <= 1.0 && d / mul <= 1.0) mul /= 10.0;
  for (i = 0, t = mul * 5; i < width; i++) t /= 10.0;

  d += t;

  // Calculate exponent, and 'mul' for scientific representation
  mul = 1.0;
  while (d >= 10.0 && d / mul >= 10.0) mul *= 10.0, e++;
  while (d < 1.0 && d / mul < 1.0) mul /= 10.0, e--;
  // printf(" --> %g %d %g %g\n", saved, e, t, mul);

  if (tz && (e >= width || e <= -width) && width > 1) {
    char exp[6];
    int ne;
    n = (int) mg_dtoa(buf + s, sizeof(buf) - (size_t) s, saved / mul, width, tz);
    // printf(" --> %.*g %d [%.*s]\n", 10, d / t, e, n, buf);
    ne = addexp(exp, e < 0 ? -e : e, e < 0 ? '-' : '+');
    if (s + n + ne >= (int) sizeof(buf))
      n = (int) sizeof(buf) - s - ne - 1;
    memcpy(buf + s + n, exp, (size_t) ne);
    n += ne;
    return mg_snprintf(dst, dstlen, "%.*s", s + n, buf);
  } else {
    int targ_width = width;
    for (i = 0, t = mul; t >= 1.0 && s + n < (int) sizeof(buf); i++) {
      int ch = (int) (d / t);
      if (n > 0 || ch > 0) buf[s + n++] = (char) (ch + '0');
      d -= ch * t;
      t /= 10.0;
    }
    // printf(" --> [%g] -> %g %g (%d) [%.*s]\n", saved, d, t, n, s + n, buf);
    if (n == 0) buf[s++] = '0';
    while (t >= 1.0 && n + s < (int) sizeof(buf)) buf[n++] = '0', t /= 10.0;
    if (s + n < (int) sizeof(buf)) buf[n + s++] = '.';
    // printf(" 1--> [%g] -> [%.*s]\n", saved, s + n, buf);
    if (!tz && n > 0) targ_width = width + n;
    for (i = 0, t = 0.1; s + n < (int) sizeof(buf) && n < targ_width; i++) {
      int ch = (int) (d / t);
      buf[s + n++] = (char) (ch + '0');
      d -= ch * t;
      t /= 10.0;
    }
  }

  while (tz && n > 0 && buf[s + n - 1] == '0') n--;  // Trim trailing zeroes
  if (tz && n > 0 && buf[s + n - 1] == '.') n--;     // Trim trailing dot
  n += s;
  if (n >= (int) sizeof(buf)) n = (int) sizeof(buf) - 1;
  buf[n] = '\0';
  return mg_snprintf(dst, dstlen, "%s", buf);
}

static size_t mg_lld(char *buf, int64_t val, bool is_signed, bool is_hex) {
  const char *letters = "0123456789abcdef";
  uint64_t v = (uint64_t) val;
  size_t s = 0, n, i;
  if (is_signed && val < 0) buf[s++] = '-', v = (uint64_t) (-val);
  // This loop prints a number in reverse order. I guess this is because we
  // write numbers from right to left: least significant digit comes last.
  // Maybe because we use Arabic numbers, and Arabs write RTL?
  if (is_hex) {
    for (n = 0; v; v >>= 4) buf[s + n++] = letters[v & 15];
  } else {
    for (n = 0; v; v /= 10) buf[s + n++] = letters[v % 10];
  }
  // Reverse a string
  for (i = 0; i < n / 2; i++) {
    char t = buf[s + i];
    buf[s + i] = buf[s + n - i - 1], buf[s + n - i - 1] = t;
  }
  if (val == 0) buf[n++] = '0';  // Handle special case
  return n + s;
}

static size_t scpy(void (*out)(char, void *), void *ptr, char *buf,
                   size_t len) {
  size_t i = 0;
  while (i < len && buf[i] != '\0') out(buf[i++], ptr);
  return i;
}

size_t mg_xprintf(void (*out)(char, void *), void *ptr, const char *fmt, ...) {
  size_t len = 0;
  va_list ap;
  va_start(ap, fmt);
  len = mg_vxprintf(out, ptr, fmt, &ap);
  va_end(ap);
  return len;
}

size_t mg_vxprintf(void (*out)(char, void *), void *param, const char *fmt,
                   va_list *ap) {
  size_t i = 0, n = 0;
  while (fmt[i] != '\0') {
    if (fmt[i] == '%') {
      size_t j, k, x = 0, is_long = 0, w = 0 /* width */, pr = ~0U /* prec */;
      char pad = ' ', minus = 0, c = fmt[++i];
      if (c == '#') x++, c = fmt[++i];
      if (c == '-') minus++, c = fmt[++i];
      if (c == '0') pad = '0', c = fmt[++i];
      while (is_digit(c)) w *= 10, w += (size_t) (c - '0'), c = fmt[++i];
      if (c == '.') {
        c = fmt[++i];
        if (c == '*') {
          pr = (size_t) va_arg(*ap, int);
          c = fmt[++i];
        } else {
          pr = 0;
          while (is_digit(c)) pr *= 10, pr += (size_t) (c - '0'), c = fmt[++i];
        }
      }
      while (c == 'h') c = fmt[++i];  // Treat h and hh as int
      if (c == 'l') {
        is_long++, c = fmt[++i];
        if (c == 'l') is_long++, c = fmt[++i];
      }
      if (c == 'z') is_long = 1, c = fmt[++i];
      if (c == 'p') x = 1, is_long = 1;
      if (c == 'd' || c == 'u' || c == 'x' || c == 'X' || c == 'p' ||
          c == 'g' || c == 'f') {
        bool s = (c == 'd'), h = (c == 'x' || c == 'X' || c == 'p');
        char tmp[40];
        size_t xl = x ? 2 : 0;
        if (c == 'g' || c == 'f') {
          double v = va_arg(*ap, double);
          if (pr == ~0U) pr = 6;
          k = mg_dtoa(tmp, sizeof(tmp), v, (int) pr, c == 'g');
        } else if (is_long == 2) {
          int64_t v = va_arg(*ap, int64_t);
          k = mg_lld(tmp, v, s, h);
        } else if (is_long == 1) {
          long v = va_arg(*ap, long);
          k = mg_lld(tmp, s ? (int64_t) v : (int64_t) (unsigned long) v, s, h);
        } else {
          int v = va_arg(*ap, int);
          k = mg_lld(tmp, s ? (int64_t) v : (int64_t) (unsigned) v, s, h);
        }
        for (j = 0; j < xl && w > 0; j++) w--;
        for (j = 0; pad == ' ' && !minus && k < w && j + k < w; j++)
          n += scpy(out, param, &pad, 1);
        n += scpy(out, param, (char *) "0x", xl);
        for (j = 0; pad == '0' && k < w && j + k < w; j++)
          n += scpy(out, param, &pad, 1);
        n += scpy(out, param, tmp, k);
        for (j = 0; pad == ' ' && minus && k < w && j + k < w; j++)
          n += scpy(out, param, &pad, 1);
      } else if (c == 'm' || c == 'M') {
        mg_pm_t f = va_arg(*ap, mg_pm_t);
        if (c == 'm') out('"', param);
        n += f(out, param, ap);
        if (c == 'm') n += 2, out('"', param);
      } else if (c == 'c') {
        int ch = va_arg(*ap, int);
        out((char) ch, param);
        n++;
      } else if (c == 's') {
        char *p = va_arg(*ap, char *);
        if (pr == ~0U) pr = p == NULL ? 0 : strlen(p);
        for (j = 0; !minus && pr < w && j + pr < w; j++)
          n += scpy(out, param, &pad, 1);
        n += scpy(out, param, p, pr);
        for (j = 0; minus && pr < w && j + pr < w; j++)
          n += scpy(out, param, &pad, 1);
      } else if (c == '%') {
        out('%', param);
        n++;
      } else {
        out('%', param);
        out(c, param);
        n += 2;
      }
      i++;
    } else {
      out(fmt[i], param), n++, i++;
    }
  }
  return n;
}

#ifdef MG_ENABLE_LINES
#line 1 "src/fs.c"
#endif






struct mg_fd *mg_fs_open(struct mg_fs *fs, const char *path, int flags) {
  struct mg_fd *fd = (struct mg_fd *) mg_calloc(1, sizeof(*fd));
  if (fd != NULL) {
    fd->fd = fs->op(path, flags);
    fd->fs = fs;
    if (fd->fd == NULL) {
      mg_free(fd);
      fd = NULL;
    }
  }
  return fd;
}

void mg_fs_close(struct mg_fd *fd) {
  if (fd != NULL) {
    fd->fs->cl(fd->fd);
    mg_free(fd);
  }
}

struct mg_str mg_file_read(struct mg_fs *fs, const char *path) {
  struct mg_str result = {NULL, 0};
  void *fp;
  fs->st(path, &result.len, NULL);
  if ((fp = fs->op(path, MG_FS_READ)) != NULL) {
    result.buf = (char *) mg_calloc(1, result.len + 1);
    if (result.buf != NULL &&
        fs->rd(fp, (void *) result.buf, result.len) != result.len) {
      mg_free((void *) result.buf);
      result.buf = NULL;
    }
    fs->cl(fp);
  }
  if (result.buf == NULL) result.len = 0;
  return result;
}

bool mg_file_write(struct mg_fs *fs, const char *path, const void *buf,
                   size_t len) {
  bool result = false;
  struct mg_fd *fd;
  char tmp[MG_PATH_MAX], rnd[10];
  size_t path_len = mg_snprintf(tmp, sizeof(tmp), "%s..%s", path,
                                mg_random_str(rnd, sizeof(rnd)));
  fs->rm(tmp);
  if (path_len < sizeof(tmp) &&
      (fd = mg_fs_open(fs, tmp, MG_FS_WRITE | MG_FS_EXCL)) != NULL) {
    result = fs->wr(fd->fd, buf, len) == len;
    mg_fs_close(fd);
    if (result) {
      fs->rm(path);
      result = fs->mv(tmp, path);
      MG_DEBUG(("mv %s -> %s result=%d", tmp, path, result));
    }
    fs->rm(tmp);
  } else {
    MG_ERROR(("mg_fs_open(%s) failed, path_len=%zu", tmp, path_len));
  }
  return result;
}

bool mg_file_printf(struct mg_fs *fs, const char *path, const char *fmt, ...) {
  va_list ap;
  char *data;
  bool result = false;
  va_start(ap, fmt);
  data = mg_vmprintf(fmt, &ap);
  va_end(ap);
  if (data != NULL) {
    result = mg_file_write(fs, path, data, strlen(data));
    mg_free(data);
  }
  return result;
}

// This helper function allows to scan a filesystem in a sequential way,
// without using callback function:
//      char buf[100] = "";
//      while (mg_fs_ls(&mg_fs_posix, "./", buf, sizeof(buf))) {
//        ...
static void mg_fs_ls_fn(const char *filename, void *param) {
  struct mg_str *s = (struct mg_str *) param;
  if (s->buf[0] == '\0') {
    mg_snprintf((char *) s->buf, s->len, "%s", filename);
  } else if (strcmp(s->buf, filename) == 0) {
    ((char *) s->buf)[0] = '\0';  // Fetch next file
  }
}

bool mg_fs_ls(struct mg_fs *fs, const char *path, char *buf, size_t len) {
  struct mg_str s;
  s.buf = buf, s.len = len;
  fs->ls(path, mg_fs_ls_fn, &s);
  return buf[0] != '\0';
}

#ifdef MG_ENABLE_LINES
#line 1 "src/fs_fat.c"
#endif





#if MG_ENABLE_FATFS
#include <ff.h>

static time_t ff_time_to_epoch(uint16_t fdate, uint16_t ftime) {
  unsigned int sec = (ftime << 1) & 0x3e;
  unsigned int min = (ftime >> 5) & 0x3f;
  unsigned int hour = (ftime >> 11) & 0x1f;
  unsigned int day = fdate & 0x1f;
  unsigned int month = (fdate >> 5) & 0x0f;
  unsigned int year = ((fdate >> 9) & 0x7f) + 1980;
  return (time_t) mg_timegm(year, month, day, hour, min, sec);
}

static int ff_stat(const char *path, size_t *size, time_t *mtime) {
  FILINFO fi;
  if (path[0] == '\0') {
    if (size) *size = 0;
    if (mtime) *mtime = 0;
    return MG_FS_DIR;
  } else if (f_stat(path, &fi) == 0) {
    if (size) *size = (size_t) fi.fsize;
    if (mtime) *mtime = ff_time_to_epoch(fi.fdate, fi.ftime);
    return MG_FS_READ | MG_FS_WRITE | ((fi.fattrib & AM_DIR) ? MG_FS_DIR : 0);
  } else {
    return 0;
  }
}

static void ff_list(const char *dir, void (*fn)(const char *, void *),
                    void *userdata) {
  DIR d;
  FILINFO fi;
  if (f_opendir(&d, dir) == FR_OK) {
    while (f_readdir(&d, &fi) == FR_OK && fi.fname[0] != '\0') {
      if (!strcmp(fi.fname, ".") || !strcmp(fi.fname, "..")) continue;
      fn(fi.fname, userdata);
    }
    f_closedir(&d);
  }
}

static void *ff_open(const char *path, int flags) {
  FIL *fp = NULL;
  unsigned char mode = FA_READ;
  if (flags & MG_FS_WRITE) {
    mode |= FA_WRITE;
    if (flags & MG_FS_EXCL) {
      mode |= FA_CREATE_NEW;
    } else {
      mode |= FA_OPEN_ALWAYS | FA_OPEN_APPEND;
    }
  }
  if ((fp = mg_calloc(1, sizeof(*fp))) != NULL &&
      f_open(fp, path, mode) != FR_OK) {
    mg_free(fp);
    fp = NULL;
  }
  return fp;
}

static void ff_close(void *fp) {
  if (fp != NULL) {
    f_close((FIL *) fp);
    mg_free(fp);
  }
}

static size_t ff_read(void *fp, void *buf, size_t len) {
  UINT n = 0, misalign = ((size_t) buf) & 3;
  if (misalign) {
    char aligned[4];
    f_read((FIL *) fp, aligned, len > misalign ? misalign : len, &n);
    memcpy(buf, aligned, n);
  } else {
    f_read((FIL *) fp, buf, len, &n);
  }
  return n;
}

static size_t ff_write(void *fp, const void *buf, size_t len) {
  UINT n = 0;
  return f_write((FIL *) fp, (char *) buf, len, &n) == FR_OK ? n : 0;
}

static size_t ff_seek(void *fp, size_t offset) {
  f_lseek((FIL *) fp, offset);
  return offset;
}

static bool ff_rename(const char *from, const char *to) {
  return f_rename(from, to) == FR_OK;
}

static bool ff_remove(const char *path) {
  return f_unlink(path) == FR_OK;
}

static bool ff_mkdir(const char *path) {
  return f_mkdir(path) == FR_OK;
}

struct mg_fs mg_fs_fat = {ff_stat,  ff_list, ff_open,   ff_close,  ff_read,
                          ff_write, ff_seek, ff_rename, ff_remove, ff_mkdir};
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/fs_packed.c"
#endif





struct packed_file {
  const char *data;
  size_t size;
  size_t pos;
};

const struct mg_mem_file *mg_mem_files;

static int mg_scmp(const char *a, const char *b) {
  while (*a && (*a == *b)) a++, b++;
  return *(const unsigned char *) a - *(const unsigned char *) b;
}
static const char *mg_unlist(size_t no) {
  return mg_mem_files == NULL ? NULL : mg_mem_files[no].path;
}
static const char *mg_unpack(const char *path, size_t *size, time_t *mtime) {
  const struct mg_mem_file *p;
  for (p = mg_mem_files; p != NULL && p->path != NULL; p++) {
    if (mg_scmp(p->path, path) != 0) continue;
    if (size != NULL) *size = p->size;
    if (mtime != NULL) *mtime = p->mtime;
    return (const char *) p->data;
  }
  return NULL;
}

struct mg_str mg_unpacked(const char *path) {
  size_t len = 0;
  const char *buf = mg_unpack(path, &len, NULL);
  return mg_str_n(buf, len);
}

static int is_dir_prefix(const char *prefix, size_t n, const char *path) {
  // MG_INFO(("[%.*s] [%s] %c", (int) n, prefix, path, path[n]));
  return n < strlen(path) && strncmp(prefix, path, n) == 0 &&
         (n == 0 || path[n] == '/' || path[n - 1] == '/');
}

static int packed_stat(const char *path, size_t *size, time_t *mtime) {
  const char *p;
  size_t i, n = strlen(path);
  if (mg_unpack(path, size, mtime)) return MG_FS_READ;  // Regular file
  // Scan all files. If `path` is a dir prefix for any of them, it's a dir
  for (i = 0; (p = mg_unlist(i)) != NULL; i++) {
    if (is_dir_prefix(path, n, p)) return MG_FS_DIR;
  }
  return 0;
}

static void packed_list(const char *dir, void (*fn)(const char *, void *),
                        void *userdata) {
  char buf[MG_PATH_MAX], tmp[sizeof(buf)];
  const char *path, *begin, *end;
  size_t i, n = strlen(dir);
  tmp[0] = '\0';  // Previously listed entry
  for (i = 0; (path = mg_unlist(i)) != NULL; i++) {
    if (!is_dir_prefix(dir, n, path)) continue;
    begin = &path[n + 1];
    end = strchr(begin, '/');
    if (end == NULL) end = begin + strlen(begin);
    mg_snprintf(buf, sizeof(buf), "%.*s", (int) (end - begin), begin);
    buf[sizeof(buf) - 1] = '\0';
    // If this entry has been already listed, skip
    // NOTE: we're assuming that file list is sorted alphabetically
    if (strcmp(buf, tmp) == 0) continue;
    fn(buf, userdata);  // Not yet listed, call user function
    strcpy(tmp, buf);   // And save this entry as listed
  }
}

static void *packed_open(const char *path, int flags) {
  size_t size = 0;
  const char *data = mg_unpack(path, &size, NULL);
  struct packed_file *fp = NULL;
  if (data == NULL) return NULL;
  if (flags & MG_FS_WRITE) return NULL;
  if ((fp = (struct packed_file *) mg_calloc(1, sizeof(*fp))) != NULL) {
    fp->size = size;
    fp->data = data;
  }
  return (void *) fp;
}

static void packed_close(void *fp) {
  if (fp != NULL) mg_free(fp);
}

static size_t packed_read(void *fd, void *buf, size_t len) {
  struct packed_file *fp = (struct packed_file *) fd;
  if (fp->pos + len > fp->size) len = fp->size - fp->pos;
  memcpy(buf, &fp->data[fp->pos], len);
  fp->pos += len;
  return len;
}

static size_t packed_write(void *fd, const void *buf, size_t len) {
  (void) fd, (void) buf, (void) len;
  return 0;
}

static size_t packed_seek(void *fd, size_t offset) {
  struct packed_file *fp = (struct packed_file *) fd;
  fp->pos = offset;
  if (fp->pos > fp->size) fp->pos = fp->size;
  return fp->pos;
}

static bool packed_rename(const char *from, const char *to) {
  (void) from, (void) to;
  return false;
}

static bool packed_remove(const char *path) {
  (void) path;
  return false;
}

static bool packed_mkdir(const char *path) {
  (void) path;
  return false;
}

struct mg_fs mg_fs_packed = {
    packed_stat,  packed_list, packed_open,   packed_close,  packed_read,
    packed_write, packed_seek, packed_rename, packed_remove, packed_mkdir};

#ifdef MG_ENABLE_LINES
#line 1 "src/fs_posix.c"
#endif


#if MG_ENABLE_POSIX_FS

#ifndef MG_STAT_STRUCT
#define MG_STAT_STRUCT stat
#endif

#ifndef MG_STAT_FUNC
#define MG_STAT_FUNC stat
#endif

#if MG_ENABLE_LFS
#include <dirent.h>
#endif

static int p_stat(const char *path, size_t *size, time_t *mtime) {
#if !defined(S_ISDIR)
  MG_ERROR(("stat() API is not supported. %p %p %p", path, size, mtime));
  return 0;
#else
#if MG_ARCH == MG_ARCH_WIN32
  struct _stati64 st;
  wchar_t tmp[MG_PATH_MAX];
  MultiByteToWideChar(CP_UTF8, 0, path, -1, tmp, sizeof(tmp) / sizeof(tmp[0]));
  if (_wstati64(tmp, &st) != 0) return 0;
  // If path is a symlink, windows reports 0 in st.st_size.
  // Get a real file size by opening it and jumping to the end
  if (st.st_size == 0 && (st.st_mode & _S_IFREG)) {
    FILE *fp = _wfopen(tmp, L"rb");
    if (fp != NULL) {
      fseek(fp, 0, SEEK_END);
      if (ftell(fp) > 0) st.st_size = ftell(fp);  // Use _ftelli64 on win10+
      fclose(fp);
    }
  }
#else
  struct MG_STAT_STRUCT st;
  if (MG_STAT_FUNC(path, &st) != 0) return 0;
#endif
  if (size) *size = (size_t) st.st_size;
  if (mtime) *mtime = st.st_mtime;
  return MG_FS_READ | MG_FS_WRITE | (S_ISDIR(st.st_mode) ? MG_FS_DIR : 0);
#endif
}

#if MG_ARCH == MG_ARCH_WIN32
struct dirent {
  char d_name[MAX_PATH];
};

typedef struct win32_dir {
  HANDLE handle;
  WIN32_FIND_DATAW info;
  struct dirent result;
} DIR;

#if 0
int gettimeofday(struct timeval *tv, void *tz) {
  FILETIME ft;
  unsigned __int64 tmpres = 0;

  if (tv != NULL) {
    GetSystemTimeAsFileTime(&ft);
    tmpres |= ft.dwHighDateTime;
    tmpres <<= 32;
    tmpres |= ft.dwLowDateTime;
    tmpres /= 10;  // convert into microseconds
    tmpres -= (int64_t) 11644473600000000;
    tv->tv_sec = (long) (tmpres / 1000000UL);
    tv->tv_usec = (long) (tmpres % 1000000UL);
  }
  (void) tz;
  return 0;
}
#endif

static int to_wchar(const char *path, wchar_t *wbuf, size_t wbuf_len) {
  int ret;
  char buf[MAX_PATH * 2], buf2[MAX_PATH * 2], *p;
  strncpy(buf, path, sizeof(buf));
  buf[sizeof(buf) - 1] = '\0';
  // Trim trailing slashes. Leave backslash for paths like "X:\"
  p = buf + strlen(buf) - 1;
  while (p > buf && p[-1] != ':' && (p[0] == '\\' || p[0] == '/')) *p-- = '\0';
  memset(wbuf, 0, wbuf_len * sizeof(wchar_t));
  ret = MultiByteToWideChar(CP_UTF8, 0, buf, -1, wbuf, (int) wbuf_len);
  // Convert back to Unicode. If doubly-converted string does not match the
  // original, something is fishy, reject.
  WideCharToMultiByte(CP_UTF8, 0, wbuf, (int) wbuf_len, buf2, sizeof(buf2),
                      NULL, NULL);
  if (strcmp(buf, buf2) != 0) {
    wbuf[0] = L'\0';
    ret = 0;
  }
  return ret;
}

DIR *opendir(const char *name) {
  DIR *d = NULL;
  wchar_t wpath[MAX_PATH];
  DWORD attrs;
  size_t n;

  if (name == NULL) {
    SetLastError(ERROR_BAD_ARGUMENTS);
  } else if ((d = (DIR *) mg_calloc(1, sizeof(*d))) == NULL) {
    SetLastError(ERROR_NOT_ENOUGH_MEMORY);
  } else {
    to_wchar(name, wpath, sizeof(wpath) / sizeof(wpath[0]));
    attrs = GetFileAttributesW(wpath);
    if (attrs != 0Xffffffff && (attrs & FILE_ATTRIBUTE_DIRECTORY)) {
      n = wcslen(wpath);
      if (n <= (sizeof(wpath) / sizeof(wpath[0])) - 3) {
        (void) wcscat(wpath, L"\\*");
        d->handle = FindFirstFileW(wpath, &d->info);
        d->result.d_name[0] = '\0';
      } else {
        mg_free(d);
        d = NULL;
        SetLastError(ERROR_BUFFER_OVERFLOW);
      }
    } else {
      mg_free(d);
      d = NULL;
    }
  }
  return d;
}

int closedir(DIR *d) {
  int result = 0;
  if (d != NULL) {
    if (d->handle != INVALID_HANDLE_VALUE)
      result = FindClose(d->handle) ? 0 : -1;
    mg_free(d);
  } else {
    result = -1;
    SetLastError(ERROR_BAD_ARGUMENTS);
  }
  return result;
}

struct dirent *readdir(DIR *d) {
  struct dirent *result = NULL;
  if (d != NULL) {
    memset(&d->result, 0, sizeof(d->result));
    if (d->handle != INVALID_HANDLE_VALUE) {
      result = &d->result;
      WideCharToMultiByte(CP_UTF8, 0, d->info.cFileName, -1, result->d_name,
                          sizeof(result->d_name), NULL, NULL);
      if (!FindNextFileW(d->handle, &d->info)) {
        FindClose(d->handle);
        d->handle = INVALID_HANDLE_VALUE;
      }
    } else {
      SetLastError(ERROR_FILE_NOT_FOUND);
    }
  } else {
    SetLastError(ERROR_BAD_ARGUMENTS);
  }
  return result;
}
#endif

static void p_list(const char *dir, void (*fn)(const char *, void *),
                   void *userdata) {
  struct dirent *dp;
  DIR *dirp;
  if ((dirp = (opendir(dir))) == NULL) return;
  while ((dp = readdir(dirp)) != NULL) {
    if (!strcmp(dp->d_name, ".") || !strcmp(dp->d_name, "..")) continue;
    fn(dp->d_name, userdata);
  }
  closedir(dirp);
}

static void *p_open(const char *path, int flags) {
#if MG_ARCH == MG_ARCH_WIN32
  const char *mode = flags == MG_FS_READ    ? "rb"
                     : (flags & MG_FS_EXCL) ? "wxb"
                                            : "a+b";
  wchar_t b1[MG_PATH_MAX], b2[10];
  MultiByteToWideChar(CP_UTF8, 0, path, -1, b1, sizeof(b1) / sizeof(b1[0]));
  MultiByteToWideChar(CP_UTF8, 0, mode, -1, b2, sizeof(b2) / sizeof(b2[0]));
  return (void *) _wfopen(b1, b2);
#else
  const char *mode = flags == MG_FS_READ    ? "rbe"
                     : (flags & MG_FS_EXCL) ? "wxbe"
                                            : "a+be"; // e for CLOSEXEC
  return (void *) fopen(path, mode);
#endif
}

static void p_close(void *fp) {
  fclose((FILE *) fp);
}

static size_t p_read(void *fp, void *buf, size_t len) {
  return fread(buf, 1, len, (FILE *) fp);
}

static size_t p_write(void *fp, const void *buf, size_t len) {
  return fwrite(buf, 1, len, (FILE *) fp);
}

static size_t p_seek(void *fp, size_t offset) {
#if (defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS == 64) ||  \
    (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) || \
    (defined(_XOPEN_SOURCE) && _XOPEN_SOURCE >= 600)
  if (fseeko((FILE *) fp, (off_t) offset, SEEK_SET) != 0) (void) 0;
#else
  if (fseek((FILE *) fp, (long) offset, SEEK_SET) != 0) (void) 0;
#endif
  return (size_t) ftell((FILE *) fp);
}

static bool p_rename(const char *from, const char *to) {
  return rename(from, to) == 0;
}

static bool p_remove(const char *path) {
  return remove(path) == 0;
}

static bool p_mkdir(const char *path) {
  return mkdir(path, 0775) == 0;
}

#else

static int p_stat(const char *path, size_t *size, time_t *mtime) {
  (void) path, (void) size, (void) mtime;
  return 0;
}
static void p_list(const char *path, void (*fn)(const char *, void *),
                   void *userdata) {
  (void) path, (void) fn, (void) userdata;
}
static void *p_open(const char *path, int flags) {
  (void) path, (void) flags;
  return NULL;
}
static void p_close(void *fp) {
  (void) fp;
}
static size_t p_read(void *fd, void *buf, size_t len) {
  (void) fd, (void) buf, (void) len;
  return 0;
}
static size_t p_write(void *fd, const void *buf, size_t len) {
  (void) fd, (void) buf, (void) len;
  return 0;
}
static size_t p_seek(void *fd, size_t offset) {
  (void) fd, (void) offset;
  return (size_t) ~0;
}
static bool p_rename(const char *from, const char *to) {
  (void) from, (void) to;
  return false;
}
static bool p_remove(const char *path) {
  (void) path;
  return false;
}
static bool p_mkdir(const char *path) {
  (void) path;
  return false;
}
#endif

struct mg_fs mg_fs_posix = {p_stat,  p_list, p_open,   p_close,  p_read,
                            p_write, p_seek, p_rename, p_remove, p_mkdir};

#ifdef MG_ENABLE_LINES
#line 1 "src/health.c"
#endif



// The one health record. Lives in RAM that survives a warm reset, see
// MG_HEALTH_RAM and the .mg_health region in the linker script
struct mg_health mg_health_record MG_HEALTH_RAM;

// STM32 H723 support. Needs the _estack symbol from the linker script
#if MG_ENABLE_HEALTH && (defined(STM32H7_DEV_ID) || defined(STM32H5))
// Fault handler body. Runs in exception context: no printf, no malloc, no
// blocking calls. Copies the unwind registers and raw stack into the health
// record, then resets.
// "used" keeps the linker from garbage-collecting this section: the only
// reference is the "b fault_c" branch in the naked handler below
__attribute__((used, noinline)) static void fault_c(uint32_t *frame,
                                                    uint32_t exc_return) {
  extern uint32_t _estack;
  uint32_t *stack = frame + 8 + ((exc_return & (1U << 4)) ? 0 : 18);
  uint32_t *r = mg_health_record.data;

  if (frame[7] & (1U << 9)) stack++;    // Eight-byte stack alignment padding
  r[0] = (uint32_t) (uintptr_t) stack;  // SP
  r[1] = frame[5];                      // LR
  r[2] = frame[6];                      // PC
  for (size_t i = 0; i < MG_HEALTH_DATA_SIZE - 3; i++) {  // Raw stack
    r[3 + i] = stack + i < &_estack ? stack[i] : 0;
  }
  mg_health_record.cpuid = SCB->CPUID;
  mg_health_record.reset_reason = MG_HEALTH_RESET_FAULT;
  NVIC_SystemReset();
}

// Common fault entry. EXC_RETURN bit 2 tells which stack was in use:
// 0 = MSP, 1 = PSP. Load the faulting SP into r0 and hand it to fault_c()
__attribute__((naked)) void HardFault_Handler(void) {
  __asm volatile(
      "mov r1, lr\n\t"
      "tst lr, #4\n\t"
      "ite eq\n\t"
      "mrseq r0, msp\n\t"
      "mrsne r0, psp\n\t"
      "b fault_c\n\t");
}

// Route the other fault types through the same entry point
void MemManage_Handler(void) __attribute__((alias("HardFault_Handler")));
void BusFault_Handler(void) __attribute__((alias("HardFault_Handler")));
void UsageFault_Handler(void) __attribute__((alias("HardFault_Handler")));
#endif

// Start a new boot: validate the record, count the boot, store the reset
// reason. The current boot moves to prev_* slots first, so the previous
// boot's uptime and reason stay readable. Called by MG_HEALTH_INIT()
void mg_health_init(void) {
  char magic[] = MG_HEALTH_MAGIC;
  // struct mg_health *h = mg_health_get();
  if (!mg_health_valid()) {  // First boot, or RAM lost its contents
    memset(&mg_health_record, 0, sizeof(mg_health_record));
    memcpy(mg_health_record.magic, magic, sizeof(mg_health_record.magic));
  }

#if 0 && MG_ARCH == MG_ARCH_ESP32 && CONFIG_ESP_COREDUMP_ENABLE_TO_FLASH
  {
    // After a crash, copy the backtrace from the core dump in flash: the same
    // addresses the panic handler prints as "Backtrace: PC:SP PC:SP ...".
    // The core dump stays in flash, so check the reset reason, otherwise an
    // old crash is reported on every boot
    static esp_core_dump_summary_t s;  // Big on RISC-V, keep it off the stack
    esp_reset_reason_t rr = esp_reset_reason();
    uint32_t *out = mg_health_record.data;
    size_t n = 0, max = MG_HEALTH_DATA_SIZE;
    if (rr != ESP_RST_PANIC && rr != ESP_RST_INT_WDT && rr != ESP_RST_TASK_WDT)
      return;  // Not a crash that went through the panic handler
    if (max == 0 || esp_core_dump_get_summary(&s) != ESP_OK) return;

    mg_health_record.cpuid = 42;  // ESP32
    mg_health_record.reset_reason = MG_HEALTH_RESET_FAULT;

    memset(out, 0, sizeof(mg_health_record.data));
    out[n++] = s.exc_pc;
#if CONFIG_IDF_TARGET_ARCH_XTENSA
    for (uint32_t i = 0; i < s.exc_bt_info.depth && n < max; i++) {
      if (s.exc_bt_info.bt[i] != s.exc_pc) out[n++] = s.exc_bt_info.bt[i];
    }
#elif CONFIG_IDF_TARGET_ARCH_RISCV
    // No unwinder: take the return address, then words on the stack that
    // point into executable memory (a heuristic, may include stale values)
    if (n < max) out[n++] = s.ex_info.ra;
    for (uint32_t i = 0; i + 4 <= s.exc_bt_info.dump_size && n < max; i += 4) {
      uint32_t w;
      memcpy(&w, &s.exc_bt_info.stackdump[i], sizeof(w));  // Byte array
      if (esp_ptr_executable((void *) (uintptr_t) w)) out[n++] = w;
    }
#endif
  }
#endif
}

static size_t print_uint32(void (*fn)(char, void *), void *arg, va_list *ap) {
  size_t n = va_arg(*ap, size_t);
  uint32_t *p = va_arg(*ap, uint32_t *);
  size_t len = 0;
  while (n--) len += mg_xprintf(fn, arg, "%s%lu", len == 0 ? "" : ",", *p++);
  return len;
}

size_t mg_print_crash_record(void (*fn)(char, void *), void *arg, va_list *ap) {
  size_t len = 0;
  if (mg_health_valid() &&
      mg_health_record.reset_reason == MG_HEALTH_RESET_FAULT) {
    size_t n = sizeof(mg_health_record.data) / sizeof(mg_health_record.data[0]);
    len += mg_xprintf(fn, arg, "{%m:%hhu,%m:%u,%m:[%M]}",            //
                      MG_ESC("version"), mg_health_record.magic[3],  //
                      MG_ESC("cpuid"), mg_health_record.cpuid,       //
                      MG_ESC("data"), print_uint32, n, mg_health_record.data);
  } else {
    len += mg_xprintf(fn, arg, "{}");
  }
  (void) ap;
  return len;
}

#ifdef MG_ENABLE_LINES
#line 1 "src/http.c"
#endif










static int mg_ncasecmp(const char *s1, const char *s2, size_t len) {
  int diff = 0;
  if (len > 0) do {
      int c = *s1++, d = *s2++;
      if (c >= 'A' && c <= 'Z') c += 'a' - 'A';
      if (d >= 'A' && d <= 'Z') d += 'a' - 'A';
      diff = c - d;
    } while (diff == 0 && s1[-1] != '\0' && --len > 0);
  return diff;
}

bool mg_to_size_t(struct mg_str str, size_t *val);
bool mg_to_size_t(struct mg_str str, size_t *val) {
  size_t i = 0, max = (size_t) -1, max2 = max / 10, result = 0, ndigits = 0;
  while (i < str.len && (str.buf[i] == ' ' || str.buf[i] == '\t')) i++;
  if (i < str.len && str.buf[i] == '-') return false;
  while (i < str.len && str.buf[i] >= '0' && str.buf[i] <= '9') {
    size_t digit = (size_t) (str.buf[i] - '0');
    if (result > max2) return false;  // Overflow
    result *= 10;
    if (result > max - digit) return false;  // Overflow
    result += digit;
    i++, ndigits++;
  }
  while (i < str.len && (str.buf[i] == ' ' || str.buf[i] == '\t')) i++;
  if (ndigits == 0) return false;  // #2322: Content-Length = 1 * DIGIT
  if (i != str.len) return false;  // Ditto
  *val = (size_t) result;
  return true;
}

// Chunk deletion marker is the MSB in the "processed" counter
#define MG_DMARK ((size_t) 1 << (sizeof(size_t) * 8 - 1))

// Multipart POST example:
// --xyz
// Content-Disposition: form-data; name="val"
//
// abcdef
// --xyz
// Content-Disposition: form-data; name="foo"; filename="a.txt"
// Content-Type: text/plain
//
// hello world
//
// --xyz--
size_t mg_http_next_multipart(struct mg_str body, size_t ofs,
                              struct mg_http_part *part) {
  struct mg_str cd = mg_str_n("Content-Disposition", 19);
  const char *s = body.buf;
  size_t b = ofs, h1, h2, b1, b2, max = body.len;

  // Init part params
  if (part != NULL) part->name = part->filename = part->body = mg_str_n(0, 0);

  // Skip boundary
  while (b + 2 < max && !(s[b] == '\r' && s[b + 1] == '\n')) b++;
  if (b <= ofs || b + 2 >= max) return 0;
  // MG_INFO(("B: %zu %zu [%.*s]", ofs, b - ofs, (int) (b - ofs), s));

  // Skip headers
  h1 = h2 = b + 2;
  for (;;) {
    while (h2 + 2 < max && !(s[h2] == '\r' && s[h2 + 1] == '\n')) h2++;
    if (h2 == h1) break;
    if (h2 + 2 >= max) return 0;
    // MG_INFO(("Header: [%.*s]", (int) (h2 - h1), &s[h1]));
    if (part != NULL && h1 + cd.len + 2 < h2 && s[h1 + cd.len] == ':' &&
        mg_ncasecmp(&s[h1], cd.buf, cd.len) == 0) {
      struct mg_str v = mg_str_n(&s[h1 + cd.len + 2], h2 - (h1 + cd.len + 2));
      part->name = mg_http_get_header_var(v, mg_str_n("name", 4));
      part->filename = mg_http_get_header_var(v, mg_str_n("filename", 8));
    }
    h1 = h2 = h2 + 2;
  }
  b1 = b2 = h2 + 2;
  while (b2 + 2 + (b - ofs) + 2 < max && !(s[b2] == '\r' && s[b2 + 1] == '\n' &&
                                           memcmp(&s[b2 + 2], s, b - ofs) == 0))
    b2++;

  if (b2 + 2 >= max) return 0;
  if (part != NULL) part->body = mg_str_n(&s[b1], b2 - b1);
  // MG_INFO(("Body: [%.*s]", (int) (b2 - b1), &s[b1]));
  return b2 + 2;
}

void mg_http_bauth(struct mg_connection *c, const char *user,
                   const char *pass) {
  struct mg_str u = mg_str(user), p = mg_str(pass);
  size_t need = c->send.len + 36 + (u.len + p.len) * 2;
  if (c->send.size < need) mg_iobuf_resize(&c->send, need);
  if (c->send.size >= need) {
    size_t i, n = 0;
    char *buf = (char *) &c->send.buf[c->send.len];
    memcpy(buf, "Authorization: Basic ", 21);  // DON'T use mg_send!
    for (i = 0; i < u.len; i++) {
      n = mg_base64_update(((unsigned char *) u.buf)[i], buf + 21, n);
    }
    if (p.len > 0) {
      n = mg_base64_update(':', buf + 21, n);
      for (i = 0; i < p.len; i++) {
        n = mg_base64_update(((unsigned char *) p.buf)[i], buf + 21, n);
      }
    }
    n = mg_base64_final(buf + 21, n);
    c->send.len += 21 + (size_t) n + 2;
    memcpy(&c->send.buf[c->send.len - 2], "\r\n", 2);
  } else {
    MG_ERROR(("%lu oom %d->%d ", c->id, (int) c->send.size, (int) need));
  }
}

struct mg_str mg_http_var(struct mg_str buf, struct mg_str name) {
  struct mg_str entry, k, v, result = mg_str_n(NULL, 0);
  while (mg_span(buf, &entry, &buf, '&')) {
    if (mg_span(entry, &k, &v, '=') && name.len == k.len &&
        mg_ncasecmp(name.buf, k.buf, k.len) == 0) {
      result = v;
      break;
    }
  }
  return result;
}

int mg_http_get_var(const struct mg_str *buf, const char *name, char *dst,
                    size_t dst_len) {
  int len;
  if (dst != NULL && dst_len > 0) {
    dst[0] = '\0';  // If destination buffer is valid, always nul-terminate it
  }
  if (dst == NULL || dst_len == 0) {
    len = -2;  // Bad destination
  } else if (buf->buf == NULL || name == NULL || buf->len == 0) {
    len = -1;  // Bad source
  } else {
    struct mg_str v = mg_http_var(*buf, mg_str(name));
    if (v.buf == NULL) {
      len = -4;  // Name does not exist
    } else {
      len = mg_url_decode(v.buf, v.len, dst, dst_len, 1);
      if (len < 0) len = -3;  // Failed to decode
    }
  }
  return len;
}

static bool isx(int c) {
  return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') ||
         (c >= 'A' && c <= 'F');
}

int mg_url_decode(const char *src, size_t src_len, char *dst, size_t dst_len,
                  int is_form_url_encoded) {
  size_t i, j;
  for (i = j = 0; i < src_len && j + 1 < dst_len; i++, j++) {
    if (src[i] == '%') {
      // Use `i + 2 < src_len`, not `i < src_len - 2`, note small src_len
      if (i + 2 < src_len && isx(src[i + 1]) && isx(src[i + 2])) {
        mg_str_to_num(mg_str_n(src + i + 1, 2), 16, &dst[j], sizeof(uint8_t));
        i += 2;
      } else {
        dst[0] = '\0';
        return -1;
      }
    } else if (is_form_url_encoded && src[i] == '+') {
      dst[j] = ' ';
    } else {
      dst[j] = src[i];
    }
  }
  if (j < dst_len) dst[j] = '\0';  // Null-terminate the destination
  return i >= src_len && j < dst_len ? (int) j : -1;
}

static bool isok(uint8_t c) {
  return c == '\n' || c == '\r' || c == '\t' || c >= ' ';
}

int mg_http_get_request_len(const unsigned char *buf, size_t buf_len) {
  size_t i;
  for (i = 0; i < buf_len; i++) {
    if (!isok(buf[i])) return -1;
    if ((i > 0 && buf[i] == '\n' && buf[i - 1] == '\n') ||
        (i > 3 && buf[i] == '\n' && buf[i - 1] == '\r' && buf[i - 2] == '\n'))
      return (int) i + 1;
  }
  return 0;
}
struct mg_str *mg_http_get_header(struct mg_http_message *h, const char *name) {
  size_t i, n = strlen(name), max = sizeof(h->headers) / sizeof(h->headers[0]);
  for (i = 0; i < max && h->headers[i].name.len > 0; i++) {
    struct mg_str *k = &h->headers[i].name, *v = &h->headers[i].value;
    if (n == k->len && mg_ncasecmp(k->buf, name, n) == 0) return v;
  }
  return NULL;
}

// Is it a valid utf-8 continuation byte
static bool vcb(uint8_t c) {
  return (c & 0xc0) == 0x80;
}

// Get character length (valid utf-8). Used to parse method, URI, headers
static size_t clen(const char *s, const char *end) {
  const unsigned char *u = (unsigned char *) s, c = *u;
  long n = (long) (end - s);
  if (c > ' ' && c <= '~') return 1;  // Usual ascii printed char
  if ((c & 0xe0) == 0xc0 && n > 1 && vcb(u[1])) return 2;  // 2-byte UTF8
  if ((c & 0xf0) == 0xe0 && n > 2 && vcb(u[1]) && vcb(u[2])) return 3;
  if ((c & 0xf8) == 0xf0 && n > 3 && vcb(u[1]) && vcb(u[2]) && vcb(u[3]))
    return 4;
  return 0;
}

// Skip until the newline. Return advanced `s`, or NULL on error
static const char *skiptorn(const char *s, const char *end, struct mg_str *v) {
  v->buf = (char *) s;
  while (s < end && s[0] != '\n' && s[0] != '\r') s++, v->len++;  // To newline
  if (s >= end || (s[0] == '\r' && s[1] != '\n')) return NULL;    // Stray \r
  if (s < end && s[0] == '\r') s++;                               // Skip \r
  if (s >= end || *s++ != '\n') return NULL;                      // Skip \n
  return s;
}

static bool mg_http_parse_headers(const char *s, const char *end,
                                  struct mg_http_header *h, size_t max_hdrs) {
  size_t i, n;
  int cl_count = 0, te_count = 0, auth_count = 0;
  int conn_count = 0, cookie_count = 0;
  for (i = 0; i < max_hdrs; i++) {
    struct mg_str k = {NULL, 0}, v = {NULL, 0};
    if (s >= end) return false;
    if (s[0] == '\n' || (s[0] == '\r' && s[1] == '\n')) break;
    k.buf = (char *) s;
    while (s < end && s[0] != ':' && (n = clen(s, end)) > 0) s += n, k.len += n;
    if (k.len == 0) return false;                     // Empty name
    if (s >= end || clen(s, end) == 0) return false;  // Invalid UTF-8
    if (*s++ != ':') return false;  // Invalid, not followed by :
    // if (clen(s, end) == 0) return false;        // Invalid UTF-8
    while (s < end && (s[0] == ' ' || s[0] == '\t')) s++;  // Skip spaces
    if ((s = skiptorn(s, end, &v)) == NULL) return false;
    while (v.len > 0 && (v.buf[v.len - 1] == ' ' || v.buf[v.len - 1] == '\t')) {
      v.len--;  // Trim spaces
    }
    // detect duplicated headers -> discard
    if (((mg_strcasecmp(k, mg_str("Content-Length")) == 0) &&
         (++cl_count > 1)) ||
        ((mg_strcasecmp(k, mg_str("Transfer-Encoding")) == 0) &&
         (++te_count > 1)) ||
        ((mg_strcasecmp(k, mg_str("Authorization")) == 0) &&
         (++auth_count > 1)) ||
        ((mg_strcasecmp(k, mg_str("Cookie")) == 0) && (++cookie_count > 1)) ||
        ((mg_strcasecmp(k, mg_str("Connection")) == 0) && (++conn_count > 1)))
      return false;
    // MG_INFO(("--HH [%.*s] [%.*s]", (int) k.len, k.buf, (int) v.len, v.buf));
    h[i].name = k, h[i].value = v;  // Success. Assign values
  }
  return true;
}

int mg_http_parse(const char *s, size_t len, struct mg_http_message *hm) {
  int is_response, req_len = mg_http_get_request_len((unsigned char *) s, len);
  const char *end = s == NULL ? NULL : s + req_len, *qs;  // Cannot add to NULL
  const struct mg_str *cl;
  size_t n;
  bool version_prefix_valid;

  memset(hm, 0, sizeof(*hm));
  if (req_len <= 0) return req_len;

  hm->message.buf = hm->head.buf = (char *) s;
  hm->body.buf = (char *) end;
  hm->head.len = (size_t) req_len;
  hm->message.len = hm->body.len = (size_t) -1;  // Set body length to infinite

  // Parse request line
  hm->method.buf = (char *) s;
  while (s < end && (n = clen(s, end)) > 0) s += n, hm->method.len += n;
  while (s < end && s[0] == ' ') s++;  // Skip spaces
  hm->uri.buf = (char *) s;
  while (s < end && (n = clen(s, end)) > 0) s += n, hm->uri.len += n;
  while (s < end && s[0] == ' ') s++;  // Skip spaces
  is_response =
      hm->method.len > 5 && (mg_ncasecmp(hm->method.buf, "HTTP/", 5) == 0);
  if ((s = skiptorn(s, end, &hm->proto)) == NULL) return false;
  // If we're given a version, check that it is HTTP/x.x
  version_prefix_valid =
      hm->proto.len > 5 && (mg_ncasecmp(hm->proto.buf, "HTTP/", 5) == 0);
  if (!is_response && !version_prefix_valid)
    return -1;  // no version detected in request
  if (!is_response && hm->proto.len > 0 &&
      (!version_prefix_valid || hm->proto.len != 8 ||
       (hm->proto.buf[5] < '0' || hm->proto.buf[5] > '9') ||
       (hm->proto.buf[6] != '.') ||
       (hm->proto.buf[7] < '0' || hm->proto.buf[7] > '9'))) {
    return -1;
  }

  // If URI contains '?' character, setup query string
  if ((qs = (const char *) memchr(hm->uri.buf, '?', hm->uri.len)) != NULL) {
    hm->query.buf = (char *) qs + 1;
    hm->query.len = (size_t) (&hm->uri.buf[hm->uri.len] - (qs + 1));
    hm->uri.len = (size_t) (qs - hm->uri.buf);
  }

  // Sanity check. Allow protocol/reason to be empty
  // Do this check after hm->method.len and hm->uri.len are finalised
  if (hm->method.len == 0 || hm->uri.len == 0) return -1;

  if (!mg_http_parse_headers(s, end, hm->headers,
                             sizeof(hm->headers) / sizeof(hm->headers[0])))
    return -1;  // error when parsing
  cl = mg_http_get_header(hm, "Content-Length");
  if (cl != NULL && mg_http_get_header(hm, "Transfer-Encoding") != NULL)
    return -1; // cannot contain both CL and TE
  if (cl != NULL) {
    if (mg_to_size_t(*cl, &hm->body.len) == false) return -1;
    hm->message.len = (size_t) req_len + hm->body.len;
  }

  // mg_http_parse() is used to parse both HTTP requests and HTTP
  // responses. If HTTP response does not have Content-Length set, then
  // body is read until socket is closed, i.e. body.len is infinite (~0).
  //
  // For HTTP requests though, if Content-Length is not specified
  // set body length to 0.
  if (hm->body.len == (size_t) ~0 && !is_response) {
    hm->body.len = 0;
    hm->message.len = (size_t) req_len;
  }

  // The 204 (No content) responses also have 0 body length
  if (hm->body.len == (size_t) ~0 && is_response &&
      mg_strcasecmp(hm->uri, mg_str("204")) == 0) {
    hm->body.len = 0;
    hm->message.len = (size_t) req_len;
  }
  if (hm->message.len < (size_t) req_len) return -1;  // Overflow protection

  return req_len;
}

static void mg_http_vprintf_chunk(struct mg_connection *c, const char *fmt,
                                  va_list *ap) {
  size_t len = c->send.len;
  if (!mg_send(c, "        \r\n", 10)) mg_error(c, "OOM");
  mg_vxprintf(mg_pfn_iobuf, &c->send, fmt, ap);
  if (c->send.len >= len + 10) {
    mg_snprintf((char *) c->send.buf + len, 9, "%08lx", c->send.len - len - 10);
    c->send.buf[len + 8] = '\r';
    if (c->send.len == len + 10) c->pfn_data = NULL;  // Last chunk, done
  }
  if (!mg_send(c, "\r\n", 2)) mg_error(c, "OOM");
}

void mg_http_printf_chunk(struct mg_connection *c, const char *fmt, ...) {
  va_list ap;
  va_start(ap, fmt);
  mg_http_vprintf_chunk(c, fmt, &ap);
  va_end(ap);
}

void mg_http_write_chunk(struct mg_connection *c, const char *buf, size_t len) {
  mg_printf(c, "%lx\r\n", (unsigned long) len);
  if (!mg_send(c, buf, len) || !mg_send(c, "\r\n", 2)) mg_error(c, "OOM");
  if (len == 0) c->pfn_data = NULL;  // Last chunk, response done
}

// clang-format off
static const char *mg_http_status_code_str(int status_code) {
  switch (status_code) {
    case 100: return "Continue";
    case 101: return "Switching Protocols";
    case 102: return "Processing";
    case 200: return "OK";
    case 201: return "Created";
    case 202: return "Accepted";
    case 203: return "Non-authoritative Information";
    case 204: return "No Content";
    case 205: return "Reset Content";
    case 206: return "Partial Content";
    case 207: return "Multi-Status";
    case 208: return "Already Reported";
    case 226: return "IM Used";
    case 300: return "Multiple Choices";
    case 301: return "Moved Permanently";
    case 302: return "Found";
    case 303: return "See Other";
    case 304: return "Not Modified";
    case 305: return "Use Proxy";
    case 307: return "Temporary Redirect";
    case 308: return "Permanent Redirect";
    case 400: return "Bad Request";
    case 401: return "Unauthorized";
    case 402: return "Payment Required";
    case 403: return "Forbidden";
    case 404: return "Not Found";
    case 405: return "Method Not Allowed";
    case 406: return "Not Acceptable";
    case 407: return "Proxy Authentication Required";
    case 408: return "Request Timeout";
    case 409: return "Conflict";
    case 410: return "Gone";
    case 411: return "Length Required";
    case 412: return "Precondition Failed";
    case 413: return "Payload Too Large";
    case 414: return "Request-URI Too Long";
    case 415: return "Unsupported Media Type";
    case 416: return "Requested Range Not Satisfiable";
    case 417: return "Expectation Failed";
    case 418: return "I'm a teapot";
    case 421: return "Misdirected Request";
    case 422: return "Unprocessable Entity";
    case 423: return "Locked";
    case 424: return "Failed Dependency";
    case 426: return "Upgrade Required";
    case 428: return "Precondition Required";
    case 429: return "Too Many Requests";
    case 431: return "Request Header Fields Too Large";
    case 444: return "Connection Closed Without Response";
    case 451: return "Unavailable For Legal Reasons";
    case 499: return "Client Closed Request";
    case 500: return "Internal Server Error";
    case 501: return "Not Implemented";
    case 502: return "Bad Gateway";
    case 503: return "Service Unavailable";
    case 504: return "Gateway Timeout";
    case 505: return "HTTP Version Not Supported";
    case 506: return "Variant Also Negotiates";
    case 507: return "Insufficient Storage";
    case 508: return "Loop Detected";
    case 510: return "Not Extended";
    case 511: return "Network Authentication Required";
    case 599: return "Network Connect Timeout Error";
    default: return "";
  }
}
// clang-format on

void mg_http_reply(struct mg_connection *c, int code, const char *headers,
                   const char *fmt, ...) {
  va_list ap;
  size_t len;
  mg_printf(c, "HTTP/1.1 %d %s\r\n%sContent-Length:            \r\n\r\n", code,
            mg_http_status_code_str(code), headers == NULL ? "" : headers);
  len = c->send.len;
  va_start(ap, fmt);
  mg_vxprintf(mg_pfn_iobuf, &c->send, fmt, &ap);
  va_end(ap);
  if (c->send.len > 16) {
    size_t n = mg_snprintf((char *) &c->send.buf[len - 15], 11, "%-10lu",
                           (unsigned long) (c->send.len - len));
    c->send.buf[len - 15 + n] = ' ';  // Change ending 0 to space
  }
  c->pfn_data = NULL;  // Response done
}

static void http_cb(struct mg_connection *, int, void *);
static void restore_http_cb(struct mg_connection *c) {
  mg_fs_close((struct mg_fd *) c->pfn_data);
  c->pfn_data = NULL;  // Response done
  c->pfn = http_cb;
}

char *mg_http_etag(char *buf, size_t len, size_t size, time_t mtime);
char *mg_http_etag(char *buf, size_t len, size_t size, time_t mtime) {
  mg_snprintf(buf, len, "\"%lld.%lld\"", (int64_t) mtime, (int64_t) size);
  return buf;
}

static void static_cb(struct mg_connection *c, int ev, void *ev_data) {
  if (ev == MG_EV_WRITE || ev == MG_EV_POLL) {
    struct mg_fd *fd = (struct mg_fd *) c->pfn_data;
    // Read to send IO buffer directly, avoid extra on-stack buffer
    size_t n, max = MG_IO_SIZE, space;
    size_t *cl = (size_t *) &c->data[(sizeof(c->data) - sizeof(size_t)) /
                                     sizeof(size_t) * sizeof(size_t)];
    if (c->send.size < max) mg_iobuf_resize(&c->send, max);
    if (c->send.len >= c->send.size) return;  // Rate limit
    if ((space = c->send.size - c->send.len) > *cl) space = *cl;
    n = fd->fs->rd(fd->fd, c->send.buf + c->send.len, space);
    c->send.len += n;
    *cl -= n;
    if (n == 0) restore_http_cb(c);
  } else if (ev == MG_EV_CLOSE) {
    restore_http_cb(c);
  }
  (void) ev_data;
}

// Known mime types. Keep it outside guess_content_type() function, since
// some environments don't like it defined there.
// clang-format off
#define MG_C_STR(a) { (char *) (a), sizeof(a) - 1 }
static struct mg_str s_known_types[] = {
    MG_C_STR("html"), MG_C_STR("text/html; charset=utf-8"),
    MG_C_STR("htm"), MG_C_STR("text/html; charset=utf-8"),
    MG_C_STR("css"), MG_C_STR("text/css; charset=utf-8"),
    MG_C_STR("js"), MG_C_STR("text/javascript; charset=utf-8"),
    MG_C_STR("mjs"), MG_C_STR("text/javascript; charset=utf-8"),
    MG_C_STR("gif"), MG_C_STR("image/gif"),
    MG_C_STR("png"), MG_C_STR("image/png"),
    MG_C_STR("jpg"), MG_C_STR("image/jpeg"),
    MG_C_STR("jpeg"), MG_C_STR("image/jpeg"),
    MG_C_STR("woff"), MG_C_STR("font/woff"),
    MG_C_STR("ttf"), MG_C_STR("font/ttf"),
    MG_C_STR("svg"), MG_C_STR("image/svg+xml"),
    MG_C_STR("txt"), MG_C_STR("text/plain; charset=utf-8"),
    MG_C_STR("avi"), MG_C_STR("video/x-msvideo"),
    MG_C_STR("csv"), MG_C_STR("text/csv"),
    MG_C_STR("doc"), MG_C_STR("application/msword"),
    MG_C_STR("exe"), MG_C_STR("application/octet-stream"),
    MG_C_STR("gz"), MG_C_STR("application/gzip"),
    MG_C_STR("ico"), MG_C_STR("image/x-icon"),
    MG_C_STR("json"), MG_C_STR("application/json"),
    MG_C_STR("mov"), MG_C_STR("video/quicktime"),
    MG_C_STR("mp3"), MG_C_STR("audio/mpeg"),
    MG_C_STR("mp4"), MG_C_STR("video/mp4"),
    MG_C_STR("mpeg"), MG_C_STR("video/mpeg"),
    MG_C_STR("pdf"), MG_C_STR("application/pdf"),
    MG_C_STR("shtml"), MG_C_STR("text/html; charset=utf-8"),
    MG_C_STR("tgz"), MG_C_STR("application/tar-gz"),
    MG_C_STR("wav"), MG_C_STR("audio/wav"),
    MG_C_STR("webp"), MG_C_STR("image/webp"),
    MG_C_STR("zip"), MG_C_STR("application/zip"),
    MG_C_STR("3gp"), MG_C_STR("video/3gpp"),
    {0, 0},
};
// clang-format on

static struct mg_str guess_content_type(struct mg_str path, const char *extra) {
  struct mg_str entry, k, v, s = mg_str(extra), asterisk = mg_str_n("*", 1);
  size_t i = 0;

  // Shrink path to its extension only
  while (i < path.len && path.buf[path.len - i - 1] != '.') i++;
  path.buf += path.len - i;
  path.len = i;

  // Process user-provided mime type overrides, if any
  while (mg_span(s, &entry, &s, ',')) {
    if (mg_span(entry, &k, &v, '=') &&
        (mg_strcmp(asterisk, k) == 0 || mg_strcmp(path, k) == 0))
      return v;
  }

  // Process built-in mime types
  for (i = 0; s_known_types[i].buf != NULL; i += 2) {
    if (mg_strcmp(path, s_known_types[i]) == 0) return s_known_types[i + 1];
  }

  return mg_str("text/plain; charset=utf-8");
}

static int getrange(struct mg_str *s, size_t *a, size_t *b) {
  size_t i, numparsed = 0;
  for (i = 0; i + 6 < s->len; i++) {
    struct mg_str k, v = mg_str_n(s->buf + i + 6, s->len - i - 6);
    if (memcmp(&s->buf[i], "bytes=", 6) != 0) continue;
    if (mg_span(v, &k, &v, '-')) {
      if (mg_to_size_t(k, a)) numparsed++;
      if (v.len > 0 && mg_to_size_t(v, b)) numparsed++;
    } else {
      if (mg_to_size_t(v, a)) numparsed++;
    }
    break;
  }
  return (int) numparsed;
}

void mg_http_serve_file(struct mg_connection *c, struct mg_http_message *hm,
                        const char *path,
                        const struct mg_http_serve_opts *opts) {
  char etag[64], tmp[MG_PATH_MAX];
  struct mg_fs *fs = opts && opts->fs ? opts->fs : &mg_fs_posix;
  struct mg_fd *fd = NULL;
  size_t size = 0;
  time_t mtime = 0;
  const char *mime_types = opts && opts->mime_types ? opts->mime_types : NULL;
  const char *hdrs = opts && opts->extra_headers ? opts->extra_headers : "";
  struct mg_str *inm = NULL;
  struct mg_str mime = guess_content_type(mg_str(path), mime_types);
  bool gzip = false;

  if (path != NULL) {
    // If a browser sends us "Accept-Encoding: gzip", try to open .gz first
    struct mg_str *ae = mg_http_get_header(hm, "Accept-Encoding");
    if (ae != NULL) {
      if (mg_match(*ae, mg_str("*gzip*"), NULL)) {
        mg_snprintf(tmp, sizeof(tmp), "%s.gz", path);
        fd = mg_fs_open(fs, tmp, MG_FS_READ);
        if (fd != NULL) gzip = true, path = tmp;
      }
    }
    // No luck opening .gz? Open what we've told to open
    if (fd == NULL) fd = mg_fs_open(fs, path, MG_FS_READ);
  }

  // Failed to open, and page404 is configured? Open it, then
  if (fd == NULL && opts && opts->page404) {
    fd = mg_fs_open(fs, opts->page404, MG_FS_READ);
    path = opts->page404;
    mime = guess_content_type(mg_str(path), mime_types);
  }

  if (fd == NULL || fs->st(path, &size, &mtime) == 0) {
    mg_http_reply(c, 404, hdrs, "Not found\n");
    mg_fs_close(fd);
    // NOTE: mg_http_etag() call should go first!
  } else if (mg_http_etag(etag, sizeof(etag), size, mtime) != NULL &&
             (inm = mg_http_get_header(hm, "If-None-Match")) != NULL &&
             mg_strcasecmp(*inm, mg_str(etag)) == 0) {
    mg_fs_close(fd);
    mg_http_reply(c, 304, hdrs, "");
  } else {
    int n, status = 200;
    char range[100];
    size_t r1 = 0, r2 = 0, cl = size;

    // Handle Range header
    struct mg_str *rh = mg_http_get_header(hm, "Range");
    range[0] = '\0';
    if (rh != NULL && (n = getrange(rh, &r1, &r2)) > 0) {
      // If range is specified like "400-", set second limit to content len
      if (n == 1) r2 = cl - 1;
      if (r1 > r2 || r2 >= cl) {
        status = 416;
        cl = 0;
        mg_snprintf(range, sizeof(range), "Content-Range: bytes */%lld\r\n",
                    (int64_t) size);
      } else {
        status = 206;
        cl = r2 - r1 + 1;
        mg_snprintf(range, sizeof(range),
                    "Content-Range: bytes %llu-%llu/%llu\r\n", (uint64_t) r1,
                    (uint64_t) (r1 + cl - 1), (uint64_t) size);
        fs->sk(fd->fd, r1);
      }
    }
    mg_printf(c,
              "HTTP/1.1 %d %s\r\n"
              "Content-Type: %.*s\r\n"
              "Etag: %s\r\n"
              "Content-Length: %llu\r\n"
              "%s%s%s\r\n",
              status, mg_http_status_code_str(status), (int) mime.len, mime.buf,
              etag, (uint64_t) cl, gzip ? "Content-Encoding: gzip\r\n" : "",
              range, hdrs);
    if (mg_strcasecmp(hm->method, mg_str("HEAD")) == 0 || c->is_closing) {
      c->pfn_data = NULL;  // Response done
      mg_fs_close(fd);
    } else {  // start serving static content only if not closing, see #3354
      // Track to-be-sent content length at the end of c->data, aligned
      size_t *clp = (size_t *) &c->data[(sizeof(c->data) - sizeof(size_t)) /
                                        sizeof(size_t) * sizeof(size_t)];
      c->pfn = static_cb;
      c->pfn_data = fd;
      *clp = cl;
    }
  }
}

struct printdirentrydata {
  struct mg_connection *c;
  struct mg_http_message *hm;
  const struct mg_http_serve_opts *opts;
  const char *dir;
  size_t count;
};

static void printdirentry(const char *name, void *userdata) {
  struct printdirentrydata *d = (struct printdirentrydata *) userdata;
  struct mg_fs *fs = d->opts->fs == NULL ? &mg_fs_posix : d->opts->fs;
  size_t size = 0;
  time_t t = 0;
  char path[MG_PATH_MAX];
  int flags;

  if (mg_snprintf(path, sizeof(path), "%s%c%s", d->dir, '/', name) >=
      sizeof(path)) {
    MG_ERROR(("%s truncated", name));
  } else if ((flags = fs->st(path, &size, &t)) == 0) {
    MG_ERROR(("%lu stat(%s)", d->c->id, path));
  } else {
    if (flags & MG_FS_DIR) {
      mg_printf(d->c, "%s\n  {%m: %m, %m: true}", //
                d->count == 0 ? "" : ",", //
                MG_ESC("name"), MG_ESC(name), MG_ESC("dir"));
    } else {
      mg_printf(d->c, "%s\n  {%m: %m, %m: %zu, %m: %llu, %m: false}", //
                d->count == 0 ? "" : ",", //
                MG_ESC("name"), MG_ESC(name), //
                MG_ESC("size"), size, //
                MG_ESC("modified"), (uint64_t) t, MG_ESC("dir"));
    }
    d->count++;
  }
}

static void listdir(struct mg_connection *c, struct mg_http_message *hm,
                    const struct mg_http_serve_opts *opts, char *dir) {
  struct mg_fs *fs = opts->fs == NULL ? &mg_fs_posix : opts->fs;
  struct printdirentrydata d = {c, hm, opts, dir, 0};
  char tmp[10];
  size_t off, n;
  mg_printf(c,
      "HTTP/1.1 200 OK\r\n"
      "Content-Type: application/json; charset=utf-8\r\n"
      "%s"
      "Content-Length:         \r\n\r\n",
      opts->extra_headers == NULL ? "" : opts->extra_headers);
  off = c->send.len;  // Start of body
  mg_printf(c, "[");
  fs->ls(dir, printdirentry, &d);
  mg_printf(c, "\n]\n");
  n = mg_snprintf(tmp, sizeof(tmp), "%lu", (unsigned long) (c->send.len - off));
  if (n > sizeof(tmp)) n = 0;
  memcpy(c->send.buf + off - 12, tmp, n);  // Set content length
  c->pfn_data = NULL;                      // Response done
}

// Map requested URI to the file path (buf,len). Use root directory r.
// r could be a path optionally followed by map: PATH,/PREFIX1=PATH1,...
static bool uri2path(const char *r, struct mg_str uri, char *buf, size_t len) {
  struct mg_str k, v, part, s = mg_str(r), u = {NULL, 0}, d = u;
  size_t n;
  while (mg_span(s, &part, &s, ',')) {
    if (!mg_span(part, &k, &v, '=')) k = part, v = mg_str_n(NULL, 0);
    if (v.len == 0) v = k, k = mg_str("/"), u = k, d = v;
    if (uri.len < k.len) continue;
    if (mg_strcmp(k, mg_str_n(uri.buf, k.len)) != 0) continue;
    u = k, d = v;
  }
  n = mg_snprintf(buf, len, "%.*s", (int) d.len, d.buf);
  if (len == 0 || n + 2 >= len) return false;  // Path overflow
  if (n > 0 && buf[n - 1] != '/') buf[n++] = '/', buf[n] = '\0';  // Add slash
  if (mg_url_decode(uri.buf + u.len, uri.len - u.len, buf + n, len - n, 0) < 0) {
    return false;
  }
  buf[len - 1] = '\0';  // Double-check
  n = strlen(buf);
  if (!mg_path_is_sane(mg_str_n(buf, n))) return false;
  while (n > 1 && buf[n - 1] == '/') buf[--n] = 0;  // Trim trailing slashes
  return true;
}

// Resolve requested file into `path` and return its fs->st() result
static int uri_to_file_status(struct mg_connection *c,
                              struct mg_http_message *hm,
                              struct mg_fs *fs, const char *root_dir,
                              char *path, size_t path_size) {
  int flags, tmp;
  size_t n;
  if (!uri2path(root_dir, hm->uri, path, path_size)) {
    mg_http_reply(c, 400, "", "Invalid path");
    return -1;
  }
  n = strlen(path);
  flags = mg_strcmp(hm->uri, mg_str("/")) == 0 ? MG_FS_DIR
                                               : fs->st(path, NULL, NULL);
  MG_VERBOSE(("%lu %.*s -> %s %d", c->id, (int) hm->uri.len, hm->uri.buf, path,
              flags));
  if (flags == 0) {
    // Do nothing - let's caller decide
  } else if ((flags & MG_FS_DIR) && hm->uri.len > 0 &&
             hm->uri.buf[hm->uri.len - 1] != '/') {
    mg_printf(c,
              "HTTP/1.1 301 Moved\r\n"
              "Location: %.*s/\r\n"
              "Content-Length: 0\r\n"
              "\r\n",
              (int) hm->uri.len, hm->uri.buf);
    c->pfn_data = NULL;  // Response done
    flags = -1;
  } else if (flags & MG_FS_DIR) {
    if (((mg_snprintf(path + n, path_size - n, "/" MG_HTTP_INDEX) > 0 &&
          (tmp = fs->st(path, NULL, NULL)) != 0) ||
         (mg_snprintf(path + n, path_size - n, "/index.shtml") > 0 &&
          (tmp = fs->st(path, NULL, NULL)) != 0))) {
      flags = tmp;
    } else if ((mg_snprintf(path + n, path_size - n, "/" MG_HTTP_INDEX ".gz") >
                    0 &&
                (tmp = fs->st(path, NULL, NULL)) !=
                    0)) {  // check for gzipped index
      flags = tmp;
      path[n + 1 + strlen(MG_HTTP_INDEX)] =
          '\0';  // Remove appended .gz in index file name
    } else {
      path[n] = '\0';  // Remove appended index file name
    }
  }
  return flags;
}

void mg_http_serve_dir(struct mg_connection *c, struct mg_http_message *hm,
                       const struct mg_http_serve_opts *opts) {
  char path[MG_PATH_MAX];
  struct mg_fs *fs = opts->fs == NULL ? &mg_fs_posix : opts->fs;
  const char *sp = opts->ssi_pattern;
  int flags = uri_to_file_status(c, hm, fs, opts->root_dir, path, sizeof(path));
  if (flags < 0) {
    // Do nothing: the response has already been sent by uri_to_file_status()
  } else if (flags & MG_FS_DIR) {
    if (opts->enable_dir_listing) {
      listdir(c, hm, opts, path);
    } else {
      mg_http_reply(c, 403, "", "Forbidden\n");
    }
  } else if (opts->enable_delete &&
            mg_strcasecmp(hm->method, mg_str("DELETE")) == 0) {
    mg_http_reply(c, flags && fs->rm(path) ? 200 : 404, "", "");
  } else if (opts->enable_upload &&
            (mg_strcasecmp(hm->method, mg_str("POST")) == 0 ||
             mg_strcasecmp(hm->method, mg_str("PUT")) == 0)) {
    // Small or already fully buffered body. mg_http_serve_upload() handles
    // the same thing without buffering, for big uploads, from MG_EV_HTTP_HDRS
    void *fd;
    bool ok = false;
    fs->rm(path);  // MG_FS_WRITE appends, not truncates: drop any old file
    if ((fd = fs->op(path, MG_FS_WRITE)) != NULL) {
      ok = fs->wr(fd, hm->body.buf, hm->body.len) == hm->body.len;
      fs->cl(fd);
    }
    mg_http_reply(c, ok ? 200 : 500, "", "");
  } else if (flags && sp != NULL && mg_match(mg_str(path), mg_str(sp), NULL)) {
    mg_http_serve_ssi(c, opts->root_dir, path);
  } else {
    mg_http_serve_file(c, hm, path, opts);
  }
}

// mg_http_stream_body() callback for mg_http_serve_upload(). The file is
// opened by the caller before streaming starts, so this only writes and closes
static bool serve_upload_cb(struct mg_http_message *hm, struct mg_str *data,
                            void **p) {
  struct mg_fd *fd = (struct mg_fd *) *p;
  if (hm != NULL) return fd != NULL;  // Start
  if (data != NULL) {                 // Next chunk
    return fd != NULL && fd->fs->wr(fd->fd, data->buf, data->len) == data->len;
  }
  if (fd != NULL) mg_fs_close(fd);  // End
  *p = NULL;
  return true;
}

// Starts a streaming upload for a big POST/PUT body, so it never sits fully
// buffered in memory. Companion to mg_http_serve_dir(): call this from
// MG_EV_HTTP_HDRS, and keep calling mg_http_serve_dir() from MG_EV_HTTP_MSG
// as before - it handles uploads too, for bodies this function skips.
// Does nothing unless opts->enable_upload is set, the method is POST or PUT,
// and Content-Length is known and >= 2 * MG_IO_SIZE.
void mg_http_serve_upload(struct mg_connection *c, struct mg_http_message *hm,
                          const struct mg_http_serve_opts *opts) {
  char path[MG_PATH_MAX];
  struct mg_fs *fs = opts->fs == NULL ? &mg_fs_posix : opts->fs;
  if (!opts->enable_upload) {
    // Not enabled, nothing to do
  } else if (mg_strcasecmp(hm->method, mg_str("POST")) != 0 &&
            mg_strcasecmp(hm->method, mg_str("PUT")) != 0) {
    // Not an upload request, nothing to do
  } else if (hm->body.len == (size_t) ~0 || hm->body.len < 2 * MG_IO_SIZE) {
    // Unknown length, or small enough for mg_http_serve_dir() to buffer it
  } else if (!uri2path(opts->root_dir, hm->uri, path, sizeof(path))) {
    MG_ERROR(("Invalid upload path: %.*s", (int) hm->uri.len, hm->uri.buf));
  } else if (fs->st(path, NULL, NULL) & MG_FS_DIR) {
    // Target is a directory, let mg_http_serve_dir() list it
  } else {
    struct mg_fd *fd;
    fs->rm(path);  // MG_FS_WRITE appends, not truncates: drop any old file
    if ((fd = mg_fs_open(fs, path, MG_FS_WRITE)) == NULL) {
      MG_ERROR(("Cannot open %s for upload", path));
    } else if (!mg_http_stream_body(c, hm, serve_upload_cb, fd)) {
      mg_fs_close(fd);
    }
  }
}

static bool mg_is_url_safe(int c) {
  return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') ||
         (c >= 'A' && c <= 'Z') || c == '.' || c == '_' || c == '-' || c == '~';
}

size_t mg_url_encode(const char *s, size_t sl, char *buf, size_t len) {
  size_t i, n = 0;
  for (i = 0; i < sl; i++) {
    int c = *(unsigned char *) &s[i];
    if (n + 4 >= len) return 0;
    if (mg_is_url_safe(c)) {
      buf[n++] = s[i];
    } else {
      mg_snprintf(&buf[n], 4, "%%%M", mg_print_hex, 1, &s[i]);
      n += 3;
    }
  }
  if (len > 0 && n < len - 1) buf[n] = '\0';  // Null-terminate the destination
  if (len > 0) buf[len - 1] = '\0';           // Always.
  return n;
}

void mg_http_creds(struct mg_http_message *hm, char *user, size_t userlen,
                   char *pass, size_t passlen) {
  struct mg_str *v = mg_http_get_header(hm, "Authorization");
  user[0] = pass[0] = '\0';
  if (v != NULL && v->len > 6 && memcmp(v->buf, "Basic ", 6) == 0) {
    char buf[256];
    size_t n = mg_base64_decode(v->buf + 6, v->len - 6, buf, sizeof(buf));
    const char *p = (const char *) memchr(buf, ':', n > 0 ? n : 0);
    if (p != NULL) {
      mg_snprintf(user, userlen, "%.*s", p - buf, buf);
      mg_snprintf(pass, passlen, "%.*s", n - (size_t) (p - buf) - 1, p + 1);
    }
  } else if (v != NULL && v->len > 7 && memcmp(v->buf, "Bearer ", 7) == 0) {
    mg_snprintf(pass, passlen, "%.*s", (int) v->len - 7, v->buf + 7);
  } else if ((v = mg_http_get_header(hm, "Cookie")) != NULL) {
    struct mg_str t = mg_http_get_header_var(*v, mg_str_n("access_token", 12));
    if (t.len > 0) mg_snprintf(pass, passlen, "%.*s", (int) t.len, t.buf);
  } else {
    mg_http_get_var(&hm->query, "access_token", pass, passlen);
  }
}

static struct mg_str stripquotes(struct mg_str s) {
  return s.len > 1 && s.buf[0] == '"' && s.buf[s.len - 1] == '"'
             ? mg_str_n(s.buf + 1, s.len - 2)
             : s;
}

struct mg_str mg_http_get_header_var(struct mg_str s, struct mg_str v) {
  size_t i;
  for (i = 0; v.len > 0 && i + v.len + 2 < s.len; i++) {
    if (s.buf[i + v.len] == '=' && memcmp(&s.buf[i], v.buf, v.len) == 0) {
      const char *p = &s.buf[i + v.len + 1], *b = p, *x = &s.buf[s.len];
      int q = p < x && *p == '"' ? 1 : 0;
      while (p < x &&
             (q ? p == b || *p != '"' : *p != ';' && *p != ' ' && *p != ','))
        p++;
      // MG_INFO(("[%.*s] [%.*s] [%.*s]", (int) s.len, s.buf, (int) v.len,
      // v.buf, (int) (p - b), b));
      return stripquotes(mg_str_n(b,
        (size_t) (p - b + (q && p < x && *p == '"' ? 1 : 0))));
    }
  }
  return mg_str_n(NULL, 0);
}

long mg_http_upload(struct mg_connection *c, struct mg_http_message *hm,
                    struct mg_fs *fs, const char *dir, size_t max_size) {
  char buf[20] = "0", file[MG_PATH_MAX], path[MG_PATH_MAX];
  long res = 0, offset;
  if (mg_http_get_var(&hm->query, "offset", buf, sizeof(buf)) == -3) {
    mg_http_reply(c, 400, "", "invalid offset");
    return -3;
  }
  if (mg_http_get_var(&hm->query, "file", file, sizeof(file)) == -3) {
    mg_http_reply(c, 400, "", "invalid file");
    return -2;
  }
  offset = strtol(buf, NULL, 0);
  mg_snprintf(path, sizeof(path), "%s%c%s", dir, MG_DIRSEP, file);
  if (hm->body.len == 0) {
    mg_http_reply(c, 200, "", "%ld", res);  // Nothing to write
  } else if (file[0] == '\0') {
    mg_http_reply(c, 400, "", "file required");
    res = -1;
  } else if (mg_path_is_sane(mg_str(file)) == false) {
    mg_http_reply(c, 400, "", "%s: invalid file", file);
    res = -2;
  } else if (offset < 0) {
    mg_http_reply(c, 400, "", "offset required");
    res = -3;
  } else if ((size_t) offset + hm->body.len > max_size) {
    mg_http_reply(c, 400, "", "%s: over max size of %lu", path,
                  (unsigned long) max_size);
    res = -4;
  } else {
    struct mg_fd *fd;
    size_t current_size = 0;
    MG_DEBUG(("%s -> %lu bytes @ %ld", path, hm->body.len, offset));
    if (offset == 0) fs->rm(path);  // If offset if 0, truncate file
    fs->st(path, &current_size, NULL);
    if (offset > 0 && current_size != (size_t) offset) {
      mg_http_reply(c, 400, "", "%s: offset mismatch", path);
      res = -5;
    } else if ((fd = mg_fs_open(fs, path, MG_FS_WRITE)) == NULL) {
      mg_http_reply(c, 400, "", "open(%s)", path);
      res = -6;
    } else {
      res = offset + (long) fs->wr(fd->fd, hm->body.buf, hm->body.len);
      mg_fs_close(fd);
      mg_http_reply(c, 200, "", "%ld", res);
    }
  }
  return res;
}

int mg_http_status(const struct mg_http_message *hm) {
  return atoi(hm->uri.buf);
}

// State of a streamed request or response body, kept in c->data. The callback
// is kept in c->fn_data, which is free because we own the connection
struct mg_upload_state {
  size_t expected;  // Number of body bytes to receive
  size_t received;  // Number of body bytes received so far
  void *user_data;  // Passed to cb, free for cb to use
  bool failed;      // Set when cb returned false
};

// Event handler installed by mg_http_stream_body(). Feeds the body to the
// callback, then replies (servers only). It does not restore the previous
// handler: the connection is closed after the reply.
static void mg_stream_handler(struct mg_connection *c, int ev, void *ev_data) {
  struct mg_upload_state *us = (struct mg_upload_state *) c->data;
    bool (*fn)(struct mg_http_message *, struct mg_str *, void **) =
      (bool (*)(struct mg_http_message *, struct mg_str *, void **))
      (ptrdiff_t) c->fn_data;
  (void) ev_data;
  if (ev == MG_EV_CLOSE || ev == MG_EV_ERROR) {
    if (us->received < us->expected) {
      // Connection dropped mid-upload. Let cb clean up, e.g. call mg_ota_end()
      us->received = us->expected;  // MG_EV_ERROR is followed by MG_EV_CLOSE
      fn(NULL, NULL, &us->user_data);
    }
  } else if (ev == MG_EV_READ && !c->is_draining) {
    if (c->recv.len > 0 && us->received < us->expected) {
      // Got some uploaded data. Feed by 512-byte aligned chunks, for OTA
      size_t left = us->expected - us->received, r = c->recv.len;
      size_t aligned = r < left ? MG_ROUND_DOWN(r, 512) : left;
      struct mg_str data = mg_str_n((char *) c->recv.buf, aligned);
      if (data.len > 0 && !us->failed) {
        us->failed = !fn(NULL, &data, &us->user_data);
      }
      us->received += aligned;
      mg_iobuf_del(&c->recv, 0, aligned);  // Shift the unprocessed tail
    }
    if (us->received >= us->expected) {
      // Finished upload. Always call cb, even after error, so it can clean up
      us->failed |= !fn(NULL, NULL, &us->user_data);
      if (!c->is_client) {
        mg_http_reply(c, us->failed ? 500 : 200, NULL, "%zu %s\n",
                      us->received, us->failed ? "fail" : "ok");
      }
      c->is_draining = 1;
    }
  }
}

bool mg_http_stream_body(struct mg_connection *c, struct mg_http_message *hm,
                         bool (*cb)(struct mg_http_message *, struct mg_str *,
                                    void **user_data),
                         void *user_data) {
  struct mg_upload_state *us = (struct mg_upload_state *) c->data;
  struct mg_str *te = mg_http_get_header(hm, "Transfer-Encoding");
  bool handled = false;

  // Catch upload requests early, without buffering whole body. Called on the
  // MG_EV_HTTP_HDRS event: we've received all HTTP headers but not necessarily
  // the full HTTP body
  if (hm->body.len == (size_t) ~0 ||
      (te != NULL && mg_strcasecmp(*te, mg_str("chunked")) == 0)) {
    // Unknown length or chunked, cannot stream. Leave it to the caller
  } else if (!c->is_client &&
             mg_strcmp(hm->method, mg_str("POST")) != 0 &&
             mg_strcmp(hm->method, mg_str("PUT")) != 0) {
    // Not an upload request. Leave it to the caller
  } else if (sizeof(*us) > sizeof(c->data)) {
    if (!c->is_client) {  // Nobody to reply to on a client connection
      c->pfn = NULL;
      mg_http_reply(c, 500, NULL, "data too small\n");
      c->is_draining = 1;
      cb(NULL, NULL, &user_data);  // Let cb release user_data, start skipped
      handled = true;
    }
  } else {
    c->pfn = NULL;  // Silence HTTP protocol handler
    memset(us, 0, sizeof(*us));
    us->expected = hm->body.len;  // Store number of bytes we expect
    us->user_data = user_data;
    us->failed = !cb(hm, NULL, &us->user_data);  // Call before deleting headers
    if (mg_http_get_header(hm, "Expect") != NULL) {
      mg_http_reply(c, 100, NULL, "");  // If curl expects, we continue
    }
    mg_iobuf_del(&c->recv, 0, hm->head.len);  // Delete HTTP headers
    c->fn = mg_stream_handler;                // We own the connection from now
    c->fn_data = (void *) (ptrdiff_t) cb;
    mg_call(c, MG_EV_READ, &c->recv.len);  // Process the body bytes we have
    handled = true;
  }
  return handled;
}

static bool is_hex_digit(int c) {
  return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') ||
         (c >= 'A' && c <= 'F');
}

static int skip_chunk(const char *buf, int len, int *pl, int *dl) {
  int i = 0, n = 0;  // pass int to mg_str_to_num, treats as unsigned
  if (len < 3) return 0;
  while (i < len && is_hex_digit(buf[i])) i++;
  if (i == 0) return -1;  // Error, no length specified
  if (len < i + 2 || buf[i] != '\r' || buf[i + 1] != '\n') return -1;  // Error
  if (mg_str_to_num(mg_str_n(buf, (size_t) i), 16, &n, sizeof(n)) == false)
    return -1;                    // Decode chunk length, overflow
  if (n < 0) return -1;           // Chunk length is too big <0 = >0x80000000
  if (n > len - i - 4) return 0;  // Chunk not yet fully buffered
  if (buf[i + n + 2] != '\r' || buf[i + n + 3] != '\n') return -1;  // Error
  *pl = i + 2, *dl = n;
  return i + 2 + n + 2;
}

static void http_cb(struct mg_connection *c, int ev, void *ev_data) {
  if (ev == MG_EV_READ || ev == MG_EV_CLOSE ||
      (ev == MG_EV_POLL && c->is_accepted && !c->is_draining &&
       c->recv.len > 0)) {  // see #2796
    struct mg_http_message hm;
    size_t ofs = 0;  // Parsing offset
    // c->pfn_data is the "response generation in progress" marker. While it
    // is set, don't parse the next pipelined request, to keep responses in order
    while (c->pfn_data == NULL && ofs < c->recv.len) {
      const char *buf = (char *) c->recv.buf + ofs;
      int n = mg_http_parse(buf, c->recv.len - ofs, &hm);
      struct mg_str *te;  // Transfer - encoding header
      bool is_chunked = false, is_http_1_0 = false;
      size_t old_len = c->recv.len;
      if (n < 0) {
        // We don't use mg_error() here, to avoid closing pipelined requests
        // prematurely, see #2592
        MG_ERROR(("HTTP parse, %lu bytes", c->recv.len));
        c->is_draining = 1;
        mg_hexdump(buf, c->recv.len - ofs > 16 ? 16 : c->recv.len - ofs);
        c->recv.len = 0;
        return;
      }
      if (n == 0) break;                 // Request is not buffered yet
      mg_call(c, MG_EV_HTTP_HDRS, &hm);  // Got all HTTP headers
      if (c->recv.len != old_len) {
        // User manipulated received data. Wash our hands
        MG_DEBUG(("%lu detaching HTTP handler", c->id));
        c->pfn = NULL;
        return;
      }
      if (ev == MG_EV_CLOSE) {  // If client did not set Content-Length
        hm.message.len = c->recv.len - ofs;  // and closes now, deliver MSG
        hm.body.len = hm.message.len - (size_t) (hm.body.buf - hm.message.buf);
      }
      is_http_1_0 =
          hm.proto.len == 8 && mg_ncasecmp(hm.proto.buf, "HTTP/1.0", 8) == 0;
      // HTTP/1.0 does not use "Transfer-Encoding: chunked"
      if (!is_http_1_0 &&
          (te = mg_http_get_header(&hm, "Transfer-Encoding")) != NULL) {
        if (mg_strcasecmp(*te, mg_str("chunked")) == 0) {
          is_chunked = true;
        } else {
          mg_error(c, "Invalid Transfer-Encoding");  // See #2460
          return;
        }
      } else if (mg_http_get_header(&hm, "Content-length") == NULL) {
        // #2593: HTTP packets must contain either Transfer-Encoding or
        // Content-length
        bool is_response = mg_ncasecmp(hm.method.buf, "HTTP/", 5) == 0;
        bool require_content_len = false;
        if (!is_response && (mg_strcasecmp(hm.method, mg_str("POST")) == 0 ||
                             mg_strcasecmp(hm.method, mg_str("PUT")) == 0)) {
          // POST and PUT should include an entity body. Therefore, they should
          // contain a Content-length header (unless the body length is 0, in
          // which case it can be omitted). Other requests can also contain a
          // body, but their content has no defined semantics (RFC 7231)
          // It is a non-chunked request, so the offset (ofs) is updated after
          // exiting this branch.
          if (hm.body.len != 0) require_content_len = true;
        } else if (is_response) {
          // HTTP spec 7.2 Entity body: All other responses must include a body
          // or Content-Length header field defined with a value of 0.
          int status = mg_http_status(&hm);
          require_content_len = status >= 200 && status != 204 && status != 304;
        }
        if (require_content_len) {
          if (!c->is_client) mg_http_reply(c, 411, "", "");
          MG_ERROR(("Content length missing from %s",
                    is_response ? "response" : "request"));
        }
      }

      if (is_chunked) {
        // For chunked data, strip off prefixes and suffixes from chunks
        // and relocate them right after the headers, then report a message
        char *s = (char *) c->recv.buf + ofs + n;
        int o = 0, pl, dl, cl, len = (int) (c->recv.len - ofs - (size_t) n);

        // Find zero-length chunk (the end of the body)
        while ((cl = skip_chunk(s + o, len - o, &pl, &dl)) > 0 && dl) o += cl;
        if (cl == 0) break;  // No zero-len chunk, buffer more data
        if (cl < 0) {
          mg_error(c, "Invalid chunk");
          break;
        }

        // Zero chunk found. Second pass: strip + relocate
        o = 0, hm.body.len = 0, hm.message.len = (size_t) n;
        while ((cl = skip_chunk(s + o, len - o, &pl, &dl)) > 0) {
          memmove(s + hm.body.len, s + o + pl, (size_t) dl);
          o += cl, hm.body.len += (size_t) dl, hm.message.len += (size_t) dl;
          if (dl == 0) break;
        }
        ofs += (size_t) (n + o);
      } else {  // Normal, non-chunked data
        size_t len = c->recv.len - ofs - (size_t) n;
        if (hm.body.len > len) break;  // Buffer more data
        ofs += (size_t) n + hm.body.len;
      }

      if (c->is_accepted) c->pfn_data = c;  // Response started
      mg_call(c, MG_EV_HTTP_MSG, &hm);  // Handler sets pfn_data NULL when done
      if (c->is_accepted && c->pfn_data == NULL) {
        struct mg_str *cc = mg_http_get_header(&hm, "Connection");
        if (cc != NULL && mg_strcasecmp(*cc, mg_str("close")) == 0) {
          c->is_draining = 1;  // honor "Connection: close"
          break;
        }
      }
    }
    if (ofs > 0) mg_iobuf_del(&c->recv, 0, ofs);  // Delete processed data
  }
  (void) ev_data;
}

struct mg_connection *mg_http_connect(struct mg_mgr *mgr, const char *url,
                                      mg_event_handler_t fn, void *fn_data) {
  return mg_connect_svc(mgr, url, fn, fn_data, http_cb, NULL);
}

struct mg_connection *mg_http_listen(struct mg_mgr *mgr, const char *url,
                                     mg_event_handler_t fn, void *fn_data) {
  struct mg_connection *c = mg_listen(mgr, url, fn, fn_data);
  if (c != NULL) c->pfn = http_cb;
  return c;
}

#ifdef MG_ENABLE_LINES
#line 1 "src/iobuf.c"
#endif





static size_t roundup(size_t size, size_t align) {
  return align == 0 ? size : (size + align - 1) / align * align;
}

bool mg_iobuf_resize(struct mg_iobuf *io, size_t new_size) {
  bool ok = true;
  new_size = roundup(new_size, io->align);
  if (new_size == 0) {
    mg_bzero(io->buf, io->size);
    mg_free(io->buf);
    io->buf = NULL;
    io->len = io->size = 0;
  } else if (new_size != io->size) {
    // NOTE(lsm): do not use realloc here. Use mg_calloc/mg_free only
    void *p = mg_calloc(1, new_size);
    if (p != NULL) {
      size_t len = new_size < io->len ? new_size : io->len;
      if (len > 0 && io->buf != NULL) memmove(p, io->buf, len);
      mg_bzero(io->buf, io->size);
      mg_free(io->buf);
      io->buf = (unsigned char *) p;
      io->size = new_size;
      io->len = len;
    } else {
      ok = false;
      MG_ERROR(("%lld->%lld", (uint64_t) io->size, (uint64_t) new_size));
    }
  }
  return ok;
}

bool mg_iobuf_init(struct mg_iobuf *io, size_t size, size_t align) {
  io->buf = NULL;
  io->align = align;
  io->size = io->len = 0;
  return mg_iobuf_resize(io, size);
}

size_t mg_iobuf_add(struct mg_iobuf *io, size_t ofs, const void *buf,
                    size_t len) {
  size_t new_size = roundup(io->len + len, io->align);
  mg_iobuf_resize(io, new_size);      // Attempt to resize
  if (new_size != io->size) len = 0;  // Resize failure, append nothing
  if (ofs < io->len) memmove(io->buf + ofs + len, io->buf + ofs, io->len - ofs);
  if (buf != NULL) memmove(io->buf + ofs, buf, len);
  if (ofs > io->len) io->len += ofs - io->len;
  io->len += len;
  return len;
}

size_t mg_iobuf_del(struct mg_iobuf *io, size_t ofs, size_t len) {
  if (ofs > io->len) ofs = io->len;
  if (ofs + len > io->len) len = io->len - ofs;
  if (io->buf) memmove(io->buf + ofs, io->buf + ofs + len, io->len - ofs - len);
  if (io->buf) mg_bzero(io->buf + io->len - len, len);
  io->len -= len;
  return len;
}

void mg_iobuf_free(struct mg_iobuf *io) {
  mg_iobuf_resize(io, 0);
}

#ifdef MG_ENABLE_LINES
#line 1 "src/json.c"
#endif





static const char *escapeseq(int esc) {
  return esc ? "\b\f\n\r\t\\\"" : "bfnrt\\\"";
}

static char json_esc(int c, int esc) {
  const char *p, *esc1 = escapeseq(esc), *esc2 = escapeseq(!esc);
  for (p = esc1; *p != '\0'; p++) {
    if (*p == c) return esc2[p - esc1];
  }
  return 0;
}

static int mg_pass_string(const char *s, int len) {
  int i;
  for (i = 0; i < len; i++) {
    if (s[i] == '\\' && i + 1 < len && json_esc(s[i + 1], 1)) {
      i++;
    } else if (s[i] == '\0') {
      return MG_JSON_INVALID;
    } else if (s[i] == '"') {
      return i;
    }
  }
  return MG_JSON_INVALID;
}

static double mg_atod(const char *p, int len, int *numlen) {
  double d = 0.0;
  int i = 0, sign = 1;

  // Sign
  if (i < len && *p == '-') {
    sign = -1, i++;
  } else if (i < len && *p == '+') {
    i++;
  }

  // Decimal
  for (; i < len && p[i] >= '0' && p[i] <= '9'; i++) {
    d *= 10.0;
    d += p[i] - '0';
  }
  d *= sign;

  // Fractional
  if (i < len && p[i] == '.') {
    double frac = 0.0, base = 0.1;
    i++;
    for (; i < len && p[i] >= '0' && p[i] <= '9'; i++) {
      frac += base * (p[i] - '0');
      base /= 10.0;
    }
    d += frac * sign;
  }

  // Exponential
  if (i < len && (p[i] == 'e' || p[i] == 'E')) {
    int exp = 0, minus = 0;
    i++;
    if (i < len && p[i] == '-') minus = 1, i++;
    if (i < len && p[i] == '+') i++;
    while (i < len && p[i] >= '0' && p[i] <= '9' && exp < 308)
      exp = exp * 10 + (p[i++] - '0');
    // use fast exponentiation
    // https://en.wikipedia.org/wiki/Exponentiation_by_squaring
    if (exp != 0) {
      double x = 10, y = 1;
      if (exp > 308) exp = 308;
      if (minus) x = 0.1;
      while (exp > 1) {
        if (exp & 1) {
          y *= x;
          --exp;
        }
        x *= x;
        exp >>= 1;
      }
      d *= x * y;
    }
  }

  if (numlen != NULL) *numlen = i;
  return d;
}

// Iterate over object or array elements
size_t mg_json_next(struct mg_str obj, size_t ofs, struct mg_str *key,
                    struct mg_str *val) {
  if (ofs >= obj.len) {
    ofs = 0;  // Out of boundaries, stop scanning
  } else if (obj.len < 2 || (*obj.buf != '{' && *obj.buf != '[')) {
    ofs = 0;  // Not an array or object, stop
  } else {
    struct mg_str sub = mg_str_n(obj.buf + ofs, obj.len - ofs);
    if (ofs == 0) ofs++, sub.buf++, sub.len--;
    if (*obj.buf == '[') {  // Iterate over an array
      int n = 0, o = mg_json_get(sub, "$", &n);
      if (n < 0 || o < 0 || (size_t) (o + n) > sub.len) {
        ofs = 0;  // Error parsing key, stop scanning
      } else {
        if (key) *key = mg_str_n(NULL, 0);
        if (val) *val = mg_str_n(sub.buf + o, (size_t) n);
        ofs = (size_t) (&sub.buf[o + n] - obj.buf);
      }
    } else {  // Iterate over an object
      int n = 0, o = mg_json_get(sub, "$", &n);
      if (n < 0 || o < 0 || (size_t) (o + n) > sub.len) {
        ofs = 0;  // Error parsing key, stop scanning
      } else {
        if (key) *key = mg_str_n(sub.buf + o, (size_t) n);
        sub.buf += o + n, sub.len -= (size_t) (o + n);
        while (sub.len > 0 && *sub.buf != ':') sub.len--, sub.buf++;
        if (sub.len > 0 && *sub.buf == ':') sub.len--, sub.buf++;
        n = 0, o = mg_json_get(sub, "$", &n);
        if (n < 0 || o < 0 || (size_t) (o + n) > sub.len) {
          ofs = 0;  // Error parsing value, stop scanning
        } else {
          if (val) *val = mg_str_n(sub.buf + o, (size_t) n);
          ofs = (size_t) (&sub.buf[o + n] - obj.buf);
        }
      }
    }
    // MG_INFO(("SUB ofs %u %.*s", ofs, sub.len, sub.buf));
    while (ofs && ofs < obj.len &&
           (obj.buf[ofs] == ' ' || obj.buf[ofs] == '\t' ||
            obj.buf[ofs] == '\n' || obj.buf[ofs] == '\r')) {
      ofs++;
    }
    if (ofs && ofs < obj.len && obj.buf[ofs] == ',') ofs++;
    if (ofs > obj.len) ofs = 0;
  }
  return ofs;
}

int mg_json_get(struct mg_str json, const char *path, int *toklen) {
  const char *s = json.buf;
  int len = (int) json.len;
  enum { S_VALUE, S_KEY, S_COLON, S_COMMA_OR_EOO } expecting = S_VALUE;
  unsigned char nesting[MG_JSON_MAX_DEPTH];
  int i = 0;             // Current offset in `s`
  int j = 0;             // Offset in `s` we're looking for (return value)
  int depth = 0;         // Current depth (nesting level)
  int ed = 0;            // Expected depth
  int pos = 1;           // Current position in `path`
  int ci = -1, ei = -1;  // Current and expected index in array

  if (toklen) *toklen = 0;
  if (path[0] != '$') return MG_JSON_INVALID;

#define MG_CHECKRET(x)                                  \
  do {                                                  \
    if (depth == ed && path[pos] == '\0' && ci == ei) { \
      if (toklen) *toklen = i - j + 1;                  \
      return j;                                         \
    }                                                   \
  } while (0)

// In the ascii table, the distance between `[` and `]` is 2.
// Ditto for `{` and `}`. Hence +2 in the code below.
#define MG_EOO(x)                                            \
  do {                                                       \
    if (depth == ed && ci != ei) return MG_JSON_NOT_FOUND;   \
    if (c != nesting[depth - 1] + 2) return MG_JSON_INVALID; \
    depth--;                                                 \
    MG_CHECKRET(x);                                          \
  } while (0)

  for (i = 0; i < len; i++) {
    unsigned char c = ((unsigned char *) s)[i];
    if (c == ' ' || c == '\t' || c == '\n' || c == '\r') continue;
    switch (expecting) {
      case S_VALUE:
        // p("V %s [%.*s] %d %d %d %d\n", path, pos, path, depth, ed, ci, ei);
        if (depth == ed) j = i;
        if (c == '{') {
          if (depth >= (int) sizeof(nesting)) return MG_JSON_TOO_DEEP;
          if (depth == ed && path[pos] == '.' && ci == ei) {
            // If we start the object, reset array indices
            ed++, pos++, ci = ei = -1;
          }
          nesting[depth++] = c;
          expecting = S_KEY;
          break;
        } else if (c == '[') {
          if (depth >= (int) sizeof(nesting)) return MG_JSON_TOO_DEEP;
          if (depth == ed && path[pos] == '[' && ei == ci) {
            ed++, pos++, ci = 0;
            for (ei = 0; path[pos] != ']' && path[pos] != '\0'; pos++) {
              ei *= 10;
              ei += path[pos] - '0';
            }
            if (path[pos] != 0) pos++;
          }
          nesting[depth++] = c;
          break;
        } else if (c == ']' && depth > 0) {  // Empty array
          MG_EOO(']');
        } else if (c == 't' && i + 3 < len && memcmp(&s[i], "true", 4) == 0) {
          i += 3;
        } else if (c == 'n' && i + 3 < len && memcmp(&s[i], "null", 4) == 0) {
          i += 3;
        } else if (c == 'f' && i + 4 < len && memcmp(&s[i], "false", 5) == 0) {
          i += 4;
        } else if (c == '-' || ((c >= '0' && c <= '9'))) {
          int numlen = 0;
          mg_atod(&s[i], len - i, &numlen);
          i += numlen - 1;
        } else if (c == '"') {
          int n = mg_pass_string(&s[i + 1], len - i - 1);
          if (n < 0) return n;
          i += n + 1;
        } else {
          return MG_JSON_INVALID;
        }
        MG_CHECKRET('V');
        if (depth == ed && ei >= 0) ci++;
        expecting = S_COMMA_OR_EOO;
        break;

      case S_KEY:
        if (c == '"') {
          int n = mg_pass_string(&s[i + 1], len - i - 1);
          if (n < 0) return n;
          if (i + 1 + n >= len) return MG_JSON_NOT_FOUND;
          if (depth < ed) return MG_JSON_NOT_FOUND;
          if (depth == ed && path[pos - 1] != '.') return MG_JSON_NOT_FOUND;
          // printf("K %s [%.*s] [%.*s] %d %d %d %d %d\n", path, pos, path, n,
          //        &s[i + 1], n, depth, ed, ci, ei);
          //  NOTE(cpq): in the check sequence below is important.
          //  strncmp() must go first: it fails fast if the remaining length
          //  of the path is smaller than `n`.
          if (depth == ed && path[pos - 1] == '.' &&
              strncmp(&s[i + 1], &path[pos], (size_t) n) == 0 &&
              (path[pos + n] == '\0' || path[pos + n] == '.' ||
               path[pos + n] == '[')) {
            pos += n;
          }
          i += n + 1;
          expecting = S_COLON;
        } else if (c == '}') {  // Empty object
          MG_EOO('}');
          expecting = S_COMMA_OR_EOO;
          if (depth == ed && ei >= 0) ci++;
        } else {
          return MG_JSON_INVALID;
        }
        break;

      case S_COLON:
        if (c == ':') {
          expecting = S_VALUE;
        } else {
          return MG_JSON_INVALID;
        }
        break;

      case S_COMMA_OR_EOO:
        if (depth <= 0) {
          return MG_JSON_INVALID;
        } else if (c == ',') {
          expecting = (nesting[depth - 1] == '{') ? S_KEY : S_VALUE;
        } else if (c == ']' || c == '}') {
          if (depth == ed && c == '}' && path[pos - 1] == '.')
            return MG_JSON_NOT_FOUND;
          if (depth == ed && c == ']' && path[pos - 1] == ',')
            return MG_JSON_NOT_FOUND;
          MG_EOO('O');
          if (depth == ed && ei >= 0) ci++;
        } else {
          return MG_JSON_INVALID;
        }
        break;
    }
  }
  return MG_JSON_NOT_FOUND;
}

struct mg_str mg_json_get_tok(struct mg_str json, const char *path) {
  int len = 0, ofs = mg_json_get(json, path, &len);
  return mg_str_n(ofs < 0 ? NULL : json.buf + ofs,
                  (size_t) (len < 0 ? 0 : len));
}

bool mg_json_get_num(struct mg_str json, const char *path, double *v) {
  int n, toklen, found = 0;
  if ((n = mg_json_get(json, path, &toklen)) >= 0 &&
      (json.buf[n] == '-' || (json.buf[n] >= '0' && json.buf[n] <= '9'))) {
    if (v != NULL) *v = mg_atod(json.buf + n, toklen, NULL);
    found = 1;
  }
  return found;
}

bool mg_json_get_bool(struct mg_str json, const char *path, bool *v) {
  int found = 0, off = mg_json_get(json, path, NULL);
  if (off >= 0 && (json.buf[off] == 't' || json.buf[off] == 'f')) {
    if (v != NULL) *v = json.buf[off] == 't';
    found = 1;
  }
  return found;
}

size_t mg_json_unescape(struct mg_str json, const char *path, char *to,
                        size_t n) {
  struct mg_str s = mg_json_get_tok(json, path);
  size_t res = 0, i, j;
  if (s.len > 1 && s.buf[0] == '"') {  // Is is a string?
    s.len -= 2, s.buf++;               // Trim surrounding double-quotes
    for (i = 0, j = 0; i < s.len && j < n; i++, j++) {
      if (s.buf[i] == '\\' && i + 5 < s.len && s.buf[i + 1] == 'u') {
        //  \uXXXX escape. We process simple one-byte chars \u00xx within ASCII
        //  range. More complex chars would require dragging in a UTF8 library,
        //  which is too much for us
        if (mg_str_to_num(mg_str_n(s.buf + i + 2, 4), 16, &to[j],
                          sizeof(uint8_t)) == false)
          break;
        i += 5;
      } else if (s.buf[i] == '\\' && i + 1 < s.len) {
        char c = json_esc(s.buf[i + 1], 0);
        if (c == 0) return false;
        to[j] = c;
        i++;
      } else {
        to[j] = s.buf[i];
      }
    }
    if (j < n) res = j;
    if (n > 0) to[j < n ? j : n - 1] = '\0';
  }
  return res;
}

char *mg_json_get_str(struct mg_str json, const char *path) {
  char *result = NULL;
  int len = 0, off = mg_json_get(json, path, &len);
  if (off >= 0 && len >= 2 && json.buf[off] == '"') {
    if ((result = (char *) mg_calloc(1, (size_t) len)) != NULL &&
        len > 2 && mg_json_unescape(json, path, result, (size_t) len) == 0) {
      mg_free(result);
      result = NULL;
    }
  }
  return result;
}

char *mg_json_get_b64(struct mg_str json, const char *path, int *slen) {
  char *result = NULL;
  int len = 0, off = mg_json_get(json, path, &len);
  if (off >= 0 && json.buf[off] == '"' && len > 1 &&
      (result = (char *) mg_calloc(1, (size_t) len)) != NULL) {
    size_t k = mg_base64_decode(json.buf + off + 1, (size_t) (len - 2), result,
                                (size_t) len);
    if (slen != NULL) *slen = (int) k;
  }
  return result;
}

char *mg_json_get_hex(struct mg_str json, const char *path, int *slen) {
  char *result = NULL;
  int len = 0, off = mg_json_get(json, path, &len);
  if (off >= 0 && json.buf[off] == '"' && len > 1 && (len - 2) % 2 == 0 &&
      (result = (char *) mg_calloc(1, (size_t) len / 2)) != NULL) {
    int i;
    for (i = 0; i < len - 2; i += 2) {
      if (!mg_str_to_num(mg_str_n(json.buf + off + 1 + i, 2), 16,
                         &result[i >> 1], sizeof(uint8_t))) break;
    }
    if (i < len - 2) mg_free(result), result = NULL;
    if (result != NULL) {
      result[len / 2 - 1] = '\0';
      if (slen != NULL) *slen = len / 2 - 1;
    }
  }
  return result;
}

long mg_json_get_long(struct mg_str json, const char *path, long dflt) {
  double dv;
  long result = dflt;
  if (mg_json_get_num(json, path, &dv)) result = (long) dv;
  return result;
}

#ifdef MG_ENABLE_LINES
#line 1 "src/jwt.c"
#endif








#if MG_TLS == MG_TLS_BUILTIN

#endif

static size_t jwt_split(struct mg_str jwt, struct mg_str *h, struct mg_str *p,
                        struct mg_str *s) {
  size_t i, a = jwt.len, b = jwt.len;
  for (i = 0; i < jwt.len; i++) {
    if (jwt.buf[i] == '.') {
      if (a == jwt.len) a = i;
      if (a != i) b = i;
      if (b != jwt.len) break;
    }
  }
  if (a == jwt.len || b == jwt.len || b + 1 >= jwt.len) return 0;
  *h = mg_str_n(jwt.buf, a);
  *p = mg_str_n(jwt.buf + a + 1, b - a - 1);
  *s = mg_str_n(jwt.buf + b + 1, jwt.len - b - 1);
  return b;
}

static bool jwt_alg(struct mg_str h, const char *alg) {
  char got[16];
  size_t len = h.len * 3 / 4 + 2;
  char *buf = (char *) mg_calloc(1, len);
  bool ok;
  if (buf == NULL) return false;
  ok = mg_base64url_decode(h.buf, h.len, buf, len) > 0 &&
       mg_json_unescape(mg_str(buf), "$.alg", got, sizeof(got)) > 0 &&
       strcmp(got, alg) == 0;
  mg_free(buf);
  return ok;
}

static size_t jwt_header_payload(const char *alg, const struct mg_jwt_opts *opts,
                                 char *buf, size_t len) {
  char *header, *kid = NULL;
  size_t n, m;
  if (opts->kid.len > 0) {
    kid = mg_mprintf(",%m:%m", MG_ESC("kid"), mg_print_esc,
                     (int) opts->kid.len, opts->kid.buf);
    if (kid == NULL) return 0;
  }
  header = mg_mprintf("{%m:%m,%m:%m%.*s%.*s%.*s}", MG_ESC("alg"),
                      MG_ESC(alg), MG_ESC("typ"), MG_ESC("JWT"),
                      kid == NULL ? 0 : (int) strlen(kid),
                      kid == NULL ? "" : kid,
                      opts->header.len > 0 ? 1 : 0, ",",
                      (int) opts->header.len, opts->header.buf);
  mg_free(kid);
  if (header == NULL) return 0;
  n = mg_base64url_encode((uint8_t *) header, strlen(header), buf, len);
  mg_free(header);
  if (n == 0 || n + 1 >= len) return 0;
  buf[n++] = '.';
  m = mg_base64url_encode((uint8_t *) opts->claims.buf, opts->claims.len,
                          buf + n, len - n);
  return m == 0 ? 0 : n + m;
}

size_t mg_jwt_sign_hs256(const struct mg_jwt_opts *opts, char *buf,
                         size_t len) {
  uint8_t sig[32];
  size_t n = jwt_header_payload("HS256", opts, buf, len);
  size_t m;
  if (n == 0 || n + 1 >= len) return 0;
  mg_hmac_sha256(sig, (uint8_t *) opts->secret.buf, opts->secret.len,
                 (uint8_t *) buf, n);
  buf[n++] = '.';
  m = mg_base64url_encode(sig, sizeof(sig), buf + n, len - n);
  return m == 0 ? 0 : n + m;
}

size_t mg_jwt_verify_hs256(struct mg_str jwt, const struct mg_jwt_opts *opts,
                           char *buf, size_t len) {
  struct mg_str h, p, s;
  uint8_t sig[33], digest[32];
  size_t n = jwt_split(jwt, &h, &p, &s);
  if (n == 0 || !jwt_alg(h, "HS256")) return 0;
  if (mg_base64url_decode(s.buf, s.len, (char *) sig, sizeof(sig)) !=
      sizeof(digest))
    return 0;
  mg_hmac_sha256(digest, (uint8_t *) opts->secret.buf, opts->secret.len,
                 (uint8_t *) jwt.buf, n);
  if (!mg_memeq(sig, digest, sizeof(digest))) return 0;
  return mg_base64url_decode(p.buf, p.len, buf, len);
}

#if MG_TLS == MG_TLS_BUILTIN
size_t mg_jwt_sign_es256(const struct mg_jwt_opts *opts, char *buf,
                         size_t len) {
  uint8_t hash[32], sig[64];
  size_t n = jwt_header_payload("ES256", opts, buf, len);
  size_t m;
  if (n == 0 || n + 1 >= len) return 0;
  mg_sha256(hash, (uint8_t *) buf, n);
  if (mg_p256_sign(opts->private_key, hash, sizeof(hash), sig) != 1)
    return 0;
  buf[n++] = '.';
  m = mg_base64url_encode(sig, sizeof(sig), buf + n, len - n);
  return m == 0 ? 0 : n + m;
}

size_t mg_jwt_verify_es256(struct mg_str jwt, const struct mg_jwt_opts *opts,
                           char *buf, size_t len) {
  struct mg_str h, p, s;
  uint8_t sig[65], hash[32];
  size_t n = jwt_split(jwt, &h, &p, &s);
  if (n == 0 || !jwt_alg(h, "ES256")) return 0;
  if (mg_base64url_decode(s.buf, s.len, (char *) sig, sizeof(sig)) != 64)
    return 0;
  mg_sha256(hash, (uint8_t *) jwt.buf, n);
  if (mg_p256_verify(opts->public_key, hash, sizeof(hash), sig) != 1)
    return 0;
  return mg_base64url_decode(p.buf, p.len, buf, len);
}
#else
size_t mg_jwt_sign_es256(const struct mg_jwt_opts *opts, char *buf,
                         size_t len) {
  (void) opts, (void) buf, (void) len;
  MG_ERROR(("JWT ES256 requires built-in TLS"));
  return 0;
}

size_t mg_jwt_verify_es256(struct mg_str jwt, const struct mg_jwt_opts *opts,
                           char *buf, size_t len) {
  (void) jwt, (void) opts, (void) buf, (void) len;
  return mg_jwt_sign_es256(NULL, NULL, 0);
}
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/l2.c"
#endif




#if MG_ENABLE_TCPIP

// L2 API
void mg_l2_init(struct mg_tcpip_if *ifp);
bool mg_l2_poll(struct mg_tcpip_if *ifp, bool expired_1000ms);
uint8_t *mg_l2_header(struct mg_tcpip_if *ifp, enum mg_l2proto proto,
                      uint8_t *src, uint8_t *dst, uint8_t *frame);
size_t mg_l2_trailer(struct mg_tcpip_if *ifp, size_t len, uint8_t *cur);
bool mg_l2_rx(struct mg_tcpip_if *ifp, enum mg_l2proto *proto,
              struct mg_str *pay, struct mg_str *raw);
uint8_t *mg_l2_getaddr(struct mg_tcpip_if *ifp, uint8_t *frame);
uint8_t *mg_l2_mapip(enum mg_l2type type, enum mg_l2addrtype addrtype,
                     struct mg_addr *ip);
#if MG_ENABLE_IPV6
bool mg_l2_genip6(enum mg_l2type type, uint64_t *ip6, uint8_t prefix_len,
                  uint8_t *addr);
bool mg_l2_ip6get(enum mg_l2type type, uint8_t *addr, uint8_t *opts,
                  uint8_t len);
uint8_t mg_l2_ip6put(enum mg_l2type type, uint8_t *addr, uint8_t *opts);
#endif
size_t mg_l2_driver_output(struct mg_tcpip_if *ifp, size_t len);

// clang-format off
extern void mg_l2_eth_init(struct mg_tcpip_if *);
extern bool mg_l2_eth_poll(struct mg_tcpip_if *, bool);
extern uint8_t *mg_l2_eth_header(struct mg_tcpip_if *, enum mg_l2proto, struct mg_l2addr *, struct mg_l2addr *, uint8_t *);
extern size_t mg_l2_eth_trailer(struct mg_tcpip_if *, size_t, uint8_t *);
extern bool mg_l2_eth_rx(struct mg_tcpip_if *, enum mg_l2proto *, struct mg_str *, struct mg_str *);
extern struct mg_l2addr *mg_l2_eth_getaddr(struct mg_tcpip_if *, uint8_t *);
extern struct mg_l2addr *mg_l2_eth_mapip(enum mg_l2addrtype, struct mg_addr *);
#if MG_ENABLE_IPV6
extern bool mg_l2_eth_genip6(uint64_t *, uint8_t, struct mg_l2addr *);
extern bool mg_l2_eth_ip6get(struct mg_l2addr *, uint8_t *, uint8_t);
extern uint8_t mg_l2_eth_ip6put(struct mg_l2addr *, uint8_t *);
#endif

extern void mg_l2_ppp_init(struct mg_tcpip_if *);
extern bool mg_l2_ppp_poll(struct mg_tcpip_if *, bool);
extern uint8_t *mg_l2_ppp_header(struct mg_tcpip_if *, enum mg_l2proto, struct mg_l2addr *, struct mg_l2addr *, uint8_t *);
extern size_t mg_l2_ppp_trailer(struct mg_tcpip_if *, size_t, uint8_t *);
extern bool mg_l2_ppp_rx(struct mg_tcpip_if *, enum mg_l2proto *, struct mg_str *, struct mg_str *);
extern struct mg_l2addr *mg_l2_ppp_getaddr(struct mg_tcpip_if *, uint8_t *);
extern struct mg_l2addr *mg_l2_ppp_mapip(enum mg_l2addrtype, struct mg_addr *);
#if MG_ENABLE_IPV6
extern bool mg_l2_ppp_genip6(uint64_t *, uint8_t, struct mg_l2addr *);
extern bool mg_l2_ppp_ip6get(struct mg_l2addr *, uint8_t *, uint8_t);
extern uint8_t mg_l2_ppp_ip6put(struct mg_l2addr *, uint8_t *);
#endif
extern void mg_l2_pppoe_init(struct mg_tcpip_if *);
extern bool mg_l2_pppoe_poll(struct mg_tcpip_if *, bool);
extern uint8_t *mg_l2_pppoe_header(struct mg_tcpip_if *, enum mg_l2proto, struct mg_l2addr *, struct mg_l2addr *, uint8_t *);
extern size_t mg_l2_pppoe_trailer(struct mg_tcpip_if *, size_t, uint8_t *);
extern bool mg_l2_pppoe_rx(struct mg_tcpip_if *, enum mg_l2proto *, struct mg_str *, struct mg_str *);

typedef void (*l2_init_fn)(struct mg_tcpip_if *);
typedef bool (*l2_poll_fn)(struct mg_tcpip_if *, bool);
typedef uint8_t *((*l2_header_fn)(struct mg_tcpip_if *, enum mg_l2proto, struct mg_l2addr *, struct mg_l2addr *, uint8_t *));
typedef size_t (*l2_trailer_fn)(struct mg_tcpip_if *, size_t, uint8_t *);
typedef bool (*l2_rx_fn)(struct mg_tcpip_if *, enum mg_l2proto *, struct mg_str *, struct mg_str *);
typedef struct mg_l2addr (*(*l2_getaddr_fn)(struct mg_tcpip_if *, uint8_t *));
typedef struct mg_l2addr (*(*l2_mapip_fn)(enum mg_l2addrtype, struct mg_addr *));
#if MG_ENABLE_IPV6
typedef bool (*l2_genip6_fn)(uint64_t *, uint8_t, struct mg_l2addr *);
typedef bool (*l2_ip6get_fn)(struct mg_l2addr *, uint8_t *, uint8_t);
typedef uint8_t (*l2_ip6put_fn)(struct mg_l2addr *, uint8_t *);
#endif
// clang-format on

static const l2_init_fn l2_init[] = {mg_l2_eth_init, mg_l2_ppp_init,
                                     mg_l2_pppoe_init};
static const l2_poll_fn l2_poll[] = {mg_l2_eth_poll, mg_l2_ppp_poll,
                                     mg_l2_pppoe_poll};
static const l2_header_fn l2_header[] = {mg_l2_eth_header, mg_l2_ppp_header,
                                         mg_l2_pppoe_header};
static const l2_trailer_fn l2_trailer[] = {mg_l2_eth_trailer, mg_l2_ppp_trailer,
                                           mg_l2_pppoe_trailer};
static const l2_rx_fn l2_rx[] = {mg_l2_eth_rx, mg_l2_ppp_rx, mg_l2_pppoe_rx};
static const l2_getaddr_fn l2_getaddr[] = {mg_l2_eth_getaddr, mg_l2_ppp_getaddr,
                                           mg_l2_ppp_getaddr};
static const l2_mapip_fn l2_mapip[] = {mg_l2_eth_mapip, mg_l2_ppp_mapip,
                                       mg_l2_ppp_mapip};
#if MG_ENABLE_IPV6
static const l2_genip6_fn l2_genip6[] = {mg_l2_eth_genip6, mg_l2_ppp_genip6,
                                         mg_l2_ppp_genip6};
static const l2_ip6get_fn l2_ip6get[] = {mg_l2_eth_ip6get, mg_l2_ppp_ip6get,
                                         mg_l2_ppp_ip6get};
static const l2_ip6put_fn l2_ip6put[] = {mg_l2_eth_ip6put, mg_l2_ppp_ip6put,
                                         mg_l2_ppp_ip6put};
#endif

void mg_l2_init(struct mg_tcpip_if *ifp) {
  l2_init[ifp->l2type](ifp);
}

bool mg_l2_poll(struct mg_tcpip_if *ifp, bool expired_1000ms) {
  return l2_poll[ifp->l2type](ifp, expired_1000ms);
}

uint8_t *mg_l2_header(struct mg_tcpip_if *ifp, enum mg_l2proto proto,
                      uint8_t *src, uint8_t *dst, uint8_t *frame) {
  return l2_header[ifp->l2type](ifp, proto, (struct mg_l2addr *) src,
                                (struct mg_l2addr *) dst, frame);
}

size_t mg_l2_trailer(struct mg_tcpip_if *ifp, size_t len, uint8_t *frame) {
  return l2_trailer[ifp->l2type](ifp, len, frame);
}

bool mg_l2_rx(struct mg_tcpip_if *ifp, enum mg_l2proto *proto,
              struct mg_str *pay, struct mg_str *raw) {
  return l2_rx[ifp->l2type](ifp, proto, pay, raw);
}

uint8_t *mg_l2_getaddr(struct mg_tcpip_if *ifp, uint8_t *frame) {
  return (uint8_t *) l2_getaddr[ifp->l2type](ifp, frame);
}

struct mg_l2addr s_mapip;

uint8_t *mg_l2_mapip(enum mg_l2type type, enum mg_l2addrtype addrtype,
                     struct mg_addr *ip) {
  return (uint8_t *) l2_mapip[type](addrtype, ip);
}

#if MG_ENABLE_IPV6
bool mg_l2_genip6(enum mg_l2type type, uint64_t *ip6, uint8_t prefix_len,
                  uint8_t *addr) {
  return l2_genip6[type](ip6, prefix_len, (struct mg_l2addr *) addr);
}

bool mg_l2_ip6get(enum mg_l2type type, uint8_t *addr, uint8_t *opts,
                  uint8_t len) {
  return l2_ip6get[type]((struct mg_l2addr *) addr, opts, len);
}
uint8_t mg_l2_ip6put(enum mg_l2type type, uint8_t *addr, uint8_t *opts) {
  return l2_ip6put[type]((struct mg_l2addr *) addr, opts);
}
#endif

size_t mg_l2_driver_output(struct mg_tcpip_if *ifp, size_t len) {
  size_t n = ifp->driver->tx(ifp->tx.buf, len, ifp);
  if (n == len) ifp->nsent++;
  return n;
}

#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/l2_eth.c"
#endif







#if MG_ENABLE_TCPIP

#if defined(__DCC__)
#pragma pack(1)
#else
#pragma pack(push, 1)
#endif

struct eth {
  uint8_t dst[6];  // Destination MAC address
  uint8_t src[6];  // Source MAC address
  uint16_t type;   // Ethernet type
};

struct qtag {
  uint16_t tpid;  // Ethernet 802.1Q type
  uint16_t tci;   // PCP, DEI, VLAN id
#define VLAN_ID(x) ((uint16_t) ((x) &0x0FFF))
};

#if defined(__DCC__)
#pragma pack(0)
#else
#pragma pack(pop)
#endif

static const uint16_t eth_types[] = {
    // order is vital, see l2.h
    0x800,   // IPv4
    0x86dd,  // IPv6
    0x806,   // ARP
    0x8863,  // PPPoE Discovery Stage
    0x8864   // PPPoE Session Stage
};

void mg_l2_eth_init(struct mg_tcpip_if *ifp) {
  struct mg_l2addr *l2addr = (struct mg_l2addr *) ifp->mac;
  // If MAC is not set, make a random one
  if (l2addr->addr.mac[0] == 0 && l2addr->addr.mac[1] == 0 &&
      l2addr->addr.mac[2] == 0 && l2addr->addr.mac[3] == 0 &&
      l2addr->addr.mac[4] == 0 && l2addr->addr.mac[5] == 0) {
    l2addr->addr.mac[0] = 0x02;  // Locally administered, unicast
    mg_random(&l2addr->addr.mac[1], sizeof(l2addr->addr.mac) - 1);
    MG_INFO(
        ("MAC not set. Generated random: %M", mg_print_mac, l2addr->addr.mac));
  }
  ifp->l2mtu = 1500;
  ifp->framesize = 1540;
}

bool mg_l2_eth_poll(struct mg_tcpip_if *ifp, bool expired_1000ms) {
  (void) ifp;
  (void) expired_1000ms;
  return true;
}

uint8_t *mg_l2_eth_header(struct mg_tcpip_if *ifp, enum mg_l2proto proto,
                          struct mg_l2addr *src, struct mg_l2addr *dst,
                          uint8_t *frame) {
  struct eth_data *d = &ifp->l2data.eth;
  struct eth *eth = (struct eth *) frame;
  uint8_t *hlp;
  hlp = (uint8_t *) (eth + 1);
  memcpy(eth->src, src->addr.mac, sizeof(eth->dst));
  memcpy(eth->dst, dst->addr.mac, sizeof(eth->dst));
  if (d->vlan_id == 0) {  // Traditional plain frame
    eth->type = mg_htons(eth_types[(unsigned int) proto]);
  } else {  // Add 802.1Q tag
    struct qtag *qtag = (struct qtag *) &eth->type;
    qtag->tpid = mg_htons(0x8100);
    qtag->tci = mg_htons(d->vlan_id);  // PCP = default (best-effort)
    hlp += sizeof(*qtag);
    MG_STORE_BE16(qtag + 1, eth_types[(unsigned int) proto]);
  }
  return hlp;
}

size_t mg_l2_eth_trailer(struct mg_tcpip_if *ifp, size_t len, uint8_t *cur) {
  struct eth_data *d = &ifp->l2data.eth;
  // there is no len field in Ethernet, CRC is hw-calculated; pad to 64 - CRC
  size_t ethlen =
      len + sizeof(struct eth) + (d->vlan_id != 0 ? sizeof(struct qtag) : 0);
  (void) cur;
  return ethlen >= 60 ? ethlen : 60;
}

struct mg_l2addr *mg_l2_eth_mapip(enum mg_l2addrtype addrtype,
                                  struct mg_addr *addr);

// Read an unaligned little-endian value from byte pointer p into a native integer.
// Safe on architectures that forbid unaligned access (e.g. Cortex-M0).
#define MG_LOAD_LE32(p)                           \
  ((uint32_t) (((uint32_t) MG_U8P(p)[3] << 24U) | \
               ((uint32_t) MG_U8P(p)[2] << 16U) | \
               ((uint32_t) MG_U8P(p)[1] << 8U) | MG_U8P(p)[0]))

bool mg_l2_eth_rx(struct mg_tcpip_if *ifp, enum mg_l2proto *proto,
                  struct mg_str *pay, struct mg_str *raw) {
  struct eth *eth = (struct eth *) raw->buf;
  struct eth_data *d = &ifp->l2data.eth;
  uint16_t type, len;
  unsigned int i;
  size_t hdrlen =
      sizeof(struct eth) + (d->vlan_id != 0 ? sizeof(struct qtag) : 0);
  if (raw->len < hdrlen) return false;  // Truncated - runt?
  len = (uint16_t) raw->len;
  if (d->vlan_id == 0) {  // We don't handle VLANs
    type = mg_ntohs(eth->type);
  } else {  // We do, check 802.1Q tag
    struct qtag *qtag = (struct qtag *) &eth->type;
    if (qtag->tpid != mg_htons(0x8100)) return false;  // Untagged frame
    if (VLAN_ID(mg_ntohs(qtag->tci)) != VLAN_ID(d->vlan_id))
      return false;  // Not our VLAN
    type = MG_LOAD_BE16(qtag + 1);
  }
  if (ifp->enable_mac_check &&
      memcmp(eth->dst, ifp->mac, sizeof(eth->dst)) != 0 &&
      memcmp(eth->dst, mg_l2_eth_mapip(MG_TCPIP_L2ADDR_BCAST, NULL),
             sizeof(eth->dst)) != 0)
    return false;  // TODO(): add multicast addresses
  if (ifp->enable_fcs_check && len > hdrlen + 4) {
    uint32_t crc, crc_rx;
    len -= 4;
    crc = mg_crc32(0, (const char *) raw->buf, len);
    crc_rx = MG_LOAD_LE32(raw->buf + len);
    if (crc_rx != crc)
      return false;
  }
  pay->buf = ((char *) eth) + hdrlen;
  pay->len = len - hdrlen;
  if (len > ifp->framesize) return false;  // Oversized
  for (i = 0; i < sizeof(eth_types) / sizeof(uint16_t); i++) {
    if (type == eth_types[i]) break;
  }
  if (i == sizeof(eth_types) / sizeof(eth_types[0])) {
    MG_DEBUG(("Unknown eth type %x", type));
    if (mg_log_level >= MG_LL_VERBOSE)
      mg_hexdump(raw->buf, raw->len >= 32 ? 32 : raw->len);
    return false;
  }
  *proto = (enum mg_l2proto) i;
  return true;
}

struct mg_l2addr *mg_l2_eth_getaddr(struct mg_tcpip_if *ifp, uint8_t *frame) {
  struct eth *eth = (struct eth *) frame; 
  (void) ifp; // address field is before a possible 802.1Q tag
  return (struct mg_l2addr *) &eth->src;
}

extern struct mg_l2addr s_mapip;

struct mg_l2addr *mg_l2_eth_mapip(enum mg_l2addrtype addrtype,
                                  struct mg_addr *addr) {
  switch (addrtype) {
    case MG_TCPIP_L2ADDR_BCAST:
      memset(s_mapip.addr.mac, 0xff, sizeof(s_mapip.addr.mac));
      break;
    case MG_TCPIP_L2ADDR_MCAST: {
      uint8_t *ip = (uint8_t *) &addr->addr.ip4;
      // IP multicast group MAC, RFC-1112 6.4
      s_mapip.addr.mac[0] = 0x01, s_mapip.addr.mac[1] = 0x00,
      s_mapip.addr.mac[2] = 0x5E;
      s_mapip.addr.mac[3] = ip[1] & 0x7F;  // 23 LSb
      s_mapip.addr.mac[4] = ip[2];
      s_mapip.addr.mac[5] = ip[3];
      break;
    }
    case MG_TCPIP_L2ADDR_MCAST6: {
      // IPv6 multicast address mapping, RFC-2464 7
      uint8_t *ip = (uint8_t *) &addr->addr.ip6;
      s_mapip.addr.mac[0] = 0x33, s_mapip.addr.mac[1] = 0x33;
      s_mapip.addr.mac[2] = ip[12], s_mapip.addr.mac[3] = ip[13],
      s_mapip.addr.mac[4] = ip[14], s_mapip.addr.mac[5] = ip[15];
      break;
    }
  }
  return &s_mapip;
}

#if MG_ENABLE_IPV6
static void meui64(uint8_t *addr, uint8_t *mac) {
  *addr++ = *mac++ ^ (uint8_t) 0x02, *addr++ = *mac++, *addr++ = *mac++;
  *addr++ = 0xff, *addr++ = 0xfe;
  *addr++ = *mac++, *addr++ = *mac++, *addr = *mac;
}

bool mg_l2_eth_genip6(uint64_t *ip6, uint8_t prefix_len,
                      struct mg_l2addr *l2addr) {
  if (prefix_len > 64) {
    MG_ERROR(("Prefix length > 64, UNSUPPORTED"));
    return false;
  }
  ip6[0] = 0;
  meui64(((uint8_t *) &ip6[1]), l2addr->addr.mac);  // RFC-4291 2.5.4, 2.5.1
  return true;
}

bool mg_l2_eth_ip6get(struct mg_l2addr *l2addr, uint8_t *opts, uint8_t len) {
  if (len != 1) return false;
  memcpy(l2addr->addr.mac, opts, 6);
  return true;
}

uint8_t mg_l2_eth_ip6put(struct mg_l2addr *l2addr, uint8_t *opts) {
  memcpy(opts, l2addr->addr.mac, 6);
  return 1;
}
#endif

#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/l2_ppp.c"
#endif







#if MG_ENABLE_TCPIP

#if defined(__DCC__)
#pragma pack(1)
#else
#pragma pack(push, 1)
#endif
// all in network order

struct ppp {  // RFC-1661
  uint16_t proto;
};

struct lcp {  // RFC-1661
  uint8_t code, id;
  uint16_t len;
};

struct ipcp {  // RFC-1332
  uint8_t code, id;
  uint16_t len;
};

struct ipv6cp {  // RFC-5072
  uint8_t code, id;
  uint16_t len;
};

struct hdlc_ {  // RFC-1662, "PPP in HDLC-like Framing"
  uint8_t addr, ctrl;
};

struct pppoe {  // RFC-2516, "A Method for Transmitting PPP Over Ethernet
                // (PPPoE)"
  uint8_t vertype, code;
  uint16_t id, len;
};

#if defined(__DCC__)
#pragma pack(0)
#else
#pragma pack(pop)
#endif

#define MG_PPP_ADDR 0xff
#define MG_PPP_CTRL 0x03

#define MG_PPP_PROTO_IP 0x0021
#define MG_PPP_PROTO_IPV6 0x0057
#define MG_PPP_PROTO_LCP 0xc021
#define MG_PPP_PROTO_IPCP 0x8021
#define MG_PPP_PROTO_IPV6CP 0x8057
#define MG_PPP_PROTO_PAP 0xc023
#define MG_PPP_PROTO_CHAP 0xc223

#define MG_PPP_LCP_CFG_REQ 1
#define MG_PPP_LCP_CFG_ACK 2
#define MG_PPP_LCP_CFG_NACK 3
#define MG_PPP_LCP_CFG_REJECT 4
#define MG_PPP_LCP_CFG_TERM_REQ 5
#define MG_PPP_LCP_CFG_TERM_ACK 6
#define MG_PPP_LCP_REJECT 8
#define MG_PPP_LCP_ECHO_REQ 9
#define MG_PPP_LCP_ECHO_REPLY 10

#define MG_PPP_IPCP_CFG_REQ 1
#define MG_PPP_IPCP_CFG_ACK 2
#define MG_PPP_IPCP_CFG_NACK 3
#define MG_PPP_IPCP_CFG_REJECT 4
#define MG_PPP_IPCP_OPT_IPADDR 3

#define MG_PPP_IPV6CP_CFG_REQ 1
#define MG_PPP_IPV6CP_CFG_ACK 2
#define MG_PPP_IPV6CP_CFG_NACK 3
#define MG_PPP_IPV6CP_CFG_REJECT 4
#define MG_PPP_IPV6CP_OPT_IFCID 1

#define MG_PPPoE_PADI 0x09
#define MG_PPPoE_PADO 0x07
#define MG_PPPoE_PADR 0x19
#define MG_PPPoE_PADS 0x65
#define MG_PPPoE_PADT 0xa7

#define MG_PPPoE_ST_DISC 0  // Discovery phase, see what servers are out there
#define MG_PPPoE_ST_REQ 1   // Chose a server, request a session and wait
#define MG_PPPoE_ST_SESS 2  // Session established, PPP traffic is exchanged

#define PDIFF(a, b) ((size_t) (((char *) (b)) - ((char *) (a))))

static bool s_lcpup = false;  // ************ THESE SHOULD MOVE TO A struct
                             // mg_l2data *******************************
static uint8_t s_state = MG_PPPoE_ST_DISC;
static uint16_t s_id;

void mg_l2_ppp_init(struct mg_tcpip_if *ifp) {
  ifp->l2mtu = 1500;
  ifp->framesize = ifp->l2mtu + sizeof(struct ppp) + sizeof(struct hdlc_) + 2;
}

extern void mg_l2_eth_init(struct mg_tcpip_if *);

void mg_l2_pppoe_init(struct mg_tcpip_if *ifp) {
  mg_l2_eth_init(ifp);
  ifp->l2mtu = ifp->l2mtu - (uint16_t) (sizeof(struct pppoe) +
                                    sizeof(struct ppp));  // 1500 --> 1492
}

bool mg_l2_ppp_poll(struct mg_tcpip_if *ifp, bool expired_1000ms) {
  if (expired_1000ms && ifp->state == MG_TCPIP_STATE_DOWN) s_lcpup = false;
  return s_lcpup;
}

static uint8_t *hdlc_header(uint8_t *p) {
  struct hdlc_ *hdlc = (struct hdlc_ *) p;
  hdlc->addr = MG_PPP_ADDR;
  hdlc->ctrl = MG_PPP_CTRL;
  return (uint8_t *) (hdlc + 1);
}

static uint8_t *ppp_header(uint16_t proto, uint8_t *p) {
  struct ppp *ppp = (struct ppp *) p;
  ppp->proto = mg_htons(proto);
  return (uint8_t *) (ppp + 1);
}

static uint8_t *l2_ppp_header(enum mg_l2proto proto, uint8_t *p) {
  uint16_t ppp_proto = proto == MG_TCPIP_L2PROTO_IPV4   ? MG_PPP_PROTO_IP
                       : proto == MG_TCPIP_L2PROTO_IPV4 ? MG_PPP_PROTO_IPV6
                                                        : 0;
  return ppp_header(ppp_proto, p);
}

uint8_t *mg_l2_ppp_header(struct mg_tcpip_if *ifp, enum mg_l2proto proto,
                          struct mg_l2addr *src, struct mg_l2addr *dst,
                          uint8_t *frame) {
  (void) ifp;
  (void) src;
  (void) dst;
  return l2_ppp_header(proto, hdlc_header(frame));
}

extern uint8_t *mg_l2_eth_header(struct mg_tcpip_if *ifp, enum mg_l2proto proto,
                                 struct mg_l2addr *src, struct mg_l2addr *dst,
                                 uint8_t *frame);

static uint8_t *pppoe_header(struct mg_tcpip_if *ifp, enum mg_l2proto proto,
                             uint8_t code, uint16_t id, struct mg_l2addr *src,
                             struct mg_l2addr *dst, uint8_t *frame) {
  struct pppoe *pppoe =
      (struct pppoe *) mg_l2_eth_header(ifp, proto, src, dst, frame);
  pppoe->vertype = 0x11;
  pppoe->code = code;
  pppoe->id = id;
  return (uint8_t *) (pppoe + 1);
}

uint8_t *mg_l2_pppoe_header(struct mg_tcpip_if *ifp, enum mg_l2proto proto,
                            struct mg_l2addr *src, struct mg_l2addr *dst,
                            uint8_t *frame) {
  (void) dst;
  return l2_ppp_header(proto, pppoe_header(ifp, MG_TCPIP_L2PROTO_PPPoE_SESS, 0,
                                           s_id, src, dst, frame));
}

size_t mg_l2_ppp_trailer(struct mg_tcpip_if *ifp, size_t len, uint8_t *cur) {
  uint16_t crc;
  uint8_t *frame;
  len += sizeof(struct ppp) + sizeof(struct hdlc_);
  frame = cur - len;
  crc = mg_crc16(0, (const char *) frame, len);
  *cur++ = (uint8_t) crc;  // add CRC, note the byte order
  *cur++ = (uint8_t) (crc >> 8);
  // there is no len field in PPP
  (void) ifp;
  return len + 2;
}

extern size_t mg_l2_eth_trailer(struct mg_tcpip_if *, size_t, uint8_t *);

static size_t pppoe_trailer(struct mg_tcpip_if *ifp, size_t len, uint8_t *cur) {
  struct pppoe *pppoe = (struct pppoe *) (cur - len - sizeof(struct pppoe));
  pppoe->len = mg_htons((uint16_t) len);
  return mg_l2_eth_trailer(ifp, PDIFF(pppoe, cur), cur);
}

size_t mg_l2_pppoe_trailer(struct mg_tcpip_if *ifp, size_t len, uint8_t *cur) {
  return pppoe_trailer(ifp, len + sizeof(struct ppp), cur);
}

size_t mg_l2_driver_output(struct mg_tcpip_if *ifp, size_t len);

static uint8_t *ppp_tx_frame_header(struct mg_tcpip_if *ifp, uint16_t proto) {
  uint8_t *l2p = (uint8_t *) ifp->tx.buf;
  if (ifp->l2type == MG_TCPIP_L2_PPP)
    return ppp_header(proto, hdlc_header(l2p));
  return ppp_header(proto, pppoe_header(ifp, MG_TCPIP_L2PROTO_PPPoE_SESS, 0,
                                        s_id, (struct mg_l2addr *) ifp->mac,
                                        (struct mg_l2addr *) ifp->gwmac, l2p));
}
static size_t ppp_tx_frame_trailer(struct mg_tcpip_if *ifp, size_t len,
                                   uint8_t *cur) {
  return mg_l2_driver_output(ifp, ifp->l2type == MG_TCPIP_L2_PPPoE
                                      ? mg_l2_pppoe_trailer(ifp, len, cur)
                                      : mg_l2_ppp_trailer(ifp, len, cur));
}

// Transmit a single PPP frame for the given protocol
static size_t ppp_tx_frame(struct mg_tcpip_if *ifp, uint16_t proto,
                           uint8_t *data, size_t datasz) {
  uint8_t *pay = ppp_tx_frame_header(ifp, proto);
  memcpy(pay, data, datasz);
  return ppp_tx_frame_trailer(ifp, datasz, pay + datasz);
}

static void ppp_handle_lcp(struct mg_tcpip_if *ifp, uint8_t *lcpp,
                           size_t lcpsz) {
  uint8_t id;
  uint16_t len;
  struct lcp *lcp = (struct lcp *) lcpp;
  if (lcpsz < sizeof(*lcp)) return;
  id = lcp->id;
  len = mg_ntohs(lcp->len);
  if (len < sizeof(*lcp) || len > lcpsz) return;
  switch (lcp->code) {
    case MG_PPP_LCP_CFG_REQ: {
      if (len == sizeof(*lcp)) {
        MG_DEBUG(("LCP config request of %d bytes, acknowledging...", len));
        lcp->code = MG_PPP_LCP_CFG_ACK;
        ppp_tx_frame(ifp, MG_PPP_PROTO_LCP, lcpp, len);
        lcp->code = MG_PPP_LCP_CFG_REQ;
        ppp_tx_frame(ifp, MG_PPP_PROTO_LCP, lcpp, len);
      } else {
        MG_DEBUG(("LCP config request of %d bytes, rejecting...", len));
        lcp->code = MG_PPP_LCP_CFG_REJECT;
        ppp_tx_frame(ifp, MG_PPP_PROTO_LCP, lcpp, len);
      }
    } break;
    case MG_PPP_LCP_CFG_ACK:
      s_lcpup = true;
      break;
    case MG_PPP_LCP_CFG_TERM_REQ: {
      uint8_t ack[4] = {MG_PPP_LCP_CFG_TERM_ACK, id, 0, 4};
      MG_DEBUG(("LCP termination request, acknowledging..."));
      ppp_tx_frame(ifp, MG_PPP_PROTO_LCP, ack, sizeof(ack));
      s_lcpup = false;
    } break;
    case MG_PPP_LCP_ECHO_REQ:  // RFC-1661 5.8: must respond
      MG_DEBUG(("LCP echo request of %d bytes, replying...", len));
      lcp->code = MG_PPP_LCP_ECHO_REPLY;
      ppp_tx_frame(ifp, MG_PPP_PROTO_LCP, lcpp, len);
      break;
  }
}

static bool find_opt(const uint8_t opt, const uint8_t optlen,
                     const uint8_t *opts, size_t optslen, uint8_t *dest) {
  uint8_t *p = (uint8_t *) opts;
  while (optslen >= 2) {  // parse options for requested one
    if (p[1] > optslen || p[1] < 2) return false;  // truncated / malformed
    if (p[0] == opt && p[1] == optlen) {
      memcpy(dest, p + 2, optlen - 2);
      return true;
    }
    optslen -= p[1];
    p += p[1];
  }
  return false;
}

static void ppp_handle_ipcp(struct mg_tcpip_if *ifp, uint8_t *ipcpp,
                            size_t ipcpsz) {
  uint16_t len;
  uint8_t id;
  struct ipcp *ipcp = (struct ipcp *) ipcpp;
  uint8_t req[] = {
      MG_PPP_IPCP_CFG_REQ, 0, 0, 10, MG_PPP_IPCP_OPT_IPADDR, 6, 0, 0, 0, 0};
  if (ipcpsz < sizeof(*ipcp)) return;
  id = ipcp->id;
  len = mg_ntohs(ipcp->len);
  if (len < sizeof(*ipcp) || len > ipcpsz) return;
  switch (ipcp->code) {
    case MG_PPP_IPCP_CFG_REQ:
      MG_VERBOSE(("got IPCP config request, acknowledging..."));
      if (len >= 10 &&
          find_opt(MG_PPP_IPCP_OPT_IPADDR, 6, (const uint8_t *) (ipcp + 1),
                   len - sizeof(*ipcp), (uint8_t *) &ifp->gw)) {
        MG_DEBUG(("IPCP cfg, GW IP: %M", mg_print_ip4, &ifp->gw));
        ipcp->code = MG_PPP_IPCP_CFG_ACK;
      } else if (ifp->gw == 0) {
        MG_ERROR(("Peer did not provide its IP address"));
        // NOTE: We should NACK with an added option, probably we can just store
        // the incoming address and offer a statically configured ifp->gw when
        // theirs == 0, but it is unlikely to find such a dumb PPP server and it
        // will complicate our config and this protocol state machine
        ipcp->code = MG_PPP_IPCP_CFG_REJECT;
        ppp_tx_frame(ifp, MG_PPP_PROTO_IPCP, ipcpp, len);
      }
      ppp_tx_frame(ifp, MG_PPP_PROTO_IPCP, ipcpp, len);
      req[1] = id;
      memcpy(req + 6, &ifp->ip, 4);  // Request config IP address or 0.0.0.0
      ppp_tx_frame(ifp, MG_PPP_PROTO_IPCP, req, sizeof(req));
      break;
    case MG_PPP_IPCP_CFG_ACK:
      // Our peer accepted our IP address
      MG_VERBOSE(("got IPCP config ack"));
      ifp->mask = 0xffffffff;  // send to gw
      ifp->state = MG_TCPIP_STATE_IP;
      ifp->gw_ready = true;
      break;
    case MG_PPP_IPCP_CFG_NACK:
      MG_VERBOSE(("got IPCP config nack"));
      // NACK contains our "suggested" IP address, use it
      if (len >= 10 &&
          find_opt(MG_PPP_IPCP_OPT_IPADDR, 6, (const uint8_t *) (ipcp + 1),
                   len - sizeof(*ipcp), (uint8_t *) &ifp->ip)) {
        MG_DEBUG(("IPCP cfg, IP: %M", mg_print_ip4, &ifp->ip));
        ipcp->code = MG_PPP_IPCP_CFG_REQ;
        ppp_tx_frame(ifp, MG_PPP_PROTO_IPCP, ipcpp, len);
      }
      break;
    case MG_PPP_IPCP_CFG_REJECT:
      MG_ERROR(("Peer rejected our IP address, need to properly set ifp->ip"));
      break;
  }
}

#if MG_ENABLE_IPV6
static void ppp_handle_ipv6cp(struct mg_tcpip_if *ifp, uint8_t *ipv6cpp,
                              size_t ipv6cpsz) {
  uint16_t len;
  uint8_t id;
  struct ipv6cp *ipv6cp = (struct ipv6cp *) ipv6cpp;
  uint8_t req[14];
  memset(req, 0, sizeof(req));
  req[0] = MG_PPP_IPV6CP_CFG_REQ, req[3] = 14, req[4] = MG_PPP_IPV6CP_OPT_IFCID,
  req[5] = 10;
  if (ipv6cpsz < sizeof(*ipv6cp)) return;
  id = ipv6cp->id;
  len = mg_ntohs(ipv6cp->len);
  if (len < sizeof(*ipv6cp) || len > ipv6cpsz) return;
  switch (ipv6cp->code) {
    case MG_PPP_IPV6CP_CFG_REQ:
      MG_VERBOSE(("got IPV6CP config request..."));
      if (len >= 10 &&
          find_opt(
              MG_PPP_IPV6CP_OPT_IFCID, 10, (const uint8_t *) (ipv6cp + 1),
              len - sizeof(*ipv6cp),
              (uint8_t *) &((struct mg_l2addr *) (ifp->gw6mac))->addr.ieee64)) {
        if (((struct mg_l2addr *) (ifp->gw6mac))->addr.ieee64 != 0) {
          MG_DEBUG(("IPV6CP cfg, GW IFCID: %M", mg_print_ieee64,
                    &((struct mg_l2addr *) (ifp->gw6mac))->addr.ieee64));
          ipv6cp->code = MG_PPP_IPV6CP_CFG_ACK;
          ppp_tx_frame(ifp, MG_PPP_PROTO_IPV6CP, ipv6cpp, len);
          req[1] = id;
          ifp->ip6ll[0] = 0, ifp->ip6ll[1] = 0;  // clear any former ll address
          memset(req + 6, 0, 8);                 // Inform ifc id 0
          ppp_tx_frame(ifp, MG_PPP_PROTO_IPV6CP, req, sizeof(req));
        } else {
          MG_ERROR(("Peer is not able to provide its interface id"));
          ipv6cp->code = MG_PPP_IPV6CP_CFG_REJECT;
          ppp_tx_frame(ifp, MG_PPP_PROTO_IPV6CP, ipv6cpp, len);
        }
      } else {
        MG_ERROR(("Peer did not provide its interface id"));
        // We should NACK with an added option, but we can't provide one for
        // them
        ipv6cp->code = MG_PPP_IPV6CP_CFG_REJECT;
        ppp_tx_frame(ifp, MG_PPP_PROTO_IPV6CP, ipv6cpp, len);
      }
      break;
    case MG_PPP_IPV6CP_CFG_ACK:
      // Our peer accepted our ifc id
      MG_VERBOSE(("got IPV6CP config ack"));
      break;
    case MG_PPP_IPV6CP_CFG_NACK: {
      struct mg_l2addr l2;
      MG_VERBOSE(("got IPV6CP config nack"));
      // NACK contains our "suggested" IFC id, use it
      if (len >= 10 &&
          find_opt(MG_PPP_IPV6CP_OPT_IFCID, 10, (const uint8_t *) (ipv6cp + 1),
                   len - sizeof(*ipv6cp), (uint8_t *) &l2.addr.ieee64)) {
        MG_DEBUG(("IPV6CP cfg, IFCID: %M", mg_print_ieee64, &l2.addr.ieee64));
        ifp->ip6ll[1] = l2.addr.ieee64;  // RFC-5072 5, signal L3 we're ready
        ipv6cp->code = MG_PPP_IPV6CP_CFG_REQ;
        ppp_tx_frame(ifp, MG_PPP_PROTO_IPV6CP, ipv6cpp, len);
      } else {
        MG_ERROR(("Peer is not able to offer an interface id"));
      }
    } break;
    case MG_PPP_IPV6CP_CFG_REJECT:
      MG_ERROR(("Peer rejected our interface id"));
      break;
  }
}
#endif

static bool ppp_rx(struct mg_tcpip_if *ifp, enum mg_l2proto *proto,
                   struct mg_str *pay, struct mg_str *raw) {
  struct ppp *ppp = (struct ppp *) pay->buf;
  if (pay->len < sizeof(*ppp)) return false;  // Truncated
  pay->buf = (char *) (ppp + 1);
  pay->len = pay->len - sizeof(*ppp);
  switch (mg_ntohs(ppp->proto)) {
    case MG_PPP_PROTO_LCP:
      ppp_handle_lcp(ifp, (uint8_t *) pay->buf, pay->len);
      return false;
    case MG_PPP_PROTO_IPCP:
      if (s_lcpup) ppp_handle_ipcp(ifp, (uint8_t *) pay->buf, pay->len);
      return false;
    case MG_PPP_PROTO_IP:
      if (!s_lcpup) return false;
      MG_VERBOSE(("got IP packet of %d bytes", pay->len));
      *proto = MG_TCPIP_L2PROTO_IPV4;
      break;
#if MG_ENABLE_IPV6
    case MG_PPP_PROTO_IPV6CP:
      if (s_lcpup) ppp_handle_ipv6cp(ifp, (uint8_t *) pay->buf, pay->len);
      return false;
    case MG_PPP_PROTO_IPV6:
      if (!s_lcpup) return false;
      MG_VERBOSE(("got IPv6 packet of %d bytes", pay->len));
      *proto = MG_TCPIP_L2PROTO_IPV6;
      break;
#endif
    default: {
      struct lcp rej;
      uint8_t *p;
      size_t msglen;
      MG_DEBUG(("unknown %u-byte PPP frame with proto 0x%04x:",
                pay->len + sizeof(*ppp), mg_ntohs(ppp->proto)));
      if (mg_log_level >= MG_LL_DEBUG)
        mg_hexdump(ppp, pay->len > 14 ? 16 : pay->len + sizeof(*ppp));
      if (!s_lcpup) return false;  // RFC-1661 5.7: must reject on link down
      if (pay->len > (size_t) (ifp->mtu - 20))
        pay->len = (size_t) (ifp->mtu - 20);  // truncate to some safe limit
      rej.code = MG_PPP_LCP_REJECT;
      mg_random(&rej.id, sizeof(rej.id));
      msglen = pay->len + sizeof(*ppp);
      rej.len = mg_htons((uint16_t) (msglen + sizeof(rej)));
      p = ppp_tx_frame_header(ifp, MG_PPP_PROTO_LCP);
      memmove(p, &rej, sizeof(rej));          // LCP reject
      memmove(p + sizeof(rej), ppp, msglen);  // rejected PPP message
      ppp_tx_frame_trailer(ifp, msglen + sizeof(rej), p + msglen + sizeof(rej));
    }
      return false;
  }
  (void) raw;
  return true;
}

bool mg_l2_ppp_rx(struct mg_tcpip_if *ifp, enum mg_l2proto *proto,
                  struct mg_str *pay, struct mg_str *raw) {
  struct hdlc_ *hdlc = (struct hdlc_ *) raw->buf;
  if (raw->len < sizeof(*hdlc) + 2) return false;  // Truncated
  if (hdlc->addr == MG_PPP_ADDR && hdlc->ctrl == MG_PPP_CTRL) {
    pay->buf = (char *) (hdlc + 1);
    pay->len = raw->len - sizeof(*hdlc) - 2;
  } else {  // Address-and-Control-Field-Compressed PPP header
    pay->buf = (char *) raw->buf;
    pay->len = raw->len - 2;
  }
  if (pay->len > ifp->l2mtu + sizeof(struct ppp)) return false;  // Oversized
  return ppp_rx(ifp, proto, pay, raw);
}

extern struct mg_l2addr s_mapip;

struct mg_l2addr *mg_l2_ppp_getaddr(struct mg_tcpip_if *ifp, uint8_t *frame) {
  (void) ifp;
  (void) frame;
  return &s_mapip;  // bogus
}

extern struct mg_l2addr *mg_l2_eth_mapip(enum mg_l2addrtype, struct mg_addr *);

struct mg_l2addr *mg_l2_ppp_mapip(enum mg_l2addrtype addrtype,
                                  struct mg_addr *addr) {
  return mg_l2_eth_mapip(addrtype, addr);
}

#if MG_ENABLE_IPV6
bool mg_l2_ppp_genip6(uint64_t *ip6, uint8_t prefix_len,
                      struct mg_l2addr *l2addr) {
  if (prefix_len > 64) {
    MG_ERROR(("Prefix length > 64, UNSUPPORTED"));
    return false;
  }
  ip6[0] = 0;
  ip6[1] = l2addr->addr.ieee64;
  return false;
}

bool mg_l2_ppp_ip6get(struct mg_l2addr *l2addr, uint8_t *opts, uint8_t len) {
  (void) l2addr;
  (void) opts;
  (void) len;
  return true;
}

uint8_t mg_l2_ppp_ip6put(struct mg_l2addr *l2addr, uint8_t *opts) {
  (void) l2addr;
  (void) opts;
  return 0;
}
#endif

// Transmit a single PPPoE frame for the discovery phase
static size_t pppoe_tx_frame(struct mg_tcpip_if *ifp, uint8_t code, uint16_t id,
                             uint8_t *data, size_t datasz,
                             struct mg_l2addr *dst) {
  uint8_t *l2p = (uint8_t *) ifp->tx.buf;
  uint8_t *p = pppoe_header(ifp, MG_TCPIP_L2PROTO_PPPoE_DISC, code, id,
                            (struct mg_l2addr *) ifp->mac, dst, l2p);
  memmove(p, data, datasz);
  return mg_l2_driver_output(ifp, pppoe_trailer(ifp, datasz, p + datasz));
}

bool mg_l2_pppoe_poll(struct mg_tcpip_if *ifp, bool expired_1000ms) {
  if (expired_1000ms && s_state == MG_PPPoE_ST_DISC &&
      ifp->state == MG_TCPIP_STATE_LINK_UP) {
    uint16_t tags[2];
    tags[0] = mg_htons(0x0101);  // Service Request
    tags[1] = mg_htons(0x0000);  // Any
    pppoe_tx_frame(ifp, MG_PPPoE_PADI, 0, (uint8_t *) tags, sizeof(tags),
                   mg_l2_eth_mapip(MG_TCPIP_L2ADDR_BCAST, NULL));
    MG_DEBUG(("Sent PADI"));
  } else if (expired_1000ms && (s_state != MG_PPPoE_ST_SESS ||
                                ifp->state == MG_TCPIP_STATE_DOWN)) {
    s_state = MG_PPPoE_ST_DISC;
  }
  return mg_l2_ppp_poll(ifp, expired_1000ms);
}

extern bool mg_l2_eth_rx(struct mg_tcpip_if *ifp, enum mg_l2proto *proto,
                         struct mg_str *pay, struct mg_str *raw);
extern struct mg_l2addr *mg_l2_eth_getaddr(struct mg_tcpip_if *, uint8_t *);

bool mg_l2_pppoe_rx(struct mg_tcpip_if *ifp, enum mg_l2proto *proto,
                    struct mg_str *pay, struct mg_str *raw) {
  enum mg_l2proto eth_proto;  // raw is handled by eth
  struct pppoe *pppoe;
  if (!mg_l2_eth_rx(ifp, &eth_proto, pay, raw)) return false;
  pppoe = (struct pppoe *) pay->buf;            // here we handle pay, not raw
  if (pay->len < sizeof(*pppoe)) return false;  // Truncated
  if (pay->len - sizeof(*pppoe) > ifp->l2mtu + sizeof(struct ppp))
    return false;  // Oversized
  if (eth_proto == MG_TCPIP_L2PROTO_PPPoE_DISC) {
    MG_VERBOSE(("PPPoE_DISC"));
    if (s_state == MG_PPPoE_ST_DISC && pppoe->code == MG_PPPoE_PADO &&
        pppoe->id == 0) {
      uint16_t tags[2];
      bool has_cookie = false;
      size_t len = pay->len - sizeof(*pppoe);
      uint8_t *p = (uint8_t *) (pppoe + 1);
      uint16_t taglen;
      while (len >= 4) {  // parse tags for a possible AC-Cookie
        uint16_t curtag = MG_LOAD_BE16(p);
        taglen = MG_LOAD_BE16(p + 2);
        if (taglen > len - 4) return false;  // truncated / malformed
        if (curtag == 0x0104) {
          has_cookie = true;
          break;
        }
        len -= 4 + taglen;
        p += 4 + taglen;
      }
      tags[0] = mg_htons(0x0101);  // Service Request
      tags[1] = mg_htons(0x0000);  // Any
      if (has_cookie) {  // copy tags to before AC-Cookie tag in rx buffer
        memcpy(p -= sizeof(tags), tags, sizeof(tags));  // point to all tags
        taglen += 4 + sizeof(tags);                     // account for tags
      } else {
        p = (uint8_t *) tags;
        taglen = sizeof(tags);
      }
      pppoe_tx_frame(ifp, MG_PPPoE_PADR, 0, p, taglen,
                     mg_l2_eth_getaddr(ifp, (uint8_t *) raw->buf));
      MG_DEBUG(("Sent PADR"));
      s_state = MG_PPPoE_ST_REQ;
    } else if (s_state == MG_PPPoE_ST_REQ && pppoe->code == MG_PPPoE_PADS) {
      s_id = pppoe->id;
      memcpy(&ifp->gwmac,
             mg_l2_eth_getaddr(ifp, (uint8_t *) raw->buf)->addr.mac, 6);
      MG_DEBUG(("PPPoE session 0x%04x started", mg_ntohs(s_id)));
      s_state = MG_PPPoE_ST_SESS;
    } else if (s_state == MG_PPPoE_ST_SESS && pppoe->code == MG_PPPoE_PADT &&
               pppoe->id == s_id) {
      MG_ERROR(("Got PADT"));
      s_id = 0;
      s_lcpup = false;
      s_state = MG_PPPoE_ST_DISC;
    }
  } else if (eth_proto == MG_TCPIP_L2PROTO_PPPoE_SESS &&
             s_state == MG_PPPoE_ST_SESS) {
    pay->buf = (char *) (pppoe + 1);
    pay->len = pay->len - sizeof(*pppoe);
    return ppp_rx(ifp, proto, pay, raw);
  }
  return false;
}

#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/lfs.c"
#endif
// Copyright (c) 2023 Cesanta Software Limited
// SPDX-License-Identifier: GPL-2.0-only or commercial





#if MG_ENABLE_LFS

#ifndef LFS_USE_RAM
#define LFS_USE_RAM 0  // If 1, use RAM FS. If 0, use flash
#endif

#ifndef MG_LFS_SIZE
#define MG_LFS_SIZE (64 * 1024)
#endif

#ifndef MG_LFS_BUF_SIZE
#define MG_LFS_BUF_SIZE 64  // Buffer size used for reads, writes, and cache
#endif

#ifndef MG_LFS_FD_BASE
#define MG_LFS_FD_BASE 12000  // Starting file descriptor number
#endif

#if LFS_USE_RAM && !defined(MG_LFS_BLOCK_SIZE)
#define MG_LFS_BLOCK_SIZE 8192
#endif

struct mg_lfs_fd {
  struct mg_lfs_fd *next;
  bool isopen;
  lfs_file_t file;
  lfs_dir_t dir;
  int fd;
};

static lfs_t s_lfs;
static struct mg_lfs_fd *s_fds;
static uint8_t *s_fs;
static bool s_lfs_ready;
#if LFS_USE_RAM
static uint8_t *s_ram_fs;
#endif

static int lfs_driver_read(const struct lfs_config *cfg, lfs_block_t block,
                           lfs_off_t off, void *buf, lfs_size_t len);
static int lfs_driver_prog(const struct lfs_config *cfg, lfs_block_t block,
                           lfs_off_t off, const void *buf, lfs_size_t len);
static int lfs_driver_erase(const struct lfs_config *cfg, lfs_block_t block);
static int lfs_driver_sync(const struct lfs_config *cfg);

static struct lfs_config s_cfg = {
    .read = lfs_driver_read,
    .prog = lfs_driver_prog,
    .erase = lfs_driver_erase,
    .sync = lfs_driver_sync,
    .block_cycles = 200,
    .cache_size = MG_LFS_BUF_SIZE,
    .read_size = MG_LFS_BUF_SIZE,
    .prog_size = MG_LFS_BUF_SIZE,
    .lookahead_size = MG_LFS_BUF_SIZE / 8,
};

static bool flash_write_buf(void *addr, const void *buf, size_t len) {
#if LFS_USE_RAM
  memmove(addr, buf, len);
  return true;
#else
  return mg_flash != NULL && mg_flash->write_fn(addr, buf, len);
#endif
}

static int lfs_driver_read(const struct lfs_config *cfg, lfs_block_t block,
                           lfs_off_t off, void *buf, lfs_size_t len) {
  memmove(buf, &s_fs[block * cfg->block_size + off], len);
  return 0;
}

static int lfs_driver_prog(const struct lfs_config *cfg, lfs_block_t block,
                           lfs_off_t off, const void *buf, lfs_size_t len) {
  uint8_t *dst = &s_fs[block * cfg->block_size + off];
  if (!flash_write_buf(dst, buf, len)) return LFS_ERR_IO;
  return 0;
}

static int lfs_driver_erase(const struct lfs_config *cfg, lfs_block_t block) {
  (void) cfg, (void) block;
  return 0;
}

static int lfs_driver_sync(const struct lfs_config *cfg) {
  (void) cfg;
  return 0;
}

bool mg_lfs_init(size_t size) {
  //bool ok = false;
  MG_DEBUG(("size: %zu", size));
  if (s_lfs_ready) return true;
  if (size == 0) {
    MG_ERROR(("invalidsize = 0"));
  } else {
#if LFS_USE_RAM
    s_cfg.block_size = MG_LFS_BLOCK_SIZE;
    if (s_ram_fs == NULL) s_ram_fs = (uint8_t *) mg_calloc(1, size);
    s_fs = s_ram_fs;
#else
    {
      // Some flash attributes may be initialised on ota_begin
      int lev = mg_log_level;
      mg_log_level = MG_LL_NONE;
      mg_ota_begin(1);
      mg_ota_end();
      mg_log_set(lev);
    }
    if (mg_flash == NULL || mg_flash->secsz == 0 || mg_flash->size < size) {
      MG_ERROR(("mg_flash %p secsz %zu flash_size %zu < size %zu", mg_flash, mg_flash->secsz, mg_flash->size, size));
    } else if (mg_flash->write_fn == NULL) {
      MG_ERROR(("flash write_fn == NULL"));
    } else {
      s_cfg.block_size = mg_flash->secsz;
      s_fs = (uint8_t *) mg_flash->start + mg_flash->size - size;
    }
#endif
    if (s_fs == NULL || s_cfg.block_size == 0 || size % s_cfg.block_size != 0) {
      MG_ERROR(("s_fs %p blk_size %zu size %zu", s_fs, s_cfg.block_size));
    } else {
      s_cfg.block_count = (lfs_size_t) (size / s_cfg.block_size);
      if (s_cfg.block_count == 0) {
        MG_ERROR(("blk_count == 0"));
      } else {
        if (lfs_mount(&s_lfs, &s_cfg) != 0) lfs_format(&s_lfs, &s_cfg);
        if (lfs_mount(&s_lfs, &s_cfg) != 0) {
          MG_ERROR(("lfs_mount"));
        } else {
          s_lfs_ready = true;
        }
      }
    }
  }
  return s_lfs_ready;
}

static struct mg_lfs_fd *find_fd(int fd) {
  struct mg_lfs_fd *f;
  for (f = s_fds; f != NULL; f = f->next) {
    if (f->isopen && f->fd == fd) return f;
  }
  return NULL;
}

static struct mg_lfs_fd *open_fd(void) {
  struct mg_lfs_fd *f = NULL, *p = s_fds;
  if (s_lfs_ready == false) mg_lfs_init(MG_LFS_SIZE);
  if (s_lfs_ready && (f = (struct mg_lfs_fd *) mg_calloc(1, sizeof(*f)))) {
    f->isopen = true, f->fd = MG_LFS_FD_BASE, f->next = s_fds;
    while (p != NULL) p = p->fd == f->fd ? (f->fd++, s_fds) : p->next;
    s_fds = f;
  }
  return f;
}

static int close_fd(struct mg_lfs_fd *fd) {
  struct mg_lfs_fd **f = &s_fds;
  while (*f != NULL && *f != fd) f = &(*f)->next;
  if (*f == NULL) return -1;
  *f = fd->next;
  fd->isopen = false;
  mg_free(fd);
  return 0;
}

int _open(const char *path, int flags, mode_t mode) {
  int err, lfs_flags = 0, fd = -1;
  struct mg_lfs_fd *f = open_fd();
  (void) mode;
  MG_DEBUG(("%s -> %p, lfs_ready %d", path, f, s_lfs_ready));
  if (f == NULL) return -1;
  fd = f->fd;
  if ((flags & 3) == O_RDONLY) lfs_flags |= LFS_O_RDONLY;
  if ((flags & 3) == O_WRONLY) lfs_flags |= LFS_O_WRONLY;
  if ((flags & 3) == O_RDWR) lfs_flags |= LFS_O_RDWR;
  if (flags & O_CREAT) lfs_flags |= LFS_O_CREAT;
  if (flags & O_TRUNC) lfs_flags |= LFS_O_TRUNC;
  if (flags & O_APPEND) lfs_flags |= LFS_O_APPEND;
  err = lfs_file_open(&s_lfs, &f->file, path, lfs_flags);
  if (err < 0) close_fd(f), fd = -1;
  return fd;
}

int _close(int fd) {
  struct mg_lfs_fd *f = find_fd(fd);
  if (fd < 3) return 0;
  if (f == NULL) return -1;
  lfs_file_close(&s_lfs, &f->file);
  close_fd(f);
  return 0;
}

int _write(int fd, char *ptr, int len) {
  struct mg_lfs_fd *f = find_fd(fd);
  return fd < 3 ? len
                : f == NULL ? -1 : lfs_file_write(&s_lfs, &f->file, ptr, len);
}

int _read(int fd, char *ptr, int len) {
  struct mg_lfs_fd *f = find_fd(fd);
  return fd < 3 ? 0 : f == NULL ? -1 : lfs_file_read(&s_lfs, &f->file, ptr, len);
}

int _lseek(int fd, int offset, int whence) {
  struct mg_lfs_fd *f = find_fd(fd);
  return fd < 3 ? 0
                : f == NULL ? -1 : lfs_file_seek(&s_lfs, &f->file, offset,
                                                  whence);
}

int _rename(const char *oldname, const char *newname) {
  if (s_lfs_ready == false) mg_lfs_init(MG_LFS_SIZE);
  return s_lfs_ready ? lfs_rename(&s_lfs, oldname, newname) : -1;
}

// Some libc rename() implementations have no atomic rename syscall and emulate
// it as link() + unlink() instead of calling _rename() directly. LittleFS has
// no real hard links, but forwarding to _rename() gives the same end result
// (oldname becomes newname), which is all mg_file_write()'s atomic-write
// pattern (write to a temp file, then rename it into place) needs
int _link(const char *oldname, const char *newname) {
  return _rename(oldname, newname);
}

int _unlink(const char *a) {
  if (s_lfs_ready == false) mg_lfs_init(MG_LFS_SIZE);
  return s_lfs_ready ? lfs_remove(&s_lfs, a) : -1;
}

DIR *opendir(const char *name) {
  struct mg_lfs_fd *f = open_fd();
  if (f == NULL) return NULL;
  if (lfs_dir_open(&s_lfs, &f->dir, name) != 0) {
    close_fd(f);
    f = NULL;
  }
  return (DIR *) f;
}

int closedir(DIR *dir) {
  struct mg_lfs_fd *f = (struct mg_lfs_fd *) dir;
  if (f == NULL || find_fd(f->fd) != f) return -1;
  lfs_dir_close(&s_lfs, &f->dir);
  return close_fd(f);
}

struct dirent *readdir(DIR *dir) {
  static struct dirent dirent;
  struct mg_lfs_fd *f = (struct mg_lfs_fd *) dir;
  struct lfs_info info;
  if (f == NULL || !f->isopen) return NULL;
  if (lfs_dir_read(&s_lfs, &f->dir, &info) < 1) return NULL;
  memset(&dirent, 0, sizeof(dirent));
  strncpy(dirent.d_name, info.name, sizeof(dirent.d_name) - 1);
  if (info.type == LFS_TYPE_DIR) dirent.d_type |= DT_DIR;
  if (info.type == LFS_TYPE_REG) dirent.d_type |= DT_REG;
  return &dirent;
}

int _stat(const char *path, struct stat *st) {
  struct lfs_info info;
  if (s_lfs_ready == false) mg_lfs_init(MG_LFS_SIZE);
  if (!s_lfs_ready || lfs_stat(&s_lfs, path, &info) != 0) return -1;
  memset(st, 0, sizeof(*st));
  st->st_mode = info.type == LFS_TYPE_DIR ? S_IFDIR : S_IFREG;
  st->st_size = info.size;
  return 0;
}

int _fstat(int fd, struct stat *st) {
  if (fd >= 3 && find_fd(fd) == NULL) return -1;
  st->st_mode = S_IFCHR;
  return 0;
}

int mkdir(const char *path, mode_t mode) {
  (void) mode;
  if (s_lfs_ready == false) mg_lfs_init(MG_LFS_SIZE);
  return s_lfs_ready ? lfs_mkdir(&s_lfs, path) : -1;
}
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/log.c"
#endif





int mg_log_level = MG_LL_DEBUG;
static mg_pfn_t s_log_func = mg_pfn_stdout;
static void *s_log_func_param = NULL;

void mg_log_set_fn(mg_pfn_t fn, void *param) {
  s_log_func = fn;
  s_log_func_param = param;
}

static void logc(unsigned char c) {
  s_log_func((char) c, s_log_func_param);
}

static void logs(const char *buf, size_t len) {
  size_t i;
  for (i = 0; i < len; i++) logc(((unsigned char *) buf)[i]);
}

#if MG_ENABLE_CUSTOM_LOG
// Let user define their own mg_log_prefix() and mg_log()
#else
void mg_log_prefix(int level, const char *file, int line, const char *fname) {
  const char *p = strrchr(file, '/');
  char buf[41];
  size_t n;
  if (p == NULL) p = strrchr(file, '\\');
  n = mg_snprintf(buf, sizeof(buf), "%-6llx %d %s:%d:%s", mg_millis(), level,
                  p == NULL ? file : p + 1, line, fname);
  if (n > sizeof(buf) - 2) n = sizeof(buf) - 2;
  while (n < sizeof(buf)) buf[n++] = ' ';
  logs(buf, n - 1);
}

void mg_log(const char *fmt, ...) {
  va_list ap;
  va_start(ap, fmt);
  mg_vxprintf(s_log_func, s_log_func_param, fmt, &ap);
  va_end(ap);
  logs("\r\n", 2);
}
#endif

static unsigned char nibble(unsigned c) {
  return (unsigned char) (c < 10 ? c + '0' : c + 'W');
}

#define ISPRINT(x) ((x) >= ' ' && (x) <= '~')
void mg_hexdump(const void *buf, size_t len) {
  const unsigned char *p = (const unsigned char *) buf;
  unsigned char ascii[16], alen = 0;
  size_t i;
  for (i = 0; i < len; i++) {
    if ((i % 16) == 0) {
      // Print buffered ascii chars
      if (i > 0)
        logs("  ", 2), logs((char *) ascii, 16), logs("\r\n", 2), alen = 0;
      // Print hex address, then \t
      logc(nibble((i >> 12) & 15)), logc(nibble((i >> 8) & 15)),
          logc(nibble((i >> 4) & 15)), logc('0'), logs("   ", 3);
    }
    logc(nibble(p[i] >> 4)), logc(nibble(p[i] & 15));  // Two nibbles, e.g. c5
    logc(' ');                                         // Space after hex number
    ascii[alen++] = ISPRINT(p[i]) ? p[i] : '.';        // Add to the ascii buf
  }
  while (alen < 16) logs("   ", 3), ascii[alen++] = ' ';
  logs("  ", 2), logs((char *) ascii, 16), logs("\r\n", 2);
}

#ifdef MG_ENABLE_LINES
#line 1 "src/md5.c"
#endif



//  This code implements the MD5 message-digest algorithm.
//  The algorithm is due to Ron Rivest.  This code was
//  written by Colin Plumb in 1993, no copyright is claimed.
//  This code is in the public domain; do with it what you wish.
//
//  Equivalent code is available from RSA Data Security, Inc.
//  This code has been tested against that, and is equivalent,
//  except that you don't need to include two pages of legalese
//  with every copy.
//
//  To compute the message digest of a chunk of bytes, declare an
//  MD5Context structure, pass it to MD5Init, call MD5Update as
//  needed on buffers full of bytes, and then call MD5Final, which
//  will fill a supplied 16-byte array with the digest.

#if defined(MG_ENABLE_MD5) && MG_ENABLE_MD5

static void mg_byte_reverse(unsigned char *buf, unsigned longs) {
  if (MG_BIG_ENDIAN) {
    do {
      uint32_t t = (uint32_t) ((unsigned) buf[3] << 8 | buf[2]) << 16 |
                   ((unsigned) buf[1] << 8 | buf[0]);
      *(uint32_t *) buf = t;
      buf += 4;
    } while (--longs);
  } else {
    (void) buf, (void) longs;  // Little endian. Do nothing
  }
}

#define F1(x, y, z) (z ^ (x & (y ^ z)))
#define F2(x, y, z) F1(z, x, y)
#define F3(x, y, z) (x ^ y ^ z)
#define F4(x, y, z) (y ^ (x | ~z))

#define MD5STEP(f, w, x, y, z, data, s) \
  (w += f(x, y, z) + data, w = w << s | w >> (32 - s), w += x)

/*
 * Start MD5 accumulation.  Set bit count to 0 and buffer to mysterious
 * initialization constants.
 */
void mg_md5_init(mg_md5_ctx *ctx) {
  ctx->buf[0] = 0x67452301;
  ctx->buf[1] = 0xefcdab89;
  ctx->buf[2] = 0x98badcfe;
  ctx->buf[3] = 0x10325476;

  ctx->bits[0] = 0;
  ctx->bits[1] = 0;
}

static void mg_md5_transform(uint32_t buf[4], uint32_t const in[16]) {
  uint32_t a, b, c, d;

  a = buf[0];
  b = buf[1];
  c = buf[2];
  d = buf[3];

  MD5STEP(F1, a, b, c, d, in[0] + 0xd76aa478, 7);
  MD5STEP(F1, d, a, b, c, in[1] + 0xe8c7b756, 12);
  MD5STEP(F1, c, d, a, b, in[2] + 0x242070db, 17);
  MD5STEP(F1, b, c, d, a, in[3] + 0xc1bdceee, 22);
  MD5STEP(F1, a, b, c, d, in[4] + 0xf57c0faf, 7);
  MD5STEP(F1, d, a, b, c, in[5] + 0x4787c62a, 12);
  MD5STEP(F1, c, d, a, b, in[6] + 0xa8304613, 17);
  MD5STEP(F1, b, c, d, a, in[7] + 0xfd469501, 22);
  MD5STEP(F1, a, b, c, d, in[8] + 0x698098d8, 7);
  MD5STEP(F1, d, a, b, c, in[9] + 0x8b44f7af, 12);
  MD5STEP(F1, c, d, a, b, in[10] + 0xffff5bb1, 17);
  MD5STEP(F1, b, c, d, a, in[11] + 0x895cd7be, 22);
  MD5STEP(F1, a, b, c, d, in[12] + 0x6b901122, 7);
  MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12);
  MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17);
  MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22);

  MD5STEP(F2, a, b, c, d, in[1] + 0xf61e2562, 5);
  MD5STEP(F2, d, a, b, c, in[6] + 0xc040b340, 9);
  MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14);
  MD5STEP(F2, b, c, d, a, in[0] + 0xe9b6c7aa, 20);
  MD5STEP(F2, a, b, c, d, in[5] + 0xd62f105d, 5);
  MD5STEP(F2, d, a, b, c, in[10] + 0x02441453, 9);
  MD5STEP(F2, c, d, a, b, in[15] + 0xd8a1e681, 14);
  MD5STEP(F2, b, c, d, a, in[4] + 0xe7d3fbc8, 20);
  MD5STEP(F2, a, b, c, d, in[9] + 0x21e1cde6, 5);
  MD5STEP(F2, d, a, b, c, in[14] + 0xc33707d6, 9);
  MD5STEP(F2, c, d, a, b, in[3] + 0xf4d50d87, 14);
  MD5STEP(F2, b, c, d, a, in[8] + 0x455a14ed, 20);
  MD5STEP(F2, a, b, c, d, in[13] + 0xa9e3e905, 5);
  MD5STEP(F2, d, a, b, c, in[2] + 0xfcefa3f8, 9);
  MD5STEP(F2, c, d, a, b, in[7] + 0x676f02d9, 14);
  MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20);

  MD5STEP(F3, a, b, c, d, in[5] + 0xfffa3942, 4);
  MD5STEP(F3, d, a, b, c, in[8] + 0x8771f681, 11);
  MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16);
  MD5STEP(F3, b, c, d, a, in[14] + 0xfde5380c, 23);
  MD5STEP(F3, a, b, c, d, in[1] + 0xa4beea44, 4);
  MD5STEP(F3, d, a, b, c, in[4] + 0x4bdecfa9, 11);
  MD5STEP(F3, c, d, a, b, in[7] + 0xf6bb4b60, 16);
  MD5STEP(F3, b, c, d, a, in[10] + 0xbebfbc70, 23);
  MD5STEP(F3, a, b, c, d, in[13] + 0x289b7ec6, 4);
  MD5STEP(F3, d, a, b, c, in[0] + 0xeaa127fa, 11);
  MD5STEP(F3, c, d, a, b, in[3] + 0xd4ef3085, 16);
  MD5STEP(F3, b, c, d, a, in[6] + 0x04881d05, 23);
  MD5STEP(F3, a, b, c, d, in[9] + 0xd9d4d039, 4);
  MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11);
  MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16);
  MD5STEP(F3, b, c, d, a, in[2] + 0xc4ac5665, 23);

  MD5STEP(F4, a, b, c, d, in[0] + 0xf4292244, 6);
  MD5STEP(F4, d, a, b, c, in[7] + 0x432aff97, 10);
  MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15);
  MD5STEP(F4, b, c, d, a, in[5] + 0xfc93a039, 21);
  MD5STEP(F4, a, b, c, d, in[12] + 0x655b59c3, 6);
  MD5STEP(F4, d, a, b, c, in[3] + 0x8f0ccc92, 10);
  MD5STEP(F4, c, d, a, b, in[10] + 0xffeff47d, 15);
  MD5STEP(F4, b, c, d, a, in[1] + 0x85845dd1, 21);
  MD5STEP(F4, a, b, c, d, in[8] + 0x6fa87e4f, 6);
  MD5STEP(F4, d, a, b, c, in[15] + 0xfe2ce6e0, 10);
  MD5STEP(F4, c, d, a, b, in[6] + 0xa3014314, 15);
  MD5STEP(F4, b, c, d, a, in[13] + 0x4e0811a1, 21);
  MD5STEP(F4, a, b, c, d, in[4] + 0xf7537e82, 6);
  MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10);
  MD5STEP(F4, c, d, a, b, in[2] + 0x2ad7d2bb, 15);
  MD5STEP(F4, b, c, d, a, in[9] + 0xeb86d391, 21);

  buf[0] += a;
  buf[1] += b;
  buf[2] += c;
  buf[3] += d;
}

void mg_md5_update(mg_md5_ctx *ctx, const unsigned char *buf, size_t len) {
  uint32_t t;

  t = ctx->bits[0];
  if ((ctx->bits[0] = t + ((uint32_t) len << 3)) < t) ctx->bits[1]++;
  ctx->bits[1] += (uint32_t) len >> 29;

  t = (t >> 3) & 0x3f;

  if (t) {
    unsigned char *p = (unsigned char *) ctx->in + t;

    t = 64 - t;
    if (len < t) {
      memcpy(p, buf, len);
      return;
    }
    memcpy(p, buf, t);
    mg_byte_reverse(ctx->in, 16);
    mg_md5_transform(ctx->buf, (uint32_t *) ctx->in);
    buf += t;
    len -= t;
  }

  while (len >= 64) {
    memcpy(ctx->in, buf, 64);
    mg_byte_reverse(ctx->in, 16);
    mg_md5_transform(ctx->buf, (uint32_t *) ctx->in);
    buf += 64;
    len -= 64;
  }

  memcpy(ctx->in, buf, len);
}

void mg_md5_final(mg_md5_ctx *ctx, unsigned char digest[16]) {
  unsigned count;
  unsigned char *p;
  uint32_t *a;

  count = (ctx->bits[0] >> 3) & 0x3F;

  p = ctx->in + count;
  *p++ = 0x80;
  count = 64 - 1 - count;
  if (count < 8) {
    memset(p, 0, count);
    mg_byte_reverse(ctx->in, 16);
    mg_md5_transform(ctx->buf, (uint32_t *) ctx->in);
    memset(ctx->in, 0, 56);
  } else {
    memset(p, 0, count - 8);
  }
  mg_byte_reverse(ctx->in, 14);

  a = (uint32_t *) ctx->in;
  a[14] = ctx->bits[0];
  a[15] = ctx->bits[1];

  mg_md5_transform(ctx->buf, (uint32_t *) ctx->in);
  mg_byte_reverse((unsigned char *) ctx->buf, 4);
  memcpy(digest, ctx->buf, 16);
  memset((char *) ctx, 0, sizeof(*ctx));
}
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/mdash.c"
#endif


#if MG_ENABLE_MDASH












static const char *mg_mdash_ca_pem =
    "-----BEGIN CERTIFICATE-----\n"
    "MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw\n"
    "TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh\n"
    "cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4\n"
    "WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu\n"
    "ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY\n"
    "MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc\n"
    "h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+\n"
    "0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U\n"
    "A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW\n"
    "T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH\n"
    "B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC\n"
    "B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv\n"
    "KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn\n"
    "OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn\n"
    "jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw\n"
    "qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI\n"
    "rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV\n"
    "HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq\n"
    "hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL\n"
    "ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ\n"
    "3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK\n"
    "NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5\n"
    "ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur\n"
    "TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC\n"
    "jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc\n"
    "oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq\n"
    "4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA\n"
    "mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d\n"
    "emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=\n"
    "-----END CERTIFICATE-----\n";

static struct mg_connection *mg_mdash_conn = NULL;
// static struct mg_str mg_mdash_key;
// static void mg_connection *mg_mdash_conn = NULL;

static void mg_mdash_fn(struct mg_connection *c, int ev, void *ev_data) {
  if (ev == MG_EV_CONNECT) {
    // c->is_hexdumping = true;
    if (mg_url_is_ssl(MG_MDASH_URL)) {
      struct mg_str host = mg_url_host(MG_MDASH_URL);
      struct mg_tls_opts opts;
      memset(&opts, 0, sizeof(struct mg_tls_opts));
      opts.ca = mg_str(mg_mdash_ca_pem);
      opts.name = host;
      mg_tls_init(c, &opts);
    }
  }
  if (ev == MG_EV_WS_OPEN) {
    MG_INFO(("Connected to %s", MG_MDASH_URL));
    // s_mDashState = MDASH_STATE_CLOUD_CONNECTED;
    // mDashTriggerEvent(s_mDashState, NULL);
  } else if (ev == MG_EV_WS_MSG) {
    struct mg_iobuf io = {0, 0, 0, 512};
    struct mg_ws_message *wm = (struct mg_ws_message *) ev_data;
    struct mg_str s = wm->data;
    struct mg_rpc_req r;
    while (s.len > 0 && isspace((uint8_t) s.buf[s.len - 1])) s.len--;
    MG_INFO(("RPC: %.*s", (int) s.len, s.buf));
    memset(&r, 0, sizeof(struct mg_rpc_req));
    r.head = &c->mgr->rpcs;
    r.rpc = NULL;
    r.pfn = mg_pfn_iobuf;
    r.pfn_data = &io;
    r.req_data = NULL;
    r.frame = s;
    mg_rpc_process(&r);
    if (io.buf != NULL) {
      MG_INFO(("     %s", io.buf));
      mg_ws_send(c, (const char *) io.buf, io.len, WEBSOCKET_OP_TEXT);
    } else {
      MG_DEBUG(("%s", "RPC unhandled"));
    }
    mg_iobuf_free(&io);
  } else if (ev == MG_EV_CLOSE) {
    mg_mdash_conn = NULL;
    MG_INFO(("disconnected, ram %lu", 0));
  }
  (void) c;
}

static void mg_mdash_rpc_get_info(struct mg_rpc_req *r) {
  const char *reset_reason = mg_health_reason_str(mg_health_reason());
  mg_rpc_ok(r, "{%m:%m,%m:%llu,%m:%m,%m:\"mws.%d\",%m:%M}",     //
            MG_ESC("fw_version"), MG_ESC(MG_FIRMWARE_VERSION),  //
            MG_ESC("uptime"), (uint64_t) (mg_millis() / 1000),  //
            MG_ESC("reboot_reason"), MG_ESC(reset_reason),      //
            MG_ESC("arch"), MG_ARCH,                            //
            MG_ESC("crash"), mg_print_crash_record);
}

static void mg_mdash_rpc_ota_begin(struct mg_rpc_req *r) {
  size_t total = (size_t) mg_json_get_long(r->frame, "$.params.size", -1);
  if (mg_ota_begin(total)) {
    mg_rpc_ok(r, "true");
  } else {
    mg_rpc_err(r, 500, "%m", MG_ESC("failed"));
  }
}

static void mg_mdash_rpc_ota_write(struct mg_rpc_req *r) {
  int len = 0, ofs = 0;
  if ((ofs = mg_json_get(r->frame, "$.params.data", &len)) <= 0 ||
      r->frame.buf[ofs] != '"') {
    mg_rpc_err(r, 500, "%m", MG_ESC("data required"));
  } else {
    struct mg_str s = mg_str_n(&r->frame.buf[ofs] + 1, (size_t) (len - 2));
    size_t n = mg_base64_decode(s.buf, s.len, s.buf, s.len);  // Decode in-place
    if (n == 0) {
      mg_rpc_ok(r, "%m", MG_ESC("finished"));
      mg_ota_end();
    } else if (mg_ota_write(s.buf, n)) {
      mg_rpc_ok(r, "{%m:%d}", MG_ESC("queued"), n);
    } else {
      mg_rpc_err(r, 500, "%m", MG_ESC("write failed"));
      mg_ota_end();
    }
    // mDashNotify("State.Set", "{%m:{%m:%d}}", MG_ESC("ota"),
    // MG_ESC("percent"), (int) (s_ota_written * 100 / s_ota_size));
  }
}

static void mg_mdash_rpc_ota_end(struct mg_rpc_req *r) {
  if (mg_ota_end()) {
    mg_rpc_ok(r, "true");
  } else {
    mg_rpc_err(r, 500, "%m", MG_ESC("failed"));
  }
  // mDashNotify("State.Set", "{%m:{%m:null}}", MG_ESC("ota"),
  // MG_ESC("percent"));
}

void mg_mdash_init(struct mg_mgr *mgr) {
  MG_INFO(("Initialising mdash agent, key [%s]", MG_MDASH_KEY));
  mg_rpc_add(&mgr->rpcs, mg_str("RPC.List"), mg_rpc_list, NULL);
  mg_rpc_add(&mgr->rpcs, mg_str("Sys.GetInfo"), mg_mdash_rpc_get_info, NULL);
  mg_rpc_add(&mgr->rpcs, mg_str("OTA.Begin"), mg_mdash_rpc_ota_begin, NULL);
  mg_rpc_add(&mgr->rpcs, mg_str("OTA.Write"), mg_mdash_rpc_ota_write, NULL);
  mg_rpc_add(&mgr->rpcs, mg_str("OTA.End"), mg_mdash_rpc_ota_end, NULL);
}

void mg_mdash_poll(struct mg_mgr *mgr) {
#if MG_TLS == MG_TLS_NONE
  (void) mgr, (void) mg_mdash_fn;  // mdash needs wss://, nothing to do
#else
  static uint64_t timer = 0;
  // No RPC handlers means mdash was never set up, or mg_mdash_free() ran.
  // No key means mdash would reject us, so do not connect at all
  if (mg_mdash_conn == NULL && mgr->rpcs != NULL && MG_MDASH_KEY[0] != '\0' &&
      mg_timer_expired(&timer, 1000, mg_millis())) {
    MG_INFO(("Connecting to %s", MG_MDASH_URL));
    mg_mdash_conn = mg_ws_connect(mgr, MG_MDASH_URL, mg_mdash_fn, NULL,
                                  "Origin: %s\r\n"
                                  "Authorization: Bearer %s\r\n",
                                  MG_MDASH_URL, MG_MDASH_KEY);
  }
#endif
}

void mg_mdash_free(struct mg_mgr *mgr) {
  mg_rpc_del(&mgr->rpcs, NULL);
}

#else
void mg_mdash_init(struct mg_mgr *mgr) {
  (void) mgr;
}
void mg_mdash_poll(struct mg_mgr *mgr) {
  (void) mgr;
}
void mg_mdash_free(struct mg_mgr *mgr) {
  (void) mgr;
}
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/modbus.c"
#endif





#define MG_MODBUS_MIN_PAYLOAD_LEN 12
#define MG_MODBUS_HEADER_LEN 7

static void mg_modbus_send_mbap(struct mg_connection *c, uint16_t txid,
                                uint16_t pdu_len, uint8_t unit_id) {
  uint8_t hdr[7];
  memset(hdr, 0, sizeof(hdr));
  MG_STORE_BE16(&hdr[0], txid & 0xFFFFU);
  MG_STORE_BE16(&hdr[4], (pdu_len + 1) & 0xFFFFU);
  hdr[6] = unit_id;
  mg_send(c, hdr, sizeof(hdr));
}

static void mg_modbus_send_response(struct mg_connection *c,
                                    struct mg_modbus_req *req,
                                    uint16_t txid, uint8_t unit_id) {
  uint8_t buf[5];
  uint8_t out;
  uint16_t reg;
  uint16_t i, nbytes;

  memset(buf, 0, sizeof(buf));
  if (req == NULL) return;
  if (req->error != MG_MODBUS_ERR_NONE) {
    mg_modbus_send_mbap(c, txid, 2, unit_id);
    buf[0] = (uint8_t) (req->func | 0x80);
    buf[1] = req->error;
    mg_send(c, buf, 2);
    return;
  }
  switch (req->func) {
    case MG_MODBUS_FUNC_READ_COILS:
    case MG_MODBUS_FUNC_READ_DISCRETE_INPUTS:
      if (req->u.bits == NULL) return;
      nbytes = (uint16_t) ((req->len + 7) / 8);
      mg_modbus_send_mbap(c, txid, (uint16_t) (nbytes + 2), unit_id);
      buf[0] = req->func;
      buf[1] = (uint8_t) nbytes;
      mg_send(c, buf, 2);
      memset(buf, 0, sizeof(buf));
      for (i = 0; i < req->len; i++) {
        if (i % 8 == 0) out = 0;
        if (req->u.bits[i]) out |= (uint8_t) (1 << (i % 8));
        if ((i % 8) == 7 || i + 1 == req->len) {
          mg_send(c, &out, 1);
        }
      }
      break;
    case MG_MODBUS_FUNC_READ_HOLDING_REGISTERS:
    case MG_MODBUS_FUNC_READ_INPUT_REGISTERS:
      if (req->u.regs == NULL) return;
      nbytes = (uint16_t) (req->len * 2);
      mg_modbus_send_mbap(c, txid, (uint16_t) (nbytes + 2), unit_id);
      buf[0] = req->func;
      buf[1] = (uint8_t) nbytes;
      mg_send(c, buf, 2);
      for (i = 0; i < req->len; i++) {
        MG_STORE_BE16(&reg, req->u.regs[i] & 0xFFFFU);
        mg_send(c, &reg, 2);
      }
      break;
    case MG_MODBUS_FUNC_WRITE_SINGLE_COIL:
      if (req->len != 1 || req->u.bits == NULL) return;
      mg_modbus_send_mbap(c, txid, 5, unit_id);
      buf[0] = req->func;
      MG_STORE_BE16(&buf[1], req->addr & 0xFFFFu);
      MG_STORE_BE16(&buf[3], req->u.bits[0] ? 0xFF00U : 0x0000U);
      mg_send(c, buf, 5);
      break;
    case MG_MODBUS_FUNC_WRITE_SINGLE_REGISTER:
      if (req->len != 1 || req->u.regs == NULL) return;
      mg_modbus_send_mbap(c, txid, 5, unit_id);
      buf[0] = req->func;
      MG_STORE_BE16(&buf[1], req->addr  & 0xFFFFU);
      MG_STORE_BE16(&buf[3], req->u.regs[0]  & 0xFFFFU);
      mg_send(c, buf, 5);
      break;
    case MG_MODBUS_FUNC_WRITE_MULTIPLE_COILS:
    case MG_MODBUS_FUNC_WRITE_MULTIPLE_REGISTERS:
      mg_modbus_send_mbap(c, txid, 5, unit_id);
      buf[0] = req->func;
      MG_STORE_BE16(&buf[1], req->addr & 0xFFFFU);
      MG_STORE_BE16(&buf[3], req->len & 0xFFFFU);
      mg_send(c, buf, 5);
      break;
    default:
      break;
  }
}

static void handle_pdu(struct mg_connection *c, uint8_t *buf, size_t len) {
  uint16_t max_len, val, txid, addr, quantity;
  uint8_t func, unit_id, byte_count, packed;
  int i;
  size_t pdu_len = len - MG_MODBUS_HEADER_LEN, len_check;
  struct mg_modbus_req mr;

  if (len < MG_MODBUS_MIN_PAYLOAD_LEN) return;
  memset(&mr, 0, sizeof(mr));
  txid = MG_LOAD_BE16(&buf[0]);
  unit_id = buf[6];
  func = buf[7];
  addr = MG_LOAD_BE16(&buf[8]);
  mr.func = func;
  mr.addr = addr;
  mr.error = MG_MODBUS_ERR_NONE;
  mr.len = 0;
  mr.u.bits = NULL;
  switch (func) {
    case MG_MODBUS_FUNC_READ_COILS:
    case MG_MODBUS_FUNC_READ_DISCRETE_INPUTS:
      if (pdu_len != 5) goto modbus_illegal_value;
      quantity = MG_LOAD_BE16(&buf[10]);
      max_len = 0x07D0;  // 2000 bits
      if (quantity == 0 || quantity > max_len) goto modbus_illegal_value;
      mr.len = quantity;
      mr.u.bits = (bool *) mg_calloc((size_t) mr.len, sizeof(bool));
      if (mr.u.bits == NULL) goto modbus_oom;
      break;
    case MG_MODBUS_FUNC_READ_HOLDING_REGISTERS:
    case MG_MODBUS_FUNC_READ_INPUT_REGISTERS:
      if (pdu_len != 5) goto modbus_illegal_value;
      quantity = MG_LOAD_BE16(&buf[10]);
      max_len = 0x007D;  // 125 registers
      if (quantity == 0 || quantity > max_len) goto modbus_illegal_value;
      mr.len = quantity;
      mr.u.regs = (uint16_t *) mg_calloc((size_t) mr.len, sizeof(uint16_t));
      if (mr.u.regs == NULL) goto modbus_oom;
      break;
    case MG_MODBUS_FUNC_WRITE_SINGLE_COIL:
      if (pdu_len != 5) goto modbus_illegal_value;
      mr.len = 1;
      mr.u.bits = (bool *) mg_calloc(1, sizeof(bool));
      if (mr.u.bits == NULL) goto modbus_oom;
      val = MG_LOAD_BE16(&buf[10]);
      if (val != 0x0000 && val != 0xFF00) goto modbus_illegal_value;
      mr.u.bits[0] = (val == 0xFF00);
      break;
    case MG_MODBUS_FUNC_WRITE_SINGLE_REGISTER:
      if (pdu_len != 5) goto modbus_illegal_value;
      mr.len = 1;
      mr.u.regs = (uint16_t *) mg_calloc(1, sizeof(uint16_t));
      if (mr.u.regs == NULL) goto modbus_oom;
      mr.u.regs[0] = MG_LOAD_BE16(&buf[10]);
      break;
    case MG_MODBUS_FUNC_WRITE_MULTIPLE_COILS: {
      quantity = MG_LOAD_BE16(&buf[10]);
      if (quantity == 0 || quantity > 0x07B0) goto modbus_illegal_value;
      if (pdu_len < 6) goto modbus_illegal_value;
      byte_count = buf[12];
      if (byte_count != (uint8_t) ((quantity + 7) / 8)) goto modbus_illegal_value;
      len_check = (size_t) (6 + byte_count);
      if (len_check != pdu_len) goto modbus_illegal_value;
      mr.len = quantity;
      mr.u.bits = (bool *) mg_calloc((size_t) mr.len, sizeof(bool));
      if (mr.u.bits == NULL) goto modbus_oom;
      for (i = 0; i < mr.len; i++) {
        packed = buf[13 + (i / 8)];
        mr.u.bits[i] = ((packed >> (i % 8)) & 1) != 0;
      }
      break;
    }
    case MG_MODBUS_FUNC_WRITE_MULTIPLE_REGISTERS: {
      quantity = MG_LOAD_BE16(&buf[10]);
      if (quantity == 0 || quantity > 0x007B) goto modbus_illegal_value;
      if (pdu_len < 6) goto modbus_illegal_value;
      byte_count = buf[12];
      if (byte_count != (uint8_t) (quantity * 2)) goto modbus_illegal_value;
      len_check = (size_t) (6 + quantity * 2);
      if (len_check != pdu_len) goto modbus_illegal_value;
      mr.len = quantity;
      mr.u.regs = (uint16_t *) mg_calloc((size_t) mr.len, sizeof(uint16_t));
      if (mr.u.regs == NULL) goto modbus_oom;
      for (i = 0; i < mr.len; i++) {
        mr.u.regs[i] = MG_LOAD_BE16(&buf[13 + i * 2]);
      }
      break;
    }
    default:
      MG_ERROR(("Unsupported modbus function"));
      mr.error = MG_MODBUS_ERR_ILLEGAL_FUNCTION;
      mg_modbus_send_response(c, &mr, txid, unit_id);
      return;
  }
  mg_call(c, MG_EV_MODBUS_REQ, &mr);
  goto modbus_exit;

modbus_illegal_value:
  MG_ERROR(("Invalid data"));
  mr.error = MG_MODBUS_ERR_ILLEGAL_VALUE;
  goto modbus_exit;
modbus_oom:
  MG_ERROR(("OOM"));
  mr.error = MG_MODBUS_ERR_DEVICE_FAILURE;
modbus_exit:
  mg_modbus_send_response(c, &mr, txid, unit_id);
  if (mr.u.bits != NULL) mg_free(mr.u.bits);
}

static void modbus_ev_handler(struct mg_connection *c, int ev, void *ev_data) {
  if (ev == MG_EV_READ) {
    while (c->recv.len >= MG_MODBUS_MIN_PAYLOAD_LEN) {
      uint16_t len = MG_LOAD_BE16(&c->recv.buf[4]);  // PDU length
      uint16_t proto_id = MG_LOAD_BE16(&c->recv.buf[2]);
      if (len < 6 || proto_id != 0) {
        mg_error(c, "invalid pdu");
        break;
      } else if (c->recv.len < len + 6U) {
        break;  // Partial frame, buffer more
      } else {
        handle_pdu(c, c->recv.buf, len + 6);  // Parse PDU and call user
        mg_iobuf_del(&c->recv, 0, len + 6);   // Delete received PDU
      }
    }
  }
  (void) ev_data;
}

struct mg_connection *mg_modbus_listen(struct mg_mgr *mgr, const char *url,
                                       mg_event_handler_t fn, void *fn_data) {
  struct mg_connection *c = mg_listen(mgr, url, fn, fn_data);
  if (c != NULL) c->pfn = modbus_ev_handler;
  return c;
}

#ifdef MG_ENABLE_LINES
#line 1 "src/mqtt.c"
#endif








#define MQTT_CLEAN_SESSION 0x02
#define MQTT_HAS_WILL 0x04
#define MQTT_WILL_RETAIN 0x20
#define MQTT_HAS_PASSWORD 0x40
#define MQTT_HAS_USER_NAME 0x80

struct mg_mqtt_pmap {
  uint8_t id;
  uint8_t type;
};

static const struct mg_mqtt_pmap s_prop_map[] = {
    {MQTT_PROP_PAYLOAD_FORMAT_INDICATOR, MQTT_PROP_TYPE_BYTE},
    {MQTT_PROP_MESSAGE_EXPIRY_INTERVAL, MQTT_PROP_TYPE_INT},
    {MQTT_PROP_CONTENT_TYPE, MQTT_PROP_TYPE_STRING},
    {MQTT_PROP_RESPONSE_TOPIC, MQTT_PROP_TYPE_STRING},
    {MQTT_PROP_CORRELATION_DATA, MQTT_PROP_TYPE_BINARY_DATA},
    {MQTT_PROP_SUBSCRIPTION_IDENTIFIER, MQTT_PROP_TYPE_VARIABLE_INT},
    {MQTT_PROP_SESSION_EXPIRY_INTERVAL, MQTT_PROP_TYPE_INT},
    {MQTT_PROP_ASSIGNED_CLIENT_IDENTIFIER, MQTT_PROP_TYPE_STRING},
    {MQTT_PROP_SERVER_KEEP_ALIVE, MQTT_PROP_TYPE_SHORT},
    {MQTT_PROP_AUTHENTICATION_METHOD, MQTT_PROP_TYPE_STRING},
    {MQTT_PROP_AUTHENTICATION_DATA, MQTT_PROP_TYPE_BINARY_DATA},
    {MQTT_PROP_REQUEST_PROBLEM_INFORMATION, MQTT_PROP_TYPE_BYTE},
    {MQTT_PROP_WILL_DELAY_INTERVAL, MQTT_PROP_TYPE_INT},
    {MQTT_PROP_REQUEST_RESPONSE_INFORMATION, MQTT_PROP_TYPE_BYTE},
    {MQTT_PROP_RESPONSE_INFORMATION, MQTT_PROP_TYPE_STRING},
    {MQTT_PROP_SERVER_REFERENCE, MQTT_PROP_TYPE_STRING},
    {MQTT_PROP_REASON_STRING, MQTT_PROP_TYPE_STRING},
    {MQTT_PROP_RECEIVE_MAXIMUM, MQTT_PROP_TYPE_SHORT},
    {MQTT_PROP_TOPIC_ALIAS_MAXIMUM, MQTT_PROP_TYPE_SHORT},
    {MQTT_PROP_TOPIC_ALIAS, MQTT_PROP_TYPE_SHORT},
    {MQTT_PROP_MAXIMUM_QOS, MQTT_PROP_TYPE_BYTE},
    {MQTT_PROP_RETAIN_AVAILABLE, MQTT_PROP_TYPE_BYTE},
    {MQTT_PROP_USER_PROPERTY, MQTT_PROP_TYPE_STRING_PAIR},
    {MQTT_PROP_MAXIMUM_PACKET_SIZE, MQTT_PROP_TYPE_INT},
    {MQTT_PROP_WILDCARD_SUBSCRIPTION_AVAILABLE, MQTT_PROP_TYPE_BYTE},
    {MQTT_PROP_SUBSCRIPTION_IDENTIFIER_AVAILABLE, MQTT_PROP_TYPE_BYTE},
    {MQTT_PROP_SHARED_SUBSCRIPTION_AVAILABLE, MQTT_PROP_TYPE_BYTE}};

static bool mqtt_send_header(struct mg_connection *c, uint8_t cmd,
                             uint8_t flags, uint32_t len) {
  uint8_t buf[1 + sizeof(len)], *vlen = &buf[1];
  buf[0] = (uint8_t) ((cmd << 4) | flags);
  do {
    *vlen = len % 0x80;
    len /= 0x80;
    if (len > 0) *vlen |= 0x80;
    vlen++;
  } while (len > 0 && vlen < &buf[sizeof(buf)]);
  return mg_send(c, buf, (size_t) (vlen - buf));
}

void mg_mqtt_send_header(struct mg_connection *c, uint8_t cmd, uint8_t flags,
                         uint32_t len) {
  if (!mqtt_send_header(c, cmd, flags, len)) mg_error(c, "OOM");
}

static bool mg_send_u16(struct mg_connection *c, uint16_t value) {
  return mg_send(c, &value, sizeof(value));
}

static bool mg_send_u32(struct mg_connection *c, uint32_t value) {
  return mg_send(c, &value, sizeof(value));
}

static uint8_t varint_size(size_t length) {
  uint8_t bytes_needed = 0;
  do {
    bytes_needed++;
    length /= 0x80;
  } while (length > 0);
  return bytes_needed;
}

static size_t encode_varint(uint8_t *buf, size_t value) {
  size_t len = 0;
  do {
    uint8_t b = (uint8_t) (value % 128);
    value /= 128;
    if (value > 0) b |= 0x80;
    buf[len++] = b;
  } while (value > 0);
  return len;
}

static size_t decode_varint(const uint8_t *buf, size_t len, uint32_t *value) {
  uint32_t mul = 1, ofs;
  *value = 0;
  for (ofs = 0; ofs < 4 && ofs < len; ofs++) {
    uint8_t enc_byte = buf[ofs];
    *value += (enc_byte & 0x7f) * mul;
    mul *= 128;
    if ((enc_byte & 0x80) == 0) return ofs + 1;
  }
  return 0;
}

static int mqtt_prop_type_by_id(uint8_t prop_id) {
  size_t i, num_properties = sizeof(s_prop_map) / sizeof(s_prop_map[0]);
  for (i = 0; i < num_properties; ++i) {
    if (s_prop_map[i].id == prop_id) return s_prop_map[i].type;
  }
  return -1;  // Property ID not found
}

// Returns the size of the properties section, without the
// size of the content's length
static size_t get_properties_length(struct mg_mqtt_prop *props, size_t count) {
  size_t i, size = 0;
  for (i = 0; i < count; i++) {
    size++;  // identifier
    switch (mqtt_prop_type_by_id(props[i].id)) {
      case MQTT_PROP_TYPE_STRING_PAIR:
        size += (uint32_t) (props[i].val.len + props[i].key.len +
                            2 * sizeof(uint16_t));
        break;
      case MQTT_PROP_TYPE_STRING:
        size += (uint32_t) (props[i].val.len + sizeof(uint16_t));
        break;
      case MQTT_PROP_TYPE_BINARY_DATA:
        size += (uint32_t) (props[i].val.len + sizeof(uint16_t));
        break;
      case MQTT_PROP_TYPE_VARIABLE_INT:
        size += varint_size((uint32_t) props[i].iv);
        break;
      case MQTT_PROP_TYPE_INT: size += (uint32_t) sizeof(uint32_t); break;
      case MQTT_PROP_TYPE_SHORT: size += (uint32_t) sizeof(uint16_t); break;
      case MQTT_PROP_TYPE_BYTE: size += (uint32_t) sizeof(uint8_t); break;
      default: return size;  // cannot parse further down
    }
  }

  return size;
}

// returns the entire size of the properties section, including the
// size of the variable length of the content
static size_t get_props_size(struct mg_mqtt_prop *props, size_t count) {
  size_t size = get_properties_length(props, count);
  size += varint_size(size);
  return size;
}

static bool mg_send_mqtt_properties(struct mg_connection *c,
                                    struct mg_mqtt_prop *props, size_t nprops) {
  size_t total_size = get_properties_length(props, nprops);
  uint8_t buf_v[4] = {0, 0, 0, 0};
  uint8_t buf[4] = {0, 0, 0, 0};
  size_t i, len = encode_varint(buf, total_size);

  if (!mg_send(c, buf, (size_t) len)) return false;
  for (i = 0; i < nprops; i++) {
    if (!mg_send(c, &props[i].id, sizeof(props[i].id))) return false;
    switch (mqtt_prop_type_by_id(props[i].id)) {
      case MQTT_PROP_TYPE_STRING_PAIR:
        if (!mg_send_u16(c, mg_htons((uint16_t) props[i].key.len)) ||
            !mg_send(c, props[i].key.buf, props[i].key.len) ||
            !mg_send_u16(c, mg_htons((uint16_t) props[i].val.len)) ||
            !mg_send(c, props[i].val.buf, props[i].val.len))
          return false;
        break;
      case MQTT_PROP_TYPE_BYTE:
        if (!mg_send(c, &props[i].iv, sizeof(uint8_t))) return false;
        break;
      case MQTT_PROP_TYPE_SHORT:
        if (!mg_send_u16(c, mg_htons((uint16_t) props[i].iv))) return false;
        break;
      case MQTT_PROP_TYPE_INT:
        if (!mg_send_u32(c, mg_htonl((uint32_t) props[i].iv))) return false;
        break;
      case MQTT_PROP_TYPE_STRING:
        if (!mg_send_u16(c, mg_htons((uint16_t) props[i].val.len)) ||
            !mg_send(c, props[i].val.buf, props[i].val.len))
          return false;
        break;
      case MQTT_PROP_TYPE_BINARY_DATA:
        if (!mg_send_u16(c, mg_htons((uint16_t) props[i].val.len)) ||
            !mg_send(c, props[i].val.buf, props[i].val.len))
          return false;
        break;
      case MQTT_PROP_TYPE_VARIABLE_INT:
        len = encode_varint(buf_v, props[i].iv);
        if (!mg_send(c, buf_v, (size_t) len)) return false;
        break;
    }
  }
  return true;
}

size_t mg_mqtt_next_prop(struct mg_mqtt_message *msg, struct mg_mqtt_prop *prop,
                         size_t ofs) {
  uint8_t *props = (uint8_t *) msg->dgram.buf + msg->props_start;
  uint8_t *props_end = props + msg->props_size;
  uint8_t *i = props + ofs;
  size_t new_pos = ofs, len;
  
  if (msg->props_start > msg->dgram.len ||
    msg->props_size > msg->dgram.len - msg->props_start ||
    ofs >= msg->props_size)
  return 0;

  memset(prop, 0, sizeof(struct mg_mqtt_prop));
  prop->id = i[0];
  i++, new_pos++;

  switch (mqtt_prop_type_by_id(prop->id)) {
    case MQTT_PROP_TYPE_STRING_PAIR:
      if (i + 2 > props_end) return 0;
      prop->key.len = (uint16_t) ((((uint16_t) i[0]) << 8) | i[1]);
      if (i + 2 + prop->key.len > props_end) return 0;
      prop->key.buf = (char *) i + 2;
      i += 2 + prop->key.len;
      if (i + 2 > props_end) return 0;
      prop->val.len = (uint16_t) ((((uint16_t) i[0]) << 8) | i[1]);
      prop->val.buf = (char *) i + 2;
      if (i + 2 + prop->val.len > props_end) return 0;
      new_pos += 2 * sizeof(uint16_t) + prop->val.len + prop->key.len;
      break;
    case MQTT_PROP_TYPE_BYTE:
      if (i + 1 > props_end) return 0;
      prop->iv = (uint8_t) i[0];
      new_pos++;
      break;
    case MQTT_PROP_TYPE_SHORT:
      if (i + 2 > props_end) return 0;
      prop->iv = (uint16_t) ((((uint16_t) i[0]) << 8) | i[1]);
      new_pos += sizeof(uint16_t);
      break;
    case MQTT_PROP_TYPE_INT:
      if (i + 4 > props_end) return 0;
      prop->iv = ((uint32_t) i[0] << 24) | ((uint32_t) i[1] << 16) |
                 ((uint32_t) i[2] << 8) | i[3];
      new_pos += sizeof(uint32_t);
      break;
    case MQTT_PROP_TYPE_STRING:
      if (i + 2 > props_end) return 0;
      prop->val.len = (uint16_t) ((((uint16_t) i[0]) << 8) | i[1]);
      prop->val.buf = (char *) i + 2;
      if (i + 2 + prop->val.len > props_end) return 0;
      new_pos += 2 + prop->val.len;
      break;
    case MQTT_PROP_TYPE_BINARY_DATA:
      if (i + 2 > props_end) return 0;
      prop->val.len = (uint16_t) ((((uint16_t) i[0]) << 8) | i[1]);
      prop->val.buf = (char *) i + 2;
      if (i + 2 + prop->val.len > props_end) return 0;
      new_pos += 2 + prop->val.len;
      break;
    case MQTT_PROP_TYPE_VARIABLE_INT:
      len = decode_varint(i, (size_t) (props_end - i), &prop->iv);
      if (i + len > props_end) return 0;
      new_pos = (len == 0) ? 0 : new_pos + len;
      break;
    default:
      new_pos = 0;
      break;
  }

  return new_pos;
}

void mg_mqtt_login(struct mg_connection *c, const struct mg_mqtt_opts *opts) {
  char client_id[21];
  struct mg_str cid = opts->client_id;
  size_t total_len = 7 + 1 + 2 + 2;
  uint8_t hdr[8] = {0, 4, 'M', 'Q', 'T', 'T', 0, 0};
  hdr[6] = opts->version;

  if (cid.len == 0) {
    mg_random_str(client_id, sizeof(client_id) - 1);
    client_id[sizeof(client_id) - 1] = '\0';
    cid = mg_str(client_id);
  }

  if (hdr[6] == 0) hdr[6] = 4;  // If version is not set, use 4 (3.1.1)
  c->is_mqtt5 = hdr[6] == 5;    // Set version 5 flag
  hdr[7] = (uint8_t) ((opts->qos & 3) << 3);  // Connection flags
  if (opts->user.len > 0) {
    total_len += 2 + (uint32_t) opts->user.len;
    hdr[7] |= MQTT_HAS_USER_NAME;
  }
  if (opts->pass.len > 0) {
    total_len += 2 + (uint32_t) opts->pass.len;
    hdr[7] |= MQTT_HAS_PASSWORD;
  }
  if (opts->topic.len > 0) {  // allow zero-length msgs, message.len is size_t
    total_len += 4 + (uint32_t) opts->topic.len + (uint32_t) opts->message.len;
    hdr[7] |= MQTT_HAS_WILL;
  }
  if (opts->clean || cid.len == 0) hdr[7] |= MQTT_CLEAN_SESSION;
  if (opts->retain) hdr[7] |= MQTT_WILL_RETAIN;
  total_len += (uint32_t) cid.len;
  if (c->is_mqtt5) {
    total_len += get_props_size(opts->props, opts->num_props);
    if (hdr[7] & MQTT_HAS_WILL)
      total_len += get_props_size(opts->will_props, opts->num_will_props);
  }

  // keepalive == 0 means "do not disconnect us!"
  if (!mqtt_send_header(c, MQTT_CMD_CONNECT, 0, (uint32_t) total_len) ||
      !mg_send(c, hdr, sizeof(hdr)) ||
      !mg_send_u16(c, mg_htons((uint16_t) opts->keepalive)))
    goto fail;

  if (c->is_mqtt5 && !mg_send_mqtt_properties(c, opts->props, opts->num_props))
    goto fail;

  if (!mg_send_u16(c, mg_htons((uint16_t) cid.len)) ||
      !mg_send(c, cid.buf, cid.len))
    goto fail;

  if (hdr[7] & MQTT_HAS_WILL) {
    if (c->is_mqtt5 &&
        !mg_send_mqtt_properties(c, opts->will_props, opts->num_will_props))
      goto fail;

    if (!mg_send_u16(c, mg_htons((uint16_t) opts->topic.len)) ||
        !mg_send(c, opts->topic.buf, opts->topic.len) ||
        !mg_send_u16(c, mg_htons((uint16_t) opts->message.len)) ||
        !mg_send(c, opts->message.buf, opts->message.len))
      goto fail;
  }
  if (opts->user.len > 0 &&
      (!mg_send_u16(c, mg_htons((uint16_t) opts->user.len)) ||
       !mg_send(c, opts->user.buf, opts->user.len)))
    goto fail;
  if (opts->pass.len > 0 &&
      (!mg_send_u16(c, mg_htons((uint16_t) opts->pass.len)) ||
       !mg_send(c, opts->pass.buf, opts->pass.len)))
    goto fail;
  return;
fail:
  mg_error(c, "OOM");
}

uint16_t mg_mqtt_pub(struct mg_connection *c, const struct mg_mqtt_opts *opts) {
  uint16_t id = opts->retransmit_id;
  uint8_t flags = (uint8_t) (((opts->qos & 3) << 1) | (opts->retain ? 1 : 0));
  size_t len = 2 + opts->topic.len + opts->message.len;
  MG_DEBUG(("%lu [%.*s] <- [%.*s%c", c->id, (int) opts->topic.len,
            (char *) opts->topic.buf,
            (int) (opts->message.len <= 10 ? opts->message.len : 10),
            (char *) opts->message.buf, opts->message.len <= 10 ? ']' : ' '));
  if (opts->qos > 0) len += 2;
  if (c->is_mqtt5) len += get_props_size(opts->props, opts->num_props);

  if (opts->qos > 0 && id != 0) flags |= 1 << 3;
  if (!mqtt_send_header(c, MQTT_CMD_PUBLISH, flags, (uint32_t) len) ||
      !mg_send_u16(c, mg_htons((uint16_t) opts->topic.len)) ||
      !mg_send(c, opts->topic.buf, opts->topic.len))
    goto fail;
  if (opts->qos > 0) {  // need to send 'id' field
    if (id == 0) {      // generate new one if not resending
      if (++c->mgr->mqtt_id == 0) ++c->mgr->mqtt_id;
      id = c->mgr->mqtt_id;
    }
    if (!mg_send_u16(c, mg_htons(id))) goto fail;
  }

  if (c->is_mqtt5 && !mg_send_mqtt_properties(c, opts->props, opts->num_props))
    goto fail;

  if (opts->message.len > 0 &&
      !mg_send(c, opts->message.buf, opts->message.len))
    goto fail;
  return id;

fail:
  mg_error(c, "OOM");
  return id;
}

static void mg_mqtt_sub_unsub(struct mg_connection *c,
                              const struct mg_mqtt_opts *opts, uint8_t cmd) {
  uint8_t qos_ = opts->qos & 3;
  bool is_sub = cmd == MQTT_CMD_SUBSCRIBE;
  size_t plen = c->is_mqtt5 ? get_props_size(opts->props, opts->num_props) : 0;
  size_t len = 2 + opts->topic.len + 2 + (is_sub ? 1 : 0) + plen;

  if (!mqtt_send_header(c, cmd, 2, (uint32_t) len)) goto fail;
  if (++c->mgr->mqtt_id == 0) ++c->mgr->mqtt_id;
  if (!mg_send_u16(c, mg_htons(c->mgr->mqtt_id))) goto fail;

  if (c->is_mqtt5 && !mg_send_mqtt_properties(c, opts->props, opts->num_props))
    goto fail;

  if (!mg_send_u16(c, mg_htons((uint16_t) opts->topic.len)) ||
      !mg_send(c, opts->topic.buf, opts->topic.len))
    goto fail;
  if (is_sub && !mg_send(c, &qos_, sizeof(qos_))) goto fail;
  return;
fail:
  mg_error(c, "OOM");
}

void mg_mqtt_sub(struct mg_connection *c, const struct mg_mqtt_opts *opts) {
  mg_mqtt_sub_unsub(c, opts, MQTT_CMD_SUBSCRIBE);
}

void mg_mqtt_unsub(struct mg_connection *c, const struct mg_mqtt_opts *opts) {
  mg_mqtt_sub_unsub(c, opts, MQTT_CMD_UNSUBSCRIBE);
}

int mg_mqtt_parse(const uint8_t *buf, size_t len, uint8_t version,
                  struct mg_mqtt_message *m) {
  uint8_t lc = 0, *p, *end;
  uint32_t n = 0, len_len = 0, tmp;

  memset(m, 0, sizeof(*m));
  m->dgram.buf = (char *) buf;
  if (len < 2) return MQTT_INCOMPLETE;
  m->cmd = (uint8_t) (buf[0] >> 4);
  m->qos = (buf[0] >> 1) & 3;

  n = len_len = 0;
  p = (uint8_t *) buf + 1;
  while ((size_t) (p - buf) < len) {
    lc = *((uint8_t *) p++);
    n += (uint32_t) ((lc & 0x7f) << 7 * len_len);
    len_len++;
    if (!(lc & 0x80)) break;
    if (len_len >= 4) return MQTT_MALFORMED;
  }
  end = p + n;
  if ((lc & 0x80) || (end > buf + len)) return MQTT_INCOMPLETE;
  m->dgram.len = (size_t) (end - buf);

  switch (m->cmd) {
    case MQTT_CMD_CONNACK:
      if (end - p < 2) return MQTT_MALFORMED;
      m->ack = p[1];
      p += 2;
      if (version == 5) {
        if (p >= end) return MQTT_MALFORMED;
        len_len = (uint32_t) decode_varint(p, (size_t) (end - p), &tmp);
        if (!len_len) return MQTT_MALFORMED;
        m->props_size = (size_t) tmp;
        m->props_start = (size_t) (p + len_len - buf);
        p += len_len + m->props_size;
        if (p > end) return MQTT_MALFORMED;
      }
      break;
    case MQTT_CMD_PUBACK:
    case MQTT_CMD_PUBREC:
    case MQTT_CMD_PUBREL:
    case MQTT_CMD_PUBCOMP:
    case MQTT_CMD_SUBSCRIBE:
    case MQTT_CMD_SUBACK:
    case MQTT_CMD_UNSUBSCRIBE:
    case MQTT_CMD_UNSUBACK:
      if (p + 2 > end) return MQTT_MALFORMED;
      m->id = (uint16_t) ((((uint16_t) p[0]) << 8) | p[1]);
      p += 2;
      break;
    case MQTT_CMD_PUBLISH: {
      if (p + 2 > end) return MQTT_MALFORMED;
      m->topic.len = (uint16_t) ((((uint16_t) p[0]) << 8) | p[1]);
      m->topic.buf = (char *) p + 2;
      p += 2 + m->topic.len;
      if (p > end) return MQTT_MALFORMED;
      if (m->qos > 0) {
        if (p + 2 > end) return MQTT_MALFORMED;
        m->id = (uint16_t) ((((uint16_t) p[0]) << 8) | p[1]);
        p += 2;
      }
      if (p > end) return MQTT_MALFORMED;
      if (version == 5 && p + 2 < end) {
        len_len = (uint32_t) decode_varint(p, (size_t) (end - p), &tmp);
        if (!len_len) return MQTT_MALFORMED;
        m->props_size = (size_t) tmp;
        m->props_start = (size_t) (p + len_len - buf);
        p += len_len + m->props_size;
      }
      if (p > end) return MQTT_MALFORMED;
      m->data.buf = (char *) p;
      m->data.len = (size_t) (end - p);
      break;
    }
    default: break;
  }
  return MQTT_OK;
}

static void mqtt_cb(struct mg_connection *c, int ev, void *ev_data) {
  if (ev == MG_EV_READ) {
    for (;;) {
      uint8_t version = c->is_mqtt5 ? 5 : 4;
      struct mg_mqtt_message mm;
      int rc = mg_mqtt_parse(c->recv.buf, c->recv.len, version, &mm);
      if (rc == MQTT_MALFORMED) {
        MG_ERROR(("%lu MQTT malformed message", c->id));
        c->is_closing = 1;
        break;
      } else if (rc == MQTT_OK) {
        MG_VERBOSE(("%lu MQTT CMD %d len %d [%.*s]", c->id, mm.cmd,
                    (int) mm.dgram.len, (int) mm.data.len, mm.data.buf));
        switch (mm.cmd) {
          case MQTT_CMD_CONNACK:
            mg_call(c, MG_EV_MQTT_OPEN, &mm.ack);
            if (mm.ack == 0) {
              MG_DEBUG(("%lu Connected", c->id));
            } else {
              MG_ERROR(("%lu MQTT auth failed, code %d", c->id, mm.ack));
              c->is_closing = 1;
            }
            break;
          case MQTT_CMD_PUBLISH: {
            MG_DEBUG(("%lu [%.*s] -> [%.*s%c", c->id, (int) mm.topic.len,
                      mm.topic.buf,
                      (int) (mm.data.len <= 10 ? mm.data.len : 10), mm.data.buf,
                      mm.data.len <= 10 ? ']' : ' '));
            if (mm.qos > 0) {
              uint16_t id = mg_ntohs(mm.id);
              uint32_t remaining_len = sizeof(id);
              if (c->is_mqtt5) remaining_len += 2;  // 3.4.2

              if (!mqtt_send_header(c,
                                    (uint8_t) (mm.qos == 2 ? MQTT_CMD_PUBREC
                                                           : MQTT_CMD_PUBACK),
                                    0, remaining_len) ||
                  !mg_send(c, &id, sizeof(id)))
                goto fail;

              if (c->is_mqtt5) {
                uint16_t zero = 0;
                if (!mg_send(c, &zero, sizeof(zero))) goto fail;
              }
            }
            mg_call(c, MG_EV_MQTT_MSG, &mm);  // let the app handle qos stuff
            break;
          }
          case MQTT_CMD_PUBREC: {  // MQTT5: 3.5.2-1 TODO(): variable header rc
            uint16_t id = mg_ntohs(mm.id);
            uint32_t remaining_len = sizeof(id);  // MQTT5 3.6.2-1
            if (!mqtt_send_header(c, MQTT_CMD_PUBREL, 2,
                                  remaining_len)  // MQTT5 3.6.1-1, flags = 2
                || !mg_send(c, &id, sizeof(id)))
              goto fail;
            break;
          }
          case MQTT_CMD_PUBREL: {  // MQTT5: 3.6.2-1 TODO(): variable header rc
            uint16_t id = mg_ntohs(mm.id);
            uint32_t remaining_len = sizeof(id);  // MQTT5 3.7.2-1
            if (!mqtt_send_header(c, MQTT_CMD_PUBCOMP, 0, remaining_len) ||
                !mg_send(c, &id, sizeof(id)))
              goto fail;
            break;
          }
        }
        mg_call(c, MG_EV_MQTT_CMD, &mm);
        mg_iobuf_del(&c->recv, 0, mm.dgram.len);
      } else {
        break;
      }
    }
  }
  (void) ev_data;
  return;
fail:
  mg_error(c, "OOM");
}

void mg_mqtt_ping(struct mg_connection *nc) {
  mg_mqtt_send_header(nc, MQTT_CMD_PINGREQ, 0, 0);
}

void mg_mqtt_pong(struct mg_connection *nc) {
  mg_mqtt_send_header(nc, MQTT_CMD_PINGRESP, 0, 0);
}

void mg_mqtt_disconnect(struct mg_connection *c,
                        const struct mg_mqtt_opts *opts) {
  size_t len = 0;
  if (c->is_mqtt5) len = 1 + get_props_size(opts->props, opts->num_props);
  if (!mqtt_send_header(c, MQTT_CMD_DISCONNECT, 0, (uint32_t) len)) goto fail;

  if (c->is_mqtt5) {
    uint8_t zero = 0;
    if (!mg_send(c, &zero, sizeof(zero))  // reason code
        || !mg_send_mqtt_properties(c, opts->props, opts->num_props))
      goto fail;
  }
  return;
fail:
  mg_error(c, "OOM");
}

struct mg_connection *mg_mqtt_connect(struct mg_mgr *mgr, const char *url,
                                      const struct mg_mqtt_opts *opts,
                                      mg_event_handler_t fn, void *fn_data) {
  struct mg_connection *c =
      mg_connect_svc(mgr, url, fn, fn_data, mqtt_cb, NULL);
  if (c != NULL) {
    struct mg_mqtt_opts empty;
    memset(&empty, 0, sizeof(empty));
    mg_mqtt_login(c, opts == NULL ? &empty : opts);
  }
  return c;
}

struct mg_connection *mg_mqtt_listen(struct mg_mgr *mgr, const char *url,
                                     mg_event_handler_t fn, void *fn_data) {
  struct mg_connection *c = mg_listen(mgr, url, fn, fn_data);
  if (c != NULL) c->pfn = mqtt_cb, c->pfn_data = mgr;
  return c;
}

#ifdef MG_ENABLE_LINES
#line 1 "src/net.c"
#endif










size_t mg_vprintf(struct mg_connection *c, const char *fmt, va_list *ap) {
  size_t old = c->send.len;
  size_t expected = mg_vxprintf(mg_pfn_iobuf, &c->send, fmt, ap);
  size_t actual = c->send.len - old;
  if (actual != expected) {
    mg_error(c, "OOM");
    c->send.len = old;
    actual = 0;
  }
  return actual;
}

size_t mg_printf(struct mg_connection *c, const char *fmt, ...) {
  size_t len = 0;
  va_list ap;
  va_start(ap, fmt);
  len = mg_vprintf(c, fmt, &ap);
  va_end(ap);
  return len;
}

bool mg_dscp(struct mg_connection *c, uint8_t dscp) {
#if MG_ENABLE_SOCKET
#if MG_ENABLE_FREERTOS_TCP || MG_ARCH == MG_ARCH_THREADX || \
    MG_ARCH == MG_ARCH_TIRTOS || MG_ARCH == MG_ARCH_WIN32
  (void) c;
  (void) dscp;
#else
  int tos = dscp << 2, level = IPPROTO_IP, optname = IP_TOS;
  c->dscp = (unsigned) (dscp & 63);
#if MG_ENABLE_IPV6 && !MG_ENABLE_LWIP
  if (c->loc.is_ip6) level = IPPROTO_IPV6, optname = IPV6_TCLASS;
#endif
  return setsockopt((MG_SOCKET_TYPE) (size_t) c->fd, level, optname,
                    (char *) &tos, sizeof(tos)) == 0;
#endif
#elif MG_ENABLE_TCPIP
  c->dscp = (unsigned) (dscp & 63);
  return true;
#endif
  return false;
}

static bool mg_atonl(struct mg_str str, struct mg_addr *addr) {
  uint32_t localhost = mg_htonl(0x7f000001);
  if (mg_strcasecmp(str, mg_str("localhost")) != 0) return false;
  memcpy(addr->addr.ip, &localhost, sizeof(uint32_t));
  addr->is_ip6 = false;
  return true;
}

static bool mg_atone(struct mg_str str, struct mg_addr *addr) {
  if (str.len > 0) return false;
  memset(addr->addr.ip, 0, sizeof(addr->addr.ip));
  addr->is_ip6 = false;
  return true;
}

static bool mg_aton4(struct mg_str str, struct mg_addr *addr) {
  uint8_t data[4] = {0, 0, 0, 0};
  size_t i, num_dots = 0;
  for (i = 0; i < str.len; i++) {
    if (str.buf[i] >= '0' && str.buf[i] <= '9') {
      int octet = data[num_dots] * 10 + (str.buf[i] - '0');
      if (octet > 255) return false;
      data[num_dots] = (uint8_t) octet;
    } else if (str.buf[i] == '.') {
      if (num_dots >= 3 || i == 0 || str.buf[i - 1] == '.') return false;
      num_dots++;
    } else {
      return false;
    }
  }
  if (num_dots != 3 || str.buf[i - 1] == '.') return false;
  memcpy(&addr->addr.ip, data, sizeof(data));
  addr->is_ip6 = false;
  return true;
}

static bool mg_v4mapped(struct mg_str str, struct mg_addr *addr) {
  int i;
  uint32_t ipv4;
  if (str.len < 14) return false;
  if (str.buf[0] != ':' || str.buf[1] != ':' || str.buf[6] != ':') return false;
  for (i = 2; i < 6; i++) {
    if (str.buf[i] != 'f' && str.buf[i] != 'F') return false;
  }
  // struct mg_str s = mg_str_n(&str.buf[7], str.len - 7);
  if (!mg_aton4(mg_str_n(&str.buf[7], str.len - 7), addr)) return false;
  memcpy(&ipv4, addr->addr.ip, sizeof(ipv4));
  memset(addr->addr.ip, 0, sizeof(addr->addr.ip));
  addr->addr.ip[10] = addr->addr.ip[11] = 255;
  memcpy(&addr->addr.ip[12], &ipv4, 4);
  addr->is_ip6 = true;
  return true;
}

static bool mg_aton6(struct mg_str str, struct mg_addr *addr) {
  size_t i, j = 0, n = 0, dc = 42;
  uint16_t val = 0;
  bool have = false;
  addr->scope_id = 0;
  if (str.len > 2 && str.buf[0] == '[') str.buf++, str.len -= 2;
  if (mg_v4mapped(str, addr)) return true;  // sets addr->is_ip6
  for (i = 0; i < str.len; i++) {
    int x = str.buf[i] >= '0' && str.buf[i] <= '9'   ? str.buf[i] - '0'
            : str.buf[i] >= 'a' && str.buf[i] <= 'f' ? str.buf[i] - 'a' + 10
            : str.buf[i] >= 'A' && str.buf[i] <= 'F' ? str.buf[i] - 'A' + 10
                                                     : -1;
    if (x >= 0) {
      if (i > j + 3) return false;
      val = (uint16_t) ((val << 4) | (uint16_t) x);
      have = true;
      continue;
    }
    if (have) {
      addr->addr.ip[n] = (uint8_t) (val >> 8),
      addr->addr.ip[n + 1] = (uint8_t) val;
      val = 0, have = false;
    }
    if (str.buf[i] == ':') {
      j = i + 1;
      if (i > 0 && str.buf[i - 1] == ':') {
        if (dc != 42) return false;
        dc = n;  // Double colon
        if (i > 1 && str.buf[i - 2] == ':') return false;
      } else if (i > 0) {
        n += 2;
      }
      if (n > 14) return false;
      addr->addr.ip[n] = addr->addr.ip[n + 1] = 0;  // For trailing ::
    } else if (str.buf[i] == '%') {                 // Scope ID, last in string
      if (mg_str_to_num(mg_str_n(&str.buf[i + 1], str.len - i - 1), 10,
                        &addr->scope_id, sizeof(uint8_t))) {
        addr->is_ip6 = true;
        return true;
      } else {
        return false;
      }
    } else {
      return false;
    }
  }
  if (have)
    addr->addr.ip[n] = (uint8_t) (val >> 8),
    addr->addr.ip[n + 1] = (uint8_t) val;
  if (n < 14 && dc == 42) return false;
  if (n < 14) {
    memmove(&addr->addr.ip[dc + (14 - n)], &addr->addr.ip[dc], n - dc + 2);
    memset(&addr->addr.ip[dc], 0, 14 - n);
  }
  addr->is_ip6 = true;
  return true;
}

bool mg_aton_(struct mg_str str, struct mg_addr *addr);
bool mg_aton_(struct mg_str str, struct mg_addr *addr) {
  return mg_aton4(str, addr) || mg_aton6(str, addr);
}

bool mg_aton(struct mg_str str, struct mg_addr *addr) {
  return mg_atone(str, addr) || mg_atonl(str, addr) || mg_aton_(str, addr);
}

struct mg_connection *mg_alloc_conn(struct mg_mgr *mgr) {
  struct mg_connection *c =
      (struct mg_connection *) mg_calloc(1, sizeof(*c) + mgr->extraconnsize);
  if (c != NULL) {
    c->mgr = mgr;
    c->send.align = c->recv.align = c->rtls.align = MG_IO_SIZE;
    c->id = ++mgr->nextid;
    MG_PROF_INIT(c);
  }
  return c;
}

void mg_close_conn(struct mg_connection *c) {
  mg_resolve_cancel(c);  // Close any pending DNS query
  LIST_DELETE(struct mg_connection, &c->mgr->conns, c);
  if (c == c->mgr->dns4.c) c->mgr->dns4.c = NULL;
  if (c == c->mgr->dns6.c) c->mgr->dns6.c = NULL;
  // Order of operations is important. `MG_EV_CLOSE` event must be fired
  // before we deallocate received data, see #1331
  mg_call(c, MG_EV_CLOSE, NULL);
  MG_DEBUG(("%lu %ld closed", c->id, c->fd));
  MG_PROF_DUMP(c);
  MG_PROF_FREE(c);

  mg_tls_free(c);
  mg_iobuf_free(&c->recv);
  mg_iobuf_free(&c->send);
  mg_iobuf_free(&c->rtls);
  mg_bzero((unsigned char *) c, sizeof(*c));
  mg_free(c);
}

struct mg_connection *mg_connect_svc(struct mg_mgr *mgr, const char *url,
                                     mg_event_handler_t fn, void *fn_data,
                                     mg_event_handler_t pfn, void *pfn_data) {
  struct mg_connection *c = NULL;
  if (url == NULL || url[0] == '\0') {
    MG_ERROR(("null url"));
#if MG_ENABLE_TCPIP
  } else if (mgr->ifp != NULL && mgr->ifp->state != MG_TCPIP_STATE_READY) {
    MG_ERROR(("Network is down"));
#endif
  } else if ((c = mg_alloc_conn(mgr)) == NULL) {
    MG_ERROR(("OOM"));
  } else {
    LIST_ADD_HEAD(struct mg_connection, &mgr->conns, c);
    c->is_udp = (strncmp(url, "udp:", 4) == 0);
    c->fd = (void *) (size_t) MG_INVALID_SOCKET;
    c->fn = fn;
    c->is_client = true;
    c->fn_data = fn_data;
    c->is_tls = (mg_url_is_ssl(url) != 0);
    c->pfn = pfn;
    c->pfn_data = pfn_data;
    mg_call(c, MG_EV_OPEN, (void *) url);
    MG_DEBUG(("%lu %ld %s", c->id, c->fd, url));
    mg_resolve(c, url);
  }
  return c;
}

struct mg_connection *mg_connect(struct mg_mgr *mgr, const char *url,
                                 mg_event_handler_t fn, void *fn_data) {
  return mg_connect_svc(mgr, url, fn, fn_data, NULL, NULL);
}

struct mg_connection *mg_listen(struct mg_mgr *mgr, const char *url,
                                mg_event_handler_t fn, void *fn_data) {
  struct mg_connection *c = NULL;
  if ((c = mg_alloc_conn(mgr)) == NULL) {
    MG_ERROR(("OOM %s", url));
  } else if (!mg_open_listener(c, url)) {
    MG_ERROR(("Failed: %s", url));
    MG_PROF_FREE(c);
    mg_free(c);
    c = NULL;
  } else {
    c->is_listening = 1;
    c->is_udp = strncmp(url, "udp:", 4) == 0;
    LIST_ADD_HEAD(struct mg_connection, &mgr->conns, c);
    c->fn = fn;
    c->fn_data = fn_data;
    c->is_tls = (mg_url_is_ssl(url) != 0);
    mg_call(c, MG_EV_OPEN, NULL);
    MG_DEBUG(("%lu %ld %s", c->id, c->fd, url));
  }
  return c;
}

struct mg_connection *mg_wrapfd(struct mg_mgr *mgr, int fd,
                                mg_event_handler_t fn, void *fn_data) {
  struct mg_connection *c = mg_alloc_conn(mgr);
  if (c != NULL) {
    c->fd = (void *) (size_t) fd;
    c->fn = fn;
    c->fn_data = fn_data;
    MG_EPOLL_ADD(c);
    mg_call(c, MG_EV_OPEN, NULL);
    LIST_ADD_HEAD(struct mg_connection, &mgr->conns, c);
  }
  return c;
}

struct mg_timer *mg_timer_add(struct mg_mgr *mgr, uint64_t milliseconds,
                              unsigned flags, void (*fn)(void *), void *arg) {
  struct mg_timer *t = (struct mg_timer *) mg_calloc(1, sizeof(*t));
  if (t != NULL) {
    flags |= MG_TIMER_AUTODELETE;  // We have alloc'ed it, so autodelete
    mg_timer_init(&mgr->timers, t, milliseconds, flags, fn, arg);
  }
  return t;
}

long mg_io_recv(struct mg_connection *c, void *buf, size_t len) {
  if (c->rtls.len == 0) return MG_IO_WAIT;
  if (len > c->rtls.len) len = c->rtls.len;
  memcpy(buf, c->rtls.buf, len);
  mg_iobuf_del(&c->rtls, 0, len);
  return (long) len;
}

void mg_mgr_free(struct mg_mgr *mgr) {
  struct mg_connection *c;
  struct mg_timer *tmp, *t = mgr->timers;
  while (t != NULL) tmp = t->next, mg_free(t), t = tmp;
  mgr->timers = NULL;  // Important. Next call to poll won't touch timers
  for (c = mgr->conns; c != NULL; c = c->next) c->is_closing = 1;
#if MG_ENABLE_MDASH
  mg_mdash_free(mgr);  // Before the final poll, so mdash does not reconnect
#endif
  mg_mgr_poll(mgr, 0);
#if MG_ENABLE_FREERTOS_TCP
  FreeRTOS_DeleteSocketSet(mgr->ss);
#endif
  MG_DEBUG(("All connections closed"));
#if MG_ENABLE_EPOLL
  if (mgr->epoll_fd >= 0) close(mgr->epoll_fd), mgr->epoll_fd = -1;
#endif
  mg_tls_ctx_free(mgr);
#if MG_ENABLE_TCPIP
  if (mgr->ifp) mg_tcpip_free(mgr->ifp);
#endif
}

void mg_mgr_init(struct mg_mgr *mgr) {
  memset(mgr, 0, sizeof(*mgr));
#if MG_ENABLE_EPOLL
  if ((mgr->epoll_fd = epoll_create1(EPOLL_CLOEXEC)) < 0)
    MG_ERROR(("epoll_create1 errno %d", errno));
#else
  mgr->epoll_fd = -1;
#endif
#if MG_ARCH == MG_ARCH_WIN32 && MG_ENABLE_WINSOCK
  // clang-format off
  { WSADATA data; WSAStartup(MAKEWORD(2, 2), &data); }
  // clang-format on
#elif MG_ENABLE_FREERTOS_TCP
  mgr->ss = FreeRTOS_CreateSocketSet();
#elif MG_ARCH == MG_ARCH_UNIX
  // Ignore SIGPIPE signal, so if client cancels the request, it
  // won't kill the whole process.
  signal(SIGPIPE, SIG_IGN);
#elif MG_ENABLE_TCPIP_DRIVER_INIT && defined(MG_TCPIP_DRIVER_INIT)
  MG_TCPIP_DRIVER_INIT(mgr);
#endif
#if MG_ENABLE_BSD_SOCKETS
  mgr->pipe.q = NULL;
#else
  mgr->pipe.fd = MG_INVALID_SOCKET;
#endif
  mgr->dnstimeout = 3000;
  mgr->dns4.url = "udp://8.8.8.8:53";
  mgr->dns6.url = "udp://[2001:4860:4860::8888]:53";
  mg_tls_ctx_init(mgr);
  MG_DEBUG(("MG_IO_SIZE: %lu, TLS: %s, conn size: %zu, mgr size: %zu",
            MG_IO_SIZE,
            MG_TLS == MG_TLS_NONE      ? "none"
            : MG_TLS == MG_TLS_MBED    ? "MbedTLS"
            : MG_TLS == MG_TLS_OPENSSL ? "OpenSSL"
            : MG_TLS == MG_TLS_BUILTIN ? "builtin"
            : MG_TLS == MG_TLS_WOLFSSL ? "WolfSSL"
                                       : "custom",
            sizeof(struct mg_connection), sizeof(struct mg_mgr)));
#if MG_ENABLE_MDASH
  mg_mdash_init(mgr);
#endif
}

#if MG_ENABLE_TCPIP
void mg_tcpip_mapip(struct mg_connection *, struct mg_addr *);
#endif
void mg_multicast_restore(struct mg_connection *c, uint8_t *from) {
  memcpy(&c->rem, from, sizeof(c->rem));
#if MG_ENABLE_TCPIP
  mg_tcpip_mapip(c, &c->rem);
#endif
}

#ifdef MG_ENABLE_LINES
#line 1 "src/net_builtin.c"
#endif




#if MG_ENABLE_TCPIP
#define MG_EPHEMERAL_PORT_BASE 32768
#define PDIFF(a, b) ((size_t) (((char *) (b)) - ((char *) (a))))

#ifndef MG_TCPIP_KEEPALIVE_MS
#define MG_TCPIP_KEEPALIVE_MS 45000  // TCP keep-alive period, ms
#endif

#define MG_TCPIP_ACK_MS 150    // Timeout for ACKing
#define MG_TCPIP_ARP_MS 100    // Timeout for ARP response
#define MG_TCPIP_SYN_MS 15000  // Timeout for connection establishment
#define MG_TCPIP_FIN_MS 1000   // Timeout for closing connection
#define MG_TCPIP_RTX_MS 1000   // Initial retransmission timeout (RFC-6298, 2.1)
#define MG_TCPIP_RTX_MAX_MS 60000  // Minimum permitted RTO cap (RFC-6298, 2.5)
#define MG_TCPIP_RTX_R1 3  // Delivery-problem threshold (RFC-9293, 3.8.3)
// No R2: MG_TCPIP_KEEPALIVE_MS eventually closes non-responsive peers

#ifndef MG_TCPIP_WIN
#define MG_TCPIP_WIN 6000  // TCP window size
#endif

#ifndef MG_TCPIP_MIN_MSS
#define MG_TCPIP_MIN_MSS 64  // Minimum peer MSS
#endif

#ifndef MG_TCPIP_TXQ_ALIGN
#define MG_TCPIP_TXQ_ALIGN 1024  // Retransmit queue allocation quantum
#endif

struct connstate {
  uint32_t seq, ack;   // TCP seq/ack counters
  uint64_t timer;      // TCP timer (see 'ttype' below)
  uint64_t txq_timer;  // TCP retransmission timer (RFC-6298, 5)
  uint32_t acked;      // Last ACK-ed number
  size_t unacked;      // Not acked bytes
  uint32_t maxseq;     // Max send seq (ack + window)
  uint32_t txq_seq;    // Sequence of first txq record (RFC-9293, 3.8)
  uint32_t txq_una;    // Oldest unacknowledged sequence (RFC-9293, 3.4)
  uint16_t win;        // destination current window size
  uint16_t dmss;       // destination MSS (from TCP opts)
  uint8_t mac[sizeof(struct mg_l2addr)];  // Peer hw address
  uint8_t ttype;                          // Timer type:
#define MIP_TTYPE_KEEPALIVE 0  // Connection is idle for long, send keepalive
#define MIP_TTYPE_ACK 1        // Peer sent us data, we have to ack it soon
#define MIP_TTYPE_ARP 2        // ARP resolve sent, waiting for response
#define MIP_TTYPE_SYN 3        // SYN sent, waiting for response
#define MIP_TTYPE_FIN 4  // FIN sent, waiting until terminating the connection
  uint8_t tmiss;         // Number of keep-alive misses
  uint8_t txq_retries;   // Retransmissions since ACK progress (RFC-9293, 3.8.3)
  bool fin_rcvd;         // We have received FIN from the peer
  bool twclosure;        // 3-way closure done
  bool retransmit;       // Retain sent data until acknowledged
  bool is_full;          // Advertise a zero receive window
  bool is_closing;       // Finished, remove connection (TCP after FIN)
  bool send_rst;         // Send RST before removing a TCP connection
  struct mg_iobuf txq;   // Length-prefixed sent TCP segments (RFC-9293, 3.8)
};

#if defined(__DCC__)
#pragma pack(1)
#else
#pragma pack(push, 1)
#endif

struct ip {
  uint8_t ver;    // Version
  uint8_t tos;    // DS field
  uint16_t len;   // Datagram length
  uint16_t id;    // Unused
  uint16_t frag;  // Fragmentation
#define IP_FRAG_OFFSET_MSK 0x1fff
#define IP_MORE_FRAGS_MSK 0x2000
  uint8_t ttl;    // Time to live
  uint8_t proto;  // Upper level protocol
  uint16_t csum;  // Checksum
  uint32_t src;   // Source IP
  uint32_t dst;   // Destination IP
};

struct ip6 {
  uint8_t ver;       // Version
  uint8_t label[3];  // Flow label
  uint16_t plen;     // Payload length
  uint8_t next;      // Upper level protocol
  uint8_t hops;      // Hop limit
  uint64_t src[2];   // Source IP
  uint64_t dst[2];   // Destination IP
};

struct icmp {
  uint8_t type;
  uint8_t code;
  uint16_t csum;
};

struct icmp6 {
  uint8_t type;
  uint8_t code;
  uint16_t csum;
};

struct ndp_na {
  uint8_t res[4];    // R S O, reserved
  uint64_t addr[2];  // Target address
};

struct ndp_ra {
  uint8_t cur_hop_limit;
  uint8_t flags;  // M,O,Prf,Resvd
  uint16_t router_lifetime;
  uint32_t reachable_time;
  uint32_t retrans_timer;
};

struct arp {
  uint16_t fmt;    // Format of hardware address
  uint16_t pro;    // Format of protocol address
  uint8_t hlen;    // Length of hardware address
  uint8_t plen;    // Length of protocol address
  uint16_t op;     // Operation
  uint8_t sha[6];  // Sender hardware address
  uint32_t spa;    // Sender protocol address
  uint8_t tha[6];  // Target hardware address
  uint32_t tpa;    // Target protocol address
};

struct tcp {
  uint16_t sport;  // Source port
  uint16_t dport;  // Destination port
  uint32_t seq;    // Sequence number
  uint32_t ack;    // Acknowledgement number
  uint8_t off;     // Data offset
  uint8_t flags;   // TCP flags
#define TH_FIN 0x01
#define TH_SYN 0x02
#define TH_RST 0x04
#define TH_PUSH 0x08
#define TH_ACK 0x10
#define TH_URG 0x20
#define TH_STDFLAGS 0x3f
  // #define TH_ECE 0x40 // not part of TCP but RFC-3168 (ECN)
  // #define TH_CWR 0x80
  uint16_t win;   // Window
  uint16_t csum;  // Checksum
  uint16_t urp;   // Urgent pointer
};

struct udp {
  uint16_t sport;  // Source port
  uint16_t dport;  // Destination port
  uint16_t len;    // UDP length
  uint16_t csum;   // UDP checksum
};

struct dhcp {
  uint8_t op, htype, hlen, hops;
  uint32_t xid;
  uint16_t secs, flags;
  uint32_t ciaddr, yiaddr, siaddr, giaddr;
  uint8_t hwaddr[208];
  uint32_t magic;
  uint8_t options[30 + sizeof(((struct mg_tcpip_if *) 0)->dhcp_name)];
};

struct dhcp6 {
  union {
    uint8_t type;
    uint32_t xid;
  };
  uint8_t options[30 + sizeof(((struct mg_tcpip_if *) 0)->dhcp_name)];
};

struct pseudoip {
  uint32_t src;  // Source IP
  uint32_t dst;  // Destination IP
  uint8_t zero;
  uint8_t proto;  // Upper level protocol
  uint16_t len;   // Datagram length
};

struct pseudoip6 {
  uint64_t src[2];  // Source IP
  uint64_t dst[2];  // Destination IP
  uint32_t plen;    // Payload length
  uint8_t zero[3];
  uint8_t next;  // Upper level protocol
};

#if defined(__DCC__)
#pragma pack(0)
#else
#pragma pack(pop)
#endif

// pkt is 8-bit aligned, pointers to headers hint compilers to generate
// byte-copy code for micros with alignment constraints
struct pkt {
  struct mg_str raw;  // Raw packet data
  struct mg_str pay;  // Payload data
  uint8_t *l2;        // Ethernet, PPP [, etc] frame data
  struct arp *arp;
  struct ip *ip;
  struct ip6 *ip6;
  struct icmp *icmp;
  struct icmp6 *icmp6;
  struct tcp *tcp;
  struct udp *udp;
  struct dhcp *dhcp;
  struct dhcp6 *dhcp6;
};

// L2 API
void mg_l2_init(struct mg_tcpip_if *ifp);
uint8_t *mg_l2_header(struct mg_tcpip_if *ifp, enum mg_l2proto proto,
                      uint8_t *src, uint8_t *dst, uint8_t *frame);
size_t mg_l2_trailer(struct mg_tcpip_if *ifp, size_t len, uint8_t *cur);
bool mg_l2_rx(struct mg_tcpip_if *ifp, enum mg_l2proto *proto,
              struct mg_str *pay, struct mg_str *raw);
uint8_t *mg_l2_getaddr(struct mg_tcpip_if *ifp, uint8_t *frame);
uint8_t *mg_l2_mapip(enum mg_l2type type, enum mg_l2addrtype addrtype,
                     struct mg_addr *ip);
#if MG_ENABLE_IPV6
bool mg_l2_genip6(enum mg_l2type type, uint64_t *ip6, uint8_t prefix_len,
                  uint8_t *addr);
bool mg_l2_ip6get(enum mg_l2type type, uint8_t *addr, uint8_t *opts,
                  uint8_t len);
uint8_t mg_l2_ip6put(enum mg_l2type type, uint8_t *addr, uint8_t *opts);
#endif
bool mg_l2_poll(struct mg_tcpip_if *ifp, bool expired_1000ms);

static void mg_tcpip_call(struct mg_tcpip_if *ifp, int ev, void *ev_data) {
#if 0 && MG_ENABLE_PROFILE
  const char *names[] = {"TCPIP_EV_STATE_CHANGE",  "TCPIP_EV_DHCP_DNS",
                         "TCPIP_EV_DHCP_SNTP",     "TCPIP_EV_ARP",
                         "TCPIP_EV_TIMER_1S",      "TCPIP_EV_WIFI_SCAN_RESULT",
                         "TCPIP_EV_WIFI_SCAN_END", "TCPIP_EV_WIFI_CONNECT_ERR",
                         "TCPIP_EV_DRIVER",        "TCPIP_EV_STATE6_CHANGE",
                         "TCPIP_EV_USER"};
  if (ev != MG_TCPIP_EV_TIMER_1S && ev < (int) (sizeof(names) / sizeof(names[0]))) {
    MG_PROF_ADD(ifp, names[ev]); // TODO(): call MG_PROF_DUMP() MG_PROF_FREE()
  }
#endif
  // Fire protocol handler first, user handler second. See #2559
  if (ifp->pfn != NULL) ifp->pfn(ifp, ev, ev_data);
  if (ifp->fn != NULL) ifp->fn(ifp, ev, ev_data);
}

static void send_syn(struct mg_connection *c);

static void mkpay(struct pkt *pkt, void *p) {
  pkt->pay =
      mg_str_n((char *) p, (size_t) (&pkt->pay.buf[pkt->pay.len] - (char *) p));
}

// NOTE(): DOES NOT handle reentries after odd length, use last
static uint32_t csumup(uint32_t sum, const void *buf, size_t len) {
  size_t i;
  const uint8_t *p = (const uint8_t *) buf;
  for (i = 0; i < len; i++) sum += i & 1 ? p[i] : ((uint32_t) p[i]) << 8;
  return sum;
}

static uint16_t csumfin(uint32_t sum) {
  while (sum >> 16) sum = (sum & 0xffff) + (sum >> 16);
  return mg_htons((uint16_t) ((uint16_t) ~sum & 0xffff));
}

static uint16_t ipcsum(const void *buf, size_t len) {
  uint32_t sum = csumup(0, buf, len);
  return csumfin(sum);
}

static bool ipcsum_ok(const void *d) {
  struct ip *ip = (struct ip *) d;
  return (ipcsum(d, (ip->ver & 0x0F) * 4) == 0);
}

static bool icmpcsum_ok(const void *d, size_t len) {
  return (ipcsum(d, len) == 0);
}

static uint16_t pcsum(void *d, void *p, size_t plen) {
  uint32_t sum;
  struct ip *ip = (struct ip *) d;
#if defined(__DCC__)
  volatile /* Makes PPC & Diab4.3 happy */
#endif
      struct pseudoip pip;
  pip.src = ip->src;
  pip.dst = ip->dst;
  pip.zero = 0;
  pip.proto = ip->proto;
  pip.len = mg_htons((uint16_t) plen);
  sum = csumup(0, &pip, sizeof(pip));  // even length
  sum = csumup(sum, p, plen);          // possibly odd length: last
  return csumfin(sum);
}

static bool udpcsum_ok(void *d, void *u) {
  struct udp *udp = (struct udp *) u;
  if (udp->csum == 0) return true;
  if (udp->csum == 0xFFFF) udp->csum = 0;
  return (pcsum(d, u, (size_t) mg_ntohs(udp->len)) == 0);
}

static bool tcpcsum_ok(void *d, void *t) {
  struct ip *ip = (struct ip *) d;
  return (pcsum(d, t, (size_t) (mg_ntohs(ip->len) - (ip->ver & 0x0F) * 4)) ==
          0);
}

#if MG_ENABLE_IPV6
static uint16_t p6csum(void *d, void *p, uint8_t proto, size_t plen) {
  uint32_t sum;
  struct ip6 *ip6 = (struct ip6 *) d;
#if defined(__DCC__)
  volatile /* Makes PPC & Diab4.3 happy */
#endif
      struct pseudoip6 pip6;
  pip6.src[0] = ip6->src[0], pip6.src[1] = ip6->src[1];
  pip6.dst[0] = ip6->dst[0], pip6.dst[1] = ip6->dst[1];
  pip6.zero[0] = 0, pip6.zero[1] = 0, pip6.zero[2] = 0;
  pip6.plen = mg_htonl((uint32_t) plen);
  pip6.next = proto;
  sum = csumup(0, &pip6, sizeof(pip6));  // even length
  sum = csumup(sum, p, plen);            // possibly odd length: last
  return csumfin(sum);
}

static bool udp6csum_ok(void *d, void *u) {
  struct udp *udp = (struct udp *) u;
  if (udp->csum == 0) return false;  // mandatory in IPv6
  if (udp->csum == 0xFFFF) udp->csum = 0;
  return (p6csum(d, u, 17, (size_t) mg_ntohs(udp->len)) == 0);
}
static bool tcp6csum_ok(void *d, void *t, size_t len) {
  return p6csum(d, t, 6, len) == 0;
}

static bool icmp6csum_ok(void *d, void *i, size_t len) {
  return p6csum(d, i, 58, len) == 0;
}
static void ip6sn(uint64_t *addr, uint64_t *sn_addr) {
  // Build solicited-node multicast address from a given unicast IP
  // RFC-4291 2.7
  uint8_t *sn = (uint8_t *) sn_addr;
  memset(sn_addr, 0, 16);
  sn[0] = 0xff;
  sn[1] = 0x02;
  sn[11] = 0x01;
  sn[12] = 0xff;
  sn[13] = ((uint8_t *) addr)[13];
  sn[14] = ((uint8_t *) addr)[14];
  sn[15] = ((uint8_t *) addr)[15];
}

static const struct mg_addr ip6_allrouters = {
    {{0xFF, 0x02, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x02}}, 0, 0, true};
static const struct mg_addr ip6_allnodes = {
    {{0xFF, 0x02, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x01}}, 0, 0, true};

#define MG_IP6MATCH(a, b) (a[0] == b[0] && a[1] == b[1])
#endif

static void settmout(struct mg_connection *c, uint8_t type) {
  struct mg_tcpip_if *ifp = c->mgr->ifp;
  struct connstate *s = (struct connstate *) (c + 1);
  unsigned n = type == MIP_TTYPE_ACK   ? MG_TCPIP_ACK_MS
               : type == MIP_TTYPE_ARP ? MG_TCPIP_ARP_MS
               : type == MIP_TTYPE_SYN ? MG_TCPIP_SYN_MS
               : type == MIP_TTYPE_FIN ? MG_TCPIP_FIN_MS
                                       : MG_TCPIP_KEEPALIVE_MS;
  if (s->ttype == MIP_TTYPE_FIN) return;  // skip if 3-way closing
  s->timer = ifp->now + n;
  s->ttype = type;
  MG_VERBOSE(("%lu %d -> %llx", c->id, type, s->timer));
}

static size_t driver_output(struct mg_tcpip_if *ifp, size_t len) {
  size_t n = ifp->driver->tx(ifp->tx.buf, len, ifp);
  if (n == len) ifp->nsent++;
  return n;
}

// RFC826, ARP assumes Ethernet MAC addresses
void mg_tcpip_arp_request(struct mg_tcpip_if *ifp, uint32_t ip, uint8_t *mac) {
  uint8_t *l2p = (uint8_t *) ifp->tx.buf;
  struct arp *arp = (struct arp *) mg_l2_header(
      ifp, MG_TCPIP_L2PROTO_ARP, ifp->mac,
      mg_l2_mapip(ifp->l2type, MG_TCPIP_L2ADDR_BCAST, NULL), l2p);
  memset(arp, 0, sizeof(*arp));
  arp->fmt = mg_htons(1), arp->pro = mg_htons(0x800), arp->hlen = 6,
  arp->plen = 4;
  arp->op = mg_htons(1), arp->tpa = ip, arp->spa = ifp->ip;
  memcpy(arp->sha, ifp->mac, sizeof(arp->sha));
  if (mac != NULL) memcpy(arp->tha, mac, sizeof(arp->tha));
  driver_output(ifp, mg_l2_trailer(ifp, sizeof(*arp), (uint8_t *) (arp + 1)));
}

static void onstatechange(struct mg_tcpip_if *ifp) {
  if (ifp->state == MG_TCPIP_STATE_IP &&
      (ifp->l2type == MG_TCPIP_L2_PPP || ifp->l2type == MG_TCPIP_L2_PPPoE))
    ifp->state = MG_TCPIP_STATE_READY;
  if (ifp->state == MG_TCPIP_STATE_READY) {
    MG_INFO(("READY, IP: %M", mg_print_ip4, &ifp->ip));
    MG_INFO(("       GW: %M", mg_print_ip4, &ifp->gw));
    if (ifp->l2type == MG_TCPIP_L2_ETH ||
        ifp->l2type == MG_TCPIP_L2_PPPoE)  // TODO(): print other l2
      MG_INFO(("      MAC: %M", mg_print_mac, ifp->mac));
    if (ifp->l2type == MG_TCPIP_L2_ETH)
      mg_tcpip_arp_request(ifp, ifp->ip, ifp->mac);  // gratuitous ARP annc
    if (ifp->is_ip_changed) {
      struct mg_connection *c;
      for (c = ifp->mgr->conns; c != NULL; c = c->next) {
        if (!c->is_listening && !c->is_udp) c->is_closing = 1;
      }
      ifp->is_ip_changed = false;
    }
  } else if (ifp->state == MG_TCPIP_STATE_IP) {
    if (ifp->gw != 0 && ifp->l2type == MG_TCPIP_L2_ETH)
      mg_tcpip_arp_request(ifp, ifp->gw, NULL);  // unsolicited GW ARP request
  } else if (ifp->state == MG_TCPIP_STATE_UP) {
    srand((unsigned int) mg_millis());
  }
  mg_tcpip_call(ifp, MG_TCPIP_EV_STATE_CHANGE, &ifp->state);
}

static struct ip *tx_ip(struct mg_tcpip_if *ifp, uint8_t *l2_dst, uint8_t proto,
                        uint8_t dscp, uint32_t ip_src, uint32_t ip_dst,
                        size_t plen) {
  // ifp->tx.buf is 8-bit aligned, keep other headers as pointers, see pkt
  uint8_t *l2p = (uint8_t *) ifp->tx.buf;
  struct ip *ip = (struct ip *) mg_l2_header(ifp, MG_TCPIP_L2PROTO_IPV4,
                                             ifp->mac, l2_dst, l2p);
  memset(ip, 0, sizeof(*ip));
  ip->ver = 0x45;  // Version 4, header length 5 words
  ip->tos = (uint8_t) (dscp << 2);
  ip->frag = mg_htons(0x4000);  // Don't fragment
  ip->len = mg_htons((uint16_t) (sizeof(*ip) + plen));
  ip->ttl = 64;
  ip->proto = proto;
  ip->src = ip_src;
  ip->dst = ip_dst;
  ip->csum = ipcsum(ip, sizeof(*ip));
  return ip;
}

#if MG_ENABLE_IPV6
static struct ip6 *tx_ip6(struct mg_tcpip_if *ifp, uint8_t *l2_dst,
                          uint8_t next, uint8_t dscp, uint64_t *ip_src,
                          uint64_t *ip_dst, size_t plen);
#endif

static bool tx_udp(struct mg_tcpip_if *ifp, uint8_t *l2_dst,
                   struct mg_addr *ip_src, struct mg_addr *ip_dst, uint8_t dscp,
                   const void *buf, size_t len) {
  uint8_t *l3p;
  size_t l2_len;
  struct ip *ip = NULL;
  struct udp *udp;
#if MG_ENABLE_IPV6
  struct ip6 *ip6 = NULL;
  if (ip_dst->is_ip6) {
    ip6 = tx_ip6(ifp, l2_dst, 17, dscp, ip_src->addr.ip6, ip_dst->addr.ip6,
                 len + sizeof(struct udp));
    udp = (struct udp *) (ip6 + 1);
    l2_len = sizeof(*ip6) + sizeof(*udp) + len;
    l3p = (uint8_t *) ip6;
  } else
#endif
  {
    ip = tx_ip(ifp, l2_dst, 17, dscp, ip_src->addr.ip4, ip_dst->addr.ip4,
               len + sizeof(struct udp));
    udp = (struct udp *) (ip + 1);
    l2_len = sizeof(*ip) + sizeof(*udp) + len;
    l3p = (uint8_t *) ip;
  }
  udp->sport = ip_src->port;
  udp->dport = ip_dst->port;
  udp->len = mg_htons((uint16_t) (sizeof(*udp) + len));
  udp->csum = 0;
  memmove(udp + 1, buf, len);
#if MG_ENABLE_IPV6
  if (ip_dst->is_ip6) {
    udp->csum = p6csum(ip6, udp, 17, sizeof(*udp) + len);
  } else
#endif
  {
    udp->csum = pcsum(ip, udp, sizeof(*udp) + len);
  }
  l2_len = mg_l2_trailer(ifp, l2_len, l3p + l2_len);
  return (driver_output(ifp, l2_len) == l2_len);
}

static bool tx_udp4(struct mg_tcpip_if *ifp, uint8_t *l2_dst, uint32_t ip_src,
                    uint16_t sport, uint32_t ip_dst, uint16_t dport,
                    const void *buf, size_t len) {
  struct mg_addr ips, ipd;
  memset(&ips, 0, sizeof(ips));
  ips.addr.ip4 = ip_src;
  ips.port = sport;
  memset(&ipd, 0, sizeof(ipd));
  ipd.addr.ip4 = ip_dst;
  ipd.port = dport;
  return tx_udp(ifp, l2_dst, &ips, &ipd, 0, buf, len);
}

static void tx_dhcp(struct mg_tcpip_if *ifp, uint8_t *l2_dst, uint32_t ip_src,
                    uint32_t ip_dst, uint8_t *opts, size_t optslen,
                    bool ciaddr) {
  // https://datatracker.ietf.org/doc/html/rfc2132#section-9.6
  // NOTE(): assumes Ethernet: htype=1 hlen=6, copy 6 bytes
  struct dhcp dhcp = {1, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, {0}, 0, {0}};
  dhcp.magic = mg_htonl(0x63825363);
  memcpy(&dhcp.hwaddr, ifp->mac, 6);
  memcpy(&dhcp.xid, ifp->mac + 2, sizeof(dhcp.xid));
  memcpy(&dhcp.options, opts, optslen);
  if (ciaddr) dhcp.ciaddr = ip_src;
  tx_udp4(ifp, l2_dst, ip_src, mg_htons(68), ip_dst, mg_htons(67), &dhcp,
          sizeof(dhcp));
}

// RFC-2131 #4.3.6, #4.4.1; RFC-2132 #9.8
static void tx_dhcp_request_sel(struct mg_tcpip_if *ifp, uint32_t ip_req,
                                uint32_t ip_srv) {
  uint8_t extra = (uint8_t) ((ifp->enable_req_dns ? 1 : 0) +
                             (ifp->enable_req_sntp ? 1 : 0));
  size_t len = strlen(ifp->dhcp_name);
  size_t olen = 21 + len + extra + 2 + 1;  // Total length of options
#define OPTS_MAXLEN (21 + sizeof(ifp->dhcp_name) + 2 + 2 + 1)
  uint8_t opts[OPTS_MAXLEN];  // Allocate options (max size possible)
  uint8_t *p = opts;
  assert(olen <= sizeof(opts));
  memset(opts, 0, sizeof(opts));
  *p++ = 53, *p++ = 1, *p++ = 3;                       // Type: DHCP request
  *p++ = 54, *p++ = 4, memcpy(p, &ip_srv, 4), p += 4;  // DHCP server ID
  *p++ = 50, *p++ = 4, memcpy(p, &ip_req, 4), p += 4;  // Requested IP
  *p++ = 12, *p++ = (uint8_t) (len & 255);             // DHCP host
  memcpy(p, ifp->dhcp_name, len), p += len;            // name
  *p++ = 55, *p++ = 2 + extra, *p++ = 1, *p++ = 3;     // GW, MASK
  if (ifp->enable_req_dns) *p++ = 6;                   // DNS
  if (ifp->enable_req_sntp) *p++ = 42;                 // SNTP
  *p++ = 255;                                          // End of options
  // assert((size_t) (p - opts) < olen);
  tx_dhcp(ifp, mg_l2_mapip(ifp->l2type, MG_TCPIP_L2ADDR_BCAST, NULL), 0,
          0xffffffff, opts, olen, 0);
  MG_DEBUG(("DHCP req sent"));
}

// RFC-2131 #4.3.6, #4.4.5 (renewing: unicast, rebinding: bcast)
static void tx_dhcp_request_re(struct mg_tcpip_if *ifp, uint8_t *l2_dst,
                               uint32_t ip_src, uint32_t ip_dst) {
  uint8_t opts[] = {
      53, 1, 3,  // Type: DHCP request
      255        // End of options
  };
  tx_dhcp(ifp, l2_dst, ip_src, ip_dst, opts, sizeof(opts), true);
  MG_DEBUG(("DHCP req sent"));
}

static void tx_dhcp_discover(struct mg_tcpip_if *ifp) {
  uint8_t opts[] = {
      53, 1, 1,     // Type: DHCP discover
      55, 2, 1, 3,  // Parameters: ip, mask
      255           // End of options
  };
  tx_dhcp(ifp, mg_l2_mapip(ifp->l2type, MG_TCPIP_L2ADDR_BCAST, NULL), 0,
          0xffffffff, opts, sizeof(opts), false);
  MG_DEBUG(("DHCP discover sent. Our MAC: %M", mg_print_mac, ifp->mac));
}

static struct mg_connection *getpeer(struct mg_mgr *mgr, struct pkt *pkt,
                                     bool lsn) {
  struct mg_connection *c = NULL;
  for (c = mgr->conns; c != NULL; c = c->next) {
    if (c->is_arplooking && pkt->arp && pkt->arp->spa == c->rem.addr.ip4) break;
#if MG_ENABLE_IPV6
    if (c->is_arplooking && pkt->icmp6 && pkt->icmp6->type == 136) {
      struct ndp_na *na = (struct ndp_na *) (pkt->icmp6 + 1);
      if (MG_IP6MATCH(na->addr, c->rem.addr.ip6)) break;
    }
#endif
    if (c->is_udp && pkt->udp && c->loc.port == pkt->udp->dport &&
        !(c->loc.is_ip6 ^ (pkt->ip6 != NULL)))  // IP or IPv6 to same dest
      break;
    if (!c->is_udp && pkt->tcp && c->loc.port == pkt->tcp->dport &&
        ((lsn && c->is_listening && !(c->loc.is_ip6 ^ (pkt->ip6 != NULL))) ||
         (!lsn && !c->is_listening && c->rem.port == pkt->tcp->sport &&
          ((!c->loc.is_ip6 && c->rem.addr.ip4 == pkt->ip->src)
#if MG_ENABLE_IPV6
           || (c->loc.is_ip6 && MG_IP6MATCH(c->rem.addr.ip6, pkt->ip6->src))
#endif
               ))))  // validate addr for established (not listening) conns
      break;
  }
  return c;
}

static void l2addr_resolved(struct mg_connection *c);
static uint8_t *get_return_l2addr(struct mg_tcpip_if *ifp, struct mg_addr *rem,
                                  bool is_udp, struct pkt *pkt);

// RFC826, ARP assumes Ethernet MAC addresses
static void rx_arp(struct mg_tcpip_if *ifp, struct pkt *pkt) {
  if (pkt->arp->op == mg_htons(1) && pkt->arp->tpa == ifp->ip) {
    // ARP request. Make a response, then send
    // MG_VERBOSE(("ARP req from %M", mg_print_ip4, &pkt->arp->spa));
    uint8_t *l2p = (uint8_t *) ifp->tx.buf;
    struct arp *arp = (struct arp *) mg_l2_header(
        ifp, MG_TCPIP_L2PROTO_ARP, ifp->mac, mg_l2_getaddr(ifp, pkt->l2), l2p);
    *arp = *pkt->arp;
    arp->op = mg_htons(2);
    memcpy(arp->tha, pkt->arp->sha, sizeof(pkt->arp->tha));
    memcpy(arp->sha, ifp->mac, sizeof(pkt->arp->sha));
    arp->tpa = pkt->arp->spa;
    arp->spa = ifp->ip;
    MG_DEBUG(("ARP: tell %M we're %M", mg_print_ip4, &arp->tpa, mg_print_mac,
              ifp->mac));
    driver_output(ifp, mg_l2_trailer(ifp, sizeof(*arp), (uint8_t *) (arp + 1)));
  } else if (pkt->arp->op == mg_htons(2)) {
    if (memcmp(pkt->arp->tha, ifp->mac, sizeof(pkt->arp->tha)) != 0) return;
    // MG_VERBOSE(("ARP resp from %M", mg_print_ip4, &pkt->arp->spa));
    if (pkt->arp->spa == ifp->gw) {
      // Got response for the GW ARP request. Set ifp->gwmac and IP -> READY
      memcpy(ifp->gwmac, pkt->arp->sha, sizeof(ifp->gwmac));
      ifp->gw_ready = true;
      if (ifp->state == MG_TCPIP_STATE_IP) {
        ifp->state = MG_TCPIP_STATE_READY;
        onstatechange(ifp);
      }
    } else {
      struct mg_connection *c = getpeer(ifp->mgr, pkt, false);
      if (c != NULL && c->is_arplooking) {
        struct connstate *s = (struct connstate *) (c + 1);
        memcpy(s->mac, pkt->arp->sha, sizeof(s->mac));
        MG_DEBUG(("%lu ARP resolved %M -> %M", c->id, mg_print_ip4,
                  &c->rem.addr.ip4, mg_print_mac, s->mac));
        c->is_arplooking = 0;
        l2addr_resolved(c);
      }
    }
  }
}

static void rx_icmp(struct mg_tcpip_if *ifp, struct pkt *pkt) {
  size_t plen = pkt->pay.len;
  if (!icmpcsum_ok(pkt->icmp, sizeof(struct icmp) + plen)) return;
  if (pkt->icmp->type == 8 && pkt->ip != NULL && pkt->ip->dst == ifp->ip) {
    size_t l2_max_overhead = ifp->framesize - ifp->l2mtu;
    size_t hlen = sizeof(struct ip) + sizeof(struct icmp);
    size_t room = ifp->tx.len - hlen - l2_max_overhead;
    uint8_t *l2addr;
    struct ip *ip;
    struct icmp *icmp;
    struct mg_addr ips;
    ips.addr.ip4 = pkt->ip->src;
    ips.is_ip6 = false;
    if ((l2addr = get_return_l2addr(ifp, &ips, false, pkt)) == NULL)
      return;  // safety net for lousy networks
    if (plen > room) plen = room;
    ip = tx_ip(ifp, l2addr, 1, 0, ifp->ip, pkt->ip->src, sizeof(*icmp) + plen);
    icmp = (struct icmp *) (ip + 1);
    memset(icmp, 0, sizeof(*icmp));        // Set csum, type, code to 0
    memcpy(icmp + 1, pkt->pay.buf, plen);  // Copy RX payload to TX
    icmp->csum = ipcsum(icmp, sizeof(*icmp) + plen);
    driver_output(
        ifp, mg_l2_trailer(ifp, hlen + plen, (uint8_t *) ip + hlen + plen));
  }
}

static void setdns4(struct mg_tcpip_if *ifp, uint32_t *ip);

static bool dhcp_opt_len_ok(uint8_t len, uint8_t *p, uint8_t *end) {
  return (len >= 4 && (len & 3) == 0 && p + 6 < end);
}

static void rx_dhcp_client(struct mg_tcpip_if *ifp, struct pkt *pkt) {
  uint32_t ip = 0, gw = 0, mask = 0, lease = 0, dns = 0, sntp = 0, owner = 0;
  uint8_t msgtype = 0, state = ifp->state;
  // perform size check first, then access fields
  uint8_t *p = (uint8_t *) pkt->pay.buf,
          *end = (uint8_t *) &pkt->pay.buf[pkt->pay.len];
  // min header length checked at payload calculation, options are optional
  if (memcmp(&pkt->dhcp->xid, ifp->mac + 2, sizeof(pkt->dhcp->xid))) return;
  while (p < end && p[0] != 255) {  // RFC-2132 9
    if (p[0] == 0) { p++; continue; }  // Pad
    if ((size_t) (end - p) < 2 || (size_t) (end - p) < 2U + p[1]) break;
    if (p[0] == 1) {  // Mask, 3.3
      if (p[1] != 4) break;
      memcpy(&mask, p + 2, sizeof(mask));
    } else if (p[0] == 3) {  // GW, 3.5
      if (!dhcp_opt_len_ok(p[1], p, end)) break;
      memcpy(&gw, p + 2, sizeof(gw));
      ip = pkt->dhcp->yiaddr;
    } else if (ifp->enable_req_dns && p[0] == 6) {  // DNS, 3.8
      if (!dhcp_opt_len_ok(p[1], p, end)) break;
      memcpy(&dns, p + 2, sizeof(dns));
    } else if (ifp->enable_req_sntp && p[0] == 42) {  // SNTP, 8.3
      if (!dhcp_opt_len_ok(p[1], p, end)) break;
      memcpy(&sntp, p + 2, sizeof(sntp));
    } else if (p[0] == 51) {  // Lease
      if (p[1] != 4) break;
      memcpy(&lease, p + 2, sizeof(lease));
      lease = mg_ntohl(lease);
    } else if (p[0] == 53) {  // Msg Type
      if (p[1] != 1) break;
      msgtype = p[2];
    } else if (p[0] == 54) {  // Server id 9.7
      if (p[1] != 4) break;
      memcpy(&owner, p + 2, sizeof(sntp));  // This is the lease owner
    }
    p += p[1] + 2;
  }
  // Process message type, RFC-1533 (9.4); RFC-2131 (3.1, 4)
  if (msgtype == 6 && ifp->ip == ip) {  // DHCPNACK, release IP
    ifp->state = MG_TCPIP_STATE_UP, ifp->ip = 0;
  } else if (msgtype == 2 && ifp->state == MG_TCPIP_STATE_UP && ip && gw &&
             lease) {  // DHCPOFFER
    // select a server (2131 4.4.1, 2132 9.7): lease owner takes precedence,
    // otherwise use siaddr (fallback to IP source addr on foul play).
    // This is broadcast, otherwise siaddr would be the destination.
    tx_dhcp_request_sel(ifp, ip,
                        owner               ? owner
                        : pkt->dhcp->siaddr ? pkt->dhcp->siaddr
                                            : pkt->ip->src);
    ifp->state = MG_TCPIP_STATE_REQ;  // REQUESTING state
  } else if (msgtype == 5) {          // DHCPACK
    if (ifp->state == MG_TCPIP_STATE_REQ && ip && gw && lease) {  // got an IP
      uint64_t rand;
      ifp->lease_expire = ifp->now + (uint64_t) lease * 1000;
      MG_INFO(("Lease: %u sec (%lld)", lease, ifp->lease_expire / 1000));
      // assume DHCP server = router until ARP resolves
      memcpy(ifp->gwmac, mg_l2_getaddr(ifp, pkt->l2), sizeof(ifp->gwmac));
      ifp->gw_ready = true;  // NOTE(): actual gw ARP won't retry now
      if (ifp->ip != ip) ifp->is_ip_changed = true;
      ifp->ip = ip, ifp->gw = gw, ifp->mask = mask;
      ifp->state = MG_TCPIP_STATE_IP;  // BOUND state
      mg_random(&rand, sizeof(rand));
      srand((unsigned int) (rand + mg_millis()));
      if (ifp->enable_req_dns && dns != 0) {
        setdns4(ifp, &dns);
        mg_tcpip_call(ifp, MG_TCPIP_EV_DHCP_DNS, &dns);
      }
      if (ifp->enable_req_sntp && sntp != 0)
        mg_tcpip_call(ifp, MG_TCPIP_EV_DHCP_SNTP, &sntp);
    } else if (ifp->state == MG_TCPIP_STATE_READY && ifp->ip == ip) {  // renew
      ifp->lease_expire = ifp->now + (uint64_t) lease * 1000;
      MG_INFO(("Lease: %u sec (%lld)", lease, ifp->lease_expire / 1000));
    }  // TODO(): accept provided T1/T2 and store server IP for renewal (4.4)
  }
  if (ifp->state != state) onstatechange(ifp);
}

// Simple DHCP server that assigns a next IP address: ifp->ip + 1
static void rx_dhcp_server(struct mg_tcpip_if *ifp, struct pkt *pkt) {
  uint8_t *mac;
  uint8_t op = 0, *p = (uint8_t *) pkt->pay.buf,
          *end = (uint8_t *) &pkt->pay.buf[pkt->pay.len];
  // NOTE(): assumes Ethernet: htype=1 hlen=6, copy 6 bytes
  struct dhcp res = {2, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, {0}, 0, {0}};
  // min header length checked at payload calculation, options are optional
  res.yiaddr = ifp->ip;
  ((uint8_t *) (&res.yiaddr))[3]++;                // Offer our IP + 1
  while (p < end && p[0] != 255) {                 // Parse options
    if (p[0] == 0) { p++; continue; }              // Pad
    if ((size_t) (end - p) < 2 || (size_t) (end - p) < 2U + p[1]) break;
    if (p[0] == 53) {  // Message type
      if (p[1] != 1) break;
      op = p[2];
    }
    p += p[1] + 2;
  }
  if (op == 1 || op == 3) {         // DHCP Discover or DHCP Request
    uint8_t msg = op == 1 ? 2 : 5;  // Message type: DHCP OFFER or DHCP ACK
    uint8_t opts[] = {
        53, 1, 0,                   // Message type
        1,  4, 0,   0,   0,   0,    // Subnet mask
        54, 4, 0,   0,   0,   0,    // Server ID
        12, 3, 'm', 'i', 'p',       // Host name: "mip"
        51, 4, 255, 255, 255, 255,  // Lease time
        255                         // End of options
    };
    opts[2] = msg;
    memcpy(&res.hwaddr, pkt->dhcp->hwaddr, 6);
    memcpy(opts + 5, &ifp->mask, sizeof(ifp->mask));
    memcpy(opts + 11, &ifp->ip, sizeof(ifp->ip));
    memcpy(&res.options, opts, sizeof(opts));
    res.magic = pkt->dhcp->magic;
    res.xid = pkt->dhcp->xid;
    mac = mg_l2_getaddr(ifp, pkt->l2);
    if (ifp->enable_get_gateway) {
      ifp->gw = res.yiaddr;  // set gw IP, best-effort gwmac as DHCP server's
      memcpy(ifp->gwmac, mac, sizeof(ifp->gwmac));
    }
    tx_udp4(ifp, mac, ifp->ip, mg_htons(67), op == 1 ? ~0U : res.yiaddr,
            mg_htons(68), &res, sizeof(res));
  }
}

#if MG_ENABLE_IPV6
static struct ip6 *tx_ip6(struct mg_tcpip_if *ifp, uint8_t *l2_dst,
                          uint8_t next, uint8_t dscp, uint64_t *ip_src,
                          uint64_t *ip_dst, size_t plen) {
  // ifp->tx.buf is 8-bit aligned, keep other headers as pointers, see pkt
  uint8_t *l2p = (uint8_t *) ifp->tx.buf;
  struct ip6 *ip6 = (struct ip6 *) mg_l2_header(ifp, MG_TCPIP_L2PROTO_IPV6,
                                                ifp->mac, l2_dst, l2p);
  memset(ip6, 0, sizeof(*ip6));
  ip6->ver = (uint8_t) (0x60 | (dscp >> 2));  // Version 6, traffic class
  ip6->label[0] = (uint8_t) (dscp << 6);
  ip6->plen = mg_htons((uint16_t) plen);
  ip6->next = next;
  ip6->hops = 255;  // NDP requires max
  ip6->src[0] = *ip_src++;
  ip6->src[1] = *ip_src;
  ip6->dst[0] = *ip_dst++;
  ip6->dst[1] = *ip_dst;
  return ip6;
}

static void tx_icmp6(struct mg_tcpip_if *ifp, uint8_t *l2_dst, uint64_t *ip_src,
                     uint64_t *ip_dst, uint8_t type, uint8_t code,
                     const void *buf, size_t len) {
  struct ip6 *ip6;
  struct icmp6 *icmp6;
  ip6 = tx_ip6(ifp, l2_dst, 58, 0, ip_src, ip_dst, sizeof(*icmp6) + len);
  icmp6 = (struct icmp6 *) (ip6 + 1);
  memset(icmp6, 0, sizeof(*icmp6));  // Set csum to 0
  icmp6->type = type;
  icmp6->code = code;
  memcpy(icmp6 + 1, buf, len);  // Copy payload
  icmp6->csum = 0;              // RFC-4443 2.3, RFC-8200 8.1
  icmp6->csum = p6csum(ip6, icmp6, 58, sizeof(*icmp6) + len);
  driver_output(ifp,
                mg_l2_trailer(ifp, sizeof(*ip6) + sizeof(*icmp6) + len,
                              (uint8_t *) (ip6 + 1) + sizeof(*icmp6) + len));
}

// Neighbor Discovery Protocol, RFC-4861
// Neighbor Advertisement, 4.4
static void tx_ndp_na(struct mg_tcpip_if *ifp, uint8_t *l2_dst,
                      uint64_t *ip_src, uint64_t *ip_dst, bool solicited,
                      uint8_t *l2) {
  uint8_t data[20 + 16];  // NOTE(): optional len upto 2 hw addr
  memset(data, 0, sizeof(data));
  data[0] = solicited ? 0x60 : 0x20;                    // O + S
  memcpy(data + 4, ip_src, 16);                         // Target address
  data[20] = 2;                                         // 4.6.1, target hwaddr
  data[21] = mg_l2_ip6put(ifp->l2type, l2, data + 22);  // option length / 8
  tx_icmp6(ifp, l2_dst, ip_src, ip_dst, 136, 0, data,
           20 + (size_t) (8 * data[21]));
}

static void onstate6change(struct mg_tcpip_if *ifp);

static void rx_ndp_na(struct mg_tcpip_if *ifp, struct pkt *pkt) {
  struct ndp_na *na = (struct ndp_na *) pkt->pay.buf;
  uint8_t *opts = (uint8_t *) (na + 1),
          *endp = opts + pkt->pay.len - sizeof(*na);
  if (pkt->pay.len < (sizeof(*na) + 2)) return;  // first 2 bytes in opts
  if ((na->res[0] & 0x40) == 0) return;          // not "solicited"
  if (*opts++ != 2) return;                      // no target hwaddr, must have
  MG_VERBOSE(("NDP NA resp from %M", mg_print_ip6, (char *) &na->addr));
  if (MG_IP6MATCH(na->addr, ifp->gw6)) {
    // Got response for the GW NS request. Set ifp->gw6mac and IP6 -> READY
    uint8_t len = *opts++;                    // check valid hwaddr and get it
    if ((opts + 8 * len - 2) > endp) return;  // truncated
    if (!mg_l2_ip6get(ifp->l2type, ifp->gw6mac, opts, len)) return;
    ifp->gw6_ready = true;
    if (ifp->state6 == MG_TCPIP_STATE_IP) {
      ifp->state6 = MG_TCPIP_STATE_READY;
      onstate6change(ifp);
    }
  } else {
    struct mg_connection *c = getpeer(ifp->mgr, pkt, false);
    if (c != NULL && c->is_arplooking) {
      struct connstate *s = (struct connstate *) (c + 1);
      uint8_t len = *opts++;                    // check valid hwaddr and get it
      if ((opts + 8 * len - 2) > endp) return;  // truncated
      if (!mg_l2_ip6get(ifp->l2type, s->mac, opts, len)) return;
      MG_DEBUG(("%lu NDP resolved %M -> %M", c->id, mg_print_ip6,
                &c->rem.addr.ip6, mg_print_l2addr, ifp->l2type, s->mac));
      c->is_arplooking = 0;
      l2addr_resolved(c);
    }
  }
}

// Neighbor Solicitation, 4.3
static void rx_ndp_ns(struct mg_tcpip_if *ifp, struct pkt *pkt) {
  struct ndp_na *ns = (struct ndp_na *) pkt->pay.buf;  // struct ndp_ns = ndp_na
  uint64_t target[2];
  if (pkt->pay.len < (sizeof(*ns) + 2)) return;  // first 2 bytes in opts
  memcpy(target, ns->addr, sizeof(target));
  if (MG_IP6MATCH(target, ifp->ip6ll) || MG_IP6MATCH(target, ifp->ip6)) {
    uint64_t req[2];  // requester address
    uint8_t l2[sizeof(struct mg_l2addr)];
    uint8_t len, *opts = (uint8_t *) (ns + 1),
                 *endp = opts + pkt->pay.len - sizeof(*ns);
    if (*opts++ != 1) return;                 // no requester hwaddr (source)
    len = *opts++;                            // check valid hwaddr and get it
    if ((opts + 8 * len - 2) > endp) return;  // truncated
    if (!mg_l2_ip6get(ifp->l2type, l2, opts, len)) return;
    req[0] = pkt->ip6->src[0], req[1] = pkt->ip6->src[1];  // align to 64-bit
    tx_ndp_na(ifp, l2, target, req, true, ifp->mac);
  }
}

// - use solicited node multicast to resolve a l2 address (l2_addr = NULL)
// - use unicast to verify presence (l2_addr = neighbor l2 address)
static void tx_ndp_ns(struct mg_tcpip_if *ifp, uint64_t *ip_dst,
                      uint8_t *l2_addr) {
  uint8_t payload[4 + 16 + 16];  // NOTE(): 16 --> optional len upto 2 hw addr
  uint64_t ip_unspec[2] = {0, 0};
  size_t payload_len = 20;
  bool mcast = (l2_addr == NULL);
  uint64_t ip_mcast[2] = {0, 0};
  uint8_t *l2 = l2_addr;

  if (ifp->l2type == MG_TCPIP_L2_PPP || ifp->l2type == MG_TCPIP_L2_PPPoE) {
    MG_DEBUG(("SKIP NS for %M", mg_print_ip6, ip_dst));
    return;
  }
  memset(payload, 0, sizeof(payload));
  memcpy(payload + 4, ip_dst, 16);
  if (mcast) {
    struct mg_addr ipd;
    ip6sn(ip_dst, ip_mcast);
    ipd.addr.ip6[0] = ip_mcast[0], ipd.addr.ip6[1] = ip_mcast[1],
    ipd.is_ip6 = true;
    l2 = mg_l2_mapip(ifp->l2type, MG_TCPIP_L2ADDR_MCAST6, &ipd);
  }
  payload_len = 20;
  // TODO(robertc2000): using only link-local IP addr for now
  // We might consider to add an option to use either link-local or global IP
  if (!MG_IP6MATCH(ifp->ip6ll, ip_unspec)) {
    payload[20] = 1;  // 4.6.1, source hwaddr; option length in 8-byte units
    payload[21] = mg_l2_ip6put(ifp->l2type, ifp->mac, payload + 22);
    payload_len += 8 * payload[21];
  }
  tx_icmp6(ifp, l2, ifp->ip6ll, mcast ? ip_mcast : ip_dst, 135, 0, payload,
           payload_len);
}

// Router Solicitation, 4.1
static void tx_ndp_rs(struct mg_tcpip_if *ifp) {
  uint8_t payload[4 + 16];  // reserved + optional len upto 2 hw addr NOTE()
  size_t payload_len = 4;
  uint64_t ip_unspec[2] = {0, 0};

  memset(payload, 0, sizeof(payload));

  if (!MG_IP6MATCH(ifp->ip6ll, ip_unspec)) {
    payload[4] = 1;  // 4.6.1, source hwaddr; option length in 8-byte units
    payload[5] = mg_l2_ip6put(ifp->l2type, ifp->mac, payload + 6);
    payload_len += 8 * payload[5];
  }
  tx_icmp6(ifp,
           mg_l2_mapip(ifp->l2type, MG_TCPIP_L2ADDR_MCAST6,
                       (struct mg_addr *) &ip6_allrouters),
           ifp->ip6ll, (uint64_t *) ip6_allrouters.addr.ip6, 133, 0, payload,
           payload_len);
  MG_DEBUG(("NDP Router Solicitation sent"));
}

static void fill_prefix(uint8_t *dst, uint8_t *src, uint8_t len) {
  uint8_t full = len / 8;
  uint8_t rem = len % 8;
  if (full > 0) memcpy(dst, src, full);
  if (rem > 0) {
    uint8_t mask = (uint8_t) (0xFF << (8 - rem));
    dst[full] |= src[full] & mask;  // mg_l2_genip6() zeroes dst
  }
}

static bool match_prefix(uint8_t *newp, uint8_t *curp, uint8_t len) {
  uint8_t full = len / 8;
  uint8_t rem = len % 8;
  if (full > 0 && memcmp(curp, newp, full) != 0) return false;
  if (rem > 0) {
    uint8_t mask = (uint8_t) (0xFF << (8 - rem));
    if (curp[full] != (newp[full] & mask)) return false;
  }
  return true;
}

static bool fill_global(struct mg_tcpip_if *ifp, uint8_t *prefix,
                        uint8_t prefix_len) {
  if (!mg_l2_genip6(ifp->l2type, ifp->ip6, prefix_len, ifp->mac)) return false;
  fill_prefix((uint8_t *) ifp->ip6, prefix, prefix_len);
  fill_prefix(ifp->prefix, prefix, prefix_len);
  ifp->prefix_len = prefix_len;
  return true;
}

// Router Advertisement, 4.2
static void rx_ndp_ra(struct mg_tcpip_if *ifp, struct pkt *pkt) {
  struct ndp_ra *ra = (struct ndp_ra *) pkt->pay.buf;
  uint8_t *opts = (uint8_t *) (ra + 1);
  size_t opt_left = pkt->pay.len - sizeof(*ra);
  bool gotl2addr = false, gotprefix = false, changed = false;
  uint8_t l2[sizeof(struct mg_l2addr)];
  uint32_t mtu = 0;
  uint8_t *prefix = NULL, prefix_len = 0;

  if (pkt->pay.len < sizeof(*ra)) return;
  if (ifp->state6 == MG_TCPIP_STATE_UP) {
    MG_DEBUG(("Received NDP RA"));  // fill gw6 address
    // parse options
    while (opt_left >= 2) {
      uint8_t type = opts[0], len = opts[1];
      size_t length = (size_t) len * 8;
      if (length == 0 || length > opt_left) break;  // malformed
      if (type == 1 && length >= 8) {
        // Received router's L2 address
        if (!mg_l2_ip6get(ifp->l2type, l2, opts + 2, len)) break;
        gotl2addr = true;
      } else if (type == 5 && length >= 8) {
        // process MTU if available, ignore if it smells
        mtu = MG_LOAD_BE32(opts + 4);
        if (mtu < 1280 || mtu > ifp->l2mtu) mtu = 0;  // RFC-8200, minimum MTU
      } else if (type == 3 && length >= 32) {
        // process prefix, 4.6.2, ignore if it smells
        uint8_t pfx_flags = opts[3];  // L=0x80, A=0x40
        uint32_t valid = MG_LOAD_BE32(opts + 4);
        uint32_t pref_lifetime = MG_LOAD_BE32(opts + 8);
        prefix_len = opts[2];
        if (prefix_len >= 128) break;
        prefix = opts + 16;

        // TODO (robertc2000): handle prefix options if necessary
        (void) pfx_flags;
        (void) valid;
        (void) pref_lifetime;

        gotprefix = true;
        if (prefix_len != ifp->prefix_len ||
            !match_prefix(prefix, ifp->prefix, ifp->prefix_len))
          changed = true;
      }
      opts += length;
      opt_left -= length;
    }

    // fill prefix and global
    if (gotprefix && !fill_global(ifp, prefix, prefix_len)) return;
    if (changed) ifp->is_ip6_changed = true;
    ifp->gw6[0] = pkt->ip6->src[0], ifp->gw6[1] = pkt->ip6->src[1];
    if (gotl2addr) memcpy(ifp->gw6mac, l2, sizeof(ifp->gw6mac));
    if (gotl2addr || ifp->l2type == MG_TCPIP_L2_PPP ||
        ifp->l2type == MG_TCPIP_L2_PPPoE) {
      ifp->state6 = MG_TCPIP_STATE_READY;
      ifp->gw6_ready = true;
    }
    if (mtu != 0 && ifp->mtu != mtu) {
      MG_ERROR(
          ("got an MTU: %u, that differs from the configured one. "
           "All devices in an IPv6 network should have the same MTU, "
           "using the router's instead...",
           mtu));
      ifp->mtu = (uint16_t) mtu;
    }
    if (ifp->state6 != MG_TCPIP_STATE_READY) {
      tx_ndp_ns(ifp, ifp->gw6, NULL);  // unsolicited GW hwaddr resolution
      ifp->state6 = MG_TCPIP_STATE_IP;
    }
    onstate6change(ifp);
  }
}

static void rx_icmp6(struct mg_tcpip_if *ifp, struct pkt *pkt) {
  if (!icmp6csum_ok(pkt->ip6, pkt->icmp6, sizeof(*pkt->icmp6) + pkt->pay.len))
    return;
  switch (pkt->icmp6->type) {
    case 128: {  // Echo Request, RFC-4443 4.1
      uint64_t target[2];
      target[0] = pkt->ip6->dst[0], target[1] = pkt->ip6->dst[1];
      if (MG_IP6MATCH(target, ifp->ip6ll) || MG_IP6MATCH(target, ifp->ip6)) {
        size_t l2_max_overhead = ifp->framesize - ifp->l2mtu;
        size_t hlen = sizeof(struct ip6) + sizeof(struct icmp6);
        size_t room = ifp->tx.len - hlen - l2_max_overhead, plen = pkt->pay.len;
        struct mg_addr ips;
        uint8_t *l2addr;

        ips.addr.ip6[0] = pkt->ip6->src[0], ips.addr.ip6[1] = pkt->ip6->src[1];
        ips.is_ip6 = true;
        if ((l2addr = get_return_l2addr(ifp, &ips, false, pkt)) == NULL)
          return;                      // safety net for lousy networks
        if (plen > room) plen = room;  // Copy (truncated) RX payload to TX
        // Echo Reply, 4.2
        tx_icmp6(ifp, l2addr, target, ips.addr.ip6, 129, 0, pkt->pay.buf, plen);
      }
    } break;
    case 134:  // Router Advertisement
      rx_ndp_ra(ifp, pkt);
      break;
    case 135:  // Neighbor Solicitation
      rx_ndp_ns(ifp, pkt);
      break;
    case 136:  // Neighbor Advertisement
      rx_ndp_na(ifp, pkt);
      break;
    default:
      if (mg_log_level >= MG_LL_VERBOSE)
        mg_hexdump(pkt->icmp6, pkt->pay.len > 16 ? 16 : pkt->pay.len);
      break;
  }
}

static void onstate6change(struct mg_tcpip_if *ifp) {
  if (ifp->state6 == MG_TCPIP_STATE_READY) {
    MG_INFO(("READY, IP: %M", mg_print_ip6, &ifp->ip6));
    MG_INFO(("       GW: %M", mg_print_ip6, &ifp->gw6));
    if (ifp->l2type == MG_TCPIP_L2_ETH)  // TODO(): print other l2
      MG_INFO(("      MAC: %M", mg_print_mac, &ifp->mac));
    if (ifp->l2type == MG_TCPIP_L2_ETH) {  // gratuitous NA annc
      tx_ndp_na(ifp,                       // RFC-4861 7.2.6
                mg_l2_mapip(ifp->l2type, MG_TCPIP_L2ADDR_MCAST6,
                            (struct mg_addr *) &ip6_allnodes),
                ifp->ip6ll, (uint64_t *) ip6_allnodes.addr.ip6, false,
                ifp->mac);
      if (ifp->ip6[0] != 0 && ifp->ip6[1] != 0)
        tx_ndp_na(ifp,  // RFC-9131 4.1
                  mg_l2_mapip(ifp->l2type, MG_TCPIP_L2ADDR_MCAST6,
                              (struct mg_addr *) &ip6_allrouters),
                  ifp->ip6, (uint64_t *) ip6_allrouters.addr.ip6, false,
                  ifp->mac);
    }
    if (ifp->is_ip6_changed) {
      struct mg_connection *c;
      for (c = ifp->mgr->conns; c != NULL; c = c->next) {
        if (!c->is_listening && !c->is_udp) c->is_closing = 1;
      }
      ifp->is_ip6_changed = false;
    }
  } else if (ifp->state6 == MG_TCPIP_STATE_IP) {
    if ((ifp->gw6[0] != 0 || ifp->gw6[1] != 0) &&
        (ifp->l2type != MG_TCPIP_L2_PPP && ifp->l2type != MG_TCPIP_L2_PPPoE))
      tx_ndp_ns(ifp, ifp->gw6, NULL);  // unsolicited GW hwaddr resolution
  } else if (ifp->state6 == MG_TCPIP_STATE_UP) {
    MG_INFO(("IP: %M", mg_print_ip6, &ifp->ip6ll));
  }
  if (ifp->state6 > MG_TCPIP_STATE_UP)
    mg_tcpip_call(ifp, MG_TCPIP_EV_STATE6_CHANGE, &ifp->state6);
}
#endif

static uint8_t *tcpip_mapip(struct mg_tcpip_if *ifp, struct mg_addr *ip) {
#if MG_ENABLE_IPV6
  if (ip->is_ip6) {
    if (MG_IP6MATCH(ip->addr.ip6, ip6_allnodes.addr.ip6))  // local broadcast
      return mg_l2_mapip(ifp->l2type, MG_TCPIP_L2ADDR_MCAST6,
                         (struct mg_addr *) &ip6_allnodes);
    if (*ip->addr.ip == 0xFF)  // multicast
      return mg_l2_mapip(ifp->l2type, MG_TCPIP_L2ADDR_MCAST6, ip);
  } else
#endif
  {  // global/local broadcast
    if (ip->addr.ip4 == 0xffffffff || ip->addr.ip4 == (ifp->ip | ~ifp->mask))
      return mg_l2_mapip(ifp->l2type, MG_TCPIP_L2ADDR_BCAST, NULL);
    if ((*ip->addr.ip & 0xE0) == 0xE0)  // 224 ~ 239 = E0 ~ EF, multicast
      return mg_l2_mapip(ifp->l2type, MG_TCPIP_L2ADDR_MCAST, ip);
  }
  return NULL;
}

static uint8_t *get_return_l2addr(struct mg_tcpip_if *ifp, struct mg_addr *rem,
                                  bool is_udp, struct pkt *pkt) {
  uint8_t *l2addr;
  if (is_udp && (l2addr = tcpip_mapip(ifp, rem)) != NULL)
    return l2addr;  // broadcast or multicast
#if MG_ENABLE_IPV6
  if (rem->is_ip6) {
    if (rem->addr.ip6[0] == ifp->ip6ll[0] ||
        match_prefix((uint8_t *) rem->addr.ip6, ifp->prefix, ifp->prefix_len))
      return mg_l2_getaddr(ifp, pkt->l2);  // same LAN, get from frame
    if (ifp->gw6_ready)                    // use the router
      return ifp->gw6mac;                  // ignore source address in frame
  } else
#endif
  {
    if (ifp->ip != 0 && ((rem->addr.ip4 & ifp->mask) == (ifp->ip & ifp->mask)))
      return mg_l2_getaddr(ifp, pkt->l2);  // same LAN, get from frame
    if (ifp->gw_ready)                     // use the router
      return ifp->gwmac;                   // ignore source address in frame
  }
  MG_ERROR(("%M %s: No way back, can't respond", mg_print_ip_port, rem,
            is_udp ? "UDP" : "TCP"));
  return NULL;
}

static void proto_error(struct mg_connection *c, bool r, const char *f, ...) {
  char buf[64];
  va_list ap;
  struct connstate *s = (struct connstate *) (c + 1);
  va_start(ap, f);
  mg_vsnprintf(buf, sizeof(buf), f, &ap);
  va_end(ap);
  MG_ERROR(("%lu %ld %s", c->id, c->fd, buf));
  c->is_closing = 1;  // Stop Mongoose I/O
  s->is_closing = 1;  // Transport-terminal; do not send FIN
  if (r && !c->is_udp) s->send_rst = true;  // honot TCP RST request
  mg_call(c, MG_EV_ERROR, buf);
}

static bool rx_udp(struct mg_tcpip_if *ifp, struct pkt *pkt) {
  struct mg_connection *c = getpeer(ifp->mgr, pkt, true);
  struct connstate *s;
  uint8_t *l2addr;
  if (c == NULL) return false;  // No UDP listener on this port
  s = (struct connstate *) (c + 1);
  c->rem.port = pkt->udp->sport;
#if MG_ENABLE_IPV6
  if (c->loc.is_ip6) {  // matching of v4/v6 to dest is done bt getpeer()
    if (!udp6csum_ok(pkt->ip6, pkt->udp)) return false;
    c->rem.addr.ip6[0] = pkt->ip6->src[0],
    c->rem.addr.ip6[1] = pkt->ip6->src[1], c->rem.is_ip6 = true;
  } else
#endif
  {
    if (!udpcsum_ok(pkt->ip, pkt->udp)) return false;
    c->rem.addr.ip4 = pkt->ip->src;
  }
  if ((l2addr = get_return_l2addr(ifp, &c->rem, true, pkt)) == NULL)
    return false;  // safety net for lousy networks
  memcpy(s->mac, l2addr, sizeof(s->mac));
  if (c->recv.len >= MG_MAX_RECV_SIZE) {
    proto_error(c, false, "max_recv_buf_size reached");
  } else if (c->recv.size - c->recv.len < pkt->pay.len &&
             !mg_iobuf_resize(&c->recv, c->recv.len + pkt->pay.len)) {
    proto_error(c, false, "oom");
  } else {
    memcpy(&c->recv.buf[c->recv.len], pkt->pay.buf, pkt->pay.len);
    c->recv.len += pkt->pay.len;
    mg_call(c, MG_EV_READ, &pkt->pay.len);
  }
  return true;
}

static size_t tx_tcp(struct mg_tcpip_if *ifp, uint8_t *l2_dst,
                     struct mg_addr *ip_src, struct mg_addr *ip_dst,
                     uint8_t dscp, uint8_t flags, uint32_t seq, uint32_t ack,
                     uint16_t win, const void *buf, size_t len) {
  uint8_t *l3p;
  struct ip *ip = NULL;
  struct tcp *tcp;
  uint16_t opts[4 / 2];
  size_t hlen = sizeof(*tcp);
#if MG_ENABLE_IPV6
  struct ip6 *ip6 = NULL;
#endif

  // Handle any options first, here, to determine header size
  if (flags & TH_SYN) {  // Send MSS
    uint16_t mss;
#if MG_ENABLE_IPV6  // RFC-9293 3.7.1; RFC-6691 2
    mss = (uint16_t) (ifp->mtu - 60);
#else
    mss = (uint16_t) (ifp->mtu - 40);
#endif
    opts[0] = mg_htons(0x0204);  // RFC-9293 3.2
    opts[1] = mg_htons(mss);
    hlen += sizeof(opts);  // always whole number of 32-bit words
  }

#if MG_ENABLE_IPV6
  if (ip_dst->is_ip6) {
    ip6 = tx_ip6(ifp, l2_dst, 6, dscp, ip_src->addr.ip6, ip_dst->addr.ip6,
                 hlen + len);
    tcp = (struct tcp *) (ip6 + 1);
    l3p = (uint8_t *) ip6;
  } else
#endif
  {
    ip = tx_ip(ifp, l2_dst, 6, dscp, ip_src->addr.ip4, ip_dst->addr.ip4,
               hlen + len);
    tcp = (struct tcp *) (ip + 1);
    l3p = (uint8_t *) ip;
  }
  memset(tcp, 0, sizeof(*tcp));
  memmove(tcp + 1, opts, hlen - sizeof(*tcp));  // copy opts if any
  if (buf != NULL && len) memmove((uint8_t *) tcp + hlen, buf, len);
  tcp->sport = ip_src->port;
  tcp->dport = ip_dst->port;
  tcp->seq = seq;
  tcp->ack = ack;
  tcp->flags = flags;
  tcp->win = mg_htons(win);
  tcp->off = (uint8_t) (hlen / 4 << 4);
#if MG_ENABLE_IPV6
  if (ip_dst->is_ip6) {
    tcp->csum = p6csum(ip6, tcp, 6, hlen + len);
  } else
#endif
  {
    tcp->csum = pcsum(ip, tcp, hlen + len);
  }
  MG_VERBOSE(("TCP %M -> %M fl %x len %u", mg_print_ip_port, ip_src,
              mg_print_ip_port, ip_dst, tcp->flags, len));
  return driver_output(
      ifp, mg_l2_trailer(ifp, PDIFF(l3p, (uint8_t *) tcp + hlen + len),
                         (uint8_t *) tcp + hlen + len));
}

static size_t tx_tcp_ctrlresp(struct mg_tcpip_if *ifp, struct pkt *pkt,
                              uint8_t flags, uint32_t seqno) {
  uint32_t ackno = mg_htonl(mg_ntohl(pkt->tcp->seq) + (uint32_t) pkt->pay.len +
                            ((pkt->tcp->flags & (TH_SYN | TH_FIN)) ? 1 : 0));
  struct mg_addr ips, ipd;
  uint8_t *l2addr;
  memset(&ips, 0, sizeof(ips));
  memset(&ipd, 0, sizeof(ipd));
  if (pkt->ip != NULL) {
    ips.addr.ip4 = pkt->ip->dst;
    ipd.addr.ip4 = pkt->ip->src;
  } else {
    ips.addr.ip6[0] = pkt->ip6->dst[0], ips.addr.ip6[1] = pkt->ip6->dst[1];
    ipd.addr.ip6[0] = pkt->ip6->src[0], ipd.addr.ip6[1] = pkt->ip6->src[1];
    ips.is_ip6 = true;
    ipd.is_ip6 = true;
  }
  ips.port = pkt->tcp->dport;
  ipd.port = pkt->tcp->sport;
  if ((l2addr = get_return_l2addr(ifp, &ipd, false, pkt)) == NULL)
    return 0;  // safety net for lousy networks
  return tx_tcp(ifp, l2addr, &ips, &ipd, 0, flags, seqno, ackno, MG_TCPIP_WIN,
                NULL, 0);
}

static size_t tx_tcp_rst(struct mg_tcpip_if *ifp, struct pkt *pkt, bool toack) {
  return tx_tcp_ctrlresp(ifp, pkt,
                         (uint8_t) (toack ? TH_RST : (TH_RST | TH_ACK)),
                         toack ? pkt->tcp->ack : 0);
}

static struct mg_connection *accept_conn(struct mg_connection *lsn,
                                         struct pkt *pkt, uint16_t mss) {
  struct connstate *s;
  uint8_t *l2addr;
  struct mg_connection *c = mg_alloc_conn(lsn->mgr);
  if (c == NULL) {
    MG_ERROR(("OOM"));
    return NULL;
  }
  s = (struct connstate *) (c + 1);
  s->retransmit = lsn->mgr->ifp->enable_tcp_retransmit;
  s->txq.align = MG_TCPIP_TXQ_ALIGN;
  s->dmss = mss;  // from options in client SYN
  s->seq = mg_ntohl(pkt->tcp->ack), s->ack = mg_ntohl(pkt->tcp->seq);
  s->win = mg_ntohs(pkt->tcp->win), s->maxseq = (uint32_t) (s->seq + s->win);
#if MG_ENABLE_IPV6
  if (lsn->loc.is_ip6) {
    c->rem.addr.ip6[0] = pkt->ip6->src[0],
    c->rem.addr.ip6[1] = pkt->ip6->src[1], c->rem.is_ip6 = true;
    c->loc.addr.ip6[0] = c->mgr->ifp->ip6[0],
    c->loc.addr.ip6[1] = c->mgr->ifp->ip6[1], c->loc.is_ip6 = true;
    // TODO(): compare lsn to link-local, or rem as link-local: use ll instead
  } else
#endif
  {
    c->rem.addr.ip4 = pkt->ip->src;
    c->loc.addr.ip4 = c->mgr->ifp->ip;
  }
  c->rem.port = pkt->tcp->sport;
  c->loc.port = lsn->loc.port;
  if ((l2addr = get_return_l2addr(lsn->mgr->ifp, &c->rem, false, pkt)) ==
      NULL) {
    free(c);      // safety net for lousy networks, not actually needed
    return NULL;  // as path has already been checked at SYN (sending SYN+ACK)
  }
  memcpy(s->mac, l2addr, sizeof(s->mac));
  settmout(c, MIP_TTYPE_KEEPALIVE);
  MG_DEBUG(("%lu accepted %M", c->id, mg_print_ip_port, &c->rem));
  LIST_ADD_HEAD(struct mg_connection, &lsn->mgr->conns, c);
  c->is_accepted = 1;
  c->is_hexdumping = lsn->is_hexdumping;
  c->pfn = lsn->pfn;
  c->pfn_data = lsn->pfn_data;
  c->fn = lsn->fn;
  c->fn_data = lsn->fn_data;
  c->is_tls = lsn->is_tls;
  mg_call(c, MG_EV_OPEN, NULL);
  mg_call(c, MG_EV_ACCEPT, NULL);
  if (!c->is_tls_hs) c->is_tls = 0;  // user did not call mg_tls_init()
  return c;
}

static size_t trim_len(struct mg_connection *c, size_t len) {
  struct mg_tcpip_if *ifp = c->mgr->ifp;
  size_t l2_max_overhead = ifp->framesize - ifp->l2mtu;
  size_t ip_max_h_len = c->rem.is_ip6 ? 40 : 24;  // we don't send options
  size_t tcp_max_h_len = 60 /* RFC-9293 3.7.1; RFC-6691 2 */, udp_h_len = 8;
  size_t max_headers_len =
      ip_max_h_len + (c->is_udp ? udp_h_len : tcp_max_h_len);
  size_t min_mtu = c->rem.is_ip6 ? 1280 /* RFC-8200, IPv6 minimum */
                   : c->is_udp   ? 68   /* RFC-791, IP minimum */
                                 : max_headers_len /* fit full TCP header */;
  // NOTE(): We are effectively reducing transmitted TCP segment length by 20,
  // accounting for possible options; though we currently don't send options
  // except for SYN.

  // If the frame exceeds the available buffer, trim the length.
  if (len + max_headers_len + l2_max_overhead > ifp->tx.len)
    len = ifp->tx.len - max_headers_len - l2_max_overhead;
  // Ensure the MTU isn't lower than the minimum allowed value
  if (ifp->mtu < min_mtu) {
    MG_ERROR(("MTU is lower than minimum, raising to %lu", min_mtu));
    ifp->mtu = (uint16_t) min_mtu;
  }
  // If the total packet size exceeds the MTU, trim the length
  if (len + max_headers_len > ifp->mtu) {
    len = ifp->mtu - max_headers_len;
    if (c->is_udp) MG_ERROR(("UDP datagram exceeds MTU. Truncating it."));
  }

  return len;
}

static bool udp_send(struct mg_connection *c, const void *buf, size_t len) {
  struct mg_tcpip_if *ifp = c->mgr->ifp;
  struct connstate *s = (struct connstate *) (c + 1);
  struct mg_addr ips;
  memset(&ips, 0, sizeof(ips));
#if MG_ENABLE_IPV6
  if (c->loc.is_ip6) {
    ips.addr.ip6[0] = ifp->ip6[0], ips.addr.ip6[1] = ifp->ip6[1],
    ips.is_ip6 = true;
    // TODO(): detect link-local (c->rem) and use it
  } else
#endif
  {
    ips.addr.ip4 = ifp->ip;
  }
  ips.port = c->loc.port;
  return tx_udp(ifp, s->mac, &ips, &c->rem, c->dscp, buf, len);
}

static size_t txq_next(struct connstate *s, uint8_t **buf) {
  uint32_t len;
  if (s->txq.len == 0) return 0;
  assert(s->txq.len >= sizeof(len));
  memcpy(&len, s->txq.buf, sizeof(len));
  assert(len > 0 && len <= s->txq.len - sizeof(len));
  if (buf != NULL) *buf = s->txq.buf + sizeof(len);
  return len;
}

static bool txq_add(struct connstate *s, const void *buf, size_t len) {
  uint32_t n = (uint32_t) len;
  size_t off = s->txq.len, total = sizeof(n) + len;
  assert(len > 0);
  if (mg_iobuf_add(&s->txq, off, NULL, total) != total) return false;
  memcpy(s->txq.buf + off, &n, sizeof(n));
  memcpy(s->txq.buf + off + sizeof(n), buf, len);
  return true;
}

long mg_io_send(struct mg_connection *c, const void *buf, size_t len) {
  struct connstate *s = (struct connstate *) (c + 1);
  len = trim_len(c, len);
  if (c->is_udp) {
    if (!udp_send(c, buf, len)) return MG_IO_WAIT;
  } else {  // TCP, cap to peer's MSS and check window
    struct mg_tcpip_if *ifp = c->mgr->ifp;
    size_t sent;
    uint32_t room = s->maxseq - s->seq;
    bool was_empty = s->txq.len == 0;
    if (s->retransmit) {
      uint32_t in_flight = s->txq.len > 0 ? s->seq - s->txq_una : 0;
      uint32_t qroom = in_flight < MG_TCPIP_WIN ? MG_TCPIP_WIN - in_flight : 0;
      room = s->win > in_flight ? s->win - in_flight : 0;
      if (room > qroom) room = qroom;
    }
    if (room == 0) return MG_IO_WAIT;
    if (len > s->dmss) len = s->dmss;  // RFC-6691, reduce if sending opts
    if (len > room) len = room;
    if (s->retransmit) {
      if (was_empty) s->txq_seq = s->txq_una = s->seq;
      if (!txq_add(s, buf, len)) return MG_IO_WAIT;
    }
    sent = tx_tcp(ifp, s->mac, &c->loc, &c->rem, c->dscp, TH_PUSH | TH_ACK,
                  mg_htonl(s->seq), mg_htonl(s->ack),
                  s->is_full ? 0 : MG_TCPIP_WIN, buf, len);
    if (sent == 0 || sent == (size_t) -1) {
      if (s->retransmit)
        mg_iobuf_del(&s->txq, s->txq.len - len - sizeof(uint32_t),
                     len + sizeof(uint32_t));
      return sent == 0 ? MG_IO_WAIT : MG_IO_ERR;
    }
    s->seq += (uint32_t) len;
    if (s->retransmit && was_empty) {
      s->txq_timer = ifp->now + MG_TCPIP_RTX_MS;
      s->txq_retries = 0;
    }
    if (s->ttype == MIP_TTYPE_ACK) settmout(c, MIP_TTYPE_KEEPALIVE);
  }
  return (long) len;
}

static void handle_tls_recv(struct mg_connection *c) {
  size_t avail = mg_tls_pending(c);  // will change after mg_tls_recv()
  size_t min = avail > MG_MAX_RECV_SIZE ? MG_MAX_RECV_SIZE : avail;
  struct mg_iobuf *io = &c->recv;  // allocated on first avail > 0
  if (io->size - io->len < min && !mg_iobuf_resize(io, io->len + min)) {
    proto_error(c, true, "oom");
  } else {
    // Decrypt data directly into c->recv. If io->buf = NULL or
    // io->len = io->size (no room), there can be outstanding data that can't be
    // moved (mg_tls_pending() > 0 after mg_tls_recv() returns). So
    // mg_tls_recv() takes care of returning 0 in this case (commented as "MIP")
    long n = mg_tls_recv(c, io->buf != NULL ? &io->buf[io->len] : io->buf,
                         io->size - io->len);
    if (n == MG_IO_ERR) {
      proto_error(c, true, "TLS recv error");
    } else if (n > 0) {
      // Decrypted successfully - trigger MG_EV_READ
      io->len += (size_t) n;
      mg_call(c, MG_EV_READ, &n);
    }  // else n < 0: outstanding data to be moved to c->recv
  }
}

static void handle_ack(struct connstate *s, uint32_t ackno, uint16_t win,
                       uint64_t now) {
  if (ackno > s->seq) return;
  if (s->retransmit && s->txq.len > 0) {
    if (ackno < s->txq_una) return;
    if (ackno > s->txq_una) {
      size_t len;
      // Remove only entirely acknowledged segments (RFC-9293, 3.10.7.4)
      while ((len = txq_next(s, NULL)) > 0 && ackno >= s->txq_seq + len) {
        mg_iobuf_del(&s->txq, 0, len + sizeof(uint32_t));
        s->txq_seq += (uint32_t) len;
      }
      s->txq_una = ackno;
      s->txq_retries = 0;
      // Restart RTO when an ACK acknowledges new data (RFC-6298, 5.3)
      s->txq_timer = now + MG_TCPIP_RTX_MS;
    }  // else: dup ack, ignore it but handle its window upddate
  } else if (ackno < (s->seq - s->win)) {
    return;
  }
  s->maxseq = (uint32_t) (ackno + win);
  s->win = win;
}

static void retransmit(struct mg_connection *c) {
  struct connstate *s = (struct connstate *) (c + 1);
  struct mg_tcpip_if *ifp = c->mgr->ifp;
  uint8_t *buf, i;
  uint64_t rto;
  size_t len = txq_next(s, &buf);
  // Send the front retransmission-queue segment only (RFC-9293, 3.10.8)
  if (len == 0 ||
      tx_tcp(ifp, s->mac, &c->loc, &c->rem, c->dscp, TH_PUSH | TH_ACK,
             mg_htonl(s->txq_seq), mg_htonl(s->ack),
             s->is_full ? 0 : MG_TCPIP_WIN, buf, len) == (size_t) -1) {
    proto_error(c, false, "retransmit"); // should RST, but sending just failed
    return;
  }
  // Back off RTO after every timeout (RFC-6298, 5.5)
  for (i = 0, rto = MG_TCPIP_RTX_MS;
       i < s->txq_retries && rto < MG_TCPIP_RTX_MAX_MS; i++) {
    rto *= 2;
    if (rto > MG_TCPIP_RTX_MAX_MS) rto = MG_TCPIP_RTX_MAX_MS;
  }
  s->txq_timer = ifp->now + rto;
}

static void read_conn(struct mg_connection *c, struct pkt *pkt) {
  struct connstate *s = (struct connstate *) (c + 1);
  struct mg_iobuf *io = c->is_tls ? &c->rtls : &c->recv;
  uint32_t seq = mg_ntohl(pkt->tcp->seq);
  // Evaluation order is important: 1) process FIN, 2) process keep-alive,
  // 3) discard invalid SEG.SEQ, 4) process ACK and payload.
  if (pkt->tcp->flags & TH_FIN) {
    uint8_t flags = TH_ACK;
    if (mg_ntohl(pkt->tcp->seq) != s->ack) {
      MG_VERBOSE(("ignoring FIN, %x != %x", mg_ntohl(pkt->tcp->seq), s->ack));
      tx_tcp(c->mgr->ifp, s->mac, &c->loc, &c->rem, c->dscp, TH_ACK,
             mg_htonl(s->seq), mg_htonl(s->ack), s->is_full ? 0 : MG_TCPIP_WIN,
             "", 0);
      return;
    }
    // If we initiated the closure, we reply with ACK upon receiving FIN
    // If we didn't initiate it, we reply with FIN as part of the normal TCP
    // closure process
    s->ack = (uint32_t) (mg_htonl(pkt->tcp->seq) + pkt->pay.len + 1);
    s->fin_rcvd = true;
    if (c->is_draining && s->ttype == MIP_TTYPE_FIN) {
      // Simultaneous closure ?
      s->twclosure = true;
    } else {
      // Peer closed first: send ACK only, enter CLOSE_WAIT.
      // The connection loop will initiate closure after pending send data
      // is flushed, then send our FIN.
      c->is_draining = 1;
    }
    tx_tcp(c->mgr->ifp, s->mac, &c->loc, &c->rem, c->dscp, flags,
           mg_htonl(s->seq), mg_htonl(s->ack), s->is_full ? 0 : MG_TCPIP_WIN,
           "", 0);
  } else if (pkt->pay.len <= 1 && mg_ntohl(pkt->tcp->seq) == s->ack - 1) {
    // Keep-Alive (RFC-9293 3.8.4, allow erroneous implementations)
    MG_VERBOSE(("%lu keepalive ACK", c->id));
    tx_tcp(c->mgr->ifp, s->mac, &c->loc, &c->rem, c->dscp, TH_ACK,
           mg_htonl(s->seq), mg_htonl(s->ack), s->is_full ? 0 : MG_TCPIP_WIN,
           NULL, 0);
    return;  // RFC-9293 3.10.7.4 discard (incorrect) payload, ACK, window
  } else if (seq != s->ack) {
    uint32_t ack = (uint32_t) (mg_htonl(pkt->tcp->seq) + pkt->pay.len);
    if (s->ack == ack) {
      MG_VERBOSE(("ignoring duplicate pkt"));
    } else {
      MG_VERBOSE(("SEQ != ACK: %x %x %x", seq, s->ack, ack));
    }
    tx_tcp(c->mgr->ifp, s->mac, &c->loc, &c->rem, c->dscp, TH_ACK,
           mg_htonl(s->seq), mg_htonl(s->ack), s->is_full ? 0 : MG_TCPIP_WIN,
           "", 0);
    return;  // drop it, RFC-9293 3.10.7.4: ignore ACKno
  }
  // Now process the segment for ACK and payload
  if (pkt->tcp->flags & TH_ACK) {
    handle_ack(s, mg_ntohl(pkt->tcp->ack), mg_ntohs(pkt->tcp->win),
               c->mgr->ifp->now);
    if (pkt->pay.len == 0 && s->fin_rcvd && s->ttype == MIP_TTYPE_FIN)
      s->twclosure = true;
  }
  if (pkt->pay.len == 0) return;
  if (!c->is_closing && io->size - io->len < pkt->pay.len &&
      !mg_iobuf_resize(io, io->len + pkt->pay.len)) {
    proto_error(c, true, "oom");
    return;  // drop it
  }
  MG_VERBOSE(("%lu SEQ %x -> %x", c->id, mg_htonl(pkt->tcp->seq), s->ack));
  if (!(pkt->tcp->flags & TH_FIN)) {  // FIN already advanced s->ack
    // Advance ACK counter
    s->ack = (uint32_t) (mg_htonl(pkt->tcp->seq) + pkt->pay.len);
    s->unacked += pkt->pay.len;
  }
  // size_t diff = s->acked <= s->ack ? s->ack - s->acked : s->ack;
  if ((s->unacked > MG_TCPIP_WIN / 2 && s->acked != s->ack) ||
      s->ttype == MIP_TTYPE_FIN) {
    // Send ACK immediately
    MG_VERBOSE(("%lu imm ACK %lu", c->id, s->acked));
    tx_tcp(c->mgr->ifp, s->mac, &c->loc, &c->rem, c->dscp, TH_ACK,
           mg_htonl(s->seq), mg_htonl(s->ack), s->is_full ? 0 : MG_TCPIP_WIN,
           NULL, 0);
    s->unacked = 0;
    s->acked = s->ack;
    if (s->ttype != MIP_TTYPE_KEEPALIVE) settmout(c, MIP_TTYPE_KEEPALIVE);
  } else {
    // if not already running, setup a timer to send an ACK later
    if (s->ttype != MIP_TTYPE_ACK) settmout(c, MIP_TTYPE_ACK);
  }
  if (c->is_closing) return; // we have a "closed socket", ignore data
  // Copy TCP payload into the IO buffer. If the connection is plain text,
  // we copy to c->recv. If the connection is TLS, this data is encrypted,
  // therefore we copy that encrypted data to the c->rtls iobuffer instead,
  // and then call mg_tls_recv() to decrypt it. NOTE: mg_tls_recv() will
  // call back mg_io_recv() which grabs raw data from c->rtls
  memcpy(&io->buf[io->len], pkt->pay.buf, pkt->pay.len);
  io->len += pkt->pay.len;
  if (c->is_tls) {
    c->is_tls_hs ? mg_tls_handshake(c) : handle_tls_recv(c);
  } else {
    // Plain text connection, data is already in c->recv, trigger MG_EV_READ
    mg_call(c, MG_EV_READ, &pkt->pay.len);
  }
}

// TCP backlog
struct mg_backlog {
  uint16_t port, mss;  // use port=0 for available entries
  uint8_t age;
};

static int backlog_insert(struct mg_connection *c, uint16_t port,
                          uint16_t mss) {
  struct mg_backlog *p = (struct mg_backlog *) c->data;
  size_t i;
  for (i = 0; i < sizeof(c->data) / sizeof(*p); i++) {
    if (p[i].port != 0) continue;
    p[i].age = 2;  // remove after two calls, average 1.5 call rate
    p[i].port = port, p[i].mss = mss;
    return (int) i;
  }
  return -1;
}

static struct mg_backlog *backlog_retrieve(struct mg_connection *c,
                                           uint16_t key, uint16_t port) {
  struct mg_backlog *p = (struct mg_backlog *) c->data;
  if (key >= sizeof(c->data) / sizeof(*p)) return NULL;
  if (p[key].port != port) return NULL;
  p += key;
  return p;
}

static void backlog_remove(struct mg_connection *c, uint16_t key) {
  struct mg_backlog *p = (struct mg_backlog *) c->data;
  p[key].port = 0;
}

static void backlog_maintain(struct mg_connection *c) {
  struct mg_backlog *p = (struct mg_backlog *) c->data;
  size_t i;  // dec age and remove those where it reaches 0
  for (i = 0; i < sizeof(c->data) / sizeof(*p); i++) {
    if (p[i].port == 0) continue;
    if (p[i].age != 0) --p[i].age;
    if (p[i].age == 0) p[i].port = 0;
  }
}

static void backlog_poll(struct mg_mgr *mgr) {
  struct mg_connection *c = NULL;
  for (c = mgr->conns; c != NULL; c = c->next) {
    if (!c->is_udp && c->is_listening) backlog_maintain(c);
  }
}

// process options (MSS)
static bool handle_opt(struct connstate *s, struct tcp *tcp, bool ip6) {
  uint8_t *opts = (uint8_t *) (tcp + 1);
  int len = 4 * ((int) (tcp->off >> 4) - ((int) sizeof(*tcp) / 4));
  s->dmss = ip6 ? 1220 : 536;  // assume default, RFC-9293 3.7.1
  while (len > 0) {            // RFC-9293 3.1 3.2
    uint8_t kind = opts[0], optlen = 1;
    if (kind != 1) {         // No-Operation
      if (kind == 0) break;  // End of Option List
      if (len < 2 || opts[1] == 0 || opts[1] > len) return false;  // Malformed
      optlen = opts[1];
      if (kind == 2 && optlen == 4) {  // set received MSS
        uint16_t mss = (uint16_t) (((uint16_t) opts[2] << 8) + opts[3]);
        if (mss == 0 || mss < MG_TCPIP_MIN_MSS) return false;
        s->dmss = mss;
      }
    }
    MG_VERBOSE(("kind: %u, optlen: %u, len: %d\n", kind, optlen, len));
    opts += optlen;
    len -= optlen;
  }
  return true;
}

static void rx_tcp(struct mg_tcpip_if *ifp, struct pkt *pkt) {
  struct mg_connection *c = getpeer(ifp->mgr, pkt, false);
  struct connstate *s = c == NULL ? NULL : (struct connstate *) (c + 1);
#if MG_ENABLE_IPV6  // matching of v4/v6 to dest is done by getpeer()
  if (pkt->ip6 != NULL &&
      !tcp6csum_ok(pkt->ip6, pkt->tcp,
                   PDIFF(pkt->tcp, pkt->pay.buf) + pkt->pay.len))
    return;
#endif
  if (pkt->ip != NULL && !tcpcsum_ok(pkt->ip, pkt->tcp)) return;
  pkt->tcp->flags &= TH_STDFLAGS;  // tolerate creative usage (ECN, ?)
  // Order is VERY important; RFC-9293 3.5.2
  // - check clients (Group 1) and established connections (Group 3)
  if (c != NULL && c->is_connecting && pkt->tcp->flags == (TH_SYN | TH_ACK)) {
    // client got a server connection accept
    if (!handle_opt(s, pkt->tcp, pkt->ip6 != NULL))
      return;  // process options (MSS)
    s->seq = mg_ntohl(pkt->tcp->ack), s->ack = mg_ntohl(pkt->tcp->seq) + 1;
    s->win = mg_ntohs(pkt->tcp->win), s->maxseq = (uint32_t) (s->seq + s->win);
    tx_tcp_ctrlresp(ifp, pkt, TH_ACK, pkt->tcp->ack);
    c->is_connecting = 0;  // Client connected
    settmout(c, MIP_TTYPE_KEEPALIVE);
    mg_call(c, MG_EV_CONNECT, NULL);  // Let user know
    if (c->is_tls_hs) mg_tls_handshake(c);
    if (!c->is_tls_hs) c->is_tls = 0;  // user did not call mg_tls_init()
  } else if (c != NULL && c->is_connecting && pkt->tcp->flags != TH_ACK) {
    proto_error(c, false, "connection refused");
  } else if (c != NULL && pkt->tcp->flags & TH_RST) {
    uint32_t seqno = mg_ntohl(pkt->tcp->seq);
    if (seqno >= s->ack && seqno < (s->ack + MG_TCPIP_WIN))  // RFC-9293 3.5.3
      proto_error(c, false, "peer RST");  // RFC-1122 4.2.2.13
  } else if (c != NULL) {
    // process segment
    s->tmiss = 0;                         // Reset missed keep-alive counter
    if (s->ttype == MIP_TTYPE_KEEPALIVE)  // Advance keep-alive timer
      settmout(c,
               MIP_TTYPE_KEEPALIVE);  // unless a former ACK timeout is pending
    read_conn(c, pkt);  // Override timer with ACK timeout if needed
    if (c->is_full && !s->is_full) {
      // The receive handler paused the connection: advertise zero window now
      s->is_full = true;
      tx_tcp(ifp, s->mac, &c->loc, &c->rem, c->dscp, TH_ACK, mg_htonl(s->seq),
             mg_htonl(s->ack), 0, NULL, 0);
    }
  } else
    // - we don't listen on that port; RFC-9293 3.5.2 Group 1
    // - check listening connections; RFC-9293 3.5.2 Group 2
    if ((c = getpeer(ifp->mgr, pkt, true)) == NULL) {
      // not listening on that port
      if (!(pkt->tcp->flags & TH_RST)) {
        tx_tcp_rst(ifp, pkt, pkt->tcp->flags & TH_ACK);
      }  // else silently discard
    } else if (pkt->tcp->flags == TH_SYN) {
      // listener receives a connection request
      struct connstate cs;  // At this point, s = NULL, there is no connection
      int key;
      uint32_t isn;
      if (pkt->tcp->sport != 0) {
        if (!handle_opt(&cs, pkt->tcp, pkt->ip6 != NULL))
          return;  // process options (MSS)
        key = backlog_insert(c, pkt->tcp->sport,
                             cs.dmss);  // backlog options (MSS)
        if (key < 0) return;  // no room in backlog, discard SYN, client retries
        // Use peer's src port and bl key as ISN, to later identify the
        // handshake
        isn = (mg_htonl(((uint32_t) key << 16) | mg_ntohs(pkt->tcp->sport)));
        if (tx_tcp_ctrlresp(ifp, pkt, TH_SYN | TH_ACK, isn) == 0)
          backlog_remove(c, (uint16_t) key);  // safety net for lousy networks
      }  // what should we do when port=0 ? Linux takes port 0 as any other
         // port
    } else if (pkt->tcp->flags == TH_ACK) {
      // listener receives an ACK
      struct mg_backlog *b = NULL;
      if ((uint16_t) (mg_htonl(pkt->tcp->ack) - 1) ==
          mg_htons(pkt->tcp->sport)) {
        uint16_t key = (uint16_t) ((mg_htonl(pkt->tcp->ack) - 1) >> 16);
        b = backlog_retrieve(c, key, pkt->tcp->sport);
        if (b != NULL) {                // ACK is a response to a SYN+ACK
          accept_conn(c, pkt, b->mss);  // pass options
          backlog_remove(c, key);
        }  // else not an actual match, reset
      }
      if (b == NULL) tx_tcp_rst(ifp, pkt, true);
    } else if (pkt->tcp->flags & TH_RST) {
      // silently discard
    } else if (pkt->tcp->flags & TH_ACK) {  // ACK + something else != RST
      tx_tcp_rst(ifp, pkt, true);
    } else if (pkt->tcp->flags & TH_SYN) {  // SYN + something else != ACK
      tx_tcp_rst(ifp, pkt, false);
    }  // else  silently discard
}

static void rx_ip(struct mg_tcpip_if *ifp, struct pkt *pkt) {
  uint8_t ihl;
  uint16_t frag, len;
  if (pkt->pay.len < sizeof(*pkt->ip)) return;  // Truncated
  if ((pkt->ip->ver >> 4) != 4) return;         // Not IP
  ihl = pkt->ip->ver & 0x0F;
  if (ihl < 5) return;                              // bad IHL
  if (pkt->pay.len < (uint16_t) (ihl * 4)) return;  // Truncated / malformed
  // There can be link padding, take length from IP header
  len = mg_ntohs(pkt->ip->len);  // IP datagram length
  if (len < (uint16_t) (ihl * 4) || len > pkt->pay.len) return;  // malformed
  pkt->pay.len = len;                      // strip padding
  mkpay(pkt, (uint32_t *) pkt->ip + ihl);  // account for opts
  if (!ipcsum_ok(pkt->ip)) return;
  frag = mg_ntohs(pkt->ip->frag);
  if (frag & IP_MORE_FRAGS_MSK || frag & IP_FRAG_OFFSET_MSK) {
    struct mg_connection *c;
    if (pkt->ip->proto == 17) pkt->udp = (struct udp *) (pkt->pay.buf);
    if (pkt->ip->proto == 6) pkt->tcp = (struct tcp *) (pkt->pay.buf);
    c = getpeer(ifp->mgr, pkt, false);
    if (c) proto_error(c, false, "Received fragmented packet");
  } else if (pkt->ip->proto == 1) {
    pkt->icmp = (struct icmp *) (pkt->pay.buf);
    if (pkt->pay.len < sizeof(*pkt->icmp)) return;
    mkpay(pkt, pkt->icmp + 1);
    rx_icmp(ifp, pkt);
  } else if (pkt->ip->proto == 17) {
    pkt->udp = (struct udp *) (pkt->pay.buf);
    if (pkt->pay.len < sizeof(*pkt->udp)) return;  // truncated
    // Take length from UDP header
    len = mg_ntohs(pkt->udp->len);  // UDP datagram length
    if (len < sizeof(*pkt->udp) || len > pkt->pay.len) return;  // malformed
    pkt->pay.len = len;  // strip excess data
    mkpay(pkt, pkt->udp + 1);
    MG_VERBOSE(("UDP %M:%hu -> %M:%hu len %u", mg_print_ip4, &pkt->ip->src,
                mg_ntohs(pkt->udp->sport), mg_print_ip4, &pkt->ip->dst,
                mg_ntohs(pkt->udp->dport), (int) pkt->pay.len));
    if (ifp->enable_dhcp_client && pkt->udp->dport == mg_htons(68) &&
        len >= offsetof(struct dhcp, options)) {
      pkt->dhcp = (struct dhcp *) (pkt->udp + 1);
      mkpay(pkt, &pkt->dhcp->options);
      rx_dhcp_client(ifp, pkt);
    } else if (ifp->enable_dhcp_server && pkt->udp->dport == mg_htons(67) &&
               len >= offsetof(struct dhcp, options)) {
      pkt->dhcp = (struct dhcp *) (pkt->udp + 1);
      mkpay(pkt, &pkt->dhcp->options);
      rx_dhcp_server(ifp, pkt);
    } else if (!rx_udp(ifp, pkt)) {
      // Should send ICMP Destination Unreachable for unicasts, but keep
      // silent
    }
  } else if (pkt->ip->proto == 6) {
    uint8_t off;
    pkt->tcp = (struct tcp *) (pkt->pay.buf);
    if (pkt->pay.len < sizeof(*pkt->tcp)) return;
    off = pkt->tcp->off >> 4;  // account for opts
    if (off < sizeof(*pkt->tcp) / 4 || pkt->pay.len < (uint16_t) (4 * off))
      return;
    mkpay(pkt, (uint32_t *) pkt->tcp + off);
    MG_VERBOSE(("TCP %M:%hu -> %M:%hu len %u", mg_print_ip4, &pkt->ip->src,
                mg_ntohs(pkt->tcp->sport), mg_print_ip4, &pkt->ip->dst,
                mg_ntohs(pkt->tcp->dport), (int) pkt->pay.len));
    rx_tcp(ifp, pkt);
  } else {
    MG_DEBUG(("Unknown IP proto %x", (int) pkt->ip->proto));
    if (mg_log_level >= MG_LL_VERBOSE)
      mg_hexdump(pkt->ip, pkt->pay.len >= 32 ? 32 : pkt->pay.len);
  }
}

#if MG_ENABLE_IPV6
static void rx_ip6(struct mg_tcpip_if *ifp, struct pkt *pkt) {
  uint16_t len = 0, plen;
  uint8_t next, *nhdr;
  bool loop = true;
  if (pkt->pay.len < sizeof(*pkt->ip6)) return;  // Truncated
  if ((pkt->ip6->ver >> 4) != 0x6) return;       // Not IPv6
  plen = mg_ntohs(pkt->ip6->plen);
  if (plen > (pkt->pay.len - sizeof(*pkt->ip6))) return;  // malformed
  next = pkt->ip6->next;
  nhdr = (uint8_t *) (pkt->ip6 + 1);
  while (loop) {
    uint16_t hlen;
    switch (next) {
      case 0:   // Hop-by-Hop 4.3
      case 43:  // Routing 4.4
      case 60:  // Destination Options 4.6
      case 51:  // Authentication RFC-4302
        MG_INFO(("IPv6 extension header %d", (int) next));
        if (((uint32_t) len + 2) > plen) return;  // nhdr[0, 1]; malformed
        next = nhdr[0];
        hlen = (uint16_t) (8 * (nhdr[1] + 1));
        if (((uint32_t) len + hlen) > plen) return;  // malformed
        len += hlen;
        nhdr += hlen;
        break;
      case 44:  // Fragment 4.5
      {
        struct mg_connection *c;
        if (((uint32_t) len + 2) > plen) return;  // nhdr[0, 1]; malformed
        if (nhdr[0] == 17) pkt->udp = (struct udp *) (pkt->pay.buf);
        if (nhdr[0] == 6) pkt->tcp = (struct tcp *) (pkt->pay.buf);
        c = getpeer(ifp->mgr, pkt, false);
        if (c) proto_error(c, false, "Received fragmented packet");
      }
        return;
      case 59:  // No Next Header 4.7
        return;
      case 50:  // IPsec ESP RFC-4303, unsupported
      default:
        loop = false;
        break;
    }
  }
  // There can be link padding, take payload length from IPv6 header - options
  pkt->pay.buf = (char *) nhdr;
  pkt->pay.len = plen - len;
  if (next == 58) {
    pkt->icmp6 = (struct icmp6 *) (pkt->pay.buf);
    if (pkt->pay.len < sizeof(*pkt->icmp6)) return;
    mkpay(pkt, pkt->icmp6 + 1);
    MG_DEBUG(("ICMPv6 %M -> %M len %u", mg_print_ip6, &pkt->ip6->src,
              mg_print_ip6, &pkt->ip6->dst, (int) pkt->pay.len));
    rx_icmp6(ifp, pkt);
  } else if (next == 17) {
    pkt->udp = (struct udp *) (pkt->pay.buf);
    if (pkt->pay.len < sizeof(*pkt->udp)) return;
    // Take length from UDP header
    len = mg_ntohs(pkt->udp->len);  // UDP datagram length
    if (len < sizeof(*pkt->udp) || len > pkt->pay.len) return;  // malformed
    pkt->pay.len = len;  // strip excess data
    mkpay(pkt, pkt->udp + 1);
    MG_DEBUG(("UDP %M:%hu -> %M:%hu len %u", mg_print_ip6, &pkt->ip6->src,
              mg_ntohs(pkt->udp->sport), mg_print_ip6, &pkt->ip6->dst,
              mg_ntohs(pkt->udp->dport), (int) pkt->pay.len));
    if (ifp->enable_dhcp6_client && pkt->udp->dport == mg_htons(546)) {
      pkt->dhcp6 = (struct dhcp6 *) (pkt->udp + 1);
      mkpay(pkt, pkt->dhcp6 + 1);
      // rx_dhcp6_client(ifp, pkt);
#if 0
    } else if (ifp->enable_dhcp_server && pkt->udp->dport == mg_htons(547)) {
      pkt->dhcp6 = (struct dhcp6 *) (pkt->udp + 1);
      mkpay(pkt, pkt->dhcp6 + 1);
      rx_dhcp6_server(ifp, pkt);
#endif
    } else if (!rx_udp(ifp, pkt)) {
      // Should send ICMPv6 Destination Unreachable for unicasts, keep silent
    }
  } else if (next == 6) {
    uint8_t off;
    pkt->tcp = (struct tcp *) (pkt->pay.buf);
    if (pkt->pay.len < sizeof(*pkt->tcp)) return;
    off = pkt->tcp->off >> 4;  // account for opts
    if (off < sizeof(*pkt->tcp) / 4 || pkt->pay.len < (uint16_t) (4 * off))
      return;
    mkpay(pkt, (uint32_t *) pkt->tcp + off);
    MG_DEBUG(("TCP %M:%hu -> %M:%hu len %u", mg_print_ip6, &pkt->ip6->src,
              mg_ntohs(pkt->tcp->sport), mg_print_ip6, &pkt->ip6->dst,
              mg_ntohs(pkt->tcp->dport), (int) pkt->pay.len));
    rx_tcp(ifp, pkt);
  } else {
    MG_DEBUG(("Unknown IPv6 next hdr %x", (int) next));
    if (mg_log_level >= MG_LL_VERBOSE)
      mg_hexdump(pkt->ip6, pkt->pay.len >= 32 ? 32 : pkt->pay.len);
  }
}
#else
#define rx_ip6(x, y)
#endif

static void mg_tcpip_rx(struct mg_tcpip_if *ifp, void *buf, size_t len) {
  struct pkt pkt;
  enum mg_l2proto proto;
  memset(&pkt, 0, sizeof(pkt));
  pkt.raw.buf = (char *) buf;
  pkt.raw.len = len;
  pkt.l2 = (uint8_t *) pkt.raw.buf;
  if (!mg_l2_rx(ifp, &proto, &pkt.pay, &pkt.raw)) return;
  if (ifp->state < MG_TCPIP_STATE_UP) return;  // discard while L2 is not up
  if (proto == MG_TCPIP_L2PROTO_ARP) {
    pkt.arp = (struct arp *) (pkt.pay.buf);
    if (pkt.pay.len < sizeof(*pkt.arp)) return;  // Truncated
    mg_tcpip_call(ifp, MG_TCPIP_EV_ARP, &pkt.raw);
    rx_arp(ifp, &pkt);
  } else if (proto == MG_TCPIP_L2PROTO_IPV6) {
    pkt.ip6 = (struct ip6 *) (pkt.pay.buf);
    rx_ip6(ifp, &pkt);
  } else if (proto == MG_TCPIP_L2PROTO_IPV4) {
    pkt.ip = (struct ip *) (pkt.pay.buf);
    rx_ip(ifp, &pkt);
  }
}

static void mg_ip_poll(struct mg_tcpip_if *ifp, bool s1) {
  if (ifp->state < MG_TCPIP_STATE_UP) return;
  // DHCP RFC-2131 (4.4)
  if (ifp->enable_dhcp_client && s1) {
    if (ifp->state == MG_TCPIP_STATE_UP) {
      tx_dhcp_discover(ifp);  // INIT (4.4.1)
    } else if (ifp->state == MG_TCPIP_STATE_READY &&
               ifp->lease_expire > 0) {  // BOUND / RENEWING / REBINDING
      if (ifp->now >= ifp->lease_expire) {
        ifp->state = MG_TCPIP_STATE_UP, ifp->ip = 0;  // expired, release IP
        onstatechange(ifp);
      } else if (ifp->now + 30UL * 60UL * 1000UL > ifp->lease_expire &&
                 ((ifp->now / 1000) % 60) == 0) {
        // hack: 30 min before deadline, try to rebind (4.3.6) every min
        tx_dhcp_request_re(
            ifp, mg_l2_mapip(ifp->l2type, MG_TCPIP_L2ADDR_BCAST, NULL), ifp->ip,
            0xffffffff);
      }  // TODO(): Handle T1 (RENEWING) and T2 (REBINDING) (4.4.5)
    }
  }
}
static void mg_ip_link(struct mg_tcpip_if *ifp, bool drv_up, bool l2_up) {
  bool cur_drv = (ifp->state != MG_TCPIP_STATE_DOWN);
  bool cur_l2 = (ifp->state >= MG_TCPIP_STATE_UP);
  if (!l2_up && ifp->enable_dhcp_client) ifp->ip = 0;
  if (drv_up != cur_drv || l2_up != cur_l2) {  // link/L2 state has changed
    ifp->state = !drv_up  ? MG_TCPIP_STATE_DOWN
                 : !l2_up ? MG_TCPIP_STATE_LINK_UP
                 : ifp->enable_dhcp_client || ifp->ip == 0 ? MG_TCPIP_STATE_UP
                                                           : MG_TCPIP_STATE_IP;
    onstatechange(ifp);
  } else if (!ifp->enable_dhcp_client && ifp->state == MG_TCPIP_STATE_UP &&
             ifp->ip) {
    ifp->state = MG_TCPIP_STATE_IP;  // ifp->fn has set an IP
    onstatechange(ifp);
  }
}

#if MG_ENABLE_IPV6
static void mg_ip6_poll(struct mg_tcpip_if *ifp, bool s1) {
  if (ifp->state6 < MG_TCPIP_STATE_UP) return;
  if (ifp->enable_slaac && s1 && ifp->state6 == MG_TCPIP_STATE_UP)
    tx_ndp_rs(ifp);
}
static void mg_ip6_link(struct mg_tcpip_if *ifp, bool drv_up, bool l2_up) {
  bool cur_drv = (ifp->state6 != MG_TCPIP_STATE_DOWN);
  bool cur_l2 = (ifp->state6 >= MG_TCPIP_STATE_UP);
  const uint8_t px[8] = {0xfe, 0x80, 0, 0, 0, 0, 0, 0};  // RFC-4291 2.5.6
  if (drv_up != cur_drv || l2_up != cur_l2) {  // link/L2 state has changed
    if (ifp->l2type == MG_TCPIP_L2_ETH && l2_up && ifp->ip6ll[0] == 0 &&
        ifp->ip6ll[1] == 0) {  // gen ll address
      mg_l2_genip6(ifp->l2type, ifp->ip6ll, 64, ifp->mac);
      memcpy(ifp->ip6ll, px, 8);  // RFC-2464 5
    }  // just got our link local address if we didn't have one.
    // If static configuration is used, global addresses,
    // prefix length, and gw are already filled at this point.
    if (ifp->ip6[0] == 0 && ifp->ip6[1] == 0) ifp->enable_slaac = true;
    if (!l2_up && ifp->enable_slaac) ifp->ip6[0] = ifp->ip6[1] = 0;
    ifp->state6 = !drv_up  ? MG_TCPIP_STATE_DOWN
                  : !l2_up ? MG_TCPIP_STATE_LINK_UP
                  : ifp->ip6ll[0] == 0 || ifp->enable_slaac || ifp->ip6[0] == 0
                      ? MG_TCPIP_STATE_UP
                      : MG_TCPIP_STATE_IP;
    onstate6change(ifp);
  } else if (!ifp->enable_slaac && ifp->state6 == MG_TCPIP_STATE_UP &&
             ifp->ip6ll[0] != 0 && ifp->ip6[0] != 0) {
    ifp->state6 = MG_TCPIP_STATE_IP;  // ifp->fn has set an IP
    onstate6change(ifp);
  }
  if ((ifp->l2type == MG_TCPIP_L2_PPP || ifp->l2type == MG_TCPIP_L2_PPPoE) &&
      ifp->state6 == MG_TCPIP_STATE_UP && ifp->ip6ll[0] == 0 &&
      ifp->ip6ll[1] != 0) {     // IPV6CP has got an IFCID, gen ll address
    memcpy(ifp->ip6ll, px, 8);  // RFC-5072 5
    onstate6change(ifp);
  }  // just got our link local address if we didn't have one.
}
#else
#define mg_ip6_poll(x, y)
#define mg_ip6_link(x, y, z)
#endif

static void mg_tcpip_poll(struct mg_tcpip_if *ifp, uint64_t now) {
  struct mg_connection *c;
  bool expired_1000ms = mg_timer_expired(&ifp->timer_1000ms, 1000, now);
  ifp->now = now;

  if (expired_1000ms) {
#if MG_ENABLE_TCPIP_PRINT_DEBUG_STATS
    const char *names[] = {"down", "lnk up", "lnk rdy", "req", "ip", "ready"};
    size_t max = sizeof(names) / sizeof(char *);
    unsigned int state = ifp->state >= max ? max - 1 : ifp->state;
    MG_INFO(("Status: %s, IP: %M, rx:%u, tx:%u, dr:%u, er:%u", names[state],
             mg_print_ip4, &ifp->ip, ifp->nrecv, ifp->nsent, ifp->ndrop,
             ifp->nerr));
#if MG_ENABLE_IPV6
    state = ifp->state6 >= max ? max - 1 : ifp->state6;
    if (state > MG_TCPIP_STATE_UP)
      MG_INFO(("Status: %s, IPv6: %M", names[state], mg_print_ip6, &ifp->ip6));
#endif
#endif
    backlog_poll(ifp->mgr);
  }
  // Handle gw ARP request timeout, order is important
  if (expired_1000ms && ifp->state == MG_TCPIP_STATE_IP) {
    ifp->state = MG_TCPIP_STATE_READY;  // keep best-effort MAC or poison mark
    onstatechange(ifp);
  }
  if (expired_1000ms && ifp->state == MG_TCPIP_STATE_READY && !ifp->gw_ready &&
      ifp->gw != 0)
    mg_tcpip_arp_request(ifp, ifp->gw, NULL);  // retry GW ARP request
#if MG_ENABLE_IPV6
  // Handle gw NS/NA req/resp timeout, order is important
  if (expired_1000ms && ifp->state6 == MG_TCPIP_STATE_IP) {
    ifp->state6 = MG_TCPIP_STATE_READY;  // keep best-effort MAC or poison mark
    onstate6change(ifp);
  }
  if (expired_1000ms && ifp->state == MG_TCPIP_STATE_READY && !ifp->gw6_ready &&
      (ifp->gw6[0] != 0 || ifp->gw6[1] != 0))
    tx_ndp_ns(ifp, ifp->gw6, NULL);  // retry GW hwaddr resolution
#endif

  {  // poll driver and let L2 do its work, if any
    // Handle physical interface up/down status
    bool drv_up, l2_up;
    drv_up = ifp->driver->poll ? ifp->driver->poll(ifp, expired_1000ms) : true;
    l2_up = mg_l2_poll(ifp, expired_1000ms);  // Handle L2 up/down link status;
    if (expired_1000ms) {                     // ifp->state rules over state6
      mg_ip_link(ifp, drv_up, l2_up);         // Handle IPv4
      mg_ip6_link(ifp, drv_up, l2_up);        // Handle IPv6
      if (ifp->state < MG_TCPIP_STATE_UP) MG_ERROR(("Network is down"));
      mg_tcpip_call(ifp, MG_TCPIP_EV_TIMER_1S, NULL);
    }
  }

  mg_ip_poll(ifp, expired_1000ms);   // Handle IPv4
  mg_ip6_poll(ifp, expired_1000ms);  // Handle IPv6

  // Read data from the network (mg_tcpip_rx() will discard for us)
  if (ifp->driver->rx != NULL) {  // Simple polling driver, returns one frame
    size_t len =
        ifp->driver->rx(ifp->recv_queue.buf, ifp->recv_queue.size, ifp);
    if (len > 0) {
      ifp->nrecv++;
      mg_tcpip_rx(ifp, ifp->recv_queue.buf, len);
    }
  } else {  // Complex poll / Interrupt-based driver. Queues recvd frames
    char *buf;
    size_t len, cnt = 7;  // Max 7 packets to fetch
    while (cnt-- > 0 && (len = mg_queue_next(&ifp->recv_queue, &buf)) > 0) {
      mg_tcpip_rx(ifp, buf, len);
      mg_queue_del(&ifp->recv_queue, len);
    }
  }
  if (ifp->state < MG_TCPIP_STATE_UP) return;  // need to let L2 do its job

  // Process timeouts
  for (c = ifp->mgr->conns; c != NULL; c = c->next) {
    struct connstate *s = (struct connstate *) (c + 1);
    if ((c->is_udp && !c->is_arplooking) || c->is_listening || c->is_resolving)
      continue;
    if (!c->is_full && s->is_full) {
      // Application resumed the connection: advertise the restored window
      s->is_full = false;
      tx_tcp(ifp, s->mac, &c->loc, &c->rem, c->dscp, TH_ACK, mg_htonl(s->seq),
             mg_htonl(s->ack), MG_TCPIP_WIN, NULL, 0);
    }
    if (s->retransmit && s->txq.len > 0 && ifp->now > s->txq_timer) {
      if (s->txq_retries < 255) s->txq_retries++;
      // R1 reports a delivery problem; it does not close (RFC-9293, 3.8.3).
      if (s->txq_retries == MG_TCPIP_RTX_R1)
        MG_ERROR(("%lu retransmit", c->id));
      retransmit(c);
    }
    if (ifp->now > s->timer) {
      if (s->ttype == MIP_TTYPE_ARP) {
        proto_error(c, false, "ARP timeout");
      } else if (c->is_udp) {
        continue;
      } else if (s->ttype == MIP_TTYPE_ACK && s->acked != s->ack) {
        MG_VERBOSE(("%lu ack %x %x", c->id, s->seq, s->ack));
        tx_tcp(ifp, s->mac, &c->loc, &c->rem, c->dscp, TH_ACK, mg_htonl(s->seq),
               mg_htonl(s->ack), s->is_full ? 0 : MG_TCPIP_WIN, NULL, 0);
        s->acked = s->ack;
      } else if (s->ttype == MIP_TTYPE_SYN) {
        proto_error(c, false, "Connection timeout");
      } else if (s->ttype == MIP_TTYPE_FIN) {
        s->is_closing = 1;
        continue;
      } else {
        if (s->tmiss++ > 2) {
          proto_error(c, false, "keepalive");
        } else {
          MG_VERBOSE(("%lu keepalive", c->id));
          tx_tcp(ifp, s->mac, &c->loc, &c->rem, c->dscp, TH_ACK,
                 mg_htonl(s->seq - 1), mg_htonl(s->ack),
                 s->is_full ? 0 : MG_TCPIP_WIN, NULL, 0);
        }
      }

      settmout(c, MIP_TTYPE_KEEPALIVE);
    }
  }
}

// This function executes in interrupt context, thus it should copy data
// somewhere fast. Note that newlib's malloc is not thread safe, thus use
// our lock-free queue with preallocated buffer to copy data and return asap
void mg_tcpip_qwrite(void *buf, size_t len, struct mg_tcpip_if *ifp) {
  char *p;
  if (mg_queue_book(&ifp->recv_queue, &p, len) >= len) {
    memcpy(p, buf, len);
    mg_queue_add(&ifp->recv_queue, len);
    ifp->nrecv++;
  } else {
    ifp->ndrop++;
  }
}

void mg_tcpip_init(struct mg_mgr *mgr, struct mg_tcpip_if *ifp) {
  // If L2 address is not set, make a random one; fill MTU
  mg_l2_init(ifp);
  ifp->mtu = ifp->l2mtu;
  if (MG_ENABLE_TCPIP_TCPRTX) ifp->enable_tcp_retransmit = true;

  if (ifp->dhcp_name[0] == '\0')  // If DHCP name is not set, use "mip"
    memcpy(ifp->dhcp_name, "mip", 4);
  ifp->dhcp_name[sizeof(ifp->dhcp_name) - 1] = '\0';  // Just in case

  if (ifp->driver->init && !ifp->driver->init(ifp)) {
    MG_ERROR(("driver init failed"));
  } else {
    ifp->tx.buf = (char *) mg_calloc(1, ifp->framesize),
    ifp->tx.len = ifp->framesize;
    if (ifp->recv_queue.size == 0)
      ifp->recv_queue.size = ifp->driver->rx ? ifp->framesize : 8192;
    ifp->recv_queue.buf = (char *) mg_calloc(1, ifp->recv_queue.size);
    ifp->timer_1000ms = mg_millis();
    mgr->ifp = ifp;
    ifp->mgr = mgr;
    mgr->extraconnsize = sizeof(struct connstate);
    if (ifp->ip == 0 && ifp->l2type != MG_TCPIP_L2_PPP &&
        ifp->l2type != MG_TCPIP_L2_PPPoE)
      ifp->enable_dhcp_client = true;
    mg_random(&ifp->eport, sizeof(ifp->eport));  // Random from 0 to 65535
    ifp->eport |= MG_EPHEMERAL_PORT_BASE;        // Random from
                                           // MG_EPHEMERAL_PORT_BASE to 65535
    if (ifp->tx.buf == NULL || ifp->recv_queue.buf == NULL) MG_ERROR(("OOM"));
    MG_DEBUG(("Builtin TCP/IP init done. Queue size: %zu, extra conn size: %zu",
              ifp->recv_queue.size, mgr->extraconnsize));
  }
  mg_bsd_init(mgr);
}

static void close_conn(struct mg_connection *c) {
  struct connstate *s = (struct connstate *) (c + 1);
  mg_iobuf_free(&s->txq);
  mg_close_conn(c);
}

void mg_tcpip_free(struct mg_tcpip_if *ifp) {
  while (ifp->mgr->conns != NULL) close_conn(ifp->mgr->conns); // TCP FIN wait
  mg_free(ifp->recv_queue.buf);
  mg_free(ifp->tx.buf);
  mg_free(ifp->dns4_url);
}

static void send_syn(struct mg_connection *c) {
  struct connstate *s = (struct connstate *) (c + 1);
  uint32_t isn = mg_htonl((uint32_t) mg_ntohs(c->loc.port));
  tx_tcp(c->mgr->ifp, s->mac, &c->loc, &c->rem, c->dscp, TH_SYN, isn, 0,
         MG_TCPIP_WIN, NULL, 0);
}

static void l2addr_resolved(struct mg_connection *c) {
  if (c->is_udp) {
    c->is_connecting = 0;
    mg_call(c, MG_EV_CONNECT, NULL);
  } else {
    send_syn(c);
    settmout(c, MIP_TTYPE_SYN);
  }
}

void mg_connect_resolved(struct mg_connection *c) {
  struct mg_tcpip_if *ifp = c->mgr->ifp;
  uint8_t *l2addr;
  struct connstate *s = (struct connstate *) (c + 1);
  s->retransmit = ifp->enable_tcp_retransmit;
  s->txq.align = MG_TCPIP_TXQ_ALIGN;
  c->is_resolving = 0;
  if (ifp->eport < MG_EPHEMERAL_PORT_BASE) ifp->eport = MG_EPHEMERAL_PORT_BASE;
  c->loc.port = mg_htons(ifp->eport++);
#if MG_ENABLE_IPV6
  if (c->rem.is_ip6) {
    if (c->rem.addr.ip6[0] == ifp->ip6ll[0]) {  // same local LAN, use ll
      c->loc.addr.ip6[0] = ifp->ip6ll[0], c->loc.addr.ip6[1] = ifp->ip6ll[1];
    } else {  // use global address
      c->loc.addr.ip6[0] = ifp->ip6[0], c->loc.addr.ip6[1] = ifp->ip6[1];
    }
    c->loc.is_ip6 = true;
  } else
#endif
  {
    c->loc.addr.ip4 = ifp->ip;
  }
  MG_DEBUG(("%lu %M -> %M", c->id, mg_print_ip_port, &c->loc, mg_print_ip_port,
            &c->rem));
  mg_call(c, MG_EV_RESOLVE, NULL);
  c->is_connecting = 1;
  if (c->is_udp && (l2addr = tcpip_mapip(ifp, &c->rem)) != NULL) {
    memcpy(s->mac, l2addr, sizeof(s->mac));
    l2addr_resolved(c);  // broadcast or multicast
#if MG_ENABLE_IPV6
  } else if (c->rem.is_ip6) {
    if (match_prefix((uint8_t *) c->rem.addr.ip6, ifp->prefix,
                     ifp->prefix_len)                       // same global LAN
        || (c->rem.addr.ip6[0] == ifp->ip6ll[0]             // same local LAN
            && !MG_IP6MATCH(c->rem.addr.ip6, ifp->gw6))) {  // and not gw
      // If we're in the same LAN, fire a Neighbor Solicitation
      MG_DEBUG(("%lu NS lookup...", c->id));
      tx_ndp_ns(ifp, c->rem.addr.ip6, NULL);  // RFC-4861 4.3, requesting
      settmout(c, MIP_TTYPE_ARP);
      c->is_arplooking = 1;
    } else if (ifp->gw6_ready) {
      memcpy(s->mac, ifp->gw6mac, sizeof(s->mac));
      l2addr_resolved(c);
    } else {
      mg_error(c, "No IPv6 gateway, can't connect");
    }
#endif
  } else {
    uint32_t rem_ip = c->rem.addr.ip4;
    if (ifp->ip && ((rem_ip & ifp->mask) == (ifp->ip & ifp->mask)) &&
        rem_ip != ifp->gw) {  // skip if gw (onstatechange -> ARP)
      // If we're in the same LAN, fire an ARP lookup.
      MG_DEBUG(("%lu ARP lookup...", c->id));
      mg_tcpip_arp_request(ifp, rem_ip, NULL);
      settmout(c, MIP_TTYPE_ARP);
      c->is_arplooking = 1;
    } else if (ifp->gw_ready) {
      memcpy(s->mac, ifp->gwmac, sizeof(s->mac));
      l2addr_resolved(c);
    } else {
      mg_error(c, "No gateway, can't connect");
    }
  }
}

bool mg_open_listener(struct mg_connection *c, const char *url) {
  c->loc.port = mg_htons(mg_url_port(url));
  if (!mg_aton(mg_url_host(url), &c->loc)) {
    MG_ERROR(("invalid listening URL: %s", url));
    return false;
  }
  return true;
}

static void write_conn(struct mg_connection *c) {
  long len = c->is_tls ? mg_tls_send(c, c->send.buf, c->send.len)
                       : mg_io_send(c, c->send.buf, c->send.len);
  // TODO(): mg_tls_send() may return 0 forever on steady OOM
  if (len == MG_IO_ERR) {
    proto_error(c, false, "tx err");
  } else if (len > 0) {
    mg_iobuf_del(&c->send, 0, (size_t) len);
    mg_call(c, MG_EV_WRITE, &len);
  }
}

static bool can_write(struct mg_connection *c) {
  return c->is_connecting == 0 && c->is_resolving == 0 && c->send.len > 0 &&
         c->is_tls_hs == 0 && c->is_arplooking == 0;
}

void mg_mgr_poll(struct mg_mgr *mgr, int ms) {
  struct mg_connection *c, *tmp;
  uint64_t now = mg_millis();
  mg_timer_poll(&mgr->timers, now);
  mg_ota_poll(mgr);
  mg_mdash_poll(mgr);
  if (mgr->ifp == NULL || mgr->ifp->driver == NULL) return;
  mg_tcpip_poll(mgr->ifp, now);
  for (c = mgr->conns; c != NULL; c = tmp) {
    struct connstate *s = (struct connstate *) (c + 1);
    long flush = 0;
    bool is_tls = c->is_tls && !c->is_resolving && !c->is_arplooking &&
                  !c->is_listening && !c->is_connecting;
    tmp = c->next;
    mg_call(c, MG_EV_POLL, &now);
    MG_VERBOSE(("%lu .. %c%c%c%c%c%c %lu %lu", c->id, c->is_tls ? 'T' : 't',
                c->is_connecting ? 'C' : 'c', c->is_tls_hs ? 'H' : 'h',
                c->is_resolving ? 'R' : 'r', c->is_closing ? 'C' : 'c',
                s->is_closing ? 'F' : 'f', mg_tls_pending(c), c->rtls.len));
    if (!c->is_closing) {
      // order is important, TLS conn close with > 1 record in buffer (below)
      if (is_tls && (c->rtls.len > 0 || mg_tls_pending(c) > 0))
      c->is_tls_hs ? mg_tls_handshake(c) : handle_tls_recv(c);
      if (can_write(c)) write_conn(c);
      if (is_tls && c->send.len == 0) flush = mg_tls_flush(c);
    }
    if (flush == MG_IO_ERR) proto_error(c, false, "tx err"); // should RST but
    if (c->is_draining && c->send.len == 0 &&
        (c->is_udp ||
         (flush == 0 && s->txq.len == 0 && s->ttype != MIP_TTYPE_FIN)))
      c->is_closing = 1;
    // For non-TLS, close immediately upon completing the 3-way closure
    // For TLS, handle any pending data (above) until MIP_TTYPE_FIN expires
    if (s->twclosure &&
        (!c->is_tls || (c->rtls.len == 0 && mg_tls_pending(c) == 0))) {
      s->is_closing = 1;
    } else if (c->is_closing && !s->is_closing) {
      if (c->is_udp || c->is_listening || c->is_resolving || c->is_arplooking ||
          c->is_connecting) {
        close_conn(c);
        continue;
      } else if (s->ttype != MIP_TTYPE_FIN) {  // active TCP: initiate closure
        size_t tx = tx_tcp(c->mgr->ifp, s->mac, &c->loc, &c->rem, c->dscp,
                           TH_FIN | TH_ACK, mg_htonl(s->seq), mg_htonl(s->ack),
                           s->is_full ? 0 : MG_TCPIP_WIN, NULL, 0);
        if (tx == (size_t) -1) {
          s->is_closing = 1;
        } else if (tx > 0) {
          s->seq++;
          settmout(c, MIP_TTYPE_FIN);
        }  // else tx = 0, should recover later or we have worse problems
      }
    }
    if (s->send_rst)  // proto_error requested TCP RST, send it and close
      tx_tcp(c->mgr->ifp, s->mac, &c->loc, &c->rem, c->dscp, TH_RST,
             mg_htonl(s->seq), 0, 0, NULL, 0);
    if (s->is_closing) close_conn(c);  // then removes the conn
  }
  mg_bsd_poll(mgr);
  (void) ms;
}

bool mg_send(struct mg_connection *c, const void *buf, size_t len) {
  struct mg_tcpip_if *ifp = c->mgr->ifp;
  bool res = false;
  if (!c->loc.is_ip6 && (ifp->ip == 0 || ifp->state != MG_TCPIP_STATE_READY)) {
    proto_error(c, false, "net down");
#if MG_ENABLE_IPV6
  } else if (c->loc.is_ip6 && ifp->state6 != MG_TCPIP_STATE_READY) {
    proto_error(c, false, "net down");
#endif
  } else if (c->is_udp && (c->is_arplooking || c->is_resolving)) {
    // Fail to send, no target MAC or IP
    MG_VERBOSE(("still resolving..."));
  } else if (c->is_udp) {
    len = trim_len(c, len);  // Trimming length if necessary
    res = udp_send(c, buf, len);
  } else {
    res = len == 0 || mg_iobuf_add(&c->send, c->send.len, buf, len) > 0;
    // returning 0 means an OOM condition (iobuf couldn't resize), yet this is
    // so far recoverable, let the caller decide
  }
  return res;
}

uint8_t mcast_addr[6] = {0x01, 0x00, 0x5e, 0x00, 0x00, 0xfb};
void mg_multicast_add(struct mg_connection *c, char *ip) {
  (void) ip;  // ip4/6_mcastmac(mcast_mac, &ip); ipv6 param
  // TODO(): actual IP -> MAC; check database, update
  c->mgr->ifp->update_mac_hash_table = true;  // mark dirty
}

bool mg_dnsc_init(struct mg_mgr *mgr, struct mg_dns *dnsc);

static void setdns4(struct mg_tcpip_if *ifp, uint32_t *ip) {
  struct mg_dns *dnsc;
  mg_free(ifp->dns4_url);
  ifp->dns4_url = mg_mprintf("udp://%M:53", mg_print_ip4, ip);
  dnsc = &ifp->mgr->dns4;
  dnsc->url = (const char *) ifp->dns4_url;
  MG_DEBUG(("Set DNS URL to %s", dnsc->url));
  if (ifp->mgr->use_dns6) return;
  if (dnsc->c != NULL) mg_close_conn(dnsc->c);
  if (!mg_dnsc_init(ifp->mgr, dnsc))  // create DNS connection
    MG_ERROR(("DNS connection creation failed"));
}

void mg_tcpip_mapip(struct mg_connection *c, struct mg_addr *ip) {
  struct connstate *s = (struct connstate *) (c + 1);
  uint8_t *l2addr = tcpip_mapip(c->mgr->ifp, ip);
  if (l2addr == NULL) return;
  memcpy(s->mac, l2addr, sizeof(s->mac));
}

#endif  // MG_ENABLE_TCPIP

#ifdef MG_ENABLE_LINES
#line 1 "src/ota.c"
#endif









#ifndef MG_OTA_MAX_URL_LEN
#define MG_OTA_MAX_URL_LEN 256
#endif

// Scannable version tag embedded in every firmware binary, for server-side
// version extraction. The version string starts after the "MG_VERSION:" prefix.
static const char s_fw_version[] = "MG_VERSION:" MG_FIRMWARE_VERSION;

static bool s_autocommit_ok;  // True after OTA server confirms "same version"

static struct mg_ota_state {
  char json_url[MG_OTA_MAX_URL_LEN];
  char url[MG_OTA_MAX_URL_LEN];
  size_t size;
  uint8_t sha256[32];
  void (*fn)(const char *error_message);
  size_t received;  // Firmware bytes written so far
  bool open;        // mg_ota_begin() succeeded, mg_ota_end() is pending
  const char *err;  // First error during download, NULL if none
} *s_ota;

static void s_firmware_fn(struct mg_connection *c, int ev, void *ev_data);

static bool s_tls_init(struct mg_connection *c, const char *url) {
  struct mg_tls_opts opts;
  memset(&opts, 0, sizeof(opts));
  opts.ca = mg_str(MG_OTA_TLS_CA);
  opts.name = mg_url_host(url);
  if (opts.ca.len == 0) {
    mg_error(c, "MG_OTA_TLS_CA is not configured");
  } else {
    mg_tls_init(c, &opts);
  }
  return c->is_tls_hs != 0 && !c->is_closing;
}

#if MG_ENABLE_CUSTOM_DEVICE_ID
#else
void mg_ota_device_id(char *buf, size_t len) {
#if defined(UID_BASE) &&                                               \
    (defined(__SYSTEM_STM32F4XX_H) || defined(__SYSTEM_STM32F7XX_H) || \
     defined(SYSTEM_STM32H5XX_H) || defined(SYSTEM_STM32H7XX_H) ||     \
     defined(SYSTEM_STM32N6XX_H) || defined(SYSTEM_STM32U5XX_H))
  uint32_t *p = (uint32_t *) UID_BASE;
  mg_snprintf(buf, len, "stm32_%08x%08x%08x", p[0], p[1], p[2]);
#elif MG_ARCH == MG_ARCH_PICOSDK
  pico_unique_board_id_t x = {0};
  pico_get_unique_board_id(&x);
  mg_snprintf(buf, len, "rp_%02x%02x%02x%02x%02x%02x%02x%02x", x.id[0], x.id[1],
              x.id[2], x.id[3], x.id[4], x.id[5], x.id[6], x.id[7]);
#elif defined(OCOTP_BASE) && OCOTP_BASE == 0x401F4000u
  mg_snprintf(buf, len, "rt10xx_%08x%08x", OCOTP->CFG0, OCOTP->CFG1);
#elif defined(OCOTP_BASE) && OCOTP_BASE == 0x40CAC000u
  mg_snprintf(buf, len, "rt11xx_%08x%08x", OCOTP->FUSEN[16].FUSE,
              OCOTP->FUSEN[17].FUSE);
#elif MG_ARCH == MG_ARCH_ESP32
  uint8_t mac[6] = {0};
  esp_efuse_mac_get_default(mac);
  mg_snprintf(buf, len, "esp32_%02x%02x%02x%02x%02x%02x", mac[0], mac[1],
              mac[2], mac[3], mac[4], mac[5]);
#else
  mg_snprintf(buf, len, "%d", 0);
#endif
}
#endif

static void s_version_fn(struct mg_connection *c, int ev, void *ev_data) {
  uint64_t expiration = *(uint64_t *) c->data;
  if (s_ota == NULL) {
    c->fn = NULL;
    c->is_closing = 1;
    return;
  }
  if (ev == MG_EV_POLL) {
    if (mg_millis() > expiration) mg_error(c, "Metadata timeout");
  } else if (ev == MG_EV_CONNECT) {
    char id[40];
    struct mg_str host = mg_url_host(s_ota->json_url);
    const char *uri = mg_url_uri(s_ota->json_url);
    const char *sep = strchr(uri, '?') == NULL ? "?" : "&";
    if (mg_url_is_ssl(s_ota->json_url)) {
      if (!s_tls_init(c, s_ota->json_url)) return;
    }
    mg_ota_device_id(id, sizeof(id));
    id[sizeof(id) - 1] = '\0';
    mg_printf(
        c,
        "GET %s%sarch=%d&version=%s&id=%s&interval=%d&boot=%d HTTP/1.1\r\n"
        "Host: %.*s\r\n"
        "Connection: close\r\n\r\n",
        uri, sep, MG_ARCH, s_fw_version + 11, id, MG_OTA_PULL_INTERVAL_SECONDS,
        MG_OTA_STATE_GET(), host.len, host.buf);
  } else if (ev == MG_EV_HTTP_MSG) {
    struct mg_http_message *hm = (struct mg_http_message *) ev_data;
    char version[MG_OTA_MAX_VERSION_LEN];
    double result;
    MG_DEBUG(("Got metadata: %.*s", hm->body.len, hm->body.buf));
    if (mg_http_status(hm) != 200 || mg_json_get(hm->body, "$", NULL) != 0 ||
        !mg_json_unescape(hm->body, "$.version", version, sizeof(version)) ||
        !mg_json_unescape(hm->body, "$.url", s_ota->url, sizeof(s_ota->url)) ||
        !mg_json_get_num(hm->body, "$.size", &result) || result <= 0) {
      char buf[100];
      mg_snprintf(buf, sizeof(buf), "Bad metadata: %.*s", hm->body.len,
                  hm->body.buf);
      s_ota->fn(buf);
      mg_free(s_ota);
      s_ota = NULL;
    } else if (strcmp(version, s_fw_version + 11) == 0) {
      s_autocommit_ok = true;
      s_ota->fn("Same version");
      mg_free(s_ota);
      s_ota = NULL;
    } else {
      struct mg_connection *fc;
      s_ota->size = (size_t) result;
      // TODO (robertc2000): parse and validate sha256
      MG_DEBUG(("Firmware version: %s, url: %s, size: %ld", version, s_ota->url,
                s_ota->size));
      fc = mg_http_connect(c->mgr, s_ota->url, s_firmware_fn, NULL);
      if (fc == NULL) {
        s_ota->fn("Failed to connect");
        mg_free(s_ota);
        s_ota = NULL;
      } else {
        *(uint64_t *) fc->data = mg_millis() + 5 * 1000;  // Set expiration
      }
    }
    c->is_closing = 1;
  } else if (ev == MG_EV_ERROR) {
    s_ota->fn((char *) ev_data);
    mg_free(s_ota);
    s_ota = NULL;
  }
}

static void status_fn(const char *errmsg) {
  if (errmsg) MG_ERROR(("OTA failed: %s", errmsg));
}

// Firmware download callback for mg_http_stream_body(). Writes chunks to the
// OTA target and reports the result via s_ota->fn(). The end call also happens
// when the connection drops before the whole body arrived.
static bool s_firmware_cb(struct mg_http_message *hm, struct mg_str *data,
                          void **p) {
  bool finalized = false, ok;
  (void) p;
  if (s_ota == NULL) return false;
  if (hm != NULL) {  // Start
    s_ota->received = 0;
    s_ota->open = mg_ota_begin(hm->body.len);
    if (!s_ota->open) s_ota->err = "OTA begin failed";
    return s_ota->open;
  } else if (data != NULL) {  // Next chunk
    if (!mg_ota_write(data->buf, data->len)) {
      s_ota->err = "OTA write failed";
      return false;
    }
    s_ota->received += data->len;
    return true;
  }
  // End
  if (s_ota->open) finalized = mg_ota_end();
  if (s_ota->err == NULL && s_ota->received != s_ota->size) {
    s_ota->err = "Connection closed";
  }
  if (s_ota->err == NULL && !finalized) s_ota->err = "OTA finalize failed";
  ok = s_ota->err == NULL;
  s_ota->fn(s_ota->err);
  mg_free(s_ota);
  s_ota = NULL;
  return ok;
}

static void s_firmware_fn(struct mg_connection *c, int ev, void *ev_data) {
  uint64_t expiration = *(uint64_t *) c->data;
  if (s_ota == NULL) {
    c->fn = NULL;
    c->is_closing = 1;
    return;
  }
  if (ev == MG_EV_POLL) {
    if (mg_millis() > expiration) mg_error(c, "OTA timeout");
  } else if (ev == MG_EV_CONNECT) {
    struct mg_str host = mg_url_host(s_ota->url);
    if (mg_url_is_ssl(s_ota->url)) {
      if (!s_tls_init(c, s_ota->url)) return;
    }
    mg_printf(c,
              "GET %s HTTP/1.1\r\n"
              "Host: %.*s\r\n"
              "Connection: close\r\n\r\n",
              mg_url_uri(s_ota->url), (int) host.len, host.buf);
    *(uint64_t *) c->data = mg_millis() + 300 * 1000;  // Set expiration
  } else if (ev == MG_EV_HTTP_HDRS) {
    struct mg_http_message *hm = (struct mg_http_message *) ev_data;
    int status = mg_http_status(hm);
    if (status != 200 || hm->body.len != s_ota->size) {
      mg_error(c, "Bad HTTP response: status %d, size %lu vs %lu", status,
               (unsigned long) hm->body.len, (unsigned long) s_ota->size);
    } else {
      MG_DEBUG(("Beginning OTA (%lu bytes)", (unsigned long) s_ota->size));
      if (!mg_http_stream_body(c, hm, s_firmware_cb, NULL)) {
        mg_error(c, "Cannot stream firmware");
      }
    }
  } else if (ev == MG_EV_ERROR) {
    s_ota->fn((char *) ev_data);
    mg_free(s_ota);
    s_ota = NULL;
  }
}

void mg_ota_url_check(struct mg_mgr *mgr, const char *json_url,
                      void (*fn)(const char *error_message)) {
  if (fn == NULL) fn = status_fn;
  if (s_ota != NULL) {
    fn("OTA already in progress");
  } else if ((s_ota = (struct mg_ota_state *) mg_calloc(1, sizeof(*s_ota))) ==
             NULL) {
    fn("Out of memory");
  } else {
    struct mg_connection *c;
    mg_snprintf(s_ota->json_url, sizeof(s_ota->json_url), "%s", json_url);
    MG_DEBUG(("Connecting to %s", json_url));
    c = mg_http_connect(mgr, s_ota->json_url, s_version_fn, NULL);
    if (c == NULL) {
      mg_free(s_ota);
      s_ota = NULL;
      fn("Failed to connect");
    } else {
      s_ota->fn = fn;
      *(uint64_t *) c->data = mg_millis() + 5 * 1000;  // Set expiration
    }
  }
}

void mg_ota_poll(struct mg_mgr *mgr) {
  static uint64_t t = 5000;  // Fire first time 5 sec after boot

  static uint64_t feed_timer;  // Advances 500ms per tick; tracks elapsed time
  if (MG_OTA_STATE_GET() == MG_OTA_FAILED &&
      mg_timer_expired(&feed_timer, 500, mg_millis())) {
    if (feed_timer < (uint64_t) MG_OTA_ROLLBACK_TIMEOUT_SECONDS * 1000) {
      MG_OTA_ROLLBACK_TIMER_FEED();  // Feed watchdog every 500ms
    } else if (s_autocommit_ok) {
      MG_INFO(("Auto-committing firmware"));
      MG_OTA_STATE_SET(MG_OTA_CONFIRMED);  // Stop feeding → IWDG resets cleanly
    } else {
      MG_INFO(("No commit confirmation, rolling back"));
      // Stop feeding → IWDG fires → resets into MG_OTA_FAILED → rollback
    }
  }

  if (MG_OTA_URL != NULL &&
      mg_timer_expired(&t, MG_OTA_PULL_INTERVAL_SECONDS * 1000, mg_millis())) {
    mg_ota_url_check(mgr, MG_OTA_URL, MG_OTA_STATUS_FN);
  }
}

#ifdef MG_ENABLE_LINES
#line 1 "src/ota_ch32v307.c"
#endif




#if MG_OTA == MG_OTA_CH32V307
// RM: https://www.wch-ic.com/downloads/CH32FV2x_V3xRM_PDF.html

static bool mg_ch32v307_write(void *, const void *, size_t);
static bool mg_ch32v307_swap(void);

static struct mg_flash s_mg_flash_ch32v307 = {
    (void *) 0x08000000,  // Start
    480 * 1024,           // Size, first 320k is 0-wait
    4 * 1024,             // Sector size, 4k
    4,                    // Align, 32 bit
    mg_ch32v307_write,
    mg_ch32v307_swap,
};

#define FLASH_BASE 0x40022000
#define FLASH_ACTLR (FLASH_BASE + 0)
#define FLASH_KEYR (FLASH_BASE + 4)
#define FLASH_OBKEYR (FLASH_BASE + 8)
#define FLASH_STATR (FLASH_BASE + 12)
#define FLASH_CTLR (FLASH_BASE + 16)
#define FLASH_ADDR (FLASH_BASE + 20)
#define FLASH_OBR (FLASH_BASE + 28)
#define FLASH_WPR (FLASH_BASE + 32)

MG_IRAM static void flash_unlock(void) {
  static bool unlocked;
  if (unlocked == false) {
    MG_REG(FLASH_KEYR) = 0x45670123;
    MG_REG(FLASH_KEYR) = 0xcdef89ab;
    unlocked = true;
  }
}

MG_IRAM static void flash_wait(void) {
  while (MG_REG(FLASH_STATR) & MG_BIT(0)) (void) 0;
}

MG_IRAM static void mg_ch32v307_erase(void *addr) {
  // MG_INFO(("%p", addr));
  flash_unlock();
  flash_wait();
  MG_REG(FLASH_ADDR) = (uint32_t) addr;
  MG_REG(FLASH_CTLR) |= MG_BIT(1) | MG_BIT(6);  // PER | STRT;
  flash_wait();
}

MG_IRAM static bool is_page_boundary(const void *addr) {
  uint32_t val = (uint32_t) addr;
  return (val & (s_mg_flash_ch32v307.secsz - 1)) == 0;
}

MG_IRAM static bool mg_ch32v307_write(void *addr, const void *buf, size_t len) {
  // MG_INFO(("%p %p %lu", addr, buf, len));
  // mg_hexdump(buf, len);
  flash_unlock();
  const uint16_t *src = (uint16_t *) buf, *end = &src[len / 2];
  uint16_t *dst = (uint16_t *) addr;
  MG_REG(FLASH_CTLR) |= MG_BIT(0);  // Set PG
  // MG_INFO(("CTLR: %#lx", MG_REG(FLASH_CTLR)));
  while (src < end) {
    if (is_page_boundary(dst)) mg_ch32v307_erase(dst);
    *dst++ = *src++;
    flash_wait();
  }
  MG_REG(FLASH_CTLR) &= ~MG_BIT(0);  // Clear PG
  return true;
}

MG_IRAM bool mg_ch32v307_swap(void) {
  return true;
}

// just overwrite instead of swap
MG_IRAM static void single_bank_swap(char *p1, char *p2, size_t s, size_t ss) {
  // no stdlib calls here
  for (size_t ofs = 0; ofs < s; ofs += ss) {
    mg_ch32v307_write(p1 + ofs, p2 + ofs, ss);
  }
  *((volatile uint32_t *) 0xbeef0000) |= 1U << 7;  // NVIC_SystemReset()
}

bool mg_ota_begin(size_t new_firmware_size) {
  return mg_ota_flash_begin(new_firmware_size, &s_mg_flash_ch32v307);
}

bool mg_ota_write(const void *buf, size_t len) {
  return mg_ota_flash_write(buf, len, &s_mg_flash_ch32v307);
}

bool mg_ota_end(void) {
  if (mg_ota_flash_end(&s_mg_flash_ch32v307)) {
    // Swap partitions. Pray power does not go away
    MG_INFO(("Swapping partitions, size %u (%u sectors)",
             s_mg_flash_ch32v307.size,
             s_mg_flash_ch32v307.size / s_mg_flash_ch32v307.secsz));
    MG_INFO(("Do NOT power off..."));
    mg_log_level = MG_LL_NONE;
    // TODO() disable IRQ, s_flash_irq_disabled = true;
    // Runs in RAM, will reset when finished
    single_bank_swap(
        (char *) s_mg_flash_ch32v307.start,
        (char *) s_mg_flash_ch32v307.start + s_mg_flash_ch32v307.size / 2,
        s_mg_flash_ch32v307.size / 2, s_mg_flash_ch32v307.secsz);
  }
  return false;
}
struct mg_flash *mg_flash = &s_mg_flash_ch32v307;
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/ota_dummy.c"
#endif



#if MG_OTA == MG_OTA_NONE
bool mg_ota_begin(size_t new_firmware_size) {
  (void) new_firmware_size;
  return true;
}
bool mg_ota_write(const void *buf, size_t len) {
  (void) buf, (void) len;
  return true;
}
bool mg_ota_end(void) {
  return true;
}
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/ota_esp32.c"
#endif


#if MG_ARCH == MG_ARCH_ESP32 && MG_OTA == MG_OTA_ESP32

static const esp_partition_t *s_ota_update_partition;
static esp_ota_handle_t s_ota_update_handle;
static bool s_ota_success;

// Those empty macros do nothing, but mark places in the code which could
// potentially trigger a watchdog reboot due to the log flash erase operation
#define disable_wdt()
#define enable_wdt()

bool mg_ota_begin(size_t new_firmware_size) {
  if (s_ota_update_partition != NULL) {
    MG_ERROR(("Update in progress. Call mg_ota_end() ?"));
    return false;
  } else {
    s_ota_success = false;
    disable_wdt();
    s_ota_update_partition = esp_ota_get_next_update_partition(NULL);
    esp_err_t err = esp_ota_begin(s_ota_update_partition, new_firmware_size,
                                  &s_ota_update_handle);
    enable_wdt();
    MG_DEBUG(("esp_ota_begin(): %d", err));
    s_ota_success = (err == ESP_OK);
  }
  return s_ota_success;
}

bool mg_ota_write(const void *buf, size_t len) {
  disable_wdt();
  esp_err_t err = esp_ota_write(s_ota_update_handle, buf, len);
  enable_wdt();
  MG_INFO(("esp_ota_write(): %d", err));
  s_ota_success = err == ESP_OK;
  return s_ota_success;
}

bool mg_ota_end(void) {
  esp_err_t err = esp_ota_end(s_ota_update_handle);
  MG_DEBUG(("esp_ota_end(%p): %d", s_ota_update_handle, err));
  if (s_ota_success && err == ESP_OK) {
    err = esp_ota_set_boot_partition(s_ota_update_partition);
    s_ota_success = (err == ESP_OK);
  }
  MG_DEBUG(("Finished ESP32 OTA, success: %d", s_ota_success));
  s_ota_update_partition = NULL;
  if (s_ota_success) {
    MG_OTA_STATE_SET(MG_OTA_TESTING);
    esp_restart();
  }
  return s_ota_success;
}

#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/ota_imxrt.c"
#endif




#if MG_OTA >= MG_OTA_RT1020 && MG_OTA <= MG_OTA_RT1180

static bool mg_imxrt_write(void *, const void *, size_t);
static bool mg_imxrt_swap(void);

#if MG_OTA <= MG_OTA_RT1060
#define MG_IMXRT_FLASH_START 0x60000000
#define FLEXSPI_NOR_INSTANCE 0
#elif MG_OTA == MG_OTA_RT1064
#define MG_IMXRT_FLASH_START 0x70000000
#define FLEXSPI_NOR_INSTANCE 1
#elif MG_OTA == MG_OTA_RT1170
#define MG_IMXRT_FLASH_START 0x30000000
#define FLEXSPI_NOR_INSTANCE 1
#else  // RT1180 FlexSPI2
#define MG_IMXRT_FLASH_START 0x04000000
#define FLEXSPI_NOR_INSTANCE 2
#endif

#if MG_OTA == MG_OTA_RT1050
#define MG_IMXRT_SECTOR_SIZE (256 * 1024)
#define MG_IMXRT_PAGE_SIZE 512
#else
#define MG_IMXRT_SECTOR_SIZE (4 * 1024)
#define MG_IMXRT_PAGE_SIZE 256
#endif

// TODO(): fill at init, support more devices in a dynamic way
// TODO(): then, check alignment is <= 256, see Wizard's #251
static struct mg_flash s_mg_flash_imxrt = {
    (void *) MG_IMXRT_FLASH_START,  // Start,
    4 * 1024 * 1024,                // Size, 4mb
    MG_IMXRT_SECTOR_SIZE,           // Sector size
    MG_IMXRT_PAGE_SIZE,             // Align
    mg_imxrt_write,
    mg_imxrt_swap,
};

struct mg_flexspi_lut_seq {
  uint8_t seqNum;
  uint8_t seqId;
  uint16_t reserved;
};

struct mg_flexspi_mem_config {
  uint32_t tag;
  uint32_t version;
  uint32_t reserved0;
  uint8_t readSampleClkSrc;
  uint8_t csHoldTime;
  uint8_t csSetupTime;
  uint8_t columnAddressWidth;
  uint8_t deviceModeCfgEnable;
  uint8_t deviceModeType;
  uint16_t waitTimeCfgCommands;
  struct mg_flexspi_lut_seq deviceModeSeq;
  uint32_t deviceModeArg;
  uint8_t configCmdEnable;
  uint8_t configModeType[3];
  struct mg_flexspi_lut_seq configCmdSeqs[3];
  uint32_t reserved1;
  uint32_t configCmdArgs[3];
  uint32_t reserved2;
  uint32_t controllerMiscOption;
  uint8_t deviceType;
  uint8_t sflashPadType;
  uint8_t serialClkFreq;
  uint8_t lutCustomSeqEnable;
  uint32_t reserved3[2];
  uint32_t sflashA1Size;
  uint32_t sflashA2Size;
  uint32_t sflashB1Size;
  uint32_t sflashB2Size;
  uint32_t csPadSettingOverride;
  uint32_t sclkPadSettingOverride;
  uint32_t dataPadSettingOverride;
  uint32_t dqsPadSettingOverride;
  uint32_t timeoutInMs;
  uint32_t commandInterval;
  uint16_t dataValidTime[2];
  uint16_t busyOffset;
  uint16_t busyBitPolarity;
  uint32_t lookupTable[64];
  struct mg_flexspi_lut_seq lutCustomSeq[12];
  uint32_t reserved4[4];
};

struct mg_flexspi_nor_config {
  struct mg_flexspi_mem_config memConfig;
  uint32_t pageSize;
  uint32_t sectorSize;
  uint8_t ipcmdSerialClkFreq;
  uint8_t isUniformBlockSize;
  uint8_t reserved0[2];
  uint8_t serialNorType;
  uint8_t needExitNoCmdMode;
  uint8_t halfClkForNonReadCmd;
  uint8_t needRestoreNoCmdMode;
  uint32_t blockSize;
  uint32_t reserve2[11];
};

/* FLEXSPI memory config block related defintions */
#define MG_FLEXSPI_CFG_BLK_TAG (0x42464346UL)      // ascii "FCFB" Big Endian
#define MG_FLEXSPI_CFG_BLK_VERSION (0x56010400UL)  // V1.4.0

#define MG_FLEXSPI_LUT_SEQ(cmd0, pad0, op0, cmd1, pad1, op1)       \
  (MG_FLEXSPI_LUT_OPERAND0(op0) | MG_FLEXSPI_LUT_NUM_PADS0(pad0) | \
   MG_FLEXSPI_LUT_OPCODE0(cmd0) | MG_FLEXSPI_LUT_OPERAND1(op1) |   \
   MG_FLEXSPI_LUT_NUM_PADS1(pad1) | MG_FLEXSPI_LUT_OPCODE1(cmd1))

#define MG_CMD_SDR 0x01
#define MG_CMD_DDR 0x21
#define MG_DUMMY_SDR 0x0C
#define MG_DUMMY_DDR 0x2C
#define MG_DUMMY_RWDS_DDR 0x2D
#define MG_RADDR_SDR 0x02
#define MG_RADDR_DDR 0x22
#define MG_CADDR_DDR 0x23
#define MG_READ_SDR 0x09
#define MG_READ_DDR 0x29
#define MG_WRITE_SDR 0x08
#define MG_WRITE_DDR 0x28
#define MG_STOP 0

#define MG_FLEXSPI_1PAD 0
#define MG_FLEXSPI_2PAD 1
#define MG_FLEXSPI_4PAD 2
#define MG_FLEXSPI_8PAD 3

#define MG_FLEXSPI_QSPI_LUT                                                    \
  {                                                                            \
    [0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_SDR, MG_FLEXSPI_1PAD, 0xEB, MG_RADDR_SDR,  \
                             MG_FLEXSPI_4PAD, 0x18),                           \
    [1] = MG_FLEXSPI_LUT_SEQ(MG_DUMMY_SDR, MG_FLEXSPI_4PAD, 0x06, MG_READ_SDR, \
                             MG_FLEXSPI_4PAD, 0x04),                           \
    [4 * 1 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_SDR, MG_FLEXSPI_1PAD, 0x05,        \
                                     MG_READ_SDR, MG_FLEXSPI_1PAD, 0x04),      \
    [4 * 3 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_SDR, MG_FLEXSPI_1PAD, 0x06,        \
                                     MG_STOP, MG_FLEXSPI_1PAD, 0x0),           \
    [4 * 5 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_SDR, MG_FLEXSPI_1PAD, 0x20,        \
                                     MG_RADDR_SDR, MG_FLEXSPI_1PAD, 0x18),     \
    [4 * 8 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_SDR, MG_FLEXSPI_1PAD, 0xD8,        \
                                     MG_RADDR_SDR, MG_FLEXSPI_1PAD, 0x18),     \
    [4 * 9 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_SDR, MG_FLEXSPI_1PAD, 0x02,        \
                                     MG_RADDR_SDR, MG_FLEXSPI_1PAD, 0x18),     \
    [4 * 9 + 1] = MG_FLEXSPI_LUT_SEQ(MG_WRITE_SDR, MG_FLEXSPI_1PAD, 0x04,      \
                                     MG_STOP, MG_FLEXSPI_1PAD, 0x0),           \
    [4 * 11 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_SDR, MG_FLEXSPI_1PAD, 0x60,       \
                                      MG_STOP, MG_FLEXSPI_1PAD, 0x0),          \
  }

#define MG_FLEXSPI_HYPER_LUT                                                  \
  {                                                                           \
    [0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0xA0, MG_RADDR_DDR, \
                             MG_FLEXSPI_8PAD, 0x18),                          \
    [1] = MG_FLEXSPI_LUT_SEQ(MG_CADDR_DDR, MG_FLEXSPI_8PAD, 0x10,             \
                             MG_DUMMY_DDR, MG_FLEXSPI_8PAD, 0x0C),            \
    [2] = MG_FLEXSPI_LUT_SEQ(MG_READ_DDR, MG_FLEXSPI_8PAD, 0x04, MG_STOP,     \
                             MG_FLEXSPI_1PAD, 0x0),                           \
    [4 * 1 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0),       \
    [4 * 1 + 1] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0xAA),      \
    [4 * 1 + 2] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x05),      \
    [4 * 1 + 3] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x70),      \
    [4 * 2 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0xA0,       \
                                     MG_RADDR_DDR, MG_FLEXSPI_8PAD, 0x18),    \
    [4 * 2 + 1] =                                                             \
        MG_FLEXSPI_LUT_SEQ(MG_CADDR_DDR, MG_FLEXSPI_8PAD, 0x10,               \
                           MG_DUMMY_RWDS_DDR, MG_FLEXSPI_8PAD, 0x0B),         \
    [4 * 2 + 2] = MG_FLEXSPI_LUT_SEQ(MG_READ_DDR, MG_FLEXSPI_8PAD, 0x4,       \
                                     MG_STOP, MG_FLEXSPI_1PAD, 0x0),          \
    [4 * 3 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0),       \
    [4 * 3 + 1] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0xAA),      \
    [4 * 3 + 2] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x05),      \
    [4 * 3 + 3] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0xAA),      \
    [4 * 4 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0),       \
    [4 * 4 + 1] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x55),      \
    [4 * 4 + 2] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x02),      \
    [4 * 4 + 3] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x55),      \
    [4 * 5 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0),       \
    [4 * 5 + 1] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0xAA),      \
    [4 * 5 + 2] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x05),      \
    [4 * 5 + 3] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x80),      \
    [4 * 6 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0),       \
    [4 * 6 + 1] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0xAA),      \
    [4 * 6 + 2] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x05),      \
    [4 * 6 + 3] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0xAA),      \
    [4 * 7 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0),       \
    [4 * 7 + 1] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x55),      \
    [4 * 7 + 2] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x02),      \
    [4 * 7 + 3] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x55),      \
    [4 * 8 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_RADDR_DDR, MG_FLEXSPI_8PAD, 0x18),    \
    [4 * 8 + 1] = MG_FLEXSPI_LUT_SEQ(MG_CADDR_DDR, MG_FLEXSPI_8PAD, 0x10,     \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0),       \
    [4 * 8 + 2] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x30,       \
                                     MG_STOP, MG_FLEXSPI_1PAD, 0x0),          \
    [4 * 9 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0),       \
    [4 * 9 + 1] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0xAA),      \
    [4 * 9 + 2] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x05),      \
    [4 * 9 + 3] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,        \
                                     MG_CMD_DDR, MG_FLEXSPI_8PAD, 0xA0),      \
    [4 * 10 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,       \
                                      MG_RADDR_DDR, MG_FLEXSPI_8PAD, 0x18),   \
    [4 * 10 + 1] = MG_FLEXSPI_LUT_SEQ(MG_CADDR_DDR, MG_FLEXSPI_8PAD, 0x10,    \
                                      MG_WRITE_DDR, MG_FLEXSPI_8PAD, 0x80),   \
    [4 * 11 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,       \
                                      MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0),      \
    [4 * 11 + 1] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,       \
                                      MG_CMD_DDR, MG_FLEXSPI_8PAD, 0xAA),     \
    [4 * 11 + 2] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,       \
                                      MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x05),     \
    [4 * 11 + 3] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,       \
                                      MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x80),     \
    [4 * 12 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,       \
                                      MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0),      \
    [4 * 12 + 1] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,       \
                                      MG_CMD_DDR, MG_FLEXSPI_8PAD, 0xAA),     \
    [4 * 12 + 2] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,       \
                                      MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x05),     \
    [4 * 12 + 3] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,       \
                                      MG_CMD_DDR, MG_FLEXSPI_8PAD, 0xAA),     \
    [4 * 13 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,       \
                                      MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0),      \
    [4 * 13 + 1] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,       \
                                      MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x55),     \
    [4 * 13 + 2] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,       \
                                      MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x02),     \
    [4 * 13 + 3] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,       \
                                      MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x55),     \
    [4 * 14 + 0] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,       \
                                      MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0),      \
    [4 * 14 + 1] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,       \
                                      MG_CMD_DDR, MG_FLEXSPI_8PAD, 0xAA),     \
    [4 * 14 + 2] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,       \
                                      MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x05),     \
    [4 * 14 + 3] = MG_FLEXSPI_LUT_SEQ(MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x0,       \
                                      MG_CMD_DDR, MG_FLEXSPI_8PAD, 0x10),     \
  }

#define MG_LUT_CUSTOM_SEQ                          \
  {                                                \
    {.seqNum = 0, .seqId = 0, .reserved = 0},      \
        {.seqNum = 2, .seqId = 1, .reserved = 0},  \
        {.seqNum = 2, .seqId = 3, .reserved = 0},  \
        {.seqNum = 4, .seqId = 5, .reserved = 0},  \
        {.seqNum = 2, .seqId = 9, .reserved = 0},  \
        {.seqNum = 4, .seqId = 11, .reserved = 0}, \
  }

#define MG_FLEXSPI_LUT_OPERAND0(x) (((uint32_t) (((uint32_t) (x)))) & 0xFFU)
#define MG_FLEXSPI_LUT_NUM_PADS0(x) \
  (((uint32_t) (((uint32_t) (x)) << 8U)) & 0x300U)
#define MG_FLEXSPI_LUT_OPCODE0(x) \
  (((uint32_t) (((uint32_t) (x)) << 10U)) & 0xFC00U)
#define MG_FLEXSPI_LUT_OPERAND1(x) \
  (((uint32_t) (((uint32_t) (x)) << 16U)) & 0xFF0000U)
#define MG_FLEXSPI_LUT_NUM_PADS1(x) \
  (((uint32_t) (((uint32_t) (x)) << 24U)) & 0x3000000U)
#define MG_FLEXSPI_LUT_OPCODE1(x) \
  (((uint32_t) (((uint32_t) (x)) << 26U)) & 0xFC000000U)

#if MG_OTA == MG_OTA_RT1020 || MG_OTA == MG_OTA_RT1050
// RT102X and RT105x boards support ROM API version 1.4
struct mg_flexspi_nor_driver_interface {
  uint32_t version;
  int (*init)(uint32_t instance, struct mg_flexspi_nor_config *config);
  int (*program)(uint32_t instance, struct mg_flexspi_nor_config *config,
                 uint32_t dst_addr, const uint32_t *src);
  uint32_t reserved;
  int (*erase)(uint32_t instance, struct mg_flexspi_nor_config *config,
               uint32_t start, uint32_t lengthInBytes);
  uint32_t reserved2;
  int (*update_lut)(uint32_t instance, uint32_t seqIndex,
                    const uint32_t *lutBase, uint32_t seqNumber);
  int (*xfer)(uint32_t instance, char *xfer);
  void (*clear_cache)(uint32_t instance);
};
#elif MG_OTA <= MG_OTA_RT1064
// RT104x and RT106x support ROM API version 1.5
struct mg_flexspi_nor_driver_interface {
  uint32_t version;
  int (*init)(uint32_t instance, struct mg_flexspi_nor_config *config);
  int (*program)(uint32_t instance, struct mg_flexspi_nor_config *config,
                 uint32_t dst_addr, const uint32_t *src);
  int (*erase_all)(uint32_t instance, struct mg_flexspi_nor_config *config);
  int (*erase)(uint32_t instance, struct mg_flexspi_nor_config *config,
               uint32_t start, uint32_t lengthInBytes);
  int (*read)(uint32_t instance, struct mg_flexspi_nor_config *config,
              uint32_t *dst, uint32_t addr, uint32_t lengthInBytes);
  void (*clear_cache)(uint32_t instance);
  int (*xfer)(uint32_t instance, char *xfer);
  int (*update_lut)(uint32_t instance, uint32_t seqIndex,
                    const uint32_t *lutBase, uint32_t seqNumber);
  int (*get_config)(uint32_t instance, struct mg_flexspi_nor_config *config,
                    uint32_t *option);
};
#else
// RT117x and RT118x
struct mg_flexspi_nor_driver_interface {
  uint32_t version;
  int (*init)(uint32_t instance, struct mg_flexspi_nor_config *config);
  int (*program)(uint32_t instance, struct mg_flexspi_nor_config *config,
                 uint32_t dst_addr, const uint32_t *src);
  int (*erase_all)(uint32_t instance, struct mg_flexspi_nor_config *config);
  int (*erase)(uint32_t instance, struct mg_flexspi_nor_config *config,
               uint32_t start, uint32_t lengthInBytes);
  int (*read)(uint32_t instance, struct mg_flexspi_nor_config *config,
              uint32_t *dst, uint32_t addr, uint32_t lengthInBytes);
  uint32_t reserved;
  int (*xfer)(uint32_t instance, char *xfer);
  int (*update_lut)(uint32_t instance, uint32_t seqIndex,
                    const uint32_t *lutBase, uint32_t seqNumber);
  int (*get_config)(uint32_t instance, struct mg_flexspi_nor_config *config,
                    uint32_t *option);
  int (*erase_sector)(uint32_t instance, struct mg_flexspi_nor_config *config,
                      uint32_t address);
  int (*erase_block)(uint32_t instance, struct mg_flexspi_nor_config *config,
                     uint32_t address);
  void (*hw_reset)(uint32_t instance, uint32_t resetLogic);
  int (*wait_busy)(uint32_t instance, struct mg_flexspi_nor_config *config,
                   bool isParallelMode, uint32_t address);
  int (*set_clock_source)(uint32_t instance, uint32_t clockSrc);
  void (*config_clock)(uint32_t instance, uint32_t freqOption,
                       uint32_t sampleClkMode);
};
#endif

#if MG_OTA <= MG_OTA_RT1064
#define MG_FLEXSPI_BASE 0x402A8000
#define flexspi_nor                                                          \
  (*((struct mg_flexspi_nor_driver_interface **) (*(uint32_t *) 0x0020001c + \
                                                  16)))
#elif MG_OTA == MG_OTA_RT1170
#define MG_FLEXSPI_BASE 0x400CC000
#define flexspi_nor                                                          \
  (*((struct mg_flexspi_nor_driver_interface **) (*(uint32_t *) 0x0021001c + \
                                                  12)))
#else
#define MG_FLEXSPI_BASE 0x445E0000
#define flexspi_nor                                                          \
  (*((struct mg_flexspi_nor_driver_interface **) (*(uint32_t *) 0x1000001c + \
                                                  12)))
#endif

static bool s_flash_irq_disabled;

MG_IRAM static bool flash_page_start(volatile uint32_t *dst) {
  char *base = (char *) s_mg_flash_imxrt.start,
       *end = base + s_mg_flash_imxrt.size;
  volatile char *p = (char *) dst;
  return p >= base && p < end && ((p - base) % s_mg_flash_imxrt.secsz) == 0;
}

#if MG_OTA == MG_OTA_RT1050
// Configuration for Hyper flash memory
static struct mg_flexspi_nor_config default_config = {
    .memConfig =
        {
            .tag = MG_FLEXSPI_CFG_BLK_TAG,
            .version = MG_FLEXSPI_CFG_BLK_VERSION,
            .readSampleClkSrc = 3,  // ReadSampleClk_LoopbackFromDqsPad
            .csHoldTime = 3,
            .csSetupTime = 3,
            .columnAddressWidth = 3u,
            .controllerMiscOption =
                MG_BIT(6) | MG_BIT(4) | MG_BIT(3) | MG_BIT(0),
            .deviceType = 1,  // serial NOR
            .sflashPadType = 8,
            .serialClkFreq = 7,  // 133MHz
            .sflashA1Size = 64 * 1024 * 1024,
            .dataValidTime = {15, 0},
            .busyOffset = 15,
            .busyBitPolarity = 1,
            .lutCustomSeqEnable = 0x1,
            .lookupTable = MG_FLEXSPI_HYPER_LUT,
            .lutCustomSeq = MG_LUT_CUSTOM_SEQ,
        },
    .pageSize = 512,
    .sectorSize = 256 * 1024,
    .ipcmdSerialClkFreq = 1,
    .serialNorType = 1u,
    .blockSize = 256 * 1024,
    .isUniformBlockSize = true};
#else
// Note: this QSPI configuration works for RTs supporting QSPI
// Configuration for QSPI memory
#if MG_OTA == MG_OTA_RT1180
#define MG_FLEXSPI_QSPI_SERIAL_CLK_FREQ 5  // 100MHz
#define MG_FLEXSPI_QSPI_FLASH_SIZE (16 * 1024 * 1024)
#else
#define MG_FLEXSPI_QSPI_SERIAL_CLK_FREQ 7  // 133MHz
#define MG_FLEXSPI_QSPI_FLASH_SIZE (8 * 1024 * 1024)
#endif
static struct mg_flexspi_nor_config default_config = {
    .memConfig = {.tag = MG_FLEXSPI_CFG_BLK_TAG,
                  .version = MG_FLEXSPI_CFG_BLK_VERSION,
                  .readSampleClkSrc = 1,  // ReadSampleClk_LoopbackFromDqsPad
                  .csHoldTime = 3,
                  .csSetupTime = 3,
                  .controllerMiscOption = MG_BIT(4),
                  .deviceType = 1,  // serial NOR
                  .sflashPadType = 4,
                  .serialClkFreq = MG_FLEXSPI_QSPI_SERIAL_CLK_FREQ,
                  .sflashA1Size = MG_FLEXSPI_QSPI_FLASH_SIZE,
                  .lookupTable = MG_FLEXSPI_QSPI_LUT},
    .pageSize = 256,
    .sectorSize = 4 * 1024,
    .ipcmdSerialClkFreq = 1,
    .blockSize = 64 * 1024,
    .isUniformBlockSize = false};
#endif

// must reside in RAM, as flash will be erased
MG_IRAM static int flexspi_nor_get_config(
    struct mg_flexspi_nor_config **config) {
  *config = &default_config;
  return 0;
}

#if 0
// ROM API get_config call (ROM version >= 1.5)
MG_IRAM static int flexspi_nor_get_config(
    struct mg_flexspi_nor_config **config) {
  uint32_t options[] = {0xc0000000, 0x00};

  MG_ARM_DISABLE_IRQ();
  uint32_t status =
      flexspi_nor->get_config(FLEXSPI_NOR_INSTANCE, *config, options);
  if (!s_flash_irq_disabled) {
    MG_ARM_ENABLE_IRQ();
  }
  if (status) {
    MG_ERROR(("Failed to extract flash configuration: status %u", status));
  }
  return status;
}
#endif

#if MG_OTA == MG_OTA_RT1020 || MG_OTA == MG_OTA_RT1050
MG_IRAM static void mg_spin(volatile uint32_t count) {
  while (count--) (void) 0;
}

MG_IRAM static void flash_wait(void) {
  while ((*((volatile uint32_t *) (MG_FLEXSPI_BASE + 0xE0)) & MG_BIT(1)) == 0)
    mg_spin(1);
}
#endif

MG_IRAM static bool flash_erase(struct mg_flexspi_nor_config *config,
                                void *addr) {
  if (flash_page_start(addr) == false) {
    MG_ERROR(("%p is not on a sector boundary", addr));
    return false;
  }

  void *dst = (void *) ((char *) addr - (char *) s_mg_flash_imxrt.start);

  bool ok = (flexspi_nor->erase(FLEXSPI_NOR_INSTANCE, config, (uint32_t) dst,
                                s_mg_flash_imxrt.secsz) == 0);
  MG_DEBUG(("Sector starting at %p erasure: %s", addr, ok ? "ok" : "fail"));
  return ok;
}

#if 0
// standalone erase call
MG_IRAM static bool mg_imxrt_erase(void *addr) {
  struct mg_flexspi_nor_config config, *config_ptr = &config;
  bool ret;
  // Interrupts must be disabled before calls to ROM API in RT1020 and 1060
  MG_ARM_DISABLE_IRQ();
  ret = (flexspi_nor_get_config(&config_ptr) == 0);
  if (ret) ret = flash_erase(config_ptr, addr);
  MG_ARM_ENABLE_IRQ();
  return ret;
}
#endif

MG_IRAM static bool mg_imxrt_write(void *addr, const void *buf, size_t len) {
  struct mg_flexspi_nor_config config, *config_ptr = &config;
  bool ok = false;
  // Interrupts must be disabled before calls to ROM API in RT1020 and 1060
  MG_ARM_DISABLE_IRQ();
  if (flexspi_nor_get_config(&config_ptr) != 0) goto fwxit;
  if ((len % s_mg_flash_imxrt.align) != 0) {
    MG_ERROR(("%lu is not aligned to %lu", len, s_mg_flash_imxrt.align));
    goto fwxit;
  }
  if ((char *) addr < (char *) s_mg_flash_imxrt.start) {
    MG_ERROR(("Invalid flash write address: %p", addr));
    goto fwxit;
  }

  uint32_t *dst = (uint32_t *) addr;
  uint32_t *src = (uint32_t *) buf;
  uint32_t *end = (uint32_t *) ((char *) buf + len);
  char *flash_start = (char *) s_mg_flash_imxrt.start;
  char *flash_end = flash_start + s_mg_flash_imxrt.size;
  ok = true;

  while (ok && src < end) {
    if (flash_page_start(dst) && flash_erase(config_ptr, dst) == false) {
      ok = false;
      break;
    }
    uint32_t status = 0;
    uint32_t dst_ofs = (uint32_t) dst - (uint32_t) s_mg_flash_imxrt.start;
    if ((char *) buf >= flash_start && (char *) buf < flash_end) {
      // If we copy from FLASH to FLASH, then we first need to copy the source
      // to RAM
      size_t tmp_buf_size = s_mg_flash_imxrt.align / sizeof(uint32_t);
      uint32_t tmp[tmp_buf_size];

#if MG_OTA == MG_OTA_RT1020 || MG_OTA == MG_OTA_RT1050
      // ROM API 1.4 does not provide flexspi_nor_flash_read().
      // Manually copy byte by byte into the RAM buffer.
      for (size_t i = 0; i < tmp_buf_size; i++) {
        flash_wait();
        tmp[i] = src[i];
      }
#else
      // When supported, prefer the read API call.
      uint32_t src_ofs =
          (uint32_t) src - (uint32_t) s_mg_flash_imxrt.start;
      status = flexspi_nor->read(FLEXSPI_NOR_INSTANCE, config_ptr, tmp,
                                 src_ofs, s_mg_flash_imxrt.align);
#endif
      if (status == 0) {
        status = flexspi_nor->program(FLEXSPI_NOR_INSTANCE, config_ptr,
                                      dst_ofs, tmp);
      }
    } else {
      status = flexspi_nor->program(FLEXSPI_NOR_INSTANCE, config_ptr,
                                    (uint32_t) dst_ofs, src);
    }
    src = (uint32_t *) ((char *) src + s_mg_flash_imxrt.align);
    dst = (uint32_t *) ((char *) dst + s_mg_flash_imxrt.align);
    if (status != 0) {
      ok = false;
    }
  }
  MG_DEBUG(("Flash write %lu bytes @ %p: %s.", len, dst, ok ? "ok" : "fail"));
fwxit:
  if (!s_flash_irq_disabled) MG_ARM_ENABLE_IRQ();
  return ok;
}

MG_IRAM static void single_bank_swap(char *p1, char *p2, size_t s, size_t ss) {
  char *scratch = p2 + s;
  for (size_t ofs = 0; ofs < s; ofs += ss) {
    MG_OTA_ROLLBACK_TIMER_FEED();
    mg_imxrt_write(scratch, p1 + ofs, ss);
    MG_OTA_ROLLBACK_TIMER_FEED();
    mg_imxrt_write(p1 + ofs, p2 + ofs, ss);
    MG_OTA_ROLLBACK_TIMER_FEED();
    mg_imxrt_write(p2 + ofs, scratch, ss);
  }
  *(volatile unsigned long *) 0xe000ed0c = 0x5fa0004;
}

MG_IRAM static bool mg_imxrt_swap(void) {
  size_t ss = s_mg_flash_imxrt.secsz;
  char *p1 = (char *) s_mg_flash_imxrt.start;
  char *p2 = p1 + s_mg_flash_imxrt.size / 2;
  size_t s = s_mg_flash_imxrt.size / 2 - ss;
  MG_INFO(("Swapping partitions, %u bytes", s));
  MG_INFO(("Do NOT power off..."));
  MG_OTA_ROLLBACK_TIMER_FEED();
  mg_log_level = MG_LL_NONE;
  s_flash_irq_disabled = true;
  single_bank_swap(p1, p2, s, ss);
  return true;  // unreachable
}

bool mg_ota_begin(size_t new_firmware_size) {
  size_t max = s_mg_flash_imxrt.size / 2 - s_mg_flash_imxrt.secsz;
  if (new_firmware_size > max) {
    MG_ERROR(("Firmware %lu too big for single-bank OTA, max %lu",
              new_firmware_size, max));
    return false;
  }
  return mg_ota_flash_begin(new_firmware_size, &s_mg_flash_imxrt);
}

bool mg_ota_write(const void *buf, size_t len) {
  return mg_ota_flash_write(buf, len, &s_mg_flash_imxrt);
}

bool mg_ota_end(void) {
  mg_ota_flash_end(&s_mg_flash_imxrt);
  return false;
}
struct mg_flash *mg_flash = &s_mg_flash_imxrt;
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/ota_mcxn.c"
#endif




#if MG_OTA == MG_OTA_MCXN

// - Flash phrase: 16 bytes; smallest portion programmed in one operation.
// - Flash page: 128 bytes; largest portion programmed in one operation.
// - Flash sector: 8 KB; smallest portion that can be erased in one operation.
// - Flash API mg_flash_driver->program: "start" and "len" must be page-size
// aligned; to use 'phrase', FMU register access is needed. Using ROM

static bool mg_mcxn_write(void *, const void *, size_t);
static bool mg_mcxn_swap(void);

static struct mg_flash s_mg_flash_mcxn = {
    (void *) 0,  // Start, filled at init
    0,           // Size, filled at init
    0,           // Sector size, filled at init
    0,           // Align, filled at init
    mg_mcxn_write,
    mg_mcxn_swap,
};

struct mg_flash_config {
  uint32_t addr;
  uint32_t size;
  uint32_t blocks;
  uint32_t page_size;
  uint32_t sector_size;
  uint32_t ffr[6];
  uint32_t reserved0[5];
  uint32_t *bootctx;
  bool useahb;
};

struct mg_flash_driver_interface {
  uint32_t version;
  uint32_t (*init)(struct mg_flash_config *);
  uint32_t (*erase)(struct mg_flash_config *, uint32_t start, uint32_t len,
                    uint32_t key);
  uint32_t (*program)(struct mg_flash_config *, uint32_t start, uint8_t *src,
                      uint32_t len);
  uint32_t (*verify_erase)(struct mg_flash_config *, uint32_t start,
                           uint32_t len);
  uint32_t (*verify_program)(struct mg_flash_config *, uint32_t start,
                             uint32_t len, const uint8_t *expected,
                             uint32_t *addr, uint32_t *failed);
  uint32_t reserved1[12];
  uint32_t (*read)(struct mg_flash_config *, uint32_t start, uint8_t *dest,
                   uint32_t len);
  uint32_t reserved2[4];
  uint32_t (*deinit)(struct mg_flash_config *);
};
#define mg_flash_driver \
  ((struct mg_flash_driver_interface *) (*((uint32_t *) 0x1303fc00 + 4)))
#define MG_MCXN_FLASK_KEY (('k' << 24) | ('e' << 16) | ('f' << 8) | 'l')

MG_IRAM static bool flash_sector_start(volatile uint32_t *dst) {
  char *base = (char *) s_mg_flash_mcxn.start,
       *end = base + s_mg_flash_mcxn.size;
  volatile char *p = (char *) dst;
  return p >= base && p < end && ((p - base) % s_mg_flash_mcxn.secsz) == 0;
}

MG_IRAM static bool flash_erase(struct mg_flash_config *config, void *addr) {
  if (flash_sector_start(addr) == false) {
    MG_ERROR(("%p is not on a sector boundary", addr));
    return false;
  }
  uint32_t dst =
      (uint32_t) addr - (uint32_t) s_mg_flash_mcxn.start;  // future-proof
  uint32_t status = mg_flash_driver->erase(config, dst, s_mg_flash_mcxn.secsz,
                                           MG_MCXN_FLASK_KEY);
  bool ok = (status == 0);
  if (!ok) MG_ERROR(("Flash write error: %lu", status));
  MG_DEBUG(("Sector starting at %p erasure: %s", addr, ok ? "ok" : "fail"));
  return ok;
}

#if 0
// read-while-write, no need to disable IRQs for standalone usage
MG_IRAM static bool mg_mcxn_erase(void *addr) {
  uint32_t status;
  struct mg_flash_config config;
  if ((status = mg_flash_driver->init(&config)) != 0) {
    MG_ERROR(("Flash driver init error: %lu", status));
    return false;
  }
  bool ok = flash_erase(&config, addr);
  mg_flash_driver->deinit(&config);
  return ok;
}
#endif

MG_IRAM static bool mg_mcxn_swap(void) {
  // TODO(): no devices so far
  return true;
}

static bool s_flash_irq_disabled;

MG_IRAM static bool mg_mcxn_write(void *addr, const void *buf, size_t len) {
  bool ok = false;
  uint32_t status;
  struct mg_flash_config config;
  if ((status = mg_flash_driver->init(&config)) != 0) {
    MG_ERROR(("Flash driver init error: %lu", status));
    return false;
  }
  if ((len % s_mg_flash_mcxn.align) != 0) {
    MG_ERROR(("%lu is not aligned to %lu", len, s_mg_flash_mcxn.align));
    goto fwxit;
  }
  if ((((size_t) addr - (size_t) s_mg_flash_mcxn.start) %
       s_mg_flash_mcxn.align) != 0) {
    MG_ERROR(("%p is not on a page boundary", addr));
    goto fwxit;
  }

  uint32_t *dst = (uint32_t *) addr;
  uint32_t *src = (uint32_t *) buf;
  uint32_t *end = (uint32_t *) ((char *) buf + len);
  ok = true;

  MG_ARM_DISABLE_IRQ();
  while (ok && src < end) {
    if (flash_sector_start(dst) && flash_erase(&config, dst) == false) {
      ok = false;
      break;
    }
    uint32_t dst_ofs = (uint32_t) dst - (uint32_t) s_mg_flash_mcxn.start;
    // assume source is in RAM or in a different bank or read-while-write
    status = mg_flash_driver->program(&config, dst_ofs, (uint8_t *) src,
                                      s_mg_flash_mcxn.align);
    src = (uint32_t *) ((char *) src + s_mg_flash_mcxn.align);
    dst = (uint32_t *) ((char *) dst + s_mg_flash_mcxn.align);
    if (status != 0) {
      MG_ERROR(("Flash write error: %lu", status));
      ok = false;
    }
  }
  if (!s_flash_irq_disabled) MG_ARM_ENABLE_IRQ();
  MG_DEBUG(("Flash write %lu bytes @ %p: %s.", len, dst, ok ? "ok" : "fail"));

fwxit:
  mg_flash_driver->deinit(&config);
  return ok;
}

// try to swap (honor dual image), otherwise just overwrite
MG_IRAM static void single_bank_swap(char *p1, char *p2, size_t s, size_t ss) {
  char *tmp = mg_calloc(1, ss);
  // no stdlib calls here
  for (size_t ofs = 0; ofs < s; ofs += ss) {
    if (tmp != NULL)
      for (size_t i = 0; i < ss; i++) tmp[i] = p1[ofs + i];
    mg_mcxn_write(p1 + ofs, p2 + ofs, ss);
    if (tmp != NULL) mg_mcxn_write(p2 + ofs, tmp, ss);
  }
  *(volatile unsigned long *) 0xe000ed0c = 0x5fa0004;
}

bool mg_ota_begin(size_t new_firmware_size) {
  uint32_t status;
  struct mg_flash_config config;
  if ((status = mg_flash_driver->init(&config)) != 0) {
    MG_ERROR(("Flash driver init error: %lu", status));
    return false;
  }
  s_mg_flash_mcxn.start = (void *) config.addr;
  s_mg_flash_mcxn.size = config.size;
  s_mg_flash_mcxn.secsz = config.sector_size;
  s_mg_flash_mcxn.align = config.page_size;
  mg_flash_driver->deinit(&config);
  MG_DEBUG(
      ("%lu-byte flash @%p, using %lu-byte sectors with %lu-byte-aligned pages",
       s_mg_flash_mcxn.size, s_mg_flash_mcxn.start, s_mg_flash_mcxn.secsz,
       s_mg_flash_mcxn.align));
  return mg_ota_flash_begin(new_firmware_size, &s_mg_flash_mcxn);
}

bool mg_ota_write(const void *buf, size_t len) {
  return mg_ota_flash_write(buf, len, &s_mg_flash_mcxn);
}

bool mg_ota_end(void) {
  if (mg_ota_flash_end(&s_mg_flash_mcxn)) {
    if (0) {  // is_dualbank()
      // TODO(): no devices so far
      *(volatile unsigned long *) 0xe000ed0c = 0x5fa0004;
    } else {
      // Swap partitions. Pray power does not go away
      MG_INFO(("Swapping partitions, size %u (%u sectors)",
               s_mg_flash_mcxn.size,
               s_mg_flash_mcxn.size / s_mg_flash_mcxn.secsz));
      MG_INFO(("Do NOT power off..."));
      mg_log_level = MG_LL_NONE;
      s_flash_irq_disabled = true;
      // Runs in RAM, will reset when finished
      single_bank_swap(
          (char *) s_mg_flash_mcxn.start,
          (char *) s_mg_flash_mcxn.start + s_mg_flash_mcxn.size / 2,
          s_mg_flash_mcxn.size / 2, s_mg_flash_mcxn.secsz);
    }
  }
  return false;
}
struct mg_flash *mg_flash = &s_mg_flash_mcxn;
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/ota_picosdk.c"
#endif




#if MG_OTA == MG_OTA_PICOSDK

// Both RP2040 and RP2350 have no flash, low-level flash access support in
// bootrom, and high-level support in Pico-SDK (2.0+ for the RP2350)
// - The RP2350 in RISC-V mode is not tested
// NOTE(): See OTA design notes

static bool mg_picosdk_write(void *, const void *, size_t);
static bool mg_picosdk_swap(void);

static struct mg_flash s_mg_flash_picosdk = {
    (void *) 0x10000000,  // Start; functions handle offset
#ifdef PICO_FLASH_SIZE_BYTES
    PICO_FLASH_SIZE_BYTES,  // Size, from board definitions
#else
    0x200000,  // Size, guess... is 2M enough ?
#endif
    FLASH_SECTOR_SIZE,  // Sector size, from hardware_flash
    FLASH_PAGE_SIZE,    // Align, from hardware_flash
    mg_picosdk_write,      mg_picosdk_swap,
};

#define MG_MODULO2(x, m) ((x) & ((m) -1))

static bool __no_inline_not_in_flash_func(flash_sector_start)(
    volatile uint32_t *dst) {
  char *base = (char *) s_mg_flash_picosdk.start,
       *end = base + s_mg_flash_picosdk.size;
  volatile char *p = (char *) dst;
  return p >= base && p < end &&
         MG_MODULO2(p - base, s_mg_flash_picosdk.secsz) == 0;
}

static bool __no_inline_not_in_flash_func(flash_erase)(void *addr) {
  if (flash_sector_start(addr) == false) {
    MG_ERROR(("%p is not on a sector boundary", addr));
    return false;
  }
  void *dst = (void *) ((char *) addr - (char *) s_mg_flash_picosdk.start);
  flash_range_erase((uint32_t) dst, s_mg_flash_picosdk.secsz);
  MG_DEBUG(("Sector starting at %p erasure", addr));
  return true;
}

static bool s_flash_irq_disabled;

static bool __no_inline_not_in_flash_func(mg_picosdk_write)(void *addr,
                                                            const void *buf,
                                                            size_t len) {
  if ((len % s_mg_flash_picosdk.align) != 0) {
    MG_ERROR(("%lu is not aligned to %lu", len, s_mg_flash_picosdk.align));
    return false;
  }
  if ((((size_t) addr - (size_t) s_mg_flash_picosdk.start) %
       s_mg_flash_picosdk.align) != 0) {
    MG_ERROR(("%p is not on a page boundary", addr));
    return false;
  }

  uint32_t *dst = (uint32_t *) addr;
  uint32_t *src = (uint32_t *) buf;
  uint32_t *end = (uint32_t *) ((char *) buf + len);

#ifndef __riscv
  MG_ARM_DISABLE_IRQ();
#else
  asm volatile("csrrc zero, mstatus, %0" : : "i"(1 << 3) : "memory");
#endif
  while (src < end) {
    uint32_t dst_ofs = (uint32_t) dst - (uint32_t) s_mg_flash_picosdk.start;
    if (flash_sector_start(dst) && flash_erase(dst) == false) break;
    // flash_range_program() runs in RAM and handles writing up to
    // FLASH_PAGE_SIZE bytes. Source must not be in flash
    flash_range_program((uint32_t) dst_ofs, (uint8_t *) src,
                        s_mg_flash_picosdk.align);
    src = (uint32_t *) ((char *) src + s_mg_flash_picosdk.align);
    dst = (uint32_t *) ((char *) dst + s_mg_flash_picosdk.align);
  }
  if (!s_flash_irq_disabled) {
#ifndef __riscv
    MG_ARM_ENABLE_IRQ();
#else
    asm volatile("csrrs mstatus, %0" : : "i"(1 << 3) : "memory");
#endif
  }
  MG_DEBUG(("Flash write %lu bytes @ %p.", len, dst));
  return true;
}

// just overwrite instead of swap
static void __no_inline_not_in_flash_func(single_bank_swap)(char *p1, char *p2,
                                                            size_t s,
                                                            size_t ss) {
  char *tmp_1 = mg_calloc(1, ss); // copy from 1st partition
  if (tmp_1 == NULL) return;
  char *tmp_2 = mg_calloc(1, ss); // copy from 2nd partition
  if (tmp_2 == NULL) return;
#if PICO_RP2040
  uint32_t xip[256 / sizeof(uint32_t)];
  void *dst_1 = (void *) ((char *) p1 - (char *) s_mg_flash_picosdk.start);
  void *dst_2 = (void *) ((char *) p2- (char *) s_mg_flash_picosdk.start);
  // use SDK function calls to get BootROM function pointers
  rom_connect_internal_flash_fn connect = (rom_connect_internal_flash_fn) rom_func_lookup(ROM_FUNC_CONNECT_INTERNAL_FLASH);
  rom_flash_exit_xip_fn xit = (rom_flash_exit_xip_fn) rom_func_lookup(ROM_FUNC_FLASH_EXIT_XIP);
  rom_flash_range_program_fn program = (rom_flash_range_program_fn) rom_func_lookup(ROM_FUNC_FLASH_RANGE_PROGRAM);
  rom_flash_flush_cache_fn flush = (rom_flash_flush_cache_fn) rom_func_lookup(ROM_FUNC_FLASH_FLUSH_CACHE);
  // no stdlib calls here.
  MG_ARM_DISABLE_IRQ();
  // 2nd bootloader (XIP) is in flash, SDK functions copy it to RAM on entry
  for (size_t i = 0; i < 256 / sizeof(uint32_t); i++)
    xip[i] = ((uint32_t *) (s_mg_flash_picosdk.start))[i];
  // flash has been erased, no XIP to copy. Only BootROM calls possible
  for (uint32_t ofs = 0; ofs < s; ofs += ss) {
    for (size_t i = 0; i < ss; i++) {
      tmp_1[i] = p1[ofs + i];
      tmp_2[i] = p2[ofs + i];
    }
    flash_range_erase((uint32_t) dst_1 + ofs, ss);
    flash_range_erase((uint32_t) dst_2 + ofs, ss);
    __compiler_memory_barrier();
    connect();
    xit();
    program((uint32_t) dst_1 + ofs, tmp_2, ss);
    program((uint32_t) dst_2 + ofs, tmp_1, ss);
    flush();
    ((void (*)(void))((intptr_t) xip + 1))(); // enter XIP again
  }
  *(volatile unsigned long *) 0xe000ed0c = 0x5fa0004;  // AIRCR = SYSRESETREQ
#else
  // RP2350 has BootRAM and copies second bootloader there, SDK uses that copy,
  // It might also be able to take advantage of partition swapping
  rom_reboot_fn reboot = (rom_reboot_fn) rom_func_lookup(ROM_FUNC_REBOOT);
  for (size_t ofs = 0; ofs < s; ofs += ss) {
    for (size_t i = 0; i < ss; i++) {
      tmp_1[i] = p1[ofs + i];
      tmp_2[i] = p2[ofs + i];
    }
    mg_picosdk_write(p1 + ofs, tmp_2, ss);
    mg_picosdk_write(p2 + ofs, tmp_1, ss);
  }
  reboot(BOOT_TYPE_NORMAL | 0x100, 1, 0, 0); // 0x100: NO_RETURN_ON_SUCCESS
#endif
}

static bool __no_inline_not_in_flash_func(mg_picosdk_swap)(void) {
  // TODO(): RP2350 might have some A/B functionality (DS 5.1)
  // Swap partitions. Pray power does not go away
  MG_INFO(("Swapping partitions, size %u (%u sectors)",
            s_mg_flash_picosdk.size,
            s_mg_flash_picosdk.size / s_mg_flash_picosdk.secsz));
  MG_INFO(("Do NOT power off..."));
  mg_log_level = MG_LL_NONE;
  s_flash_irq_disabled = true;
  // Runs in RAM, will reset when finished or return on failure
  single_bank_swap(
      (char *) s_mg_flash_picosdk.start,
      (char *) s_mg_flash_picosdk.start + s_mg_flash_picosdk.size / 2,
      s_mg_flash_picosdk.size / 2, s_mg_flash_picosdk.secsz);
  return false;
}

bool mg_ota_begin(size_t new_firmware_size) {
  return mg_ota_flash_begin(new_firmware_size, &s_mg_flash_picosdk);
}

bool mg_ota_write(const void *buf, size_t len) {
  return mg_ota_flash_write(buf, len, &s_mg_flash_picosdk);
}

bool mg_ota_end(void) {
  if (mg_ota_flash_end(&s_mg_flash_picosdk));
  return false;
}
struct mg_flash *mg_flash = &s_mg_flash_picosdk;
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/ota_rw612.c"
#endif




#if MG_OTA == MG_OTA_RW612

MG_IRAM static bool mg_frdm_write(void *, const void *, size_t);
static bool mg_frdm_swap(void);

static struct mg_flash s_mg_flash_frdm = {(void *) 0x08000000,  // Start,
                                          0x200000,             // Size
                                          0x1000,               // Sector size
                                          0x100,                // Align
                                          mg_frdm_write,
                                          mg_frdm_swap};

struct mg_flexspi_lut_seq {
  uint8_t seqNum;
  uint8_t seqId;
  uint16_t reserved;
};

struct mg_flexspi_mem_config {
  uint32_t tag;
  uint32_t version;
  uint32_t reserved0;
  uint8_t readSampleClkSrc;
  uint8_t csHoldTime;
  uint8_t csSetupTime;
  uint8_t columnAddressWidth;
  uint8_t deviceModeCfgEnable;
  uint8_t deviceModeType;
  uint16_t waitTimeCfgCommands;
  struct mg_flexspi_lut_seq deviceModeSeq;
  uint32_t deviceModeArg;
  uint8_t configCmdEnable;
  uint8_t configModeType[3];
  struct mg_flexspi_lut_seq configCmdSeqs[3];
  uint32_t reserved1;
  uint32_t configCmdArgs[3];
  uint32_t reserved2;
  uint32_t controllerMiscOption;
  uint8_t deviceType;
  uint8_t sflashPadType;
  uint8_t serialClkFreq;
  uint8_t lutCustomSeqEnable;
  uint32_t reserved3[2];
  uint32_t sflashA1Size;
  uint32_t sflashA2Size;
  uint32_t sflashB1Size;
  uint32_t sflashB2Size;
  uint32_t csPadSettingOverride;
  uint32_t sclkPadSettingOverride;
  uint32_t dataPadSettingOverride;
  uint32_t dqsPadSettingOverride;
  uint32_t timeoutInMs;
  uint32_t commandInterval;
  uint16_t dataValidTime[2];
  uint16_t busyOffset;
  uint16_t busyBitPolarity;
  uint32_t lookupTable[64];
  struct mg_flexspi_lut_seq lutCustomSeq[12];
  uint32_t reserved4[4];
};

struct mg_flexspi_nor_config {
  struct mg_flexspi_mem_config memConfig;
  uint32_t pageSize;
  uint32_t sectorSize;
  uint8_t ipcmdSerialClkFreq;
  uint8_t isUniformBlockSize;
  uint8_t isDataOrderSwapped;
  uint8_t reserved0[1];
  uint8_t serialNorType;
  uint8_t needExitNoCmdMode;
  uint8_t halfClkForNonReadCmd;
  uint8_t needRestoreNoCmdMode;
  uint32_t blockSize;
  uint32_t flashStateCtx;
  uint32_t reserve2[10];
};

struct mg_flexspi_nor_driver_interface {
  uint32_t version;
  uint32_t (*init)(uint32_t instance, struct mg_flexspi_nor_config *config);
  uint32_t (*wait_busy)(uint32_t instance, struct mg_flexspi_nor_config *config,
                        uint32_t address, bool keepState);
  uint32_t (*page_program)(uint32_t instance,
                           struct mg_flexspi_nor_config *config,
                           uint32_t dstAddr, const uint32_t *src,
                           bool keepState);
  uint32_t (*erase_all)(uint32_t instance,
                        struct mg_flexspi_nor_config *config);
  uint32_t (*erase)(uint32_t instance, struct mg_flexspi_nor_config *config,
                    uint32_t start, uint32_t length);
  uint32_t (*erase_sector)(uint32_t instance,
                           struct mg_flexspi_nor_config *config,
                           uint32_t address);
  uint32_t (*erase_block)(uint32_t instance,
                          struct mg_flexspi_nor_config *config,
                          uint32_t address);
  uint32_t (*read)(uint32_t instance, struct mg_flexspi_nor_config *config,
                   uint32_t *dst, uint32_t start, uint32_t bytes);
  void (*config_clock)(uint32_t instance, uint32_t freqOption,
                       uint32_t sampleClkMode);
  uint32_t (*set_clock_source)(uint32_t clockSrc);
  uint32_t (*get_config)(uint32_t instance,
                         struct mg_flexspi_nor_config *config,
                         uint32_t *option);
  void (*hw_reset)(uint32_t instance, uint32_t reset_logic);
  uint32_t (*xfer)(uint32_t instance, char *xfer);
  uint32_t (*update_lut)(uint32_t instance, uint32_t seqIndex,
                         const uint32_t *lutBase, uint32_t numberOfSeq);
  uint32_t (*partial_program)(uint32_t instance,
                              struct mg_flexspi_nor_config *config,
                              uint32_t dstAddr, const uint32_t *src,
                              uint32_t length, bool keepState);
};

#define MG_FLEXSPI_CFG_BLK_TAG (0x42464346UL)
#define MG_FLEXSPI_BASE 0x40134000UL

#define MG_CMD_SDR 0x01
#define MG_RADDR_SDR 0x02
#define MG_WRITE_SDR 0x08
#define MG_READ_SDR 0x09
#define MG_DUMMY_SDR 0x0C
#define MG_STOP_EXE 0

#define MG_FLEXSPI_1PAD 0
#define MG_FLEXSPI_4PAD 2

#define MG_FLEXSPI_LUT_OPERAND0(x) (((x) &0xFF) << 0)
#define MG_FLEXSPI_LUT_NUM_PADS0(x) (((x) &0x3) << 8)
#define MG_FLEXSPI_LUT_OPCODE0(x) (((x) &0x3F) << 10)
#define MG_FLEXSPI_LUT_OPERAND1(x) (((x) &0xFF) << 16)
#define MG_FLEXSPI_LUT_NUM_PADS1(x) (((x) &0x3) << 24)
#define MG_FLEXSPI_LUT_OPCODE1(x) (((x) &0x3F) << 26)

#define MG_FLEXSPI_LUT_SEQ(cmd0, pad0, op0, cmd1, pad1, op1)       \
  (MG_FLEXSPI_LUT_OPERAND0(op0) | MG_FLEXSPI_LUT_NUM_PADS0(pad0) | \
   MG_FLEXSPI_LUT_OPCODE0(cmd0) | MG_FLEXSPI_LUT_OPERAND1(op1) |   \
   MG_FLEXSPI_LUT_NUM_PADS1(pad1) | MG_FLEXSPI_LUT_OPCODE1(cmd1))

struct mg_flexspi_nor_config default_config = {
    .memConfig =
        {
            .tag = MG_FLEXSPI_CFG_BLK_TAG,
            .version = 0,
            .readSampleClkSrc = 1,
            .csHoldTime = 3,
            .csSetupTime = 3,
            .deviceModeCfgEnable = 1,
            .deviceModeSeq = {.seqNum = 1, .seqId = 2},
            .deviceModeArg = 0x0740,
            .configCmdEnable = 0,
            .deviceType = 0x1,
            .sflashPadType = 4,
            .serialClkFreq = 4,
            .sflashA1Size = 0x4000000U,
            .sflashA2Size = 0,
            .sflashB1Size = 0,
            .sflashB2Size = 0,
            .lookupTable =
                {
                    [0] =
                        MG_FLEXSPI_LUT_SEQ(MG_CMD_SDR, MG_FLEXSPI_1PAD, 0xEB,
                                           MG_RADDR_SDR, MG_FLEXSPI_4PAD, 0x18),
                    [1] =
                        MG_FLEXSPI_LUT_SEQ(MG_DUMMY_SDR, MG_FLEXSPI_4PAD, 0x06,
                                           MG_READ_SDR, MG_FLEXSPI_4PAD, 0x04),
                    [4 * 1 + 0] =
                        MG_FLEXSPI_LUT_SEQ(MG_CMD_SDR, MG_FLEXSPI_1PAD, 0x05,
                                           MG_READ_SDR, MG_FLEXSPI_1PAD, 0x04),
                    [4 * 2 + 0] =
                        MG_FLEXSPI_LUT_SEQ(MG_CMD_SDR, MG_FLEXSPI_1PAD, 0x01,
                                           MG_WRITE_SDR, MG_FLEXSPI_1PAD, 0x02),
                    [4 * 3 + 0] =
                        MG_FLEXSPI_LUT_SEQ(MG_CMD_SDR, MG_FLEXSPI_1PAD, 0x06,
                                           MG_STOP_EXE, MG_FLEXSPI_1PAD, 0x00),
                    [4 * 5 + 0] =
                        MG_FLEXSPI_LUT_SEQ(MG_CMD_SDR, MG_FLEXSPI_1PAD, 0x20,
                                           MG_RADDR_SDR, MG_FLEXSPI_1PAD, 0x18),
                    [4 * 8 + 0] =
                        MG_FLEXSPI_LUT_SEQ(MG_CMD_SDR, MG_FLEXSPI_1PAD, 0x52,
                                           MG_RADDR_SDR, MG_FLEXSPI_1PAD, 0x18),
                    [4 * 9 + 0] =
                        MG_FLEXSPI_LUT_SEQ(MG_CMD_SDR, MG_FLEXSPI_1PAD, 0x02,
                                           MG_RADDR_SDR, MG_FLEXSPI_1PAD, 0x18),
                    [4 * 9 + 1] =
                        MG_FLEXSPI_LUT_SEQ(MG_WRITE_SDR, MG_FLEXSPI_1PAD, 0x00,
                                           MG_STOP_EXE, MG_FLEXSPI_1PAD, 0x00),
                    [4 * 11 + 0] =
                        MG_FLEXSPI_LUT_SEQ(MG_CMD_SDR, MG_FLEXSPI_1PAD, 0x60,
                                           MG_STOP_EXE, MG_FLEXSPI_1PAD, 0x00),
                },
        },
    .pageSize = 0x100,
    .sectorSize = 0x1000,
    .ipcmdSerialClkFreq = 0,
    .blockSize = 0x8000,
};

#define MG_FLEXSPI_NOR_INSTANCE 0
#define MG_ROMAPI_ADDRESS 0x13030000U
#define flexspi_nor                              \
  ((struct mg_flexspi_nor_driver_interface *) (( \
      (uint32_t *) MG_ROMAPI_ADDRESS)[5]))

MG_IRAM static bool flash_page_start(volatile uint32_t *dst) {
  char *base = (char *) s_mg_flash_frdm.start,
       *end = base + s_mg_flash_frdm.size;
  volatile char *p = (char *) dst;
  return p >= base && p < end && ((p - base) % s_mg_flash_frdm.secsz) == 0;
}

MG_IRAM static int flexspi_nor_get_config(
    struct mg_flexspi_nor_config *config) {
  uint32_t option = 0xc0000004;
  return flexspi_nor->get_config(MG_FLEXSPI_NOR_INSTANCE, config, &option);
}

MG_IRAM static int flash_init(void) {
  static bool initialized = false;
  if (!initialized) {
    struct mg_flexspi_nor_config config;
    memset(&config, 0, sizeof(config));
    flexspi_nor->set_clock_source(0);
    flexspi_nor->config_clock(MG_FLEXSPI_NOR_INSTANCE, 1, 0);
    if (flexspi_nor->init(MG_FLEXSPI_NOR_INSTANCE, &default_config)) {
      return 1;
    }
    flexspi_nor_get_config(&config);
    if (flexspi_nor->init(MG_FLEXSPI_NOR_INSTANCE, &config)) {
      return 1;
    }
    initialized = true;
  }
  return 0;
}

MG_IRAM static bool flash_erase(struct mg_flexspi_nor_config *config,
                                void *addr) {
  if (flash_page_start(addr) == false) {
    MG_ERROR(("%p is not on a sector boundary", addr));
    return false;
  }

  void *dst = (void *) ((char *) addr - (char *) s_mg_flash_frdm.start);
  bool ok = (flexspi_nor->erase_sector(MG_FLEXSPI_NOR_INSTANCE, config,
                                       (uint32_t) dst) == 0);
  MG_INFO(("Sector starting at %p erasure: %s", addr, ok ? "ok" : "fail"));
  return ok;
}

MG_IRAM bool mg_frdm_swap(void) {
  return true;
}

MG_IRAM static void flash_wait(void) {
  while ((*((volatile uint32_t *) (MG_FLEXSPI_BASE + 0xE0)) & MG_BIT(1)) == 0)
    (void) 0;
}

static bool s_flash_irq_disabled;

MG_IRAM static bool mg_frdm_write(void *addr, const void *buf, size_t len) {
  struct mg_flexspi_nor_config config;
  bool ok = false;
  MG_ARM_DISABLE_IRQ();
  if (flash_init() != 0) goto fwxit;
  if (flexspi_nor_get_config(&config) != 0) goto fwxit;
  if ((len % s_mg_flash_frdm.align) != 0) {
    MG_ERROR(("%lu is not aligned to %lu", len, s_mg_flash_frdm.align));
    goto fwxit;
  }
  if ((char *) addr < (char *) s_mg_flash_frdm.start) {
    MG_ERROR(("Invalid flash write address: %p", addr));
    goto fwxit;
  }

  uint32_t *dst = (uint32_t *) addr;
  uint32_t *src = (uint32_t *) buf;
  uint32_t *end = (uint32_t *) ((char *) buf + len);
  ok = true;

  while (ok && src < end) {
    if (flash_page_start(dst) && flash_erase(&config, dst) == false) {
      ok = false;
      break;
    }
    uint32_t status;
    uint32_t dst_ofs = (uint32_t) dst - (uint32_t) s_mg_flash_frdm.start;
    size_t tmp_buf_size = s_mg_flash_frdm.align / sizeof(uint32_t);
    uint32_t tmp[tmp_buf_size];
    if ((char *) buf >= (char *) s_mg_flash_frdm.start &&
        (char *) buf <
            (char *) (s_mg_flash_frdm.start + s_mg_flash_frdm.size)) {
      // If we copy from FLASH to FLASH, then we first need to copy the source
      // to RAM
      for (size_t i = 0; i < tmp_buf_size; i++) {
        flash_wait();
        tmp[i] = src[i];
      }
    } else {
      memcpy(tmp, src, sizeof(tmp));
    }
    status = flexspi_nor->page_program(MG_FLEXSPI_NOR_INSTANCE, &config,
                                       (uint32_t) dst_ofs, tmp, false);
    src = (uint32_t *) ((char *) src + s_mg_flash_frdm.align);
    dst = (uint32_t *) ((char *) dst + s_mg_flash_frdm.align);
    if (status != 0) {
      ok = false;
    }
  }
  MG_INFO(("Flash write %lu bytes @ %p: %s.", len, dst, ok ? "ok" : "fail"));
fwxit:
  if (!s_flash_irq_disabled) MG_ARM_ENABLE_IRQ();
  return ok;
}

// just overwrite instead of swap
MG_IRAM static void single_bank_swap(char *p1, char *p2, size_t s, size_t ss) {
  // no stdlib calls here
  for (size_t ofs = 0; ofs < s; ofs += ss) {
    mg_frdm_write(p1 + ofs, p2 + ofs, ss);
  }
  *(volatile unsigned long *) 0xe000ed0c = 0x5fa0004;
}

bool mg_ota_begin(size_t new_firmware_size) {
  return mg_ota_flash_begin(new_firmware_size, &s_mg_flash_frdm);
}

bool mg_ota_write(const void *buf, size_t len) {
  return mg_ota_flash_write(buf, len, &s_mg_flash_frdm);
}

bool mg_ota_end(void) {
  if (mg_ota_flash_end(&s_mg_flash_frdm)) {
    if (0) {  // is_dualbank()
      // TODO(): no devices so far
      *(volatile unsigned long *) 0xe000ed0c = 0x5fa0004;
    } else {
      // Swap partitions. Pray power does not go away
      MG_INFO(("Swapping partitions, size %u (%u sectors)",
               s_mg_flash_frdm.size,
               s_mg_flash_frdm.size / s_mg_flash_frdm.secsz));
      MG_INFO(("Do NOT power off..."));
      mg_log_level = MG_LL_NONE;
      s_flash_irq_disabled = true;
      // Runs in RAM, will reset when finished
      single_bank_swap(
          (char *) s_mg_flash_frdm.start,
          (char *) s_mg_flash_frdm.start + s_mg_flash_frdm.size / 2,
          s_mg_flash_frdm.size / 2, s_mg_flash_frdm.secsz);
    }
  }
  return false;
}
struct mg_flash *mg_flash = &s_mg_flash_frdm;
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/ota_stm32f.c"
#endif




#if MG_OTA == MG_OTA_STM32F

static bool mg_stm32f_write(void *, const void *, size_t);
static bool mg_stm32f_swap(void);

static struct mg_flash s_mg_flash_stm32f = {
    (void *) 0x08000000,  // Start
    0,                    // Size, FLASH_SIZE_REG
    0,                    // Irregular sector size
    32,                   // Align, 256 bit
    mg_stm32f_write,
    mg_stm32f_swap,
};

#define MG_FLASH_BASE 0x40023c00
#define MG_FLASH_KEYR 0x04
#define MG_FLASH_SR 0x0c
#define MG_FLASH_CR 0x10
#define MG_FLASH_OPTCR 0x14
#define MG_FLASH_SIZE_REG_F7 0x1FF0F442
#define MG_FLASH_SIZE_REG_F4 0x1FFF7A22

#define STM_DBGMCU_IDCODE 0xE0042000
#define STM_DEV_ID (MG_REG(STM_DBGMCU_IDCODE) & (MG_BIT(12) - 1))
#define SYSCFG_MEMRMP 0x40013800

#define MG_FLASH_SIZE_REG_LOCATION \
  ((STM_DEV_ID >= 0x449) ? MG_FLASH_SIZE_REG_F7 : MG_FLASH_SIZE_REG_F4)

static size_t flash_size(void) {
  return (MG_REG(MG_FLASH_SIZE_REG_LOCATION) & 0xFFFF) * 1024;
}

MG_IRAM static size_t last_sector_size(void) {
  return (STM_DEV_ID >= 0x449 ? 256 : 128) * 1024;
}

MG_IRAM static int is_dualbank(void) {
  // only F42x/F43x series (0x419) support dual bank
  return STM_DEV_ID == 0x419;
}

MG_IRAM static void flash_unlock(void) {
  if (MG_REG(MG_FLASH_BASE + MG_FLASH_CR) & MG_BIT(31)) {
    MG_REG(MG_FLASH_BASE + MG_FLASH_KEYR) = 0x45670123;
    MG_REG(MG_FLASH_BASE + MG_FLASH_KEYR) = 0xcdef89ab;
  }
}

#define MG_FLASH_CONFIG_16_64_128 1   // used by STM32F7
#define MG_FLASH_CONFIG_32_128_256 2  // used by STM32F4 and F2

MG_IRAM static bool flash_page_start(volatile uint32_t *dst) {
  char *base = (char *) s_mg_flash_stm32f.start;
  char *end = base + s_mg_flash_stm32f.size;

  if (is_dualbank() && dst >= (uint32_t *) (base + (end - base) / 2)) {
    dst = (uint32_t *) ((uint32_t) dst - (end - base) / 2);
  }

  uint32_t flash_config = MG_FLASH_CONFIG_16_64_128;
  if (STM_DEV_ID >= 0x449) {
    flash_config = MG_FLASH_CONFIG_32_128_256;
  }

  volatile char *p = (char *) dst;
  if (p >= base && p < end) {
    if (p < base + 16 * 1024 * 4 * flash_config) {
      if ((p - base) % (16 * 1024 * flash_config) == 0) return true;
    } else if (p == base + 16 * 1024 * 4 * flash_config) {
      return true;
    } else if ((p - base) % (128 * 1024 * flash_config) == 0) {
      return true;
    }
  }
  return false;
}

MG_IRAM static int flash_sector(volatile uint32_t *addr) {
  char *base = (char *) s_mg_flash_stm32f.start;
  char *end = base + s_mg_flash_stm32f.size;
  bool addr_in_bank_2 = false;
  if (is_dualbank() && addr >= (uint32_t *) (base + (end - base) / 2)) {
    addr = (uint32_t *) ((uint32_t) addr - (end - base) / 2);
    addr_in_bank_2 = true;
  }
  volatile char *p = (char *) addr;
  uint32_t flash_config = MG_FLASH_CONFIG_16_64_128;
  if (STM_DEV_ID >= 0x449) {
    flash_config = MG_FLASH_CONFIG_32_128_256;
  }
  int sector = -1;
  if (p >= base && p < end) {
    if (p < base + 16 * 1024 * 4 * flash_config) {
      sector = (p - base) / (16 * 1024 * flash_config);
    } else if (p >= base + 64 * 1024 * flash_config &&
               p < base + 128 * 1024 * flash_config) {
      sector = 4;
    } else {
      sector = (p - base) / (128 * 1024 * flash_config) + 4;
    }
  }
  if (sector == -1) return -1;
  if (addr_in_bank_2) sector += 12;  // a bank has 12 sectors
  return sector;
}

MG_IRAM static bool flash_is_err(void) {
  return MG_REG(MG_FLASH_BASE + MG_FLASH_SR) & ((MG_BIT(7) - 1) << 1);
}

MG_IRAM static void flash_wait(void) {
  while (MG_REG(MG_FLASH_BASE + MG_FLASH_SR) & (MG_BIT(16))) (void) 0;
}

MG_IRAM static void flash_clear_err(void) {
  flash_wait();                                // Wait until ready
  MG_REG(MG_FLASH_BASE + MG_FLASH_SR) = 0xf2;  // Clear all errors
}

MG_IRAM static bool mg_stm32f_erase(void *addr) {
  bool ok = false;
  if (flash_page_start(addr) == false) {
    MG_ERROR(("%p is not on a sector boundary", addr));
  } else {
    int sector = flash_sector(addr);
    if (sector < 0) return false;
    uint32_t sector_reg = sector;
    if (is_dualbank() && sector >= 12) {
      // 3.9.8 Flash control register (FLASH_CR) for F42xxx and F43xxx
      // BITS[7:3]
      sector_reg -= 12;
      sector_reg |= MG_BIT(4);
    }
    flash_unlock();
    flash_wait();
    uint32_t cr = MG_BIT(1);       // SER
    cr |= MG_BIT(16);              // STRT
    cr |= (sector_reg & 31) << 3;  // sector
    MG_REG(MG_FLASH_BASE + MG_FLASH_CR) = cr;
    ok = !flash_is_err();
    MG_DEBUG(("Erase sector %lu @ %p %s. CR %#lx SR %#lx", sector, addr,
              ok ? "ok" : "fail", MG_REG(MG_FLASH_BASE + MG_FLASH_CR),
              MG_REG(MG_FLASH_BASE + MG_FLASH_SR)));
    // After we have erased the sector, set CR flags for programming
    // 2 << 8 is word write parallelism, bit(0) is PG. RM0385, section 3.7.5
    MG_REG(MG_FLASH_BASE + MG_FLASH_CR) = MG_BIT(0) | (2 << 8);
    flash_clear_err();
  }
  return ok;
}

MG_IRAM static void single_bank_swap(char *p1, char *p2, size_t s, size_t ss);

static bool s_flash_irq_disabled;

MG_IRAM static bool mg_stm32f_write(void *addr, const void *buf, size_t len) {
  if ((len % s_mg_flash_stm32f.align) != 0) {
    MG_ERROR(("%lu is not aligned to %lu", len, s_mg_flash_stm32f.align));
    return false;
  }
  uint32_t *dst = (uint32_t *) addr;
  uint32_t *src = (uint32_t *) buf;
  uint32_t *end = (uint32_t *) ((char *) buf + len);
  bool ok = true;
  MG_ARM_DISABLE_IRQ();
  flash_unlock();
  flash_clear_err();
  MG_REG(MG_FLASH_BASE + MG_FLASH_CR) = MG_BIT(0) | MG_BIT(9);  // PG, 32-bit
  flash_wait();
  MG_DEBUG(("Writing flash @ %p, %lu bytes", addr, len));
  while (ok && src < end) {
    if (flash_page_start(dst) && mg_stm32f_erase(dst) == false) break;
    *(volatile uint32_t *) dst++ = *src++;
    MG_DSB();  // ensure flash is written with no errors
    flash_wait();
    if (flash_is_err()) ok = false;
  }
  if (!s_flash_irq_disabled) MG_ARM_ENABLE_IRQ();
  MG_DEBUG(("Flash write %lu bytes @ %p: %s. CR %#lx SR %#lx", len, dst,
            ok ? "ok" : "fail", MG_REG(MG_FLASH_BASE + MG_FLASH_CR),
            MG_REG(MG_FLASH_BASE + MG_FLASH_SR)));
  MG_REG(MG_FLASH_BASE + MG_FLASH_CR) &= ~MG_BIT(0);  // Clear programming flag
  return ok;
}

MG_IRAM static bool mg_stm32f_swap(void) {
  // STM32 F42x/F43x support dual bank, however, the memory mapping
  // change will not be carried through a hard reset. Therefore, we use
  // the single-bank scratch-sector swap for this family as well.
  size_t ss = last_sector_size();
  char *p1 = (char *) s_mg_flash_stm32f.start;
  char *p2;
  size_t s;
  s_mg_flash_stm32f.size = flash_size();
  p2 = p1 + s_mg_flash_stm32f.size / 2;
  s = s_mg_flash_stm32f.size / 2 - ss;
  MG_INFO(("Swapping partitions, %u bytes", s));
  MG_INFO(("Do NOT power off..."));
  MG_OTA_ROLLBACK_TIMER_FEED();
  mg_log_level = MG_LL_NONE;
  s_flash_irq_disabled = true;
  single_bank_swap(p1, p2, s, ss);
  return true;  // unreachable
}

// True exchange between first half [p1..p1+s) and second half [p2..p2+s),
// using the last sector of the second half as scratch. Runs from RAM and resets.
MG_IRAM static void single_bank_swap(char *p1, char *p2, size_t s, size_t ss) {
  char *scratch = p2 + s;
  for (size_t ofs = 0; ofs < s; ofs += ss) {
    MG_OTA_ROLLBACK_TIMER_FEED();
    mg_stm32f_write(scratch, p1 + ofs, ss);
    MG_OTA_ROLLBACK_TIMER_FEED();
    mg_stm32f_write(p1 + ofs, p2 + ofs, ss);
    MG_OTA_ROLLBACK_TIMER_FEED();
    mg_stm32f_write(p2 + ofs, scratch, ss);
  }
  *(volatile uint32_t *) 0xe000ed0cU = 0x5fa0004U;  // NVIC_SystemReset()
}

bool mg_ota_begin(size_t new_firmware_size) {
  size_t max;
  s_mg_flash_stm32f.size = flash_size();
#ifdef __ZEPHYR__
  *((uint32_t *)0xE000ED94) = 0;
  MG_DEBUG(("Jailbreak %s", *((uint32_t *)0xE000ED94) == 0 ? "successful" : "failed"));
#endif
  max = s_mg_flash_stm32f.size / 2 - last_sector_size();
  if (new_firmware_size > max) {
    MG_ERROR(("Firmware %lu too big for single-bank OTA, max %lu",
              new_firmware_size, max));
    return false;
  }
  return mg_ota_flash_begin(new_firmware_size, &s_mg_flash_stm32f);
}

bool mg_ota_write(const void *buf, size_t len) {
  return mg_ota_flash_write(buf, len, &s_mg_flash_stm32f);
}

bool mg_ota_end(void) {
  mg_ota_flash_end(&s_mg_flash_stm32f);
  return false;
}
struct mg_flash *mg_flash = &s_mg_flash_stm32f;
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/ota_stm32h5.c"
#endif




#if MG_OTA == MG_OTA_STM32H5

static bool mg_stm32h5_write(void *, const void *, size_t);
static bool mg_stm32h5_swap(void);

static struct mg_flash s_mg_flash_stm32h5 = {
    (void *) 0x08000000,  // Start
    0,                    // Size, FLASHSIZE_BASE
    8 * 1024,             // Sector size, 8k
    16,                   // Align, 128 bit
    mg_stm32h5_write,
    mg_stm32h5_swap,
};

#define MG_FLASH_BASE 0x40022000          // Base address of the flash controller
#define FLASH_KEYR (MG_FLASH_BASE + 0x4)  // See RM0481 7.11
#define FLASH_OPTKEYR (MG_FLASH_BASE + 0xc)
#define FLASH_OPTCR (MG_FLASH_BASE + 0x1c)
#define FLASH_NSSR (MG_FLASH_BASE + 0x20)
#define FLASH_NSCR (MG_FLASH_BASE + 0x28)
#define FLASH_NSCCR (MG_FLASH_BASE + 0x30)
#define FLASH_OPTSR_CUR (MG_FLASH_BASE + 0x50)
#define FLASH_OPTSR_PRG (MG_FLASH_BASE + 0x54)
#ifndef FLASHSIZE_BASE
#define FLASHSIZE_BASE 0x08FFF80CUL
#endif

static size_t flash_size(void) {
  uint32_t kb = MG_REG(FLASHSIZE_BASE) & 0xFFFF;
  return (kb == 0 || kb == 0xFFFF) ? 2 * 1024 * 1024 : kb * 1024;
}

static uint32_t sectors_per_bank(void) {
  return (uint32_t) (s_mg_flash_stm32h5.size / 2 / s_mg_flash_stm32h5.secsz);
}

static void flash_unlock(void) {
  if (MG_REG(FLASH_NSCR) & MG_BIT(0)) {
    MG_REG(FLASH_KEYR) = 0x45670123;
    MG_REG(FLASH_KEYR) = 0xcdef89ab;
  }

  if (MG_REG(FLASH_OPTCR) & MG_BIT(0)) {
    MG_REG(FLASH_OPTKEYR) = 0x08192a3b;
    MG_REG(FLASH_OPTKEYR) = 0x4c5d6e7f;
  }
}

static int flash_page_start(volatile uint32_t *dst) {
  char *base = (char *) s_mg_flash_stm32h5.start,
       *end = base + s_mg_flash_stm32h5.size;
  volatile char *p = (char *) dst;
  return p >= base && p < end && ((p - base) % s_mg_flash_stm32h5.secsz) == 0;
}

static bool flash_is_err(void) {
  return MG_REG(FLASH_NSSR) & ((MG_BIT(8) - 1) << 17);  // RM0481 7.11.9
}

static void flash_wait(void) {
  while ((MG_REG(FLASH_NSSR) & MG_BIT(0)) &&
         (MG_REG(FLASH_NSSR) & MG_BIT(16)) == 0) {
    (void) 0;
  }
}

static void flash_clear_err(void) {
  flash_wait();                                    // Wait until ready
  MG_REG(FLASH_NSCCR) = ((MG_BIT(9) - 1) << 16U);  // Clear all errors
}

static bool flash_bank_is_swapped(void) {
  return MG_REG(FLASH_OPTCR) & MG_BIT(31);  // RM0481 7.11.8
}

static bool mg_stm32h5_erase(void *location) {
  bool ok = false;
  if (flash_page_start(location) == false) {
    MG_ERROR(("%p is not on a sector boundary"));
  } else {
    uintptr_t diff = (char *) location - (char *) s_mg_flash_stm32h5.start;
    uint32_t sector = diff / s_mg_flash_stm32h5.secsz;
    uint32_t bank_sectors = sectors_per_bank();
    uint32_t saved_cr = MG_REG(FLASH_NSCR);  // Save CR value
    flash_unlock();
    flash_clear_err();
    MG_REG(FLASH_NSCR) = 0;
    if ((sector < bank_sectors && flash_bank_is_swapped()) ||
        (sector >= bank_sectors && !flash_bank_is_swapped())) {
      MG_REG(FLASH_NSCR) |= MG_BIT(31);  // Set FLASH_CR_BKSEL
    }
    if (sector >= bank_sectors) sector -= bank_sectors;
    MG_REG(FLASH_NSCR) |= MG_BIT(2) | (sector << 6);  // Erase | sector_num
    MG_REG(FLASH_NSCR) |= MG_BIT(5);                  // Start erasing
    flash_wait();
    ok = !flash_is_err();
    MG_DEBUG(("Erase sector %lu @ %p: %s. CR %#lx SR %#lx", sector, location,
              ok ? "ok" : "fail", MG_REG(FLASH_NSCR), MG_REG(FLASH_NSSR)));
    // mg_hexdump(location, 32);
    MG_REG(FLASH_NSCR) = saved_cr;  // Restore saved CR
  }
  return ok;
}

static bool mg_stm32h5_swap(void) {
  uint32_t desired = flash_bank_is_swapped() ? 0 : MG_BIT(31);
  flash_unlock();
  flash_clear_err();
  MG_SET_BITS(MG_REG(FLASH_OPTSR_PRG), MG_BIT(31), desired);
  MG_REG(FLASH_OPTCR) |= MG_BIT(1);  // OPTSTART; triggers auto-reset on H5
  while ((MG_REG(FLASH_OPTSR_CUR) & MG_BIT(31)) != desired) (void) 0;
  return true;
}

static bool mg_stm32h5_write(void *addr, const void *buf, size_t len) {
  if ((len % s_mg_flash_stm32h5.align) != 0) {
    MG_ERROR(("%lu is not aligned to %lu", len, s_mg_flash_stm32h5.align));
    return false;
  }
  uint32_t *dst = (uint32_t *) addr;
  uint32_t *src = (uint32_t *) buf;
  uint32_t *end = (uint32_t *) ((char *) buf + len);
  bool ok = true;
  MG_ARM_DISABLE_IRQ();
  flash_unlock();
  flash_clear_err();
  MG_REG(FLASH_NSCR) = MG_BIT(1);  // Set programming flag
  while (ok && src < end) {
    if (flash_page_start(dst) && mg_stm32h5_erase(dst) == false) {
      ok = false;
      break;
    }
    *(volatile uint32_t *) dst++ = *src++;
    flash_wait();
    if (flash_is_err()) ok = false;
  }
  MG_ARM_ENABLE_IRQ();
  MG_DEBUG(("Flash write %lu bytes @ %p: %s. CR %#lx SR %#lx", len, dst,
            flash_is_err() ? "fail" : "ok", MG_REG(FLASH_NSCR),
            MG_REG(FLASH_NSSR)));
  MG_REG(FLASH_NSCR) = 0;  // Clear flags
  return ok;
}

bool mg_ota_begin(size_t new_firmware_size) {
  s_mg_flash_stm32h5.size = flash_size();
#ifdef __ZEPHYR__
  *((uint32_t *)0xE000ED94) = 0;
  MG_DEBUG(("Jailbreak %s", *((uint32_t *)0xE000ED94) == 0 ? "successful" : "failed"));
#endif
  return mg_ota_flash_begin(new_firmware_size, &s_mg_flash_stm32h5);
}

bool mg_ota_write(const void *buf, size_t len) {
  return mg_ota_flash_write(buf, len, &s_mg_flash_stm32h5);
}

bool mg_ota_end(void) {
  if (!mg_ota_flash_end(&s_mg_flash_stm32h5)) return false;
  *(volatile uint32_t *) 0xe000ed0c = 0x5fa0004U;  // NVIC_SystemReset()
  return true;
}

struct mg_flash *mg_flash = &s_mg_flash_stm32h5;
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/ota_stm32h7.c"
#endif




#if MG_OTA == MG_OTA_STM32H7 || MG_OTA == MG_OTA_STM32H7_DUAL_CORE

// - H723/735 RM 4.3.3: Note: The application can simultaneously request a read
// and a write operation through the AXI interface.
//   - We only need IRAM for partition swapping in the H723, however, all
//   related functions must reside in IRAM for this to be possible.
// - Linker files for other devices won't define a .iram section so there's no
// associated penalty

static bool mg_stm32h7_write(void *, const void *, size_t);
static bool mg_stm32h7_swap(void);

static struct mg_flash s_mg_flash_stm32h7 = {
    (void *) 0x08000000,  // Start
    0,                    // Size, FLASH_SIZE_REG
    128 * 1024,           // Sector size, 128k
    32,                   // Align, 256 bit
    mg_stm32h7_write,
    mg_stm32h7_swap,
};

#define FLASH_BASE1 0x52002000  // Base address for bank1
#define FLASH_BASE2 0x52002100  // Base address for bank2
#define FLASH_KEYR 0x04         // See RM0433 4.9.2
#define FLASH_OPTKEYR 0x08
#define FLASH_OPTCR 0x18
#define FLASH_SR 0x10
#define FLASH_CR 0x0c
#define FLASH_CCR 0x14
#define FLASH_OPTSR_CUR 0x1c
#define FLASH_OPTSR_PRG 0x20
#define FLASH_SIZE_REG 0x1ff1e880

#define IS_DUALCORE() (MG_OTA == MG_OTA_STM32H7_DUAL_CORE)

MG_IRAM static size_t flash_size(void) {
  size_t size = MG_REG(FLASH_SIZE_REG) * 1024;
  return IS_DUALCORE() ? size / 2 : size;
}

MG_IRAM static bool is_dualbank(void) {
  if (IS_DUALCORE()) {
    // H745/H755 and H747/H757 are running on dual core.
    // Using only the 1st bank (mapped to CM7), in order not to interfere
    // with the 2nd bank (CM4), possibly causing CM4 to boot unexpectedly.
    return false;
  }
  return (s_mg_flash_stm32h7.size < 2 * 1024 * 1024) ? false : true;
}

MG_IRAM static void flash_unlock(void) {
  if (MG_REG(FLASH_BASE1 + FLASH_CR) & MG_BIT(0)) {
	  MG_REG(FLASH_BASE1 + FLASH_KEYR) = 0x45670123;
	  MG_REG(FLASH_BASE1 + FLASH_KEYR) = 0xcdef89ab;
  }
  if (is_dualbank() && MG_REG(FLASH_BASE2 + FLASH_CR) & MG_BIT(0)) {
	  MG_REG(FLASH_BASE2 + FLASH_KEYR) = 0x45670123;
	  MG_REG(FLASH_BASE2 + FLASH_KEYR) = 0xcdef89ab;
  }
  if (MG_REG(FLASH_BASE1 + FLASH_OPTCR) & MG_BIT(0)) {
	  MG_REG(FLASH_BASE1 + FLASH_OPTKEYR) = 0x08192a3b;  // opt reg is "shared"
	  MG_REG(FLASH_BASE1 + FLASH_OPTKEYR) = 0x4c5d6e7f;  // thus unlock once
  }
}

MG_IRAM static bool flash_page_start(uint32_t *dst) {
  char *base = (char *) s_mg_flash_stm32h7.start,
       *end = base + s_mg_flash_stm32h7.size;
  char *p = (char *) dst;
  return p >= base && p < end && ((p - base) % s_mg_flash_stm32h7.secsz) == 0;
}

MG_IRAM static bool flash_is_err(uint32_t bank) {
  return MG_REG(bank + FLASH_SR) & ((MG_BIT(11) - 1) << 17);  // RM0433 4.9.5
}

MG_IRAM static void flash_wait(uint32_t bank) {
  while (MG_REG(bank + FLASH_SR) & (MG_BIT(0) | MG_BIT(2))) (void) 0;
}

MG_IRAM static void flash_clear_err(uint32_t bank) {
  flash_wait(bank);                                      // Wait until ready
  MG_REG(bank + FLASH_CCR) = ((MG_BIT(11) - 1) << 16U);  // Clear all errors
}

MG_IRAM static bool flash_bank_is_swapped(uint32_t bank) {
  return MG_REG(bank + FLASH_OPTCR) & MG_BIT(31);  // RM0433 4.9.7
}

// Figure out flash bank based on the address
MG_IRAM static uint32_t flash_bank(void *addr) {
  size_t ofs = (char *) addr - (char *) s_mg_flash_stm32h7.start;
  if (!is_dualbank()) return FLASH_BASE1;
  return ofs < s_mg_flash_stm32h7.size / 2 ? FLASH_BASE1 : FLASH_BASE2;
}

// read-while-write, no need to disable IRQs for standalone usage
MG_IRAM static bool mg_stm32h7_erase(void *addr) {
  bool ok = false;
  if (flash_page_start((uint32_t *) addr) == false) {
    MG_ERROR(("%p is not on a sector boundary", addr));
  } else {
    uintptr_t diff = (char *) addr - (char *) s_mg_flash_stm32h7.start;
    uint32_t sector = diff / s_mg_flash_stm32h7.secsz;
    uint32_t bank = flash_bank(addr);
    uint32_t saved_cr = MG_REG(bank + FLASH_CR);  // Save CR value

    flash_unlock();
    if (sector > 7) sector -= 8;

    flash_clear_err(bank);
    MG_REG(bank + FLASH_CR) = MG_BIT(5);             // 32-bit write parallelism
    MG_REG(bank + FLASH_CR) |= (sector & 7U) << 8U;  // Sector to erase
    MG_REG(bank + FLASH_CR) |= MG_BIT(2);            // Sector erase bit
    MG_REG(bank + FLASH_CR) |= MG_BIT(7);            // Start erasing
    flash_wait(bank);
    ok = !flash_is_err(bank);
    MG_DEBUG(("Erase sector %lu @ %p %s. CR %#lx SR %#lx", sector, addr,
              ok ? "ok" : "fail", MG_REG(bank + FLASH_CR),
              MG_REG(bank + FLASH_SR)));
    MG_REG(bank + FLASH_CR) = saved_cr;  // Restore CR
  }
  return ok;
}

MG_IRAM static void single_bank_swap(char *p1, char *p2, size_t s, size_t ss);

static bool s_flash_irq_disabled;

MG_IRAM static bool mg_stm32h7_swap(void) {
  s_mg_flash_stm32h7.size = flash_size();
  if (!is_dualbank()) {
    // True sector-by-sector swap using the last sector of the second half as
    // scratch. Resets inside single_bank_swap, never returns.
    size_t ss = s_mg_flash_stm32h7.secsz;
    char *p1 = (char *) s_mg_flash_stm32h7.start;
    char *p2 = p1 + s_mg_flash_stm32h7.size / 2;
    size_t s = s_mg_flash_stm32h7.size / 2 - ss;
    MG_INFO(("Swapping partitions, %u bytes (%u sectors)", s, s / ss));
    MG_INFO(("Do NOT power off..."));
    MG_OTA_ROLLBACK_TIMER_FEED();
    mg_log_level = MG_LL_NONE;
    s_flash_irq_disabled = true;
    single_bank_swap(p1, p2, s, ss);
    return true;  // unreachable
  }
  uint32_t bank = FLASH_BASE1;
  uint32_t desired = flash_bank_is_swapped(bank) ? 0 : MG_BIT(31);
  flash_unlock();
  flash_clear_err(bank);
  MG_SET_BITS(MG_REG(bank + FLASH_OPTSR_PRG), MG_BIT(31), desired);
  MG_REG(bank + FLASH_OPTCR) |= MG_BIT(1);  // OPTSTART
  while ((MG_REG(bank + FLASH_OPTSR_CUR) & MG_BIT(31)) != desired) (void) 0;
  return true;
}

MG_IRAM static bool mg_stm32h7_write(void *addr, const void *buf, size_t len) {
  if ((len % s_mg_flash_stm32h7.align) != 0) {
    MG_ERROR(("%lu is not aligned to %lu", len, s_mg_flash_stm32h7.align));
    return false;
  }
  uint32_t bank = flash_bank(addr);
  uint32_t *dst = (uint32_t *) addr;
  uint32_t *src = (uint32_t *) buf;
  uint32_t *end = (uint32_t *) ((char *) buf + len);
  bool ok = true;
  MG_ARM_DISABLE_IRQ();
  flash_unlock();
  flash_clear_err(bank);
  MG_REG(bank + FLASH_CR) = MG_BIT(1);   // Set programming flag
  MG_REG(bank + FLASH_CR) |= MG_BIT(5);  // 32-bit write parallelism
  while (ok && src < end) {
    if (flash_page_start(dst) && mg_stm32h7_erase(dst) == false) {
      ok = false;
      break;
    }
    *(volatile uint32_t *) dst++ = *src++;
    flash_wait(bank);
    if (flash_is_err(bank)) ok = false;
  }
  if (!s_flash_irq_disabled) MG_ARM_ENABLE_IRQ();
  MG_DEBUG(("Flash write %lu bytes @ %p: %s. CR %#lx SR %#lx", len, addr,
            ok ? "ok" : "fail", MG_REG(bank + FLASH_CR),
            MG_REG(bank + FLASH_SR)));
  MG_REG(bank + FLASH_CR) &= ~MG_BIT(1);  // Clear programming flag
  return ok;
}

// True sector-by-sector exchange between first half [p1..p1+s) and second half
// [p2..p2+s), using the sector at p2+s as scratch. Runs from RAM. Resets on
// completion. Symmetric: calling twice returns flash to original state.
MG_IRAM static void single_bank_swap(char *p1, char *p2, size_t s, size_t ss) {
  char *scratch = p2 + s;  // Last sector of second half, reserved as scratch
  for (size_t ofs = 0; ofs < s; ofs += ss) {
    MG_OTA_ROLLBACK_TIMER_FEED();
    mg_stm32h7_write(scratch, p1 + ofs, ss);   // Save p1[i] to scratch
    MG_OTA_ROLLBACK_TIMER_FEED();
    mg_stm32h7_write(p1 + ofs, p2 + ofs, ss);  // Copy p2[i] to p1[i]
    MG_OTA_ROLLBACK_TIMER_FEED();
    mg_stm32h7_write(p2 + ofs, scratch, ss);   // Copy scratch to p2[i]
  }
  *(volatile uint32_t *) 0xe000ed0cU = 0x5fa0004U;  // NVIC_SystemReset()
}

bool mg_ota_begin(size_t new_firmware_size) {
  s_mg_flash_stm32h7.size = flash_size();
#ifdef __ZEPHYR__
  *((uint32_t *) 0xE000ED94) = 0;
  MG_DEBUG(("Jailbreak %s", *((uint32_t *) 0xE000ED94) == 0 ? "ok" : "failed"));
#endif
  if (!is_dualbank()) {
    // Last sector of the second half is reserved as swap scratch; enforce limit.
    size_t max = s_mg_flash_stm32h7.size / 2 - s_mg_flash_stm32h7.secsz;
    if (new_firmware_size > max) {
      MG_ERROR(("Firmware %lu too big for single-bank OTA, max %lu", new_firmware_size, max));
      return false;
    }
  }
  return mg_ota_flash_begin(new_firmware_size, &s_mg_flash_stm32h7);
}

bool mg_ota_write(const void *buf, size_t len) {
  return mg_ota_flash_write(buf, len, &s_mg_flash_stm32h7);
}

bool mg_ota_end(void) {
  if (mg_ota_flash_end(&s_mg_flash_stm32h7)) {
    if (is_dualbank()) {
      // Bank swap is deferred until reset, been executing in flash, reset
      *(volatile uint32_t *) 0xe000ed0cU = 0x5fa0004U;  // NVIC_SystemReset()
    }
  }
  return false;
}

struct mg_flash *mg_flash = &s_mg_flash_stm32h7;
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/printf.c"
#endif




size_t mg_queue_printf(struct mg_queue *q, const char *fmt, ...) {
  char *buf;
  size_t len;
  va_list ap1, ap2;
  va_start(ap1, fmt);
  len = mg_vsnprintf(NULL, 0, fmt, &ap1);
  va_end(ap1);
  if (len == 0 || mg_queue_book(q, &buf, len + 1) < len + 1)
    return 0;  // Nah. Not enough space
  va_start(ap2, fmt);
  len = mg_vsnprintf(buf, len + 1, fmt, &ap2);
  mg_queue_add(q, len);
  va_end(ap2);
  return len;
}

static void mg_pfn_iobuf_private(char ch, void *param, bool expand) {
  struct mg_iobuf *io = (struct mg_iobuf *) param;
  if (expand && io->len + 2 > io->size) mg_iobuf_resize(io, io->len + 2);
  if (io->len + 2 <= io->size) {
    io->buf[io->len++] = (uint8_t) ch;
    io->buf[io->len] = 0;
  } else if (io->len < io->size) {
    io->buf[io->len++] = 0;  // Guarantee to 0-terminate
  }
}

void mg_pfn_iobuf_noresize(char ch, void *param) {
  mg_pfn_iobuf_private(ch, param, false);
}

void mg_pfn_iobuf(char ch, void *param) {
  mg_pfn_iobuf_private(ch, param, true);
}

size_t mg_vsnprintf(char *buf, size_t len, const char *fmt, va_list *ap) {
  struct mg_iobuf io = {0, 0, 0, 0};
  size_t n;
  io.buf = (uint8_t *) buf, io.size = len;
  n = mg_vxprintf(mg_pfn_iobuf_noresize, &io, fmt, ap);
  if (n < len) buf[n] = '\0';
  return n;
}

size_t mg_snprintf(char *buf, size_t len, const char *fmt, ...) {
  va_list ap;
  size_t n;
  va_start(ap, fmt);
  n = mg_vsnprintf(buf, len, fmt, &ap);
  va_end(ap);
  return n;
}

char *mg_vmprintf(const char *fmt, va_list *ap) {
  struct mg_iobuf io = {0, 0, 0, 256};
  mg_vxprintf(mg_pfn_iobuf, &io, fmt, ap);
  return (char *) io.buf;
}

char *mg_mprintf(const char *fmt, ...) {
  char *s;
  va_list ap;
  va_start(ap, fmt);
  s = mg_vmprintf(fmt, &ap);
  va_end(ap);
  return s;
}

void mg_pfn_stdout(char c, void *param) {
#if MG_ENABLE_POSIX_FS
  putchar(c);
#endif
  (void) param, (void) c;
}

static size_t print_ip4(void (*out)(char, void *), void *arg, uint8_t *p) {
  return mg_xprintf(out, arg, "%d.%d.%d.%d", p[0], p[1], p[2], p[3]);
}

static size_t print_ip6(void (*out)(char, void *), void *arg, uint16_t *p) {
  return mg_xprintf(out, arg, "[%x:%x:%x:%x:%x:%x:%x:%x]", mg_ntohs(p[0]),
                    mg_ntohs(p[1]), mg_ntohs(p[2]), mg_ntohs(p[3]),
                    mg_ntohs(p[4]), mg_ntohs(p[5]), mg_ntohs(p[6]),
                    mg_ntohs(p[7]));
}

size_t mg_print_ip4(void (*out)(char, void *), void *arg, va_list *ap) {
  uint8_t *p = va_arg(*ap, uint8_t *);
  return print_ip4(out, arg, p);
}

size_t mg_print_ip6(void (*out)(char, void *), void *arg, va_list *ap) {
  uint16_t *p = va_arg(*ap, uint16_t *);
  return print_ip6(out, arg, p);
}

size_t mg_print_ip(void (*out)(char, void *), void *arg, va_list *ap) {
  struct mg_addr *addr = va_arg(*ap, struct mg_addr *);
  if (addr->is_ip6) return print_ip6(out, arg, (uint16_t *) addr->addr.ip);
  return print_ip4(out, arg, (uint8_t *) &addr->addr.ip);
}

size_t mg_print_ip_port(void (*out)(char, void *), void *arg, va_list *ap) {
  struct mg_addr *a = va_arg(*ap, struct mg_addr *);
  return mg_xprintf(out, arg, "%M:%hu", mg_print_ip, a, mg_ntohs(a->port));
}

static size_t print_mac(void (*out)(char, void *), void *arg, uint8_t *p) {
  return mg_xprintf(out, arg, "%02x:%02x:%02x:%02x:%02x:%02x", p[0], p[1], p[2],
                    p[3], p[4], p[5]);
}

size_t mg_print_mac(void (*out)(char, void *), void *arg, va_list *ap) {
  uint8_t *p = va_arg(*ap, uint8_t *);
  return print_mac(out, arg, p);
}

static size_t print_ieee64(void (*out)(char, void *), void *arg, uint8_t *p) {
  return mg_xprintf(out, arg, "%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x", p[0],
                    p[1], p[2], p[3], p[4], p[5], p[6], p[7]);
}

size_t mg_print_ieee64(void (*out)(char, void *), void *arg, va_list *ap) {
  uint8_t *p = va_arg(*ap, uint8_t *);
  return print_ieee64(out, arg, p);
}

#if MG_ENABLE_TCPIP
size_t mg_print_l2addr(void (*out)(char, void *), void *arg, va_list *ap) {
  enum mg_l2type type = (enum mg_l2type) va_arg(*ap, int);
  switch (type) {
    case MG_TCPIP_L2_ETH:
    case MG_TCPIP_L2_PPPoE: {
      uint8_t *p = va_arg(*ap, uint8_t *);
      return print_mac(out, arg, p);
    } break;
    default: break;
  }
  return 0;
}
#endif

static char mg_esc(int c, bool esc) {
  const char *p, *esc1 = "\b\f\n\r\t\\\"", *esc2 = "bfnrt\\\"";
  for (p = esc ? esc1 : esc2; *p != '\0'; p++) {
    if (*p == c) return esc ? esc2[p - esc1] : esc1[p - esc2];
  }
  return 0;
}

static char mg_escape(int c) {
  return mg_esc(c, true);
}

static size_t qcpy(void (*out)(char, void *), void *ptr, char *buf,
                   size_t len) {
  size_t i = 0, extra = 0;
  for (i = 0; i < len && buf[i] != '\0'; i++) {
    char c = mg_escape(buf[i]);
    if (c) {
      out('\\', ptr), out(c, ptr), extra++;
    } else {
      out(buf[i], ptr);
    }
  }
  return i + extra;
}

static size_t bcpy(void (*out)(char, void *), void *arg, uint8_t *buf,
                   size_t len) {
  size_t i, j, n = 0;
  const char *t =
      "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
  for (i = 0; i < len; i += 3) {
    uint8_t c1 = buf[i], c2 = i + 1 < len ? buf[i + 1] : 0,
            c3 = i + 2 < len ? buf[i + 2] : 0;
    char tmp[4] = {0, 0, '=', '='};
    tmp[0] = t[c1 >> 2], tmp[1] = t[(c1 & 3) << 4 | (c2 >> 4)];
    if (i + 1 < len) tmp[2] = t[(c2 & 15) << 2 | (c3 >> 6)];
    if (i + 2 < len) tmp[3] = t[c3 & 63];
    for (j = 0; j < sizeof(tmp) && tmp[j] != '\0'; j++) out(tmp[j], arg);
    n += j;
  }
  return n;
}

size_t mg_print_hex(void (*out)(char, void *), void *arg, va_list *ap) {
  size_t bl = (size_t) va_arg(*ap, int);
  uint8_t *p = va_arg(*ap, uint8_t *);
  const char *hex = "0123456789abcdef";
  size_t j;
  for (j = 0; j < bl; j++) {
    out(hex[(p[j] >> 4) & 0x0F], arg);
    out(hex[p[j] & 0x0F], arg);
  }
  return 2 * bl;
}
size_t mg_print_base64(void (*out)(char, void *), void *arg, va_list *ap) {
  size_t len = (size_t) va_arg(*ap, int);
  uint8_t *buf = va_arg(*ap, uint8_t *);
  return bcpy(out, arg, buf, len);
}

size_t mg_print_esc(void (*out)(char, void *), void *arg, va_list *ap) {
  size_t len = (size_t) va_arg(*ap, int);
  char *p = va_arg(*ap, char *);
  if (len == 0) len = p == NULL ? 0 : strlen(p);
  return qcpy(out, arg, p, len);
}

size_t mg_print_html_esc(void (*out)(char, void *), void *arg, va_list *ap) {
  size_t i, n = 0;
  int len = va_arg(*ap, int);
  const char *s = va_arg(*ap, const char *);
  for (i = 0; i < (size_t) len; i++) {
    const char *esc = NULL;
    switch (s[i]) {
      // clang-format off
      case '&': esc = "&amp;"; break;
      case '<': esc = "&lt;"; break;
      case '>': esc = "&gt;"; break;
      case '"': esc = "&quot;"; break;
      default: break;
      // clang-format on
    }
    if (esc != NULL) {
      while (*esc != '\0') {
        out(*esc++, arg);
        n++;
      }
    } else {
      out(s[i], arg);
      n++;
    }
  }
  return n;
}

#ifdef MG_ENABLE_LINES
#line 1 "src/queue.c"
#endif



#if (defined(__GNUC__) && (__GNUC__ > 4) ||                                \
     (defined(__GNUC_MINOR__) && __GNUC__ == 4 && __GNUC_MINOR__ >= 1)) || \
    defined(__clang__)
#define MG_MEMORY_BARRIER() __sync_synchronize()
#elif defined(_MSC_VER) && _MSC_VER >= 1700
#define MG_MEMORY_BARRIER() MemoryBarrier()
#elif !defined(MG_MEMORY_BARRIER)
#define MG_MEMORY_BARRIER()
#endif

// Every message in a queue is prepended by a 32-bit message length (ML).
// If ML is 0, then it is the end, and reader must wrap to the beginning.
//
//  Queue when q->tail <= q->head:
//  |----- free -----| ML | message1 | ML | message2 |  ----- free ------|
//  ^                ^                               ^                   ^
// buf              tail                            head                len
//
//  Queue when q->tail > q->head:
//  | ML | message2 |----- free ------| ML | message1 | 0 |---- free ----|
//  ^               ^                 ^                                  ^
// buf             head              tail                               len

void mg_queue_init(struct mg_queue *q, char *buf, size_t size) {
  q->size = size;
  q->buf = buf;
  q->head = q->tail = 0;
}

static size_t mg_queue_read_len(struct mg_queue *q) {
  uint32_t n = 0;
  MG_MEMORY_BARRIER();
  memcpy(&n, q->buf + q->tail, sizeof(n));
  assert(q->tail + n + sizeof(n) <= q->size);
  return n;
}

static void mg_queue_write_len(struct mg_queue *q, size_t len) {
  uint32_t n = (uint32_t) len;
  memcpy(q->buf + q->head, &n, sizeof(n));
  MG_MEMORY_BARRIER();
}

size_t mg_queue_book(struct mg_queue *q, char **buf, size_t len) {
  size_t space = 0, hs = sizeof(uint32_t) * 2;  // *2 is for the 0 marker
  if (q->head >= q->tail && q->head + len + hs <= q->size) {
    space = q->size - q->head - hs;  // There is enough space
  } else if (q->head >= q->tail && q->tail > hs) {
    mg_queue_write_len(q, 0);  // Not enough space ahead
    q->head = 0;               // Wrap head to the beginning
  }
  if (q->head + hs + len < q->tail) space = q->tail - q->head - hs;
  if (buf != NULL) *buf = q->buf + q->head + sizeof(uint32_t);
  return space;
}

size_t mg_queue_next(struct mg_queue *q, char **buf) {
  size_t len = 0;
  if (q->tail != q->head) {
    len = mg_queue_read_len(q);
    if (len == 0) {  // Zero (head wrapped) ?
      q->tail = 0;   // Reset tail to the start
      if (q->head > q->tail) len = mg_queue_read_len(q);  // Read again
    }
  }
  if (buf != NULL) *buf = q->buf + q->tail + sizeof(uint32_t);
  assert(q->tail + len <= q->size);
  return len;
}

void mg_queue_add(struct mg_queue *q, size_t len) {
  assert(len > 0);
  mg_queue_write_len(q, len);
  assert(q->head + sizeof(uint32_t) * 2 + len <= q->size);
  q->head += len + sizeof(uint32_t);
}

void mg_queue_del(struct mg_queue *q, size_t len) {
  q->tail += len + sizeof(uint32_t);
  assert(q->tail + sizeof(uint32_t) <= q->size);
}

#ifdef MG_ENABLE_LINES
#line 1 "src/rpc.c"
#endif




void mg_rpc_add(struct mg_rpc **head, struct mg_str method,
                void (*fn)(struct mg_rpc_req *), void *fn_data) {
  struct mg_rpc *rpc = (struct mg_rpc *) mg_calloc(1, sizeof(*rpc));
  if (rpc != NULL) {
    rpc->method = mg_strdup(method);
    rpc->fn = fn;
    rpc->fn_data = fn_data;
    rpc->next = *head, *head = rpc;
  }
}

void mg_rpc_del(struct mg_rpc **head, void (*fn)(struct mg_rpc_req *)) {
  struct mg_rpc *r;
  while ((r = *head) != NULL) {
    if (r->fn == fn || fn == NULL) {
      *head = r->next;
      mg_free((void *) r->method.buf);
      mg_free(r);
    } else {
      head = &(*head)->next;
    }
  }
}

static void mg_rpc_call(struct mg_rpc_req *r, struct mg_str method) {
  struct mg_rpc *h = r->head == NULL ? NULL : *r->head;
  while (h != NULL && !mg_match(method, h->method, NULL)) h = h->next;
  if (h != NULL) {
    r->rpc = h;
    h->fn(r);
  } else {
    mg_rpc_err(r, -32601, "\"%.*s not found\"", (int) method.len, method.buf);
  }
}

void mg_rpc_process(struct mg_rpc_req *r) {
  int len, off = mg_json_get(r->frame, "$.method", &len);
  if (off > 0 && r->frame.buf[off] == '"') {
    struct mg_str method = mg_str_n(&r->frame.buf[off + 1], (size_t) len - 2);
    mg_rpc_call(r, method);
  } else if ((off = mg_json_get(r->frame, "$.result", &len)) > 0 ||
             (off = mg_json_get(r->frame, "$.error", &len)) > 0) {
    mg_rpc_call(r, mg_str(""));  // JSON response! call "" method handler
  } else {
    mg_rpc_err(r, -32700, "%m", mg_print_esc, (int) r->frame.len,
               r->frame.buf);  // Invalid
  }
}

void mg_rpc_vok(struct mg_rpc_req *r, const char *fmt, va_list *ap) {
  int len, off = mg_json_get(r->frame, "$.id", &len);
  if (off > 0) {
    mg_xprintf(r->pfn, r->pfn_data, "{%m:%.*s,%m:", mg_print_esc, 0, "id", len,
               &r->frame.buf[off], mg_print_esc, 0, "result");
    mg_vxprintf(r->pfn, r->pfn_data, fmt == NULL ? "null" : fmt, ap);
    mg_xprintf(r->pfn, r->pfn_data, "}");
  }
}

void mg_rpc_ok(struct mg_rpc_req *r, const char *fmt, ...) {
  va_list ap;
  va_start(ap, fmt);
  mg_rpc_vok(r, fmt, &ap);
  va_end(ap);
}

void mg_rpc_verr(struct mg_rpc_req *r, int code, const char *fmt, va_list *ap) {
  int len, off = mg_json_get(r->frame, "$.id", &len);
  mg_xprintf(r->pfn, r->pfn_data, "{");
  if (off > 0) {
    mg_xprintf(r->pfn, r->pfn_data, "%m:%.*s,", mg_print_esc, 0, "id", len,
               &r->frame.buf[off]);
  }
  mg_xprintf(r->pfn, r->pfn_data, "%m:{%m:%d,%m:", mg_print_esc, 0, "error",
             mg_print_esc, 0, "code", code, mg_print_esc, 0, "message");
  mg_vxprintf(r->pfn, r->pfn_data, fmt == NULL ? "null" : fmt, ap);
  mg_xprintf(r->pfn, r->pfn_data, "}}");
}

void mg_rpc_err(struct mg_rpc_req *r, int code, const char *fmt, ...) {
  va_list ap;
  va_start(ap, fmt);
  mg_rpc_verr(r, code, fmt, &ap);
  va_end(ap);
}

static size_t print_methods(mg_pfn_t pfn, void *pfn_data, va_list *ap) {
  struct mg_rpc *h, **head = (struct mg_rpc **) va_arg(*ap, void **);
  size_t len = 0;
  for (h = *head; h != NULL; h = h->next) {
    if (h->method.len == 0) continue;  // Ignore response handler
    len += mg_xprintf(pfn, pfn_data, "%s%m", h == *head ? "" : ",",
                      mg_print_esc, (int) h->method.len, h->method.buf);
  }
  return len;
}

void mg_rpc_list(struct mg_rpc_req *r) {
  mg_rpc_ok(r, "[%M]", print_methods, r->head);
}

#ifdef MG_ENABLE_LINES
#line 1 "src/sha1.c"
#endif
/* Copyright(c) By Steve Reid <steve@edmweb.com> */
/* 100% Public Domain */



union char64long16 {
  unsigned char c[64];
  uint32_t l[16];
};

#define rol(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits))))

static uint32_t blk0(union char64long16 *block, int i) {
  if (MG_BIG_ENDIAN) {
  } else {
    block->l[i] = (rol(block->l[i], 24) & 0xFF00FF00) |
                  (rol(block->l[i], 8) & 0x00FF00FF);
  }
  return block->l[i];
}

/* Avoid redefine warning (ARM /usr/include/sys/ucontext.h define R0~R4) */
#undef blk
#undef R0
#undef R1
#undef R2
#undef R3
#undef R4

#define blk(i)                                                               \
  (block->l[i & 15] = rol(block->l[(i + 13) & 15] ^ block->l[(i + 8) & 15] ^ \
                              block->l[(i + 2) & 15] ^ block->l[i & 15],     \
                          1))
#define R0(v, w, x, y, z, i)                                          \
  z += ((w & (x ^ y)) ^ y) + blk0(block, i) + 0x5A827999 + rol(v, 5); \
  w = rol(w, 30);
#define R1(v, w, x, y, z, i)                                  \
  z += ((w & (x ^ y)) ^ y) + blk(i) + 0x5A827999 + rol(v, 5); \
  w = rol(w, 30);
#define R2(v, w, x, y, z, i)                          \
  z += (w ^ x ^ y) + blk(i) + 0x6ED9EBA1 + rol(v, 5); \
  w = rol(w, 30);
#define R3(v, w, x, y, z, i)                                        \
  z += (((w | x) & y) | (w & x)) + blk(i) + 0x8F1BBCDC + rol(v, 5); \
  w = rol(w, 30);
#define R4(v, w, x, y, z, i)                          \
  z += (w ^ x ^ y) + blk(i) + 0xCA62C1D6 + rol(v, 5); \
  w = rol(w, 30);

static void mg_sha1_transform(uint32_t state[5],
                              const unsigned char *buffer) {
  uint32_t a, b, c, d, e;
  union char64long16 block[1];

  memcpy(block, buffer, 64);
  a = state[0];
  b = state[1];
  c = state[2];
  d = state[3];
  e = state[4];
  R0(a, b, c, d, e, 0);
  R0(e, a, b, c, d, 1);
  R0(d, e, a, b, c, 2);
  R0(c, d, e, a, b, 3);
  R0(b, c, d, e, a, 4);
  R0(a, b, c, d, e, 5);
  R0(e, a, b, c, d, 6);
  R0(d, e, a, b, c, 7);
  R0(c, d, e, a, b, 8);
  R0(b, c, d, e, a, 9);
  R0(a, b, c, d, e, 10);
  R0(e, a, b, c, d, 11);
  R0(d, e, a, b, c, 12);
  R0(c, d, e, a, b, 13);
  R0(b, c, d, e, a, 14);
  R0(a, b, c, d, e, 15);
  R1(e, a, b, c, d, 16);
  R1(d, e, a, b, c, 17);
  R1(c, d, e, a, b, 18);
  R1(b, c, d, e, a, 19);
  R2(a, b, c, d, e, 20);
  R2(e, a, b, c, d, 21);
  R2(d, e, a, b, c, 22);
  R2(c, d, e, a, b, 23);
  R2(b, c, d, e, a, 24);
  R2(a, b, c, d, e, 25);
  R2(e, a, b, c, d, 26);
  R2(d, e, a, b, c, 27);
  R2(c, d, e, a, b, 28);
  R2(b, c, d, e, a, 29);
  R2(a, b, c, d, e, 30);
  R2(e, a, b, c, d, 31);
  R2(d, e, a, b, c, 32);
  R2(c, d, e, a, b, 33);
  R2(b, c, d, e, a, 34);
  R2(a, b, c, d, e, 35);
  R2(e, a, b, c, d, 36);
  R2(d, e, a, b, c, 37);
  R2(c, d, e, a, b, 38);
  R2(b, c, d, e, a, 39);
  R3(a, b, c, d, e, 40);
  R3(e, a, b, c, d, 41);
  R3(d, e, a, b, c, 42);
  R3(c, d, e, a, b, 43);
  R3(b, c, d, e, a, 44);
  R3(a, b, c, d, e, 45);
  R3(e, a, b, c, d, 46);
  R3(d, e, a, b, c, 47);
  R3(c, d, e, a, b, 48);
  R3(b, c, d, e, a, 49);
  R3(a, b, c, d, e, 50);
  R3(e, a, b, c, d, 51);
  R3(d, e, a, b, c, 52);
  R3(c, d, e, a, b, 53);
  R3(b, c, d, e, a, 54);
  R3(a, b, c, d, e, 55);
  R3(e, a, b, c, d, 56);
  R3(d, e, a, b, c, 57);
  R3(c, d, e, a, b, 58);
  R3(b, c, d, e, a, 59);
  R4(a, b, c, d, e, 60);
  R4(e, a, b, c, d, 61);
  R4(d, e, a, b, c, 62);
  R4(c, d, e, a, b, 63);
  R4(b, c, d, e, a, 64);
  R4(a, b, c, d, e, 65);
  R4(e, a, b, c, d, 66);
  R4(d, e, a, b, c, 67);
  R4(c, d, e, a, b, 68);
  R4(b, c, d, e, a, 69);
  R4(a, b, c, d, e, 70);
  R4(e, a, b, c, d, 71);
  R4(d, e, a, b, c, 72);
  R4(c, d, e, a, b, 73);
  R4(b, c, d, e, a, 74);
  R4(a, b, c, d, e, 75);
  R4(e, a, b, c, d, 76);
  R4(d, e, a, b, c, 77);
  R4(c, d, e, a, b, 78);
  R4(b, c, d, e, a, 79);
  state[0] += a;
  state[1] += b;
  state[2] += c;
  state[3] += d;
  state[4] += e;
  /* Erase working structures. The order of operations is important,
   * used to ensure that compiler doesn't optimize those out. */
  memset(block, 0, sizeof(block));
  a = b = c = d = e = 0;
  (void) a;
  (void) b;
  (void) c;
  (void) d;
  (void) e;
}

void mg_sha1_init(mg_sha1_ctx *context) {
  context->state[0] = 0x67452301;
  context->state[1] = 0xEFCDAB89;
  context->state[2] = 0x98BADCFE;
  context->state[3] = 0x10325476;
  context->state[4] = 0xC3D2E1F0;
  context->count[0] = context->count[1] = 0;
}

void mg_sha1_update(mg_sha1_ctx *context, const unsigned char *data,
                    size_t len) {
  size_t i, j;

  j = context->count[0];
  if ((context->count[0] += (uint32_t) len << 3) < j) context->count[1]++;
  context->count[1] += (uint32_t) (len >> 29);
  j = (j >> 3) & 63;
  if ((j + len) > 63) {
    memcpy(&context->buffer[j], data, (i = 64 - j));
    mg_sha1_transform(context->state, context->buffer);
    for (; i + 63 < len; i += 64) {
      mg_sha1_transform(context->state, &data[i]);
    }
    j = 0;
  } else
    i = 0;
  memcpy(&context->buffer[j], &data[i], len - i);
}

void mg_sha1_final(unsigned char digest[20], mg_sha1_ctx *context) {
  unsigned i;
  unsigned char finalcount[8], c;

  for (i = 0; i < 8; i++) {
    finalcount[i] = (unsigned char) ((context->count[(i >= 4 ? 0 : 1)] >>
                                      ((3 - (i & 3)) * 8)) &
                                     255);
  }
  c = 0200;
  mg_sha1_update(context, &c, 1);
  while ((context->count[0] & 504) != 448) {
    c = 0000;
    mg_sha1_update(context, &c, 1);
  }
  mg_sha1_update(context, finalcount, 8);
  for (i = 0; i < 20; i++) {
    digest[i] =
        (unsigned char) ((context->state[i >> 2] >> ((3 - (i & 3)) * 8)) & 255);
  }
  memset(context, '\0', sizeof(*context));
  memset(&finalcount, '\0', sizeof(finalcount));
}

#ifdef MG_ENABLE_LINES
#line 1 "src/sha256.c"
#endif
// https://github.com/B-Con/crypto-algorithms
// Author:     Brad Conte (brad AT bradconte.com)
// Disclaimer: This code is presented "as is" without any guarantees.
// Details:    Defines the API for the corresponding SHA1 implementation.
// Copyright:  public domain



#define ror(x, n) (((x) >> (n)) | ((x) << (32 - (n))))
#define ch(x, y, z) (((x) & (y)) ^ (~(x) & (z)))
#define maj(x, y, z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z)))
#define ep0(x) (ror(x, 2) ^ ror(x, 13) ^ ror(x, 22))
#define ep1(x) (ror(x, 6) ^ ror(x, 11) ^ ror(x, 25))
#define sig0(x) (ror(x, 7) ^ ror(x, 18) ^ ((x) >> 3))
#define sig1(x) (ror(x, 17) ^ ror(x, 19) ^ ((x) >> 10))

static const uint32_t mg_sha256_k[64] = {
    0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1,
    0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
    0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786,
    0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
    0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147,
    0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
    0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b,
    0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
    0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a,
    0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
    0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2};

void mg_sha256_init(mg_sha256_ctx *ctx) {
  ctx->len = 0;
  ctx->bits = 0;
  ctx->state[0] = 0x6a09e667;
  ctx->state[1] = 0xbb67ae85;
  ctx->state[2] = 0x3c6ef372;
  ctx->state[3] = 0xa54ff53a;
  ctx->state[4] = 0x510e527f;
  ctx->state[5] = 0x9b05688c;
  ctx->state[6] = 0x1f83d9ab;
  ctx->state[7] = 0x5be0cd19;
}

static void mg_sha256_chunk(mg_sha256_ctx *ctx) {
  int i, j;
  uint32_t a, b, c, d, e, f, g, h;
  uint32_t m[64];
  for (i = 0, j = 0; i < 16; ++i, j += 4)
    m[i] = (uint32_t) (((uint32_t) ctx->buffer[j] << 24) |
                       ((uint32_t) ctx->buffer[j + 1] << 16) |
                       ((uint32_t) ctx->buffer[j + 2] << 8) |
                       ((uint32_t) ctx->buffer[j + 3]));
  for (; i < 64; ++i)
    m[i] = sig1(m[i - 2]) + m[i - 7] + sig0(m[i - 15]) + m[i - 16];

  a = ctx->state[0];
  b = ctx->state[1];
  c = ctx->state[2];
  d = ctx->state[3];
  e = ctx->state[4];
  f = ctx->state[5];
  g = ctx->state[6];
  h = ctx->state[7];

  for (i = 0; i < 64; ++i) {
    uint32_t t1 = h + ep1(e) + ch(e, f, g) + mg_sha256_k[i] + m[i];
    uint32_t t2 = ep0(a) + maj(a, b, c);
    h = g;
    g = f;
    f = e;
    e = d + t1;
    d = c;
    c = b;
    b = a;
    a = t1 + t2;
  }

  ctx->state[0] += a;
  ctx->state[1] += b;
  ctx->state[2] += c;
  ctx->state[3] += d;
  ctx->state[4] += e;
  ctx->state[5] += f;
  ctx->state[6] += g;
  ctx->state[7] += h;
}

void mg_sha256_update(mg_sha256_ctx *ctx, const unsigned char *data,
                      size_t len) {
  size_t i;
  for (i = 0; i < len; i++) {
    ctx->buffer[ctx->len] = data[i];
    if ((++ctx->len) == 64) {
      mg_sha256_chunk(ctx);
      ctx->bits += 512;
      ctx->len = 0;
    }
  }
}

// TODO: make final reusable (remove side effects)
void mg_sha256_final(unsigned char digest[32], mg_sha256_ctx *ctx) {
  uint32_t i = ctx->len;
  if (i < 56) {
    ctx->buffer[i++] = 0x80;
    while (i < 56) {
      ctx->buffer[i++] = 0x00;
    }
  } else {
    ctx->buffer[i++] = 0x80;
    while (i < 64) {
      ctx->buffer[i++] = 0x00;
    }
    mg_sha256_chunk(ctx);
    memset(ctx->buffer, 0, 56);
  }

  ctx->bits += ctx->len * 8;
  ctx->buffer[63] = (uint8_t) ((ctx->bits) & 0xff);
  ctx->buffer[62] = (uint8_t) ((ctx->bits >> 8) & 0xff);
  ctx->buffer[61] = (uint8_t) ((ctx->bits >> 16) & 0xff);
  ctx->buffer[60] = (uint8_t) ((ctx->bits >> 24) & 0xff);
  ctx->buffer[59] = (uint8_t) ((ctx->bits >> 32) & 0xff);
  ctx->buffer[58] = (uint8_t) ((ctx->bits >> 40) & 0xff);
  ctx->buffer[57] = (uint8_t) ((ctx->bits >> 48) & 0xff);
  ctx->buffer[56] = (uint8_t) ((ctx->bits >> 56) & 0xff);
  mg_sha256_chunk(ctx);

  for (i = 0; i < 4; ++i) {
    digest[i] = (uint8_t) ((ctx->state[0] >> (24 - i * 8)) & 0xff);
    digest[i + 4] = (uint8_t) ((ctx->state[1] >> (24 - i * 8)) & 0xff);
    digest[i + 8] = (uint8_t) ((ctx->state[2] >> (24 - i * 8)) & 0xff);
    digest[i + 12] = (uint8_t) ((ctx->state[3] >> (24 - i * 8)) & 0xff);
    digest[i + 16] = (uint8_t) ((ctx->state[4] >> (24 - i * 8)) & 0xff);
    digest[i + 20] = (uint8_t) ((ctx->state[5] >> (24 - i * 8)) & 0xff);
    digest[i + 24] = (uint8_t) ((ctx->state[6] >> (24 - i * 8)) & 0xff);
    digest[i + 28] = (uint8_t) ((ctx->state[7] >> (24 - i * 8)) & 0xff);
  }
}

void mg_sha256(uint8_t dst[32], uint8_t *data, size_t datasz) {
  mg_sha256_ctx ctx;
  mg_sha256_init(&ctx);
  mg_sha256_update(&ctx, data, datasz);
  mg_sha256_final(dst, &ctx);
}

void mg_hmac_sha256(uint8_t dst[32], uint8_t *key, size_t keysz, uint8_t *data,
                    size_t datasz) {
  mg_sha256_ctx ctx;
  uint8_t k[64] = {0};
  uint8_t o_pad[64], i_pad[64];
  unsigned int i;
  memset(i_pad, 0x36, sizeof(i_pad));
  memset(o_pad, 0x5c, sizeof(o_pad));
  if (keysz < 64) {
    if (keysz > 0) memmove(k, key, keysz);
  } else {
    mg_sha256_init(&ctx);
    mg_sha256_update(&ctx, key, keysz);
    mg_sha256_final(k, &ctx);
  }
  for (i = 0; i < sizeof(k); i++) {
    i_pad[i] ^= k[i];
    o_pad[i] ^= k[i];
  }
  mg_sha256_init(&ctx);
  mg_sha256_update(&ctx, i_pad, sizeof(i_pad));
  mg_sha256_update(&ctx, data, datasz);
  mg_sha256_final(dst, &ctx);
  mg_sha256_init(&ctx);
  mg_sha256_update(&ctx, o_pad, sizeof(o_pad));
  mg_sha256_update(&ctx, dst, 32);
  mg_sha256_final(dst, &ctx);
}

#define rotr64(x, n) (((x) >> (n)) | ((x) << (64 - (n))))
#define ep064(x) (rotr64(x, 28) ^ rotr64(x, 34) ^ rotr64(x, 39))
#define ep164(x) (rotr64(x, 14) ^ rotr64(x, 18) ^ rotr64(x, 41))
#define sig064(x) (rotr64(x, 1) ^ rotr64(x, 8) ^ ((x) >> 7))
#define sig164(x) (rotr64(x, 19) ^ rotr64(x, 61) ^ ((x) >> 6))

static const uint64_t mg_sha256_k2[80] = {
#if defined(__DCC__)
    0x428a2f98d728ae22ull, 0x7137449123ef65cdull, 0xb5c0fbcfec4d3b2full,
    0xe9b5dba58189dbbcull, 0x3956c25bf348b538ull, 0x59f111f1b605d019ull,
    0x923f82a4af194f9bull, 0xab1c5ed5da6d8118ull, 0xd807aa98a3030242ull,
    0x12835b0145706fbeull, 0x243185be4ee4b28cull, 0x550c7dc3d5ffb4e2ull,
    0x72be5d74f27b896full, 0x80deb1fe3b1696b1ull, 0x9bdc06a725c71235ull,
    0xc19bf174cf692694ull, 0xe49b69c19ef14ad2ull, 0xefbe4786384f25e3ull,
    0x0fc19dc68b8cd5b5ull, 0x240ca1cc77ac9c65ull, 0x2de92c6f592b0275ull,
    0x4a7484aa6ea6e483ull, 0x5cb0a9dcbd41fbd4ull, 0x76f988da831153b5ull,
    0x983e5152ee66dfabull, 0xa831c66d2db43210ull, 0xb00327c898fb213full,
    0xbf597fc7beef0ee4ull, 0xc6e00bf33da88fc2ull, 0xd5a79147930aa725ull,
    0x06ca6351e003826full, 0x142929670a0e6e70ull, 0x27b70a8546d22ffcull,
    0x2e1b21385c26c926ull, 0x4d2c6dfc5ac42aedull, 0x53380d139d95b3dfull,
    0x650a73548baf63deull, 0x766a0abb3c77b2a8ull, 0x81c2c92e47edaee6ull,
    0x92722c851482353bull, 0xa2bfe8a14cf10364ull, 0xa81a664bbc423001ull,
    0xc24b8b70d0f89791ull, 0xc76c51a30654be30ull, 0xd192e819d6ef5218ull,
    0xd69906245565a910ull, 0xf40e35855771202aull, 0x106aa07032bbd1b8ull,
    0x19a4c116b8d2d0c8ull, 0x1e376c085141ab53ull, 0x2748774cdf8eeb99ull,
    0x34b0bcb5e19b48a8ull, 0x391c0cb3c5c95a63ull, 0x4ed8aa4ae3418acbull,
    0x5b9cca4f7763e373ull, 0x682e6ff3d6b2b8a3ull, 0x748f82ee5defb2fcull,
    0x78a5636f43172f60ull, 0x84c87814a1f0ab72ull, 0x8cc702081a6439ecull,
    0x90befffa23631e28ull, 0xa4506cebde82bde9ull, 0xbef9a3f7b2c67915ull,
    0xc67178f2e372532bull, 0xca273eceea26619cull, 0xd186b8c721c0c207ull,
    0xeada7dd6cde0eb1eull, 0xf57d4f7fee6ed178ull, 0x06f067aa72176fbaull,
    0x0a637dc5a2c898a6ull, 0x113f9804bef90daeull, 0x1b710b35131c471bull,
    0x28db77f523047d84ull, 0x32caab7b40c72493ull, 0x3c9ebe0a15c9bebcull,
    0x431d67c49c100d4cull, 0x4cc5d4becb3e42b6ull, 0x597f299cfc657e2aull,
    0x5fcb6fab3ad6faecull, 0x6c44198c4a475817ull
#else
    0x428a2f98d728ae22, 0x7137449123ef65cd, 0xb5c0fbcfec4d3b2f,
    0xe9b5dba58189dbbc, 0x3956c25bf348b538, 0x59f111f1b605d019,
    0x923f82a4af194f9b, 0xab1c5ed5da6d8118, 0xd807aa98a3030242,
    0x12835b0145706fbe, 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2,
    0x72be5d74f27b896f, 0x80deb1fe3b1696b1, 0x9bdc06a725c71235,
    0xc19bf174cf692694, 0xe49b69c19ef14ad2, 0xefbe4786384f25e3,
    0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65, 0x2de92c6f592b0275,
    0x4a7484aa6ea6e483, 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5,
    0x983e5152ee66dfab, 0xa831c66d2db43210, 0xb00327c898fb213f,
    0xbf597fc7beef0ee4, 0xc6e00bf33da88fc2, 0xd5a79147930aa725,
    0x06ca6351e003826f, 0x142929670a0e6e70, 0x27b70a8546d22ffc,
    0x2e1b21385c26c926, 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df,
    0x650a73548baf63de, 0x766a0abb3c77b2a8, 0x81c2c92e47edaee6,
    0x92722c851482353b, 0xa2bfe8a14cf10364, 0xa81a664bbc423001,
    0xc24b8b70d0f89791, 0xc76c51a30654be30, 0xd192e819d6ef5218,
    0xd69906245565a910, 0xf40e35855771202a, 0x106aa07032bbd1b8,
    0x19a4c116b8d2d0c8, 0x1e376c085141ab53, 0x2748774cdf8eeb99,
    0x34b0bcb5e19b48a8, 0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb,
    0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3, 0x748f82ee5defb2fc,
    0x78a5636f43172f60, 0x84c87814a1f0ab72, 0x8cc702081a6439ec,
    0x90befffa23631e28, 0xa4506cebde82bde9, 0xbef9a3f7b2c67915,
    0xc67178f2e372532b, 0xca273eceea26619c, 0xd186b8c721c0c207,
    0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178, 0x06f067aa72176fba,
    0x0a637dc5a2c898a6, 0x113f9804bef90dae, 0x1b710b35131c471b,
    0x28db77f523047d84, 0x32caab7b40c72493, 0x3c9ebe0a15c9bebc,
    0x431d67c49c100d4c, 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a,
    0x5fcb6fab3ad6faec, 0x6c44198c4a475817
#endif
};

static void mg_sha384_transform(mg_sha384_ctx *ctx, const uint8_t data[]) {
  uint64_t m[80];
  uint64_t a, b, c, d, e, f, g, h;
  int i, j;

  for (i = 0, j = 0; i < 16; ++i, j += 8)
    m[i] = ((uint64_t) data[j] << 56) | ((uint64_t) data[j + 1] << 48) |
           ((uint64_t) data[j + 2] << 40) | ((uint64_t) data[j + 3] << 32) |
           ((uint64_t) data[j + 4] << 24) | ((uint64_t) data[j + 5] << 16) |
           ((uint64_t) data[j + 6] << 8) | ((uint64_t) data[j + 7]);
  for (; i < 80; ++i)
    m[i] = sig164(m[i - 2]) + m[i - 7] + sig064(m[i - 15]) + m[i - 16];

  a = ctx->state[0];
  b = ctx->state[1];
  c = ctx->state[2];
  d = ctx->state[3];
  e = ctx->state[4];
  f = ctx->state[5];
  g = ctx->state[6];
  h = ctx->state[7];

  for (i = 0; i < 80; ++i) {
    uint64_t t1 = h + ep164(e) + ch(e, f, g) + mg_sha256_k2[i] + m[i];
    uint64_t t2 = ep064(a) + maj(a, b, c);
    h = g;
    g = f;
    f = e;
    e = d + t1;
    d = c;
    c = b;
    b = a;
    a = t1 + t2;
  }

  ctx->state[0] += a;
  ctx->state[1] += b;
  ctx->state[2] += c;
  ctx->state[3] += d;
  ctx->state[4] += e;
  ctx->state[5] += f;
  ctx->state[6] += g;
  ctx->state[7] += h;
}

void mg_sha384_init(mg_sha384_ctx *ctx) {
  ctx->datalen = 0;
  ctx->bitlen[0] = 0;
  ctx->bitlen[1] = 0;
#if defined(__DCC__)
  ctx->state[0] = 0xcbbb9d5dc1059ed8ull;
  ctx->state[1] = 0x629a292a367cd507ull;
  ctx->state[2] = 0x9159015a3070dd17ull;
  ctx->state[3] = 0x152fecd8f70e5939ull;
  ctx->state[4] = 0x67332667ffc00b31ull;
  ctx->state[5] = 0x8eb44a8768581511ull;
  ctx->state[6] = 0xdb0c2e0d64f98fa7ull;
  ctx->state[7] = 0x47b5481dbefa4fa4ull;
#else
  ctx->state[0] = 0xcbbb9d5dc1059ed8;
  ctx->state[1] = 0x629a292a367cd507;
  ctx->state[2] = 0x9159015a3070dd17;
  ctx->state[3] = 0x152fecd8f70e5939;
  ctx->state[4] = 0x67332667ffc00b31;
  ctx->state[5] = 0x8eb44a8768581511;
  ctx->state[6] = 0xdb0c2e0d64f98fa7;
  ctx->state[7] = 0x47b5481dbefa4fa4;
#endif
}

void mg_sha384_update(mg_sha384_ctx *ctx, const uint8_t *data, size_t len) {
  size_t i;
  for (i = 0; i < len; ++i) {
    ctx->buffer[ctx->datalen] = data[i];
    ctx->datalen++;
    if (ctx->datalen == 128) {
      mg_sha384_transform(ctx, ctx->buffer);
      ctx->bitlen[1] += 1024;
      if (ctx->bitlen[1] < 1024) ctx->bitlen[0]++;
      ctx->datalen = 0;
    }
  }
}

void mg_sha384_final(uint8_t hash[48], mg_sha384_ctx *ctx) {
  size_t i = ctx->datalen;

  if (ctx->datalen < 112) {
    ctx->buffer[i++] = 0x80;
    while (i < 112) ctx->buffer[i++] = 0x00;
  } else {
    ctx->buffer[i++] = 0x80;
    while (i < 128) ctx->buffer[i++] = 0x00;
    mg_sha384_transform(ctx, ctx->buffer);
    memset(ctx->buffer, 0, 112);
  }

  ctx->bitlen[1] += ctx->datalen * 8;
  if (ctx->bitlen[1] < ctx->datalen * 8) ctx->bitlen[0]++;
  ctx->buffer[127] = (uint8_t) (ctx->bitlen[1]);
  ctx->buffer[126] = (uint8_t) (ctx->bitlen[1] >> 8);
  ctx->buffer[125] = (uint8_t) (ctx->bitlen[1] >> 16);
  ctx->buffer[124] = (uint8_t) (ctx->bitlen[1] >> 24);
  ctx->buffer[123] = (uint8_t) (ctx->bitlen[1] >> 32);
  ctx->buffer[122] = (uint8_t) (ctx->bitlen[1] >> 40);
  ctx->buffer[121] = (uint8_t) (ctx->bitlen[1] >> 48);
  ctx->buffer[120] = (uint8_t) (ctx->bitlen[1] >> 56);
  ctx->buffer[119] = (uint8_t) (ctx->bitlen[0]);
  ctx->buffer[118] = (uint8_t) (ctx->bitlen[0] >> 8);
  ctx->buffer[117] = (uint8_t) (ctx->bitlen[0] >> 16);
  ctx->buffer[116] = (uint8_t) (ctx->bitlen[0] >> 24);
  ctx->buffer[115] = (uint8_t) (ctx->bitlen[0] >> 32);
  ctx->buffer[114] = (uint8_t) (ctx->bitlen[0] >> 40);
  ctx->buffer[113] = (uint8_t) (ctx->bitlen[0] >> 48);
  ctx->buffer[112] = (uint8_t) (ctx->bitlen[0] >> 56);
  mg_sha384_transform(ctx, ctx->buffer);

  for (i = 0; i < 6; ++i) {
    hash[i * 8] = (uint8_t) ((ctx->state[i] >> 56) & 0xff);
    hash[i * 8 + 1] = (uint8_t) ((ctx->state[i] >> 48) & 0xff);
    hash[i * 8 + 2] = (uint8_t) ((ctx->state[i] >> 40) & 0xff);
    hash[i * 8 + 3] = (uint8_t) ((ctx->state[i] >> 32) & 0xff);
    hash[i * 8 + 4] = (uint8_t) ((ctx->state[i] >> 24) & 0xff);
    hash[i * 8 + 5] = (uint8_t) ((ctx->state[i] >> 16) & 0xff);
    hash[i * 8 + 6] = (uint8_t) ((ctx->state[i] >> 8) & 0xff);
    hash[i * 8 + 7] = (uint8_t) (ctx->state[i] & 0xff);
  }
}

void mg_sha384(uint8_t dst[48], uint8_t *data, size_t datasz) {
  mg_sha384_ctx ctx;
  mg_sha384_init(&ctx);
  mg_sha384_update(&ctx, data, datasz);
  mg_sha384_final(dst, &ctx);
}

#ifdef MG_ENABLE_LINES
#line 1 "src/sntp.c"
#endif






#define SNTP_TIME_OFFSET 2208988800U  // (1970 - 1900) in seconds
#define SNTP_MAX_FRAC 4294967295.0    // 2 ** 32 - 1

uint64_t mg_boot_timestamp_ms = 0;  // Updated by SNTP

uint64_t mg_now(void) {
  return mg_millis() + mg_boot_timestamp_ms;
}

static int64_t gettimestamp(const uint32_t *data) {
  uint32_t sec = mg_ntohl(data[0]), frac = mg_ntohl(data[1]);
  if (sec) sec -= SNTP_TIME_OFFSET;
  return ((int64_t) sec) * 1000 + (int64_t) (frac / SNTP_MAX_FRAC * 1000.0);
}

int64_t mg_sntp_parse(const unsigned char *buf, size_t len) {
  int64_t epoch_milliseconds = -1;
  int mode = len > 0 ? buf[0] & 7 : 0;
  int version = len > 0 ? (buf[0] >> 3) & 7 : 0;
  if (len < 48) {
    MG_ERROR(("%s", "corrupt packet"));
  } else if (mode != 4 && mode != 5) {
    MG_ERROR(("%s", "not a server reply"));
  } else if (buf[1] == 0) {
    MG_ERROR(("%s", "server sent a kiss of death"));
  } else if (version == 4 || version == 3) {
    // int64_t ref = gettimestamp((uint32_t *) &buf[16]);
    int64_t origin_time = gettimestamp((uint32_t *) &buf[24]);
    int64_t receive_time = gettimestamp((uint32_t *) &buf[32]);
    int64_t transmit_time = gettimestamp((uint32_t *) &buf[40]);
    int64_t now = (int64_t) mg_millis();
    int64_t latency = (now - origin_time) - (transmit_time - receive_time);
    epoch_milliseconds = transmit_time + latency / 2;
    mg_boot_timestamp_ms = (uint64_t) (epoch_milliseconds - now);
  } else {
    MG_ERROR(("unexpected version: %d", version));
  }
  return epoch_milliseconds;
}

static void sntp_cb(struct mg_connection *c, int ev, void *ev_data) {
  uint64_t *expiration_time = (uint64_t *) c->data;
  if (ev == MG_EV_OPEN) {
    *expiration_time = mg_millis() + 3000;  // Store expiration time in 3s
  } else if (ev == MG_EV_CONNECT) {
    mg_sntp_request(c);
  } else if (ev == MG_EV_READ) {
    int64_t milliseconds = mg_sntp_parse(c->recv.buf, c->recv.len);
    if (milliseconds > 0) {
      mg_boot_timestamp_ms = (uint64_t) milliseconds - mg_millis();
      mg_call(c, MG_EV_SNTP_TIME, (uint64_t *) &milliseconds);
      MG_DEBUG(("%lu got time: %lld ms from epoch", c->id, milliseconds));
    }
    // mg_iobuf_del(&c->recv, 0, c->recv.len);  // Free receive buffer
    c->is_closing = 1;
  } else if (ev == MG_EV_POLL) {
    if (mg_millis() > *expiration_time) c->is_closing = 1;
  } else if (ev == MG_EV_CLOSE) {
  }
  (void) ev_data;
}

void mg_sntp_request(struct mg_connection *c) {
  if (c->is_resolving) {
    MG_ERROR(("%lu wait until resolved", c->id));
  } else {
    int64_t now = (int64_t) mg_millis();  // Use int64_t, for vc98
    uint8_t buf[48] = {0};
    uint32_t *t = (uint32_t *) &buf[40];
    double frac = ((double) (now % 1000)) / 1000.0 * SNTP_MAX_FRAC;
    buf[0] = (0 << 6) | (4 << 3) | 3;
    t[0] = mg_htonl((uint32_t) (now / 1000) + SNTP_TIME_OFFSET);
    t[1] = mg_htonl((uint32_t) frac);
    mg_send(c, buf, sizeof(buf));
  }
}

struct mg_connection *mg_sntp_connect(struct mg_mgr *mgr, const char *url,
                                      mg_event_handler_t fn, void *fn_data) {
  if (url == NULL) url = "udp://time.google.com:123";
  return mg_connect_svc(mgr, url, fn, fn_data, sntp_cb, NULL);
}


#ifdef MG_ENABLE_LINES
#line 1 "src/sock.c"
#endif












#if MG_ENABLE_SOCKET

#ifndef closesocket
#define closesocket(x) close(x)
#endif

#define FD(c_) ((MG_SOCKET_TYPE) (size_t) (c_)->fd)
#define S2PTR(s_) ((void *) (size_t) (s_))

#ifndef MSG_NONBLOCKING
#define MSG_NONBLOCKING 0
#endif

#ifndef AF_INET6
#define AF_INET6 10
#endif

#ifndef MG_SOCK_ERR
#define MG_SOCK_ERR(errcode) ((errcode) < 0 ? errno : 0)
#endif

#ifndef MG_SOCK_INTR
#define MG_SOCK_INTR(fd) (fd == MG_INVALID_SOCKET && MG_SOCK_ERR(-1) == EINTR)
#endif

#ifndef MG_SOCK_PENDING
#define MG_SOCK_PENDING(errcode) \
  (((errcode) < 0) && (errno == EINPROGRESS || errno == EWOULDBLOCK))
#endif

#ifndef MG_SOCK_RESET
#define MG_SOCK_RESET(errcode) \
  (((errcode) < 0) && (errno == EPIPE || errno == ECONNRESET))
#endif

union usa {
  struct sockaddr sa;
  struct sockaddr_in sin;
#if MG_ENABLE_IPV6
  struct sockaddr_in6 sin6;
#endif
};

static socklen_t tousa(struct mg_addr *a, union usa *usa) {
  socklen_t len = sizeof(usa->sin);
  memset(usa, 0, sizeof(*usa));
  usa->sin.sin_family = AF_INET;
  usa->sin.sin_port = a->port;
  memcpy(&usa->sin.sin_addr, a->addr.ip, sizeof(uint32_t));
#if MG_ENABLE_IPV6
  if (a->is_ip6) {
    usa->sin.sin_family = AF_INET6;
    usa->sin6.sin6_port = a->port;
    usa->sin6.sin6_scope_id = a->scope_id;
    memcpy(&usa->sin6.sin6_addr, a->addr.ip, sizeof(a->addr.ip));
    len = sizeof(usa->sin6);
  }
#endif
  return len;
}

static void tomgaddr(union usa *usa, struct mg_addr *a, bool is_ip6) {
  a->is_ip6 = is_ip6;
#if MG_ENABLE_IPV6
  if (is_ip6) {
    memcpy(a->addr.ip, &usa->sin6.sin6_addr, sizeof(a->addr.ip));
    a->port = usa->sin6.sin6_port;
    a->scope_id = (uint8_t) usa->sin6.sin6_scope_id;
  } else
#endif
  {
    a->port = usa->sin.sin_port;
    memcpy(&a->addr.ip, &usa->sin.sin_addr, sizeof(uint32_t));
  }
}

static void setlocaddr(MG_SOCKET_TYPE fd, struct mg_addr *addr) {
  union usa usa;
  socklen_t n = sizeof(usa);
  if (getsockname(fd, &usa.sa, &n) == 0) {
    tomgaddr(&usa, addr, n != sizeof(usa.sin));
  }
}

// Get the local 'addr' the stack will use to connect to 'to'
void mg_getlocaddr(struct mg_connection *c, struct mg_addr *to,
                   struct mg_addr *addr);
void mg_getlocaddr(struct mg_connection *c, struct mg_addr *to,
                   struct mg_addr *addr) {
  union usa usa;
  socklen_t slen;
  MG_SOCKET_TYPE fd;
  int rc, af = to->is_ip6 ? AF_INET6 : AF_INET;
  fd = socket(af, SOCK_DGRAM, IPPROTO_UDP);
  if (fd == MG_INVALID_SOCKET) {
    mg_error(c, "socket(): %d", MG_SOCK_ERR(-1));
    return;
  }
  // NOTE(): TI-RTOS NDK may require binding
  slen = tousa(to, &usa);
  if ((rc = connect(fd, &usa.sa, slen)) != 0) {
    mg_error(c, "connect: %d", MG_SOCK_ERR(rc));
  } else {
    setlocaddr(fd, addr);
  }
  closesocket(fd);
}

static void iolog(struct mg_connection *c, char *buf, long n, bool r) {
  if (n == MG_IO_WAIT) {
    // Do nothing
  } else if (n <= 0) {
    c->is_closing = 1;  // Termination. Don't call mg_error(): #1529
  } else if (n > 0) {
    if (c->is_hexdumping) {
      MG_INFO(("\n-- %lu %M %s %M %ld", c->id, mg_print_ip_port, &c->loc,
               r ? "<-" : "->", mg_print_ip_port, &c->rem, n));
      mg_hexdump(buf, (size_t) n);
    }
    if (r) {
      c->recv.len += (size_t) n;
      mg_call(c, MG_EV_READ, &n);
    } else {
      mg_iobuf_del(&c->send, 0, (size_t) n);
      // if (c->send.len == 0) mg_iobuf_resize(&c->send, 0);
      if (c->send.len == 0) {
        MG_EPOLL_MOD(c, 0);
      }
      mg_call(c, MG_EV_WRITE, &n);
    }
  }
}

long mg_io_send(struct mg_connection *c, const void *buf, size_t len) {
  long n;
  if (c->is_udp) {
    union usa usa;
    socklen_t slen = tousa(&c->rem, &usa);
    n = sendto(FD(c), (char *) buf, len, 0, &usa.sa, slen);
    if (n > 0) setlocaddr(FD(c), &c->loc);
  } else {
    n = send(FD(c), (char *) buf, len, MSG_NONBLOCKING);
  }
  MG_VERBOSE(("%lu %ld %d", c->id, n, MG_SOCK_ERR(n)));
  if (MG_SOCK_PENDING(n)) return MG_IO_WAIT;
  if (MG_SOCK_RESET(n)) return MG_IO_ERR;  // See #1507, #3031
  if (n <= 0) return MG_IO_ERR;
  return n;
}

bool mg_send(struct mg_connection *c, const void *buf, size_t len) {
  if (c->is_udp) {
    long n = mg_io_send(c, buf, len);
    MG_DEBUG(("%lu %ld %lu:%lu:%lu %ld err %d", c->id, c->fd, c->send.len,
              c->recv.len, c->rtls.len, n, MG_SOCK_ERR(n)));
    iolog(c, (char *) buf, n, false);
    return n > 0;
  } else {
    return len == 0 || mg_iobuf_add(&c->send, c->send.len, buf, len) > 0;
    // returning 0 means an OOM condition (iobuf couldn't resize), yet this is
    // so far recoverable, let the caller decide
  }
}

static void mg_set_non_blocking_mode(MG_SOCKET_TYPE fd) {
#if defined(MG_CUSTOM_NONBLOCK)
  MG_CUSTOM_NONBLOCK(fd);
#elif MG_ARCH == MG_ARCH_WIN32 && MG_ENABLE_WINSOCK
  unsigned long on = 1;
  ioctlsocket(fd, FIONBIO, &on);
#elif MG_ENABLE_RL
  unsigned long on = 1;
  ioctlsocket(fd, FIONBIO, &on);
#elif MG_ENABLE_FREERTOS_TCP
  const BaseType_t off = 0;
  if (setsockopt(fd, 0, FREERTOS_SO_RCVTIMEO, &off, sizeof(off)) != 0) (void) 0;
  if (setsockopt(fd, 0, FREERTOS_SO_SNDTIMEO, &off, sizeof(off)) != 0) (void) 0;
#elif MG_ENABLE_LWIP
  lwip_fcntl(fd, F_SETFL, O_NONBLOCK);
#elif MG_ARCH == MG_ARCH_THREADX
  // NetxDuo fails to send large blocks of data to the non-blocking sockets
  (void) fd;
  // fcntl(fd, F_SETFL, O_NONBLOCK);
#elif MG_ARCH == MG_ARCH_TIRTOS
  int val = 0;
  setsockopt(fd, SOL_SOCKET, SO_BLOCKING, &val, sizeof(val));
  // SPRU524J section 3.3.3 page 63, SO_SNDLOWAT
  int sz = sizeof(val);
  getsockopt(fd, SOL_SOCKET, SO_SNDBUF, &val, &sz);
  val /= 2;  // set send low-water mark at half send buffer size
  setsockopt(fd, SOL_SOCKET, SO_SNDLOWAT, &val, sizeof(val));
#else
  fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);  // Non-blocking mode
  fcntl(fd, F_SETFD, FD_CLOEXEC);                          // Set close-on-exec
#endif
}

void mg_multicast_add(struct mg_connection *c, char *ip);
void mg_multicast_add(struct mg_connection *c, char *ip) {
#if MG_ENABLE_RL
  MG_ERROR(("unsupported"));
#elif MG_ENABLE_FREERTOS_TCP
  // TODO(): prvAllowIPPacketIPv4()
#else
  // lwIP, Unix, Windows, Zephyr 4+(, AzureRTOS ?)
#if MG_ENABLE_LWIP && !LWIP_IGMP
  MG_ERROR(("LWIP_IGMP not defined, no multicast support"));
#else
#if defined(__ZEPHYR__) && ZEPHYR_VERSION_CODE < 0x40000
  MG_ERROR(("struct ip_mreq not defined"));
#else
  struct ip_mreq mreq;
  mreq.imr_multiaddr.s_addr = inet_addr(ip);
  mreq.imr_interface.s_addr = mg_htonl(INADDR_ANY);
  setsockopt(FD(c), IPPROTO_IP, IP_ADD_MEMBERSHIP, (char *) &mreq,
             sizeof(mreq));
#endif  // !Zephyr
#endif  // !lwIP
#endif
}

bool mg_open_listener(struct mg_connection *c, const char *url) {
  MG_SOCKET_TYPE fd = MG_INVALID_SOCKET;
  bool success = false;
  c->loc.port = mg_htons(mg_url_port(url));
  if (!mg_aton(mg_url_host(url), &c->loc)) {
    MG_ERROR(("invalid listening URL: %s", url));
  } else {
    union usa usa;
    socklen_t slen = tousa(&c->loc, &usa);
    int rc, on = 1, af = c->loc.is_ip6 ? AF_INET6 : AF_INET;
    int type = strncmp(url, "udp:", 4) == 0 ? SOCK_DGRAM : SOCK_STREAM;
    int proto = type == SOCK_DGRAM ? IPPROTO_UDP : IPPROTO_TCP;
    (void) on;

    if ((fd = socket(af, type, proto)) == MG_INVALID_SOCKET) {
      MG_ERROR(("socket: %d", MG_SOCK_ERR(-1)));
#if defined(SO_EXCLUSIVEADDRUSE)
    } else if ((rc = setsockopt(fd, SOL_SOCKET, SO_EXCLUSIVEADDRUSE,
                                (char *) &on, sizeof(on))) != 0) {
      // "Using SO_REUSEADDR and SO_EXCLUSIVEADDRUSE"
      MG_ERROR(("setsockopt(SO_EXCLUSIVEADDRUSE): %d %d", on, MG_SOCK_ERR(rc)));
#elif defined(SO_REUSEADDR) && (!defined(LWIP_SOCKET) || SO_REUSE)
    } else if ((rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *) &on,
                                sizeof(on))) != 0) {
      // 1. SO_REUSEADDR semantics on UNIX and Windows is different.  On
      // Windows, SO_REUSEADDR allows to bind a socket to a port without error
      // even if the port is already open by another program. This is not the
      // behavior SO_REUSEADDR was designed for, and leads to hard-to-track
      // failure scenarios.
      //
      // 2. For LWIP, SO_REUSEADDR should be explicitly enabled by defining
      // SO_REUSE = 1 in lwipopts.h, otherwise the code below will compile but
      // won't work! (setsockopt will return EINVAL)
      MG_ERROR(("setsockopt(SO_REUSEADDR): %d", MG_SOCK_ERR(rc)));
#endif
#if MG_IPV6_V6ONLY
      // Bind only to the V6 address, not V4 address on this port
    } else if (c->loc.is_ip6 &&
               (rc = setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, (char *) &on,
                                sizeof(on))) != 0) {
      // See #2089. Allow to bind v4 and v6 sockets on the same port
      MG_ERROR(("setsockopt(IPV6_V6ONLY): %d", MG_SOCK_ERR(rc)));
#endif
    } else if ((rc = bind(fd, &usa.sa, slen)) != 0) {
      MG_ERROR(("bind: %d", MG_SOCK_ERR(rc)));
    } else if ((type == SOCK_STREAM &&
                (rc = listen(fd, MG_SOCK_LISTEN_BACKLOG_SIZE)) != 0)) {
      // NOTE(lsm): FreeRTOS uses backlog value as a connection limit
      // In case port was set to 0, get the real port number
      MG_ERROR(("listen: %d", MG_SOCK_ERR(rc)));
    } else {
      setlocaddr(fd, &c->loc);
      mg_set_non_blocking_mode(fd);
      c->fd = S2PTR(fd);
      MG_EPOLL_ADD(c);
      success = true;
    }
  }
  if (success == false && fd != MG_INVALID_SOCKET) closesocket(fd);
  return success;
}

static long recv_raw(struct mg_connection *c, void *buf, size_t len) {
  long n = 0;
  if (c->is_udp) {
    union usa usa;
    socklen_t slen = tousa(&c->rem, &usa);
    n = recvfrom(FD(c), (char *) buf, len, 0, &usa.sa, &slen);
    if (n > 0) tomgaddr(&usa, &c->rem, slen != sizeof(usa.sin));
  } else {
    n = recv(FD(c), (char *) buf, len, MSG_NONBLOCKING);
  }
  MG_VERBOSE(("%lu %ld %d", c->id, n, MG_SOCK_ERR(n)));
  if (MG_SOCK_PENDING(n)) return MG_IO_WAIT;
  if (MG_SOCK_RESET(n)) return MG_IO_ERR;  // See #1507, #3031
  if (n <= 0) return MG_IO_ERR;
  return n;
}

static bool ioalloc(struct mg_connection *c, struct mg_iobuf *io) {
  bool res = false;
  if (io->len >= MG_MAX_RECV_SIZE) {
    mg_error(c, "MG_MAX_RECV_SIZE");
  } else if (io->size <= io->len &&
             !mg_iobuf_resize(io, io->size + MG_IO_SIZE)) {
    mg_error(c, "OOM");
  } else {
    res = true;
  }
  return res;
}

// NOTE(lsm): do only one iteration of reads, cause some systems
// (e.g. FreeRTOS stack) return 0 instead of -1/EWOULDBLOCK when no data
static void read_conn(struct mg_connection *c) {
  long n = MG_IO_WAIT;
  if (ioalloc(c, &c->recv) == false) return;  // Oopsie poopsie, OOM
  if (c->is_tls) {
    size_t rtls = c->rtls.len, pending = mg_tls_pending(c);
    long m = MG_IO_WAIT;
    if (c->rtls.len < 16 * 1024 + 40) {  // TLS record, header, MAC, padding
      if (!ioalloc(c, &c->rtls)) return;
      n = recv_raw(c, (char *) &c->rtls.buf[c->rtls.len],
                   c->rtls.size - c->rtls.len);
      if (n > 0) c->rtls.len += (size_t) n;
    }
    // There can still be > 16K from last iteration, always mg_tls_recv()
    if (c->is_tls_hs) {
      if (n != MG_IO_ERR || c->rtls.len > 0 || pending > 0) {
        mg_tls_handshake(c);
      }
    } else {
      m = mg_tls_recv(c, &c->recv.buf[c->recv.len], c->recv.size - c->recv.len);
    }
    if (n == MG_IO_ERR &&
        (m == MG_IO_ERR || (c->rtls.len == 0 && mg_tls_pending(c) == 0) ||
         (c->is_tls_hs && c->rtls.len == rtls &&
          mg_tls_pending(c) == pending))) {
      // Close only when we have fully drained both rtls and TLS buffers
      c->is_closing = 1;         // or there's nothing we can do about it.
      if (m < 0) m = MG_IO_ERR;  // but return last record data, see #3104
    }
    n = m;
  } else {
    n = recv_raw(c, &c->recv.buf[c->recv.len], c->recv.size - c->recv.len);
  }
  MG_DEBUG(("%lu %ld %lu:%lu:%lu %ld err %d", c->id, c->fd, c->send.len,
            c->recv.len, c->rtls.len, n, MG_SOCK_ERR(n)));
  iolog(c, (char *) &c->recv.buf[c->recv.len], n, true);
}

static void write_conn(struct mg_connection *c) {
  char *buf = (char *) c->send.buf;
  size_t len = c->send.len;
  long n = c->is_tls ? mg_tls_send(c, buf, len) : mg_io_send(c, buf, len);
  // TODO(): mg_tls_send() may return 0 forever on steady OOM
  MG_DEBUG(("%lu %ld snd %ld/%ld rcv %ld/%ld n=%ld err=%d", c->id, c->fd,
            (long) c->send.len, (long) c->send.size, (long) c->recv.len,
            (long) c->recv.size, n, MG_SOCK_ERR(n)));
  iolog(c, buf, n, false);
}

static void close_conn(struct mg_connection *c) {
  if (FD(c) != MG_INVALID_SOCKET) {
#if MG_ENABLE_EPOLL
    epoll_ctl(c->mgr->epoll_fd, EPOLL_CTL_DEL, FD(c), NULL);
#endif
    closesocket(FD(c));
#if MG_ENABLE_FREERTOS_TCP
    FreeRTOS_FD_CLR(c->fd, c->mgr->ss, eSELECT_ALL);
#endif
  }
  mg_close_conn(c);
}

static void connect_conn(struct mg_connection *c) {
  union usa usa;
  socklen_t n = sizeof(usa);
  // Use getpeername() to test whether we have connected
  if (getpeername(FD(c), &usa.sa, &n) == 0) {
    c->is_connecting = 0;
    setlocaddr(FD(c), &c->loc);
    mg_call(c, MG_EV_CONNECT, NULL);
    MG_EPOLL_MOD(c, 0);
    if (c->is_tls_hs) mg_tls_handshake(c);
    if (!c->is_tls_hs) c->is_tls = 0;  // user did not call mg_tls_init()
  } else {
    mg_error(c, "socket error");
  }
}

static void setsockopts(struct mg_connection *c) {
#if MG_ENABLE_FREERTOS_TCP || MG_ARCH == MG_ARCH_THREADX || \
    MG_ARCH == MG_ARCH_TIRTOS
  (void) c;
#else
  int on = 1;
#if !defined(SOL_TCP)
#define SOL_TCP IPPROTO_TCP
#endif
  if (setsockopt(FD(c), SOL_TCP, TCP_NODELAY, (char *) &on, sizeof(on)) != 0)
    (void) 0;
  if (setsockopt(FD(c), SOL_SOCKET, SO_KEEPALIVE, (char *) &on, sizeof(on)) !=
      0)
    (void) 0;
#endif
}

void mg_connect_resolved(struct mg_connection *c) {
  int type = c->is_udp ? SOCK_DGRAM : SOCK_STREAM;
  int proto = type == SOCK_DGRAM ? IPPROTO_UDP : IPPROTO_TCP;
  int rc, af = c->rem.is_ip6 ? AF_INET6 : AF_INET;  // c->rem has resolved IP
  c->fd = S2PTR(socket(af, type, proto));           // Create outbound socket
  c->is_resolving = 0;                              // Clear resolving flag
  if (FD(c) == MG_INVALID_SOCKET) {
    mg_error(c, "socket(): %d", MG_SOCK_ERR(-1));
  } else if (c->is_udp) {
    MG_EPOLL_ADD(c);
#if MG_ARCH == MG_ARCH_TIRTOS
    union usa usa;  // TI-RTOS NDK requires binding to receive on UDP sockets
    socklen_t slen = tousa(&c->loc, &usa);
    if ((rc = bind(c->fd, &usa.sa, slen)) != 0)
      MG_ERROR(("bind: %d", MG_SOCK_ERR(rc)));
#endif
    setlocaddr(FD(c), &c->loc);
    mg_call(c, MG_EV_RESOLVE, NULL);
    mg_call(c, MG_EV_CONNECT, NULL);
  } else {
    union usa usa;
    socklen_t slen = tousa(&c->rem, &usa);
    mg_set_non_blocking_mode(FD(c));
    setsockopts(c);
    MG_EPOLL_ADD(c);
    mg_call(c, MG_EV_RESOLVE, NULL);
    rc = connect(FD(c), &usa.sa, slen);  // Attempt to connect
    if (rc == 0) {                       // Success
      setlocaddr(FD(c), &c->loc);
      mg_call(c, MG_EV_CONNECT, NULL);   // Send MG_EV_CONNECT to the user
      if (c->is_tls_hs) mg_tls_handshake(c);
      if (!c->is_tls_hs) c->is_tls = 0;  // user did not call mg_tls_init()
    } else if (MG_SOCK_PENDING(rc)) {    // Need to wait for TCP handshake
      MG_DEBUG(("%lu %ld -> %M pend", c->id, c->fd, mg_print_ip_port, &c->rem));
      c->is_connecting = 1;
    } else {
      mg_error(c, "connect: %d", MG_SOCK_ERR(rc));
    }
  }
}

static MG_SOCKET_TYPE raccept(MG_SOCKET_TYPE sock, union usa *usa,
                              socklen_t *len) {
  MG_SOCKET_TYPE fd = MG_INVALID_SOCKET;
  do {
    memset(usa, 0, sizeof(*usa));
    fd = accept(sock, &usa->sa, len);
#if MG_ENABLE_FREERTOS_TCP
    // FreeRTOS_accept() returns NULL to mean "no pending connection",
    // therefore we avoid retrying it forever on non-blocking listeners
    if (fd == NULL) {
      fd = MG_INVALID_SOCKET;
      break;
    }
#endif
  } while (MG_SOCK_INTR(fd));
  return fd;
}

static void accept_conn(struct mg_mgr *mgr, struct mg_connection *lsn) {
  struct mg_connection *c = NULL;
  union usa usa;
  socklen_t sa_len = sizeof(usa);
  MG_SOCKET_TYPE fd = raccept(FD(lsn), &usa, &sa_len);
  if (fd == MG_INVALID_SOCKET) {
#if MG_ARCH == MG_ARCH_THREADX || defined(__ECOS)
    // NetxDuo, in non-block socket mode can mark listening socket readable
    // even it is not. See comment for 'select' func implementation in
    // nx_bsd.c That's not an error, just should try later
    if (errno != EAGAIN)
#endif
      MG_ERROR(("%lu accept failed, errno %d", lsn->id, MG_SOCK_ERR(-1)));
#if (MG_ARCH != MG_ARCH_WIN32) && !MG_ENABLE_FREERTOS_TCP && \
    (MG_ARCH != MG_ARCH_TIRTOS) && !MG_ENABLE_POLL && !MG_ENABLE_EPOLL
  } else if ((long) fd >= FD_SETSIZE) {
    MG_ERROR(("%ld > %ld", (long) fd, (long) FD_SETSIZE));
    closesocket(fd);
#endif
  } else if ((c = mg_alloc_conn(mgr)) == NULL) {
    MG_ERROR(("%lu OOM", lsn->id));
    closesocket(fd);
  } else {
    tomgaddr(&usa, &c->rem, sa_len != sizeof(usa.sin));
    LIST_ADD_HEAD(struct mg_connection, &mgr->conns, c);
    c->fd = S2PTR(fd);
    MG_EPOLL_ADD(c);
    mg_set_non_blocking_mode(FD(c));
    setsockopts(c);
    c->is_accepted = 1;
    c->is_hexdumping = lsn->is_hexdumping;
    setlocaddr(fd, &c->loc);  // set local addr to where the client connected to
    c->pfn = lsn->pfn;
    c->pfn_data = lsn->pfn_data;
    c->fn = lsn->fn;
    c->fn_data = lsn->fn_data;
    c->is_tls = lsn->is_tls;
    MG_DEBUG(("%lu %ld accepted %M -> %M", c->id, c->fd, mg_print_ip_port,
              &c->rem, mg_print_ip_port, &c->loc));
    mg_call(c, MG_EV_OPEN, NULL);
    mg_call(c, MG_EV_ACCEPT, NULL);
    if (!c->is_tls_hs) c->is_tls = 0;  // user did not call mg_tls_init()
  }
}

static bool can_read(const struct mg_connection *c) {
  return c->is_full == false;
}

static bool can_write(const struct mg_connection *c) {
  return c->is_connecting || (c->send.len > 0 && c->is_tls_hs == 0);
}

static bool skip_iotest(const struct mg_connection *c) {
  return (c->is_closing || c->is_resolving || FD(c) == MG_INVALID_SOCKET) ||
         (can_read(c) == false && can_write(c) == false);
}

static void mg_iotest(struct mg_mgr *mgr, int ms) {
#if MG_ENABLE_FREERTOS_TCP
  struct mg_connection *c;
  for (c = mgr->conns; c != NULL; c = c->next) {
    c->is_readable = c->is_writable = 0;
    if (skip_iotest(c)) continue;
    if (can_read(c))
      FreeRTOS_FD_SET(c->fd, mgr->ss, eSELECT_READ | eSELECT_EXCEPT);
    if (can_write(c)) FreeRTOS_FD_SET(c->fd, mgr->ss, eSELECT_WRITE);
    if (c->is_closing) ms = 1;
  }
  FreeRTOS_select(mgr->ss, pdMS_TO_TICKS(ms));
  for (c = mgr->conns; c != NULL; c = c->next) {
    EventBits_t bits = FreeRTOS_FD_ISSET(c->fd, mgr->ss);
    c->is_readable = bits & (eSELECT_READ | eSELECT_EXCEPT) ? 1U : 0;
    c->is_writable = bits & eSELECT_WRITE ? 1U : 0;
    if (c->fd != MG_INVALID_SOCKET)
      FreeRTOS_FD_CLR(c->fd, mgr->ss,
                      eSELECT_READ | eSELECT_EXCEPT | eSELECT_WRITE);
  }
#elif MG_ENABLE_EPOLL
  size_t max = 1;
  for (struct mg_connection *c = mgr->conns; c != NULL; c = c->next) {
    c->is_readable = c->is_writable = 0;
    if (c->rtls.len > 0 || mg_tls_pending(c) > 0) ms = 1, c->is_readable = 1;
    if (can_write(c)) MG_EPOLL_MOD(c, 1);
    if (c->is_closing) ms = 1;
    max++;
  }
  struct epoll_event *evs = (struct epoll_event *) alloca(max * sizeof(evs[0]));
  int n = epoll_wait(mgr->epoll_fd, evs, (int) max, ms);
  for (int i = 0; i < n; i++) {
    struct mg_connection *c = (struct mg_connection *) evs[i].data.ptr;
    if (evs[i].events & EPOLLERR) {
      mg_error(c, "socket error");
    } else if (c->is_readable == 0) {
      bool rd = evs[i].events & (EPOLLIN | EPOLLHUP);
      bool wr = evs[i].events & EPOLLOUT;
      c->is_readable = can_read(c) && rd ? 1U : 0;
      c->is_writable = can_write(c) && wr ? 1U : 0;
      if (c->rtls.len > 0 || mg_tls_pending(c) > 0) c->is_readable = 1;
    }
  }
  (void) skip_iotest;
#elif MG_ENABLE_POLL
  nfds_t n = 0;
  for (struct mg_connection *c = mgr->conns; c != NULL; c = c->next) n++;
  struct pollfd *fds = (struct pollfd *) alloca(n * sizeof(fds[0]));
  memset(fds, 0, n * sizeof(fds[0]));
  n = 0;
  for (struct mg_connection *c = mgr->conns; c != NULL; c = c->next) {
    c->is_readable = c->is_writable = 0;
    if (c->is_closing) ms = 1;
    if (skip_iotest(c)) {
      // Socket not valid, ignore
    } else {
      // Don't wait if TLS is ready
      if (c->rtls.len > 0 || mg_tls_pending(c) > 0) ms = 1;
      fds[n].fd = FD(c);
      if (can_read(c)) fds[n].events |= POLLIN;
      if (can_write(c)) fds[n].events |= POLLOUT;
      n++;
    }
  }

  // MG_INFO(("poll n=%d ms=%d", (int) n, ms));
  if (poll(fds, n, ms) < 0) {
#if MG_ARCH == MG_ARCH_WIN32
    if (n == 0) Sleep(ms);  // On Windows, poll fails if no sockets
#endif
    memset(fds, 0, n * sizeof(fds[0]));
  }
  n = 0;
  for (struct mg_connection *c = mgr->conns; c != NULL; c = c->next) {
    if (skip_iotest(c)) {
      // Socket not valid, ignore
    } else {
      if (fds[n].revents & POLLERR) {
        mg_error(c, "socket error");
      } else {
        c->is_readable =
            (unsigned) (fds[n].revents & (POLLIN | POLLHUP) ? 1 : 0);
        c->is_writable = (unsigned) (fds[n].revents & POLLOUT ? 1 : 0);
        if (c->rtls.len > 0 || mg_tls_pending(c) > 0) c->is_readable = 1;
      }
      n++;
    }
  }
#else
  struct timeval tv = {ms / 1000, (ms % 1000) * 1000}, tv_1ms = {0, 1000}, *tvp;
  struct mg_connection *c;
  fd_set rset, wset, eset;
  MG_SOCKET_TYPE maxfd = 0;
  int rc;

  FD_ZERO(&rset);
  FD_ZERO(&wset);
  FD_ZERO(&eset);
  tvp = ms < 0 ? NULL : &tv;
  for (c = mgr->conns; c != NULL; c = c->next) {
    c->is_readable = c->is_writable = 0;
    if (skip_iotest(c)) continue;
    FD_SET(FD(c), &eset);
    if (can_read(c)) FD_SET(FD(c), &rset);
    if (can_write(c)) FD_SET(FD(c), &wset);
    if (c->rtls.len > 0 || mg_tls_pending(c) > 0) tvp = &tv_1ms;
    if (FD(c) > maxfd) maxfd = FD(c);
    if (c->is_closing) tvp = &tv_1ms;
  }

  if ((rc = select((int) maxfd + 1, &rset, &wset, &eset, tvp)) <= 0) {
#if MG_ARCH == MG_ARCH_WIN32
    if (maxfd == 0) Sleep(ms);  // On Windows, select fails if no sockets
#else
    if (rc < 0) MG_ERROR(("select: %d %d", rc, MG_SOCK_ERR(rc)));
#endif
    FD_ZERO(&rset);
    FD_ZERO(&wset);
    FD_ZERO(&eset);
  }

  for (c = mgr->conns; c != NULL; c = c->next) {
    if (FD(c) != MG_INVALID_SOCKET && FD_ISSET(FD(c), &eset)) {
#if MG_ARCH == MG_ARCH_THREADX
      // NetxDuo stack returns exceptions for listening connection after accept
      if (c->is_listening == 0) mg_error(c, "socket error");
#else
      mg_error(c, "socket error");
#endif
    } else {
      c->is_readable = FD(c) != MG_INVALID_SOCKET && FD_ISSET(FD(c), &rset);
      c->is_writable = FD(c) != MG_INVALID_SOCKET && FD_ISSET(FD(c), &wset);
      if (c->rtls.len > 0 || mg_tls_pending(c) > 0) c->is_readable = 1;
    }
  }
#endif
}

static bool mg_socketpair(MG_SOCKET_TYPE sp[2], union usa usa[2]) {
  socklen_t n = sizeof(usa[0].sin);
  bool success = false;

  sp[0] = sp[1] = MG_INVALID_SOCKET;
  (void) memset(&usa[0], 0, sizeof(usa[0]));
  usa[0].sin.sin_family = AF_INET;
  *(uint32_t *) &usa->sin.sin_addr = mg_htonl(0x7f000001U);  // 127.0.0.1
  usa[1] = usa[0];

  if ((sp[0] = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) != MG_INVALID_SOCKET &&
      (sp[1] = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) != MG_INVALID_SOCKET &&
      bind(sp[0], &usa[0].sa, n) == 0 &&          //
      bind(sp[1], &usa[1].sa, n) == 0 &&          //
      getsockname(sp[0], &usa[0].sa, &n) == 0 &&  //
      getsockname(sp[1], &usa[1].sa, &n) == 0 &&  //
      connect(sp[0], &usa[1].sa, n) == 0 &&       //
      connect(sp[1], &usa[0].sa, n) == 0) {       //
    success = true;
  }
  if (!success) {
    if (sp[0] != MG_INVALID_SOCKET) closesocket(sp[0]);
    if (sp[1] != MG_INVALID_SOCKET) closesocket(sp[1]);
    sp[0] = sp[1] = MG_INVALID_SOCKET;
  }
  return success;
}

// mg_wakeup() event handler
static void wufn(struct mg_connection *c, int ev, void *ev_data) {
  if (ev == MG_EV_READ) {
    unsigned long *id = (unsigned long *) c->recv.buf;
    // MG_INFO(("Got data"));
    // mg_hexdump(c->recv.buf, c->recv.len);
    if (c->recv.len >= sizeof(*id)) {
      struct mg_connection *t;
      for (t = c->mgr->conns; t != NULL; t = t->next) {
        if (t->id == *id) {
          struct mg_str data = mg_str_n((char *) c->recv.buf + sizeof(*id),
                                        c->recv.len - sizeof(*id));
          mg_call(t, MG_EV_WAKEUP, &data);
        }
      }
    }
    c->recv.len = 0;  // Consume received data
  } else if (ev == MG_EV_CLOSE) {
    closesocket(c->mgr->pipe.fd);        // When we're closing, close the other
    c->mgr->pipe.fd = MG_INVALID_SOCKET; // side of the socketpair, too
  }
  (void) ev_data;
}

bool mg_wakeup_init(struct mg_mgr *mgr) {
  bool ok = false;
  if (mgr->pipe.fd == MG_INVALID_SOCKET) {
    union usa usa[2];
    MG_SOCKET_TYPE sp[2] = {MG_INVALID_SOCKET, MG_INVALID_SOCKET};
    struct mg_connection *c = NULL;
    if (!mg_socketpair(sp, usa)) {
      MG_ERROR(("Cannot create socket pair"));
    } else if ((c = mg_wrapfd(mgr, (int) sp[1], wufn, NULL)) == NULL) {
      closesocket(sp[0]);
      closesocket(sp[1]);
      sp[0] = sp[1] = MG_INVALID_SOCKET;
    } else {
      tomgaddr(&usa[0], &c->rem, false);
      MG_DEBUG(("%lu %ld pipe %ld", c->id, c->fd, (long) sp[0]));
      mgr->pipe.fd = sp[0];
      ok = true;
    }
  }
  return ok;
}

bool mg_wakeup(struct mg_mgr *mgr, unsigned long conn_id, const void *buf,
               size_t len) {
  if (mgr->pipe.fd != MG_INVALID_SOCKET && conn_id > 0) {
    char *extended_buf = (char *) alloca(len + sizeof(conn_id));
    memcpy(extended_buf, &conn_id, sizeof(conn_id));
    memcpy(extended_buf + sizeof(conn_id), buf, len);
    send(mgr->pipe.fd, extended_buf, len + sizeof(conn_id), MSG_NONBLOCKING);
    return true;
  }
  return false;
}

void mg_mgr_poll(struct mg_mgr *mgr, int ms) {
  struct mg_connection *c, *tmp;
  uint64_t now;

  mg_iotest(mgr, ms);
  now = mg_millis();
  mg_timer_poll(&mgr->timers, now);
  mg_ota_poll(mgr);
  mg_mdash_poll(mgr);

  for (c = mgr->conns; c != NULL; c = tmp) {
    long flush = 0;
    tmp = c->next;
    mg_call(c, MG_EV_POLL, &now);
    MG_VERBOSE(("%lu %c%c %c%c%c%c%c %lu %lu", c->id,
                c->is_readable ? 'r' : '-', c->is_writable ? 'w' : '-',
                c->is_tls ? 'T' : 't', c->is_connecting ? 'C' : 'c',
                c->is_tls_hs ? 'H' : 'h', c->is_resolving ? 'R' : 'r',
                c->is_closing ? 'C' : 'c', mg_tls_pending(c), c->rtls.len));
    if (c->is_resolving || c->is_closing) {
      // Do nothing
    } else if (c->is_listening && c->is_udp == 0) {
      if (c->is_readable) accept_conn(mgr, c);
    } else if (c->is_connecting) {
      if (c->is_readable || c->is_writable) connect_conn(c);
    } else {
      if (c->is_readable) read_conn(c);
      if (c->is_writable) write_conn(c);
      if (c->is_tls && !c->is_tls_hs && c->send.len == 0)
        flush = mg_tls_flush(c);
    }

    if (flush == MG_IO_ERR) mg_error(c, "tx err");
    if (c->is_draining && c->send.len == 0 && flush == 0) c->is_closing = 1;
    if (c->is_closing) close_conn(c);
  }
}
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/ssi.c"
#endif





#ifndef MG_MAX_SSI_DEPTH
#define MG_MAX_SSI_DEPTH 5
#endif

#ifndef MG_SSI_BUFSIZ
#define MG_SSI_BUFSIZ 1024
#endif

#if MG_ENABLE_SSI
static char *mg_ssi(const char *path, const char *root, int depth) {
  struct mg_iobuf b = {NULL, 0, 0, MG_IO_SIZE};
  FILE *fp = fopen(path, "rb");
  if (fp != NULL) {
    char buf[MG_SSI_BUFSIZ], arg[sizeof(buf)];
    int ch, intag = 0;
    size_t len = 0;
    buf[0] = arg[0] = '\0';
    while ((ch = fgetc(fp)) != EOF) {
      if (intag && ch == '>' && len >= 2 && buf[len - 1] == '-' && buf[len - 2] == '-') {
        buf[len++] = (char) (ch & 0xff);
        buf[len] = '\0';
        if (sscanf(buf, "<!--#include file=\"%[^\"]", arg) > 0) {
          char tmp[MG_PATH_MAX + MG_SSI_BUFSIZ + 10],
              *p = (char *) path + strlen(path), *data;
          while (p > path && p[-1] != MG_DIRSEP && p[-1] != '/') p--;
          mg_snprintf(tmp, sizeof(tmp), "%.*s%s", (int) (p - path), path, arg);
          if (!mg_path_is_sane(mg_str(tmp))) {
            MG_ERROR(("SSI include path traversal blocked: %s", arg));
          } else if (depth < MG_MAX_SSI_DEPTH &&
              (data = mg_ssi(tmp, root, depth + 1)) != NULL) {
            size_t datalen = strlen(data);
            size_t ret = mg_iobuf_add(&b, b.len, data, datalen);
            mg_free(data);
            if (datalen > 0 && ret == 0) goto fail;
          } else {
            MG_ERROR(("%s: file=%s error or too deep", path, arg));
          } // TODO(): or OOM at recursive call
        } else if (sscanf(buf, "<!--#include virtual=\"%[^\"]", arg) > 0) {
          char tmp[MG_PATH_MAX + MG_SSI_BUFSIZ + 10], *data;
          mg_snprintf(tmp, sizeof(tmp), "%s%s", root, arg);
          if (!mg_path_is_sane(mg_str(tmp))) {
            MG_ERROR(("SSI include path traversal blocked: %s", arg));
          } else if (depth < MG_MAX_SSI_DEPTH &&
              (data = mg_ssi(tmp, root, depth + 1)) != NULL) {
            size_t datalen = strlen(data);
            size_t ret = mg_iobuf_add(&b, b.len, data, datalen);
            mg_free(data);
            if (datalen > 0 && ret == 0) goto fail;
          } else {
            MG_ERROR(("%s: virtual=%s error or too deep", path, arg));
          } // TODO(): or OOM at recursive call
        } else {
          // Unknown SSI tag
          MG_ERROR(("Unknown SSI tag: %.*s", (int) len, buf));
          if (len > 0 && mg_iobuf_add(&b, b.len, buf, len) == 0) goto fail;
        }
        intag = 0;
        len = 0;
      } else if (ch == '<') {
        intag = 1;
        if (len > 0 && mg_iobuf_add(&b, b.len, buf, len) == 0) goto fail;
        len = 0;
        buf[len++] = (char) (ch & 0xff);
      } else if (intag) {
        if (len == 5 && strncmp(buf, "<!--#", 5) != 0) {
          intag = 0;
        } else if (len >= sizeof(buf) - 2) {
          MG_ERROR(("%s: SSI tag is too large", path));
          len = 0;
        }
        buf[len++] = (char) (ch & 0xff);
      } else {
        buf[len++] = (char) (ch & 0xff);
        if (len >= sizeof(buf)) {
          if (mg_iobuf_add(&b, b.len, buf, len) == 0) goto fail;
          len = 0;
        }
      }
    }
    if (len > 0 && mg_iobuf_add(&b, b.len, buf, len) == 0) goto fail;
    if (b.len > 0 && mg_iobuf_add(&b, b.len, "", 1) == 0)  // nul-terminate
      goto fail;
    fclose(fp);
  }
  (void) depth;
  (void) root;
  return (char *) b.buf;

fail:
  fclose(fp);
  mg_iobuf_free(&b);
  return NULL;
}

void mg_http_serve_ssi(struct mg_connection *c, const char *root,
                       const char *fullpath) {
  const char *headers = "Content-Type: text/html; charset=utf-8\r\n";
  char *data = mg_ssi(fullpath, root, 0);
  if (data == NULL) {
    mg_error(c, "OOM");
    return;
  }
  mg_http_reply(c, 200, headers, "%s", data == NULL ? "" : data);
  mg_free(data);
}
#else
void mg_http_serve_ssi(struct mg_connection *c, const char *root,
                       const char *fullpath) {
  mg_http_reply(c, 501, NULL, "SSI not enabled");
  (void) root, (void) fullpath;
}
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/str.c"
#endif



struct mg_str mg_str_s(const char *s) {
  struct mg_str str;
  str.buf = (char *) s, str.len = (s == NULL) ? 0 : strlen(s);
  return str;
}

struct mg_str mg_str_n(const char *s, size_t n) {
  struct mg_str str;
  str.buf = (char *) s, str.len = n;
  return str;
}

static int mg_tolc(char c) {
  return (c >= 'A' && c <= 'Z') ? c + 'a' - 'A' : c;
}

int mg_casecmp(const char *s1, const char *s2) {
  int diff = 0;
  do {
    int c = mg_tolc(*s1++), d = mg_tolc(*s2++);
    diff = c - d;
  } while (diff == 0 && s1[-1] != '\0');
  return diff;
}

struct mg_str mg_strdup(const struct mg_str s) {
  struct mg_str r = {NULL, 0};
  if (s.len > 0 && s.buf != NULL) {
    char *sc = (char *) mg_calloc(1, s.len + 1);
    if (sc != NULL) {
      memcpy(sc, s.buf, s.len);
      sc[s.len] = '\0';
      r.buf = sc;
      r.len = s.len;
    }
  }
  return r;
}

int mg_strcmp(const struct mg_str str1, const struct mg_str str2) {
  size_t i = 0;
  while (i < str1.len && i < str2.len) {
    int c1 = str1.buf[i];
    int c2 = str2.buf[i];
    if (c1 < c2) return -1;
    if (c1 > c2) return 1;
    i++;
  }
  if (i < str1.len) return 1;
  if (i < str2.len) return -1;
  return 0;
}

int mg_strcasecmp(const struct mg_str str1, const struct mg_str str2) {
  size_t i = 0;
  while (i < str1.len && i < str2.len) {
    int c1 = mg_tolc(str1.buf[i]);
    int c2 = mg_tolc(str2.buf[i]);
    if (c1 < c2) return -1;
    if (c1 > c2) return 1;
    i++;
  }
  if (i < str1.len) return 1;
  if (i < str2.len) return -1;
  return 0;
}

bool mg_match(struct mg_str s, struct mg_str p, struct mg_str *caps) {
  size_t i = 0, j = 0, ni = 0, nj = 0;
  if (caps) caps->buf = NULL, caps->len = 0;
  while (i < p.len || j < s.len) {
    if (i < p.len && j < s.len &&
        (p.buf[i] == '?' ||
         (p.buf[i] != '*' && p.buf[i] != '#' && s.buf[j] == p.buf[i]))) {
      if (caps == NULL) {
      } else if (p.buf[i] == '?') {
        caps->buf = &s.buf[j], caps->len = 1;     // Finalize `?` cap
        caps++, caps->buf = NULL, caps->len = 0;  // Init next cap
      } else if (caps->buf != NULL && caps->len == 0) {
        caps->len = (size_t) (&s.buf[j] - caps->buf);  // Finalize current cap
        caps++, caps->len = 0, caps->buf = NULL;       // Init next cap
      }
      i++, j++;
    } else if (i < p.len && (p.buf[i] == '*' || p.buf[i] == '#')) {
      if (caps && !caps->buf) caps->len = 0, caps->buf = &s.buf[j];  // Init cap
      ni = i++, nj = j + 1;
    } else if (nj > 0 && nj <= s.len &&
               ((ni < p.len && p.buf[ni] == '#') ||
                (j < s.len && s.buf[j] != '/'))) {
      i = ni, j = nj;
      if (caps && caps->buf == NULL && caps->len == 0) {
        caps--, caps->len = 0;  // Restart previous cap
      }
    } else {
      return false;
    }
  }
  if (caps && caps->buf && caps->len == 0) {
    caps->len = (size_t) (&s.buf[j] - caps->buf);
  }
  return true;
}

bool mg_span(struct mg_str s, struct mg_str *a, struct mg_str *b, char sep) {
  if (s.len == 0 || s.buf == NULL) {
    return false;  // Empty string, nothing to span - fail
  } else {
    size_t len = 0;
    while (len < s.len && s.buf[len] != sep) len++;  // Find separator
    if (a) *a = mg_str_n(s.buf, len);                // Init a
    if (b) *b = mg_str_n(s.buf + len, s.len - len);  // Init b
    if (b && len < s.len) b->buf++, b->len--;        // Skip separator
    return true;
  }
}

bool mg_str_to_num(struct mg_str str, int base, void *val, size_t val_len) {
  size_t i = 0, ndigits = 0;
  uint64_t max = val_len == sizeof(uint8_t)    ? 0xFF
                 : val_len == sizeof(uint16_t) ? 0xFFFF
                 : val_len == sizeof(uint32_t) ? 0xFFFFFFFF
                                               : (uint64_t) ~0;
  uint64_t result = 0;
  if (max == (uint64_t) ~0 && val_len != sizeof(uint64_t)) return false;
  if (base == 0 && str.len >= 2) {
    if (str.buf[i] == '0') {
      i++;
      base = str.buf[i] == 'b' ? 2 : str.buf[i] == 'x' ? 16 : 10;
      if (base != 10) ++i;
    } else {
      base = 10;
    }
  }
  switch (base) {
    case 2:
      while (i < str.len && (str.buf[i] == '0' || str.buf[i] == '1')) {
        uint64_t digit = (uint64_t) (str.buf[i] - '0');
        if (result > max / 2) return false;  // Overflow
        result *= 2;
        if (result > max - digit) return false;  // Overflow
        result += digit;
        i++, ndigits++;
      }
      break;
    case 10:
      while (i < str.len && str.buf[i] >= '0' && str.buf[i] <= '9') {
        uint64_t digit = (uint64_t) (str.buf[i] - '0');
        if (result > max / 10) return false;  // Overflow
        result *= 10;
        if (result > max - digit) return false;  // Overflow
        result += digit;
        i++, ndigits++;
      }
      break;
    case 16:
      while (i < str.len) {
        char c = str.buf[i];
        uint64_t digit = (c >= '0' && c <= '9')   ? (uint64_t) (c - '0')
                         : (c >= 'A' && c <= 'F') ? (uint64_t) (c - '7')
                         : (c >= 'a' && c <= 'f') ? (uint64_t) (c - 'W')
                                                  : (uint64_t) ~0;
        if (digit == (uint64_t) ~0) break;
        if (result > max / 16) return false;  // Overflow
        result *= 16;
        if (result > max - digit) return false;  // Overflow
        result += digit;
        i++, ndigits++;
      }
      break;
    default: return false;
  }
  if (ndigits == 0) return false;
  if (i != str.len) return false;
  if (val_len == 1) {
    *((uint8_t *) val) = (uint8_t) result;
  } else if (val_len == 2) {
    *((uint16_t *) val) = (uint16_t) result;
  } else if (val_len == 4) {
    *((uint32_t *) val) = (uint32_t) result;
  } else {
    *((uint64_t *) val) = (uint64_t) result;
  }
  return true;
}

#ifdef MG_ENABLE_LINES
#line 1 "src/timer.c"
#endif




void mg_timer_init(struct mg_timer **head, struct mg_timer *t, uint64_t ms,
                   unsigned flags, void (*fn)(void *), void *arg) {
  t->period_ms = ms, t->expire = 0;
  t->flags = flags, t->fn = fn, t->arg = arg, t->next = *head;
  *head = t;
}

void mg_timer_free(struct mg_timer **head, struct mg_timer *t) {
  while (*head && *head != t) head = &(*head)->next;
  if (*head) *head = t->next;
}

// t: expiration time, prd: period, now: current time. Return true if expired
bool mg_timer_expired(uint64_t *t, uint64_t prd, uint64_t now) {
  if (now + prd < *t) *t = 0;                    // Time wrapped? Reset timer
  if (*t == 0) *t = now + prd;                   // Firt poll? Set expiration
  if (*t > now) return false;                    // Not expired yet, return
  *t = (now - *t) > prd ? now + prd : *t + prd;  // Next expiration time
  return true;                                   // Expired, return true
}

void mg_timer_poll(struct mg_timer **head, uint64_t now_ms) {
  struct mg_timer *t, *tmp;
  for (t = *head; t != NULL; t = tmp) {
    bool once = t->expire == 0 && (t->flags & MG_TIMER_RUN_NOW) &&
                !(t->flags & MG_TIMER_CALLED);  // Handle MG_TIMER_NOW only once
    bool expired = mg_timer_expired(&t->expire, t->period_ms, now_ms);
    tmp = t->next;
    if (!once && !expired) continue;
    if ((t->flags & MG_TIMER_REPEAT) || !(t->flags & MG_TIMER_CALLED)) {
      t->fn(t->arg);
    }
    t->flags |= MG_TIMER_CALLED;

    // If this timer is not repeating and marked AUTODELETE, remove it
    if (!(t->flags & MG_TIMER_REPEAT) && (t->flags & MG_TIMER_AUTODELETE)) {
      mg_timer_free(head, t);
      mg_free(t);
    }
  }
}

#ifdef MG_ENABLE_LINES
#line 1 "src/tls_aes128.c"
#endif
/******************************************************************************
 *
 * THIS SOURCE CODE IS HEREBY PLACED INTO THE PUBLIC DOMAIN FOR THE GOOD OF ALL
 *
 * This is a simple and straightforward implementation of the AES Rijndael
 * 128-bit block cipher designed by Vincent Rijmen and Joan Daemen. The focus
 * of this work was correctness & accuracy.  It is written in 'C' without any
 * particular focus upon optimization or speed. It should be endian (memory
 * byte order) neutral since the few places that care are handled explicitly.
 *
 * This implementation of Rijndael was created by Steven M. Gibson of GRC.com.
 *
 * It is intended for general purpose use, but was written in support of GRC's
 * reference implementation of the SQRL (Secure Quick Reliable Login) client.
 *
 * See:    http://csrc.nist.gov/archive/aes/rijndael/wsdindex.html
 *
 * NO COPYRIGHT IS CLAIMED IN THIS WORK, HOWEVER, NEITHER IS ANY WARRANTY MADE
 * REGARDING ITS FITNESS FOR ANY PARTICULAR PURPOSE. USE IT AT YOUR OWN RISK.
 *
 *******************************************************************************/




#if MG_TLS == MG_TLS_BUILTIN
/******************************************************************************/
#define AES_DECRYPTION 1  // whether AES decryption is supported
/******************************************************************************/

#define MG_ENCRYPT 1  // specify whether we're encrypting
#define MG_DECRYPT 0  // or decrypting



/******************************************************************************
 *  AES_INIT_KEYGEN_TABLES : MUST be called once before any AES use
 ******************************************************************************/
static void aes_init_keygen_tables(void);

/******************************************************************************
 *  AES_SETKEY : called to expand the key for encryption or decryption
 ******************************************************************************/
static int aes_setkey(
    aes_context *ctx,          // pointer to context
    int mode,                  // 1 or 0 for Encrypt/Decrypt
    const unsigned char *key,  // AES input key
    unsigned int keysize);     // size in bytes (must be 16, 24, 32 for
                               // 128, 192 or 256-bit keys respectively)
                               // returns 0 for success

/******************************************************************************
 *  AES_CIPHER : called to encrypt or decrypt ONE 128-bit block of data
 ******************************************************************************/
static int aes_cipher(
    aes_context *ctx,               // pointer to context
    const unsigned char input[16],  // 128-bit block to en/decipher
    unsigned char output[16]);      // 128-bit output result block
                                    // returns 0 for success

/******************************************************************************
 *  GCM_CONTEXT : GCM context / holds keytables, instance data, and AES ctx
 ******************************************************************************/
typedef struct {
  int mode;                     // cipher direction: encrypt/decrypt
  uint64_t len;                 // cipher data length processed so far
  uint64_t add_len;             // total add data length
  uint64_t HL[16];              // precalculated lo-half HTable
  uint64_t HH[16];              // precalculated hi-half HTable
  unsigned char base_ectr[16];  // first counter-mode cipher output for tag
  unsigned char y[16];          // the current cipher-input IV|Counter value
  unsigned char buf[16];        // buf working value
  aes_context aes_ctx;          // cipher context used
} gcm_context;

/******************************************************************************
 *  GCM_SETKEY : sets the GCM (and AES) keying material for use
 ******************************************************************************/
static int gcm_setkey(
    gcm_context *ctx,           // caller-provided context ptr
    const unsigned char *key,   // pointer to cipher key
    const unsigned int keysize  // size in bytes (must be 16, 24, 32 for
                                // 128, 192 or 256-bit keys respectively)
);                              // returns 0 for success

/******************************************************************************
 *
 *  GCM_CRYPT_AND_TAG
 *
 *  This either encrypts or decrypts the user-provided data and, either
 *  way, generates an authentication tag of the requested length. It must be
 *  called with a GCM context whose key has already been set with GCM_SETKEY.
 *
 *  The user would typically call this explicitly to ENCRYPT a buffer of data
 *  and optional associated data, and produce its an authentication tag.
 *
 *  To reverse the process the user would typically call the companion
 *  GCM_AUTH_DECRYPT function to decrypt data and verify a user-provided
 *  authentication tag.  The GCM_AUTH_DECRYPT function calls this function
 *  to perform its decryption and tag generation, which it then compares.
 *
 ******************************************************************************/
static int gcm_crypt_and_tag(
    gcm_context *ctx,  // gcm context with key already setup
    int mode,          // cipher direction: MG_ENCRYPT (1) or MG_DECRYPT (0)
    const unsigned char *iv,     // pointer to the 12-byte initialization vector
    size_t iv_len,               // byte length if the IV. should always be 12
    const unsigned char *add,    // pointer to the non-ciphered additional data
    size_t add_len,              // byte length of the additional AEAD data
    const unsigned char *input,  // pointer to the cipher data source
    unsigned char *output,       // pointer to the cipher data destination
    size_t length,               // byte length of the cipher data
    unsigned char *tag,          // pointer to the tag to be generated
    size_t tag_len);             // byte length of the tag to be generated

/******************************************************************************
 *
 *  GCM_START
 *
 *  Given a user-provided GCM context, this initializes it, sets the encryption
 *  mode, and preprocesses the initialization vector and additional AEAD data.
 *
 ******************************************************************************/
static int gcm_start(
    gcm_context *ctx,          // pointer to user-provided GCM context
    int mode,                  // MG_ENCRYPT (1) or MG_DECRYPT (0)
    const unsigned char *iv,   // pointer to initialization vector
    size_t iv_len,             // IV length in bytes (should == 12)
    const unsigned char *add,  // pointer to additional AEAD data (NULL if none)
    size_t add_len);           // length of additional AEAD data (bytes)

/******************************************************************************
 *
 *  GCM_UPDATE
 *
 *  This is called once or more to process bulk plaintext or ciphertext data.
 *  We give this some number of bytes of input and it returns the same number
 *  of output bytes. If called multiple times (which is fine) all but the final
 *  invocation MUST be called with length mod 16 == 0. (Only the final call can
 *  have a partial block length of < 128 bits.)
 *
 ******************************************************************************/
static int gcm_update(gcm_context *ctx,  // pointer to user-provided GCM context
                      size_t length,     // length, in bytes, of data to process
                      const unsigned char *input,  // pointer to source data
                      unsigned char *output);  // pointer to destination data

/******************************************************************************
 *
 *  GCM_FINISH
 *
 *  This is called once after all calls to GCM_UPDATE to finalize the GCM.
 *  It performs the final GHASH to produce the resulting authentication TAG.
 *
 ******************************************************************************/
static int gcm_finish(
    gcm_context *ctx,    // pointer to user-provided GCM context
    unsigned char *tag,  // ptr to tag buffer - NULL if tag_len = 0
    size_t tag_len);     // length, in bytes, of the tag-receiving buf

/******************************************************************************
 *
 *  GCM_ZERO_CTX
 *
 *  The GCM context contains both the GCM context and the AES context.
 *  This includes keying and key-related material which is security-
 *  sensitive, so it MUST be zeroed after use. This function does that.
 *
 ******************************************************************************/
static void gcm_zero_ctx(gcm_context *ctx);

/******************************************************************************
 *
 * THIS SOURCE CODE IS HEREBY PLACED INTO THE PUBLIC DOMAIN FOR THE GOOD OF ALL
 *
 * This is a simple and straightforward implementation of the AES Rijndael
 * 128-bit block cipher designed by Vincent Rijmen and Joan Daemen. The focus
 * of this work was correctness & accuracy.  It is written in 'C' without any
 * particular focus upon optimization or speed. It should be endian (memory
 * byte order) neutral since the few places that care are handled explicitly.
 *
 * This implementation of Rijndael was created by Steven M. Gibson of GRC.com.
 *
 * It is intended for general purpose use, but was written in support of GRC's
 * reference implementation of the SQRL (Secure Quick Reliable Login) client.
 *
 * See:    http://csrc.nist.gov/archive/aes/rijndael/wsdindex.html
 *
 * NO COPYRIGHT IS CLAIMED IN THIS WORK, HOWEVER, NEITHER IS ANY WARRANTY MADE
 * REGARDING ITS FITNESS FOR ANY PARTICULAR PURPOSE. USE IT AT YOUR OWN RISK.
 *
 *******************************************************************************/

static int aes_tables_inited = 0;  // run-once flag for performing key
                                   // expasion table generation (see below)
/*
 *  The following static local tables must be filled-in before the first use of
 *  the GCM or AES ciphers. They are used for the AES key expansion/scheduling
 *  and once built are read-only and thread safe. The "gcm_initialize" function
 *  must be called once during system initialization to populate these arrays
 *  for subsequent use by the AES key scheduler. If they have not been built
 *  before attempted use, an error will be returned to the caller.
 *
 *  NOTE: GCM Encryption/Decryption does NOT REQUIRE AES decryption. Since
 *  GCM uses AES in counter-mode, where the AES cipher output is XORed with
 *  the GCM input, we ONLY NEED AES encryption.  Thus, to save space AES
 *  decryption is typically disabled by setting AES_DECRYPTION to 0 in aes.h.
 */
// We always need our forward tables
static unsigned char FSb[256];  // Forward substitution box (FSb)
static uint32_t FT0[256];       // Forward key schedule assembly tables
static uint32_t FT1[256];
static uint32_t FT2[256];
static uint32_t FT3[256];

#if AES_DECRYPTION              // We ONLY need reverse for decryption
static unsigned char RSb[256];  // Reverse substitution box (RSb)
static uint32_t RT0[256];       // Reverse key schedule assembly tables
static uint32_t RT1[256];
static uint32_t RT2[256];
static uint32_t RT3[256];
#endif /* AES_DECRYPTION */

static uint32_t RCON[10];  // AES round constants

/*
 * Platform Endianness Neutralizing Load and Store Macro definitions
 * AES wants platform-neutral Little Endian (LE) byte ordering
 */
#define GET_UINT32_LE(n, b, i)                                               \
  {                                                                          \
    (n) = ((uint32_t) (b)[(i)]) | ((uint32_t) (b)[(i) + 1] << 8) |           \
          ((uint32_t) (b)[(i) + 2] << 16) | ((uint32_t) (b)[(i) + 3] << 24); \
  }

#define PUT_UINT32_LE(n, b, i)                  \
  {                                             \
    (b)[(i)] = (unsigned char) ((n));           \
    (b)[(i) + 1] = (unsigned char) ((n) >> 8);  \
    (b)[(i) + 2] = (unsigned char) ((n) >> 16); \
    (b)[(i) + 3] = (unsigned char) ((n) >> 24); \
  }

/*
 *  AES forward and reverse encryption round processing macros
 */
#define AES_FROUND(X0, X1, X2, X3, Y0, Y1, Y2, Y3)          \
  {                                                         \
    X0 = *RK++ ^ FT0[(Y0) & 0xFF] ^ FT1[(Y1 >> 8) & 0xFF] ^ \
         FT2[(Y2 >> 16) & 0xFF] ^ FT3[(Y3 >> 24) & 0xFF];   \
                                                            \
    X1 = *RK++ ^ FT0[(Y1) & 0xFF] ^ FT1[(Y2 >> 8) & 0xFF] ^ \
         FT2[(Y3 >> 16) & 0xFF] ^ FT3[(Y0 >> 24) & 0xFF];   \
                                                            \
    X2 = *RK++ ^ FT0[(Y2) & 0xFF] ^ FT1[(Y3 >> 8) & 0xFF] ^ \
         FT2[(Y0 >> 16) & 0xFF] ^ FT3[(Y1 >> 24) & 0xFF];   \
                                                            \
    X3 = *RK++ ^ FT0[(Y3) & 0xFF] ^ FT1[(Y0 >> 8) & 0xFF] ^ \
         FT2[(Y1 >> 16) & 0xFF] ^ FT3[(Y2 >> 24) & 0xFF];   \
  }

#define AES_RROUND(X0, X1, X2, X3, Y0, Y1, Y2, Y3)          \
  {                                                         \
    X0 = *RK++ ^ RT0[(Y0) & 0xFF] ^ RT1[(Y3 >> 8) & 0xFF] ^ \
         RT2[(Y2 >> 16) & 0xFF] ^ RT3[(Y1 >> 24) & 0xFF];   \
                                                            \
    X1 = *RK++ ^ RT0[(Y1) & 0xFF] ^ RT1[(Y0 >> 8) & 0xFF] ^ \
         RT2[(Y3 >> 16) & 0xFF] ^ RT3[(Y2 >> 24) & 0xFF];   \
                                                            \
    X2 = *RK++ ^ RT0[(Y2) & 0xFF] ^ RT1[(Y1 >> 8) & 0xFF] ^ \
         RT2[(Y0 >> 16) & 0xFF] ^ RT3[(Y3 >> 24) & 0xFF];   \
                                                            \
    X3 = *RK++ ^ RT0[(Y3) & 0xFF] ^ RT1[(Y2 >> 8) & 0xFF] ^ \
         RT2[(Y1 >> 16) & 0xFF] ^ RT3[(Y0 >> 24) & 0xFF];   \
  }

/*
 *  These macros improve the readability of the key
 *  generation initialization code by collapsing
 *  repetitive common operations into logical pieces.
 */
#define ROTL8(x) ((x << 8) & 0xFFFFFFFF) | (x >> 24)
#define XTIME(x) ((x << 1) ^ ((x & 0x80) ? 0x1B : 0x00))
#define MUL(x, y) ((x && y) ? pow[(log[x] + log[y]) % 255] : 0)
#define MIX(x, y)                     \
  {                                   \
    y = ((y << 1) | (y >> 7)) & 0xFF; \
    x ^= y;                           \
  }
#define CPY128     \
  {                \
    *RK++ = *SK++; \
    *RK++ = *SK++; \
    *RK++ = *SK++; \
    *RK++ = *SK++; \
  }

/******************************************************************************
 *
 *  AES_INIT_KEYGEN_TABLES
 *
 *  Fills the AES key expansion tables allocated above with their static
 *  data. This is not "per key" data, but static system-wide read-only
 *  table data. THIS FUNCTION IS NOT THREAD SAFE. It must be called once
 *  at system initialization to setup the tables for all subsequent use.
 *
 ******************************************************************************/
void aes_init_keygen_tables(void) {
  int i, x, y, z;  // general purpose iteration and computation locals
  int pow[256];
  int log[256];

  if (aes_tables_inited) return;

  // fill the 'pow' and 'log' tables over GF(2^8)
  for (i = 0, x = 1; i < 256; i++) {
    pow[i] = x;
    log[x] = i;
    x = (x ^ XTIME(x)) & 0xFF;
  }
  // compute the round constants
  for (i = 0, x = 1; i < 10; i++) {
    RCON[i] = (uint32_t) x;
    x = XTIME(x) & 0xFF;
  }
  // fill the forward and reverse substitution boxes
  FSb[0x00] = 0x63;
#if AES_DECRYPTION  // whether AES decryption is supported
  RSb[0x63] = 0x00;
#endif /* AES_DECRYPTION */

  for (i = 1; i < 256; i++) {
    x = y = pow[255 - log[i]];
    MIX(x, y);
    MIX(x, y);
    MIX(x, y);
    MIX(x, y);
    FSb[i] = (unsigned char) (x ^= 0x63);
#if AES_DECRYPTION  // whether AES decryption is supported
    RSb[x] = (unsigned char) i;
#endif /* AES_DECRYPTION */
  }
  // generate the forward and reverse key expansion tables
  for (i = 0; i < 256; i++) {
    x = FSb[i];
    y = XTIME(x) & 0xFF;
    z = (y ^ x) & 0xFF;

    FT0[i] = ((uint32_t) y) ^ ((uint32_t) x << 8) ^ ((uint32_t) x << 16) ^
             ((uint32_t) z << 24);

    FT1[i] = ROTL8(FT0[i]);
    FT2[i] = ROTL8(FT1[i]);
    FT3[i] = ROTL8(FT2[i]);

#if AES_DECRYPTION  // whether AES decryption is supported
    x = RSb[i];

    RT0[i] = ((uint32_t) MUL(0x0E, x)) ^ ((uint32_t) MUL(0x09, x) << 8) ^
             ((uint32_t) MUL(0x0D, x) << 16) ^ ((uint32_t) MUL(0x0B, x) << 24);

    RT1[i] = ROTL8(RT0[i]);
    RT2[i] = ROTL8(RT1[i]);
    RT3[i] = ROTL8(RT2[i]);
#endif /* AES_DECRYPTION */
  }
  aes_tables_inited = 1;  // flag that the tables have been generated
}  // to permit subsequent use of the AES cipher

/******************************************************************************
 *
 *  AES_SET_ENCRYPTION_KEY
 *
 *  This is called by 'aes_setkey' when we're establishing a key for
 *  subsequent encryption.  We give it a pointer to the encryption
 *  context, a pointer to the key, and the key's length in bytes.
 *  Valid lengths are: 16, 24 or 32 bytes (128, 192, 256 bits).
 *
 ******************************************************************************/
static int aes_set_encryption_key(aes_context *ctx, const unsigned char *key,
                                  unsigned int keysize) {
  unsigned int i;          // general purpose iteration local
  uint32_t *RK = ctx->rk;  // initialize our RoundKey buffer pointer

  for (i = 0; i < (keysize >> 2); i++) {
    GET_UINT32_LE(RK[i], key, i << 2);
  }

  switch (ctx->rounds) {
    case 10:
      for (i = 0; i < 10; i++, RK += 4) {
        RK[4] = RK[0] ^ RCON[i] ^ ((uint32_t) FSb[(RK[3] >> 8) & 0xFF]) ^
                ((uint32_t) FSb[(RK[3] >> 16) & 0xFF] << 8) ^
                ((uint32_t) FSb[(RK[3] >> 24) & 0xFF] << 16) ^
                ((uint32_t) FSb[(RK[3]) & 0xFF] << 24);

        RK[5] = RK[1] ^ RK[4];
        RK[6] = RK[2] ^ RK[5];
        RK[7] = RK[3] ^ RK[6];
      }
      break;

    case 12:
      for (i = 0; i < 8; i++, RK += 6) {
        RK[6] = RK[0] ^ RCON[i] ^ ((uint32_t) FSb[(RK[5] >> 8) & 0xFF]) ^
                ((uint32_t) FSb[(RK[5] >> 16) & 0xFF] << 8) ^
                ((uint32_t) FSb[(RK[5] >> 24) & 0xFF] << 16) ^
                ((uint32_t) FSb[(RK[5]) & 0xFF] << 24);

        RK[7] = RK[1] ^ RK[6];
        RK[8] = RK[2] ^ RK[7];
        RK[9] = RK[3] ^ RK[8];
        RK[10] = RK[4] ^ RK[9];
        RK[11] = RK[5] ^ RK[10];
      }
      break;

    case 14:
      for (i = 0; i < 7; i++, RK += 8) {
        RK[8] = RK[0] ^ RCON[i] ^ ((uint32_t) FSb[(RK[7] >> 8) & 0xFF]) ^
                ((uint32_t) FSb[(RK[7] >> 16) & 0xFF] << 8) ^
                ((uint32_t) FSb[(RK[7] >> 24) & 0xFF] << 16) ^
                ((uint32_t) FSb[(RK[7]) & 0xFF] << 24);

        RK[9] = RK[1] ^ RK[8];
        RK[10] = RK[2] ^ RK[9];
        RK[11] = RK[3] ^ RK[10];

        RK[12] = RK[4] ^ ((uint32_t) FSb[(RK[11]) & 0xFF]) ^
                 ((uint32_t) FSb[(RK[11] >> 8) & 0xFF] << 8) ^
                 ((uint32_t) FSb[(RK[11] >> 16) & 0xFF] << 16) ^
                 ((uint32_t) FSb[(RK[11] >> 24) & 0xFF] << 24);

        RK[13] = RK[5] ^ RK[12];
        RK[14] = RK[6] ^ RK[13];
        RK[15] = RK[7] ^ RK[14];
      }
      break;

    default:
      return -1;
  }
  return (0);
}

#if AES_DECRYPTION  // whether AES decryption is supported

/******************************************************************************
 *
 *  AES_SET_DECRYPTION_KEY
 *
 *  This is called by 'aes_setkey' when we're establishing a
 *  key for subsequent decryption.  We give it a pointer to
 *  the encryption context, a pointer to the key, and the key's
 *  length in bits. Valid lengths are: 128, 192, or 256 bits.
 *
 ******************************************************************************/
static int aes_set_decryption_key(aes_context *ctx, const unsigned char *key,
                                  unsigned int keysize) {
  int i, j;
  aes_context cty;         // a calling aes context for set_encryption_key
  uint32_t *RK = ctx->rk;  // initialize our RoundKey buffer pointer
  uint32_t *SK;
  int ret;

  cty.rounds = ctx->rounds;  // initialize our local aes context
  cty.rk = cty.buf;          // round count and key buf pointer

  if ((ret = aes_set_encryption_key(&cty, key, keysize)) != 0) return (ret);

  SK = cty.rk + cty.rounds * 4;

  CPY128  // copy a 128-bit block from *SK to *RK

      for (i = ctx->rounds - 1, SK -= 8; i > 0; i--, SK -= 8) {
    for (j = 0; j < 4; j++, SK++) {
      *RK++ = RT0[FSb[(*SK) & 0xFF]] ^ RT1[FSb[(*SK >> 8) & 0xFF]] ^
              RT2[FSb[(*SK >> 16) & 0xFF]] ^ RT3[FSb[(*SK >> 24) & 0xFF]];
    }
  }
  CPY128  // copy a 128-bit block from *SK to *RK
      memset(&cty, 0, sizeof(aes_context));  // clear local aes context
  return (0);
}

#endif /* AES_DECRYPTION */

/******************************************************************************
 *
 *  AES_SETKEY
 *
 *  Invoked to establish the key schedule for subsequent encryption/decryption
 *
 ******************************************************************************/
static int aes_setkey(aes_context *ctx,  // AES context provided by our caller
                      int mode,          // ENCRYPT or DECRYPT flag
                      const unsigned char *key,  // pointer to the key
                      unsigned int keysize)      // key length in bytes
{
  // since table initialization is not thread safe, we could either add
  // system-specific mutexes and init the AES key generation tables on
  // demand, or ask the developer to simply call "gcm_initialize" once during
  // application startup before threading begins. That's what we choose.
  if (!aes_tables_inited) return (-1);  // fail the call when not inited.

  ctx->mode = mode;    // capture the key type we're creating
  ctx->rk = ctx->buf;  // initialize our round key pointer

  switch (keysize)  // set the rounds count based upon the keysize
  {
    case 16:
      ctx->rounds = 10;
      break;  // 16-byte, 128-bit key
    case 24:
      ctx->rounds = 12;
      break;  // 24-byte, 192-bit key
    case 32:
      ctx->rounds = 14;
      break;  // 32-byte, 256-bit key
    default:
      return (-1);
  }

#if AES_DECRYPTION
  if (mode == MG_DECRYPT)  // expand our key for encryption or decryption
    return (aes_set_decryption_key(ctx, key, keysize));
  else /* MG_ENCRYPT */
#endif /* AES_DECRYPTION */
    return (aes_set_encryption_key(ctx, key, keysize));
}

/******************************************************************************
 *
 *  AES_CIPHER
 *
 *  Perform AES encryption and decryption.
 *  The AES context will have been setup with the encryption mode
 *  and all keying information appropriate for the task.
 *
 ******************************************************************************/
static int aes_cipher(aes_context *ctx, const unsigned char input[16],
                      unsigned char output[16]) {
  int i;
  uint32_t *RK, X0, X1, X2, X3, Y0, Y1, Y2, Y3;  // general purpose locals

  RK = ctx->rk;

  GET_UINT32_LE(X0, input, 0);
  X0 ^= *RK++;  // load our 128-bit
  GET_UINT32_LE(X1, input, 4);
  X1 ^= *RK++;  // input buffer in a storage
  GET_UINT32_LE(X2, input, 8);
  X2 ^= *RK++;  // memory endian-neutral way
  GET_UINT32_LE(X3, input, 12);
  X3 ^= *RK++;

#if AES_DECRYPTION  // whether AES decryption is supported

  if (ctx->mode == MG_DECRYPT) {
    for (i = (ctx->rounds >> 1) - 1; i > 0; i--) {
      AES_RROUND(Y0, Y1, Y2, Y3, X0, X1, X2, X3);
      AES_RROUND(X0, X1, X2, X3, Y0, Y1, Y2, Y3);
    }

    AES_RROUND(Y0, Y1, Y2, Y3, X0, X1, X2, X3);

    X0 = *RK++ ^ ((uint32_t) RSb[(Y0) & 0xFF]) ^
         ((uint32_t) RSb[(Y3 >> 8) & 0xFF] << 8) ^
         ((uint32_t) RSb[(Y2 >> 16) & 0xFF] << 16) ^
         ((uint32_t) RSb[(Y1 >> 24) & 0xFF] << 24);

    X1 = *RK++ ^ ((uint32_t) RSb[(Y1) & 0xFF]) ^
         ((uint32_t) RSb[(Y0 >> 8) & 0xFF] << 8) ^
         ((uint32_t) RSb[(Y3 >> 16) & 0xFF] << 16) ^
         ((uint32_t) RSb[(Y2 >> 24) & 0xFF] << 24);

    X2 = *RK++ ^ ((uint32_t) RSb[(Y2) & 0xFF]) ^
         ((uint32_t) RSb[(Y1 >> 8) & 0xFF] << 8) ^
         ((uint32_t) RSb[(Y0 >> 16) & 0xFF] << 16) ^
         ((uint32_t) RSb[(Y3 >> 24) & 0xFF] << 24);

    X3 = *RK++ ^ ((uint32_t) RSb[(Y3) & 0xFF]) ^
         ((uint32_t) RSb[(Y2 >> 8) & 0xFF] << 8) ^
         ((uint32_t) RSb[(Y1 >> 16) & 0xFF] << 16) ^
         ((uint32_t) RSb[(Y0 >> 24) & 0xFF] << 24);
  } else /* MG_ENCRYPT */
  {
#endif /* AES_DECRYPTION */

    for (i = (ctx->rounds >> 1) - 1; i > 0; i--) {
      AES_FROUND(Y0, Y1, Y2, Y3, X0, X1, X2, X3);
      AES_FROUND(X0, X1, X2, X3, Y0, Y1, Y2, Y3);
    }

    AES_FROUND(Y0, Y1, Y2, Y3, X0, X1, X2, X3);

    X0 = *RK++ ^ ((uint32_t) FSb[(Y0) & 0xFF]) ^
         ((uint32_t) FSb[(Y1 >> 8) & 0xFF] << 8) ^
         ((uint32_t) FSb[(Y2 >> 16) & 0xFF] << 16) ^
         ((uint32_t) FSb[(Y3 >> 24) & 0xFF] << 24);

    X1 = *RK++ ^ ((uint32_t) FSb[(Y1) & 0xFF]) ^
         ((uint32_t) FSb[(Y2 >> 8) & 0xFF] << 8) ^
         ((uint32_t) FSb[(Y3 >> 16) & 0xFF] << 16) ^
         ((uint32_t) FSb[(Y0 >> 24) & 0xFF] << 24);

    X2 = *RK++ ^ ((uint32_t) FSb[(Y2) & 0xFF]) ^
         ((uint32_t) FSb[(Y3 >> 8) & 0xFF] << 8) ^
         ((uint32_t) FSb[(Y0 >> 16) & 0xFF] << 16) ^
         ((uint32_t) FSb[(Y1 >> 24) & 0xFF] << 24);

    X3 = *RK++ ^ ((uint32_t) FSb[(Y3) & 0xFF]) ^
         ((uint32_t) FSb[(Y0 >> 8) & 0xFF] << 8) ^
         ((uint32_t) FSb[(Y1 >> 16) & 0xFF] << 16) ^
         ((uint32_t) FSb[(Y2 >> 24) & 0xFF] << 24);

#if AES_DECRYPTION  // whether AES decryption is supported
  }
#endif /* AES_DECRYPTION */

  PUT_UINT32_LE(X0, output, 0);
  PUT_UINT32_LE(X1, output, 4);
  PUT_UINT32_LE(X2, output, 8);
  PUT_UINT32_LE(X3, output, 12);

  return (0);
}
/* end of aes.c */
/******************************************************************************
 *
 * THIS SOURCE CODE IS HEREBY PLACED INTO THE PUBLIC DOMAIN FOR THE GOOD OF ALL
 *
 * This is a simple and straightforward implementation of AES-GCM authenticated
 * encryption. The focus of this work was correctness & accuracy. It is written
 * in straight 'C' without any particular focus upon optimization or speed. It
 * should be endian (memory byte order) neutral since the few places that care
 * are handled explicitly.
 *
 * This implementation of AES-GCM was created by Steven M. Gibson of GRC.com.
 *
 * It is intended for general purpose use, but was written in support of GRC's
 * reference implementation of the SQRL (Secure Quick Reliable Login) client.
 *
 * See:    http://csrc.nist.gov/publications/nistpubs/800-38D/SP-800-38D.pdf
 *         http://csrc.nist.gov/groups/ST/toolkit/BCM/documents/proposedmodes/
 *         gcm/gcm-revised-spec.pdf
 *
 * NO COPYRIGHT IS CLAIMED IN THIS WORK, HOWEVER, NEITHER IS ANY WARRANTY MADE
 * REGARDING ITS FITNESS FOR ANY PARTICULAR PURPOSE. USE IT AT YOUR OWN RISK.
 *
 *******************************************************************************/

/******************************************************************************
 *                      ==== IMPLEMENTATION WARNING ====
 *
 *  This code was developed for use within SQRL's fixed environmnent. Thus, it
 *  is somewhat less "general purpose" than it would be if it were designed as
 *  a general purpose AES-GCM library. Specifically, it bothers with almost NO
 *  error checking on parameter limits, buffer bounds, etc. It assumes that it
 *  is being invoked by its author or by someone who understands the values it
 *  expects to receive. Its behavior will be undefined otherwise.
 *
 *  All functions that might fail are defined to return 'ints' to indicate a
 *  problem. Most do not do so now. But this allows for error propagation out
 *  of internal functions if robust error checking should ever be desired.
 *
 ******************************************************************************/

/* Calculating the "GHASH"
 *
 * There are many ways of calculating the so-called GHASH in software, each with
 * a traditional size vs performance tradeoff.  The GHASH (Galois field hash) is
 * an intriguing construction which takes two 128-bit strings (also the cipher's
 * block size and the fundamental operation size for the system) and hashes them
 * into a third 128-bit result.
 *
 * Many implementation solutions have been worked out that use large precomputed
 * table lookups in place of more time consuming bit fiddling, and this approach
 * can be scaled easily upward or downward as needed to change the time/space
 * tradeoff. It's been studied extensively and there's a solid body of theory
 * and practice.  For example, without using any lookup tables an implementation
 * might obtain 119 cycles per byte throughput, whereas using a simple, though
 * large, key-specific 64 kbyte 8-bit lookup table the performance jumps to 13
 * cycles per byte.
 *
 * And Intel's processors have, since 2010, included an instruction which does
 * the entire 128x128->128 bit job in just several 64x64->128 bit pieces.
 *
 * Since SQRL is interactive, and only processing a few 128-bit blocks, I've
 * settled upon a relatively slower but appealing small-table compromise which
 * folds a bunch of not only time consuming but also bit twiddling into a simple
 * 16-entry table which is attributed to Victor Shoup's 1996 work while at
 * Bellcore: "On Fast and Provably Secure MessageAuthentication Based on
 * Universal Hashing."  See: http://www.shoup.net/papers/macs.pdf
 * See, also section 4.1 of the "gcm-revised-spec" cited above.
 */

/*
 *  This 16-entry table of pre-computed constants is used by the
 *  GHASH multiplier to improve over a strictly table-free but
 *  significantly slower 128x128 bit multiple within GF(2^128).
 */
static const uint64_t last4[16] = {
    0x0000, 0x1c20, 0x3840, 0x2460, 0x7080, 0x6ca0, 0x48c0, 0x54e0,
    0xe100, 0xfd20, 0xd940, 0xc560, 0x9180, 0x8da0, 0xa9c0, 0xb5e0};

/*
 * Platform Endianness Neutralizing Load and Store Macro definitions
 * GCM wants platform-neutral Big Endian (BE) byte ordering
 */
#define GET_UINT32_BE(n, b, i)                                            \
  {                                                                       \
    (n) = ((uint32_t) (b)[(i)] << 24) | ((uint32_t) (b)[(i) + 1] << 16) | \
          ((uint32_t) (b)[(i) + 2] << 8) | ((uint32_t) (b)[(i) + 3]);     \
  }

#define PUT_UINT32_BE(n, b, i)                  \
  {                                             \
    (b)[(i)] = (unsigned char) ((n) >> 24);     \
    (b)[(i) + 1] = (unsigned char) ((n) >> 16); \
    (b)[(i) + 2] = (unsigned char) ((n) >> 8);  \
    (b)[(i) + 3] = (unsigned char) ((n));       \
  }

/******************************************************************************
 *
 *  GCM_INITIALIZE
 *
 *  Must be called once to initialize the GCM library.
 *
 *  At present, this only calls the AES keygen table generator, which expands
 *  the AES keying tables for use. This is NOT A THREAD-SAFE function, so it
 *  MUST be called during system initialization before a multi-threading
 *  environment is running.
 *
 ******************************************************************************/
int mg_gcm_initialize(void) {
  aes_init_keygen_tables();
  return (0);
}

/******************************************************************************
 *
 *  GCM_MULT
 *
 *  Performs a GHASH operation on the 128-bit input vector 'x', setting
 *  the 128-bit output vector to 'x' times H using our precomputed tables.
 *  'x' and 'output' are seen as elements of GCM's GF(2^128) Galois field.
 *
 ******************************************************************************/
static void gcm_mult(
    gcm_context *ctx,           // pointer to established context
    const unsigned char x[16],  // pointer to 128-bit input vector
    unsigned char output[16])   // pointer to 128-bit output vector
{
  int i;
  unsigned char lo, hi, rem;
  uint64_t zh, zl;

  lo = (unsigned char) (x[15] & 0x0f);
  hi = (unsigned char) (x[15] >> 4);
  zh = ctx->HH[lo];
  zl = ctx->HL[lo];

  for (i = 15; i >= 0; i--) {
    lo = (unsigned char) (x[i] & 0x0f);
    hi = (unsigned char) (x[i] >> 4);

    if (i != 15) {
      rem = (unsigned char) (zl & 0x0f);
      zl = (zh << 60) | (zl >> 4);
      zh = (zh >> 4);
      zh ^= (uint64_t) last4[rem] << 48;
      zh ^= ctx->HH[lo];
      zl ^= ctx->HL[lo];
    }
    rem = (unsigned char) (zl & 0x0f);
    zl = (zh << 60) | (zl >> 4);
    zh = (zh >> 4);
    zh ^= (uint64_t) last4[rem] << 48;
    zh ^= ctx->HH[hi];
    zl ^= ctx->HL[hi];
  }
  PUT_UINT32_BE(zh >> 32, output, 0);
  PUT_UINT32_BE(zh, output, 4);
  PUT_UINT32_BE(zl >> 32, output, 8);
  PUT_UINT32_BE(zl, output, 12);
}

/******************************************************************************
 *
 *  GCM_SETKEY
 *
 *  This is called to set the AES-GCM key. It initializes the AES key
 *  and populates the gcm context's pre-calculated HTables.
 *
 ******************************************************************************/
static int gcm_setkey(
    gcm_context *ctx,            // pointer to caller-provided gcm context
    const unsigned char *key,    // pointer to the AES encryption key
    const unsigned int keysize)  // size in bytes (must be 16, 24, 32 for
                                 // 128, 192 or 256-bit keys respectively)
{
  int ret, i, j;
  uint64_t hi, lo;
  uint64_t vl, vh;
  unsigned char h[16];

  memset(ctx, 0, sizeof(gcm_context));  // zero caller-provided GCM context
  memset(h, 0, 16);                     // initialize the block to encrypt

  // encrypt the null 128-bit block to generate a key-based value
  // which is then used to initialize our GHASH lookup tables
  if ((ret = aes_setkey(&ctx->aes_ctx, MG_ENCRYPT, key, keysize)) != 0)
    return (ret);
  if ((ret = aes_cipher(&ctx->aes_ctx, h, h)) != 0) return (ret);

  GET_UINT32_BE(hi, h, 0);  // pack h as two 64-bit ints, big-endian
  GET_UINT32_BE(lo, h, 4);
  vh = (uint64_t) hi << 32 | lo;

  GET_UINT32_BE(hi, h, 8);
  GET_UINT32_BE(lo, h, 12);
  vl = (uint64_t) hi << 32 | lo;

  ctx->HL[8] = vl;  // 8 = 1000 corresponds to 1 in GF(2^128)
  ctx->HH[8] = vh;
  ctx->HH[0] = 0;  // 0 corresponds to 0 in GF(2^128)
  ctx->HL[0] = 0;

  for (i = 4; i > 0; i >>= 1) {
    uint32_t T = (uint32_t) (vl & 1) * 0xe1000000U;
    vl = (vh << 63) | (vl >> 1);
    vh = (vh >> 1) ^ ((uint64_t) T << 32);
    ctx->HL[i] = vl;
    ctx->HH[i] = vh;
  }
  for (i = 2; i < 16; i <<= 1) {
    uint64_t *HiL = ctx->HL + i, *HiH = ctx->HH + i;
    vh = *HiH;
    vl = *HiL;
    for (j = 1; j < i; j++) {
      HiH[j] = vh ^ ctx->HH[j];
      HiL[j] = vl ^ ctx->HL[j];
    }
  }
  return (0);
}

/******************************************************************************
 *
 *    GCM processing occurs four phases: SETKEY, START, UPDATE and FINISH.
 *
 *  SETKEY:
 *
 *   START: Sets the Encryption/Decryption mode.
 *          Accepts the initialization vector and additional data.
 *
 *  UPDATE: Encrypts or decrypts the plaintext or ciphertext.
 *
 *  FINISH: Performs a final GHASH to generate the authentication tag.
 *
 ******************************************************************************
 *
 *  GCM_START
 *
 *  Given a user-provided GCM context, this initializes it, sets the encryption
 *  mode, and preprocesses the initialization vector and additional AEAD data.
 *
 ******************************************************************************/
int gcm_start(
    gcm_context *ctx,          // pointer to user-provided GCM context
    int mode,                  // GCM_ENCRYPT or GCM_DECRYPT
    const unsigned char *iv,   // pointer to initialization vector
    size_t iv_len,             // IV length in bytes (should == 12)
    const unsigned char *add,  // ptr to additional AEAD data (NULL if none)
    size_t add_len)            // length of additional AEAD data (bytes)
{
  int ret;                     // our error return if the AES encrypt fails
  unsigned char work_buf[16];  // XOR source built from provided IV if len != 16
  const unsigned char *p;      // general purpose array pointer
  size_t use_len;              // byte count to process, up to 16 bytes
  size_t i;                    // local loop iterator

  // since the context might be reused under the same key
  // we zero the working buffers for this next new process
  memset(ctx->y, 0x00, sizeof(ctx->y));
  memset(ctx->buf, 0x00, sizeof(ctx->buf));
  ctx->len = 0;
  ctx->add_len = 0;

  ctx->mode = mode;                // set the GCM encryption/decryption mode
  ctx->aes_ctx.mode = MG_ENCRYPT;  // GCM *always* runs AES in ENCRYPTION mode

  if (iv_len == 12) {            // GCM natively uses a 12-byte, 96-bit IV
    memcpy(ctx->y, iv, iv_len);  // copy the IV to the top of the 'y' buff
    ctx->y[15] = 1;              // start "counting" from 1 (not 0)
  } else  // if we don't have a 12-byte IV, we GHASH whatever we've been given
  {
    memset(work_buf, 0x00, 16);               // clear the working buffer
    PUT_UINT32_BE(iv_len * 8, work_buf, 12);  // place the IV into buffer

    p = iv;
    while (iv_len > 0) {
      use_len = (iv_len < 16) ? iv_len : 16;
      for (i = 0; i < use_len; i++) ctx->y[i] ^= p[i];
      gcm_mult(ctx, ctx->y, ctx->y);
      iv_len -= use_len;
      p += use_len;
    }
    for (i = 0; i < 16; i++) ctx->y[i] ^= work_buf[i];
    gcm_mult(ctx, ctx->y, ctx->y);
  }
  if ((ret = aes_cipher(&ctx->aes_ctx, ctx->y, ctx->base_ectr)) != 0)
    return (ret);

  ctx->add_len = add_len;
  p = add;
  while (add_len > 0) {
    use_len = (add_len < 16) ? add_len : 16;
    for (i = 0; i < use_len; i++) ctx->buf[i] ^= p[i];
    gcm_mult(ctx, ctx->buf, ctx->buf);
    add_len -= use_len;
    p += use_len;
  }
  return (0);
}

/******************************************************************************
 *
 *  GCM_UPDATE
 *
 *  This is called once or more to process bulk plaintext or ciphertext data.
 *  We give this some number of bytes of input and it returns the same number
 *  of output bytes. If called multiple times (which is fine) all but the final
 *  invocation MUST be called with length mod 16 == 0. (Only the final call can
 *  have a partial block length of < 128 bits.)
 *
 ******************************************************************************/
int gcm_update(gcm_context *ctx,  // pointer to user-provided GCM context
               size_t length,     // length, in bytes, of data to process
               const unsigned char *input,  // pointer to source data
               unsigned char *output)       // pointer to destination data
{
  int ret;                 // our error return if the AES encrypt fails
  unsigned char ectr[16];  // counter-mode cipher output for XORing
  size_t use_len;          // byte count to process, up to 16 bytes
  size_t i;                // local loop iterator

  ctx->len += length;  // bump the GCM context's running length count

  while (length > 0) {
    // clamp the length to process at 16 bytes
    use_len = (length < 16) ? length : 16;

    // increment the context's 128-bit IV||Counter 'y' vector
    for (i = 16; i > 12; i--)
      if (++ctx->y[i - 1] != 0) break;

    // encrypt the context's 'y' vector under the established key
    if ((ret = aes_cipher(&ctx->aes_ctx, ctx->y, ectr)) != 0) return (ret);

    // encrypt or decrypt the input to the output
    if (ctx->mode == MG_ENCRYPT) {
      for (i = 0; i < use_len; i++) {
        // XOR the cipher's ouptut vector (ectr) with our input
        output[i] = (unsigned char) (ectr[i] ^ input[i]);
        // now we mix in our data into the authentication hash.
        // if we're ENcrypting we XOR in the post-XOR (output)
        // results, but if we're DEcrypting we XOR in the input
        // data
        ctx->buf[i] ^= output[i];
      }
    } else {
      for (i = 0; i < use_len; i++) {
        // but if we're DEcrypting we XOR in the input data first,
        // i.e. before saving to ouput data, otherwise if the input
        // and output buffer are the same (inplace decryption) we
        // would not get the correct auth tag

        ctx->buf[i] ^= input[i];

        // XOR the cipher's ouptut vector (ectr) with our input
        output[i] = (unsigned char) (ectr[i] ^ input[i]);
      }
    }
    gcm_mult(ctx, ctx->buf, ctx->buf);  // perform a GHASH operation

    length -= use_len;  // drop the remaining byte count to process
    input += use_len;   // bump our input pointer forward
    output += use_len;  // bump our output pointer forward
  }
  return (0);
}

/******************************************************************************
 *
 *  GCM_FINISH
 *
 *  This is called once after all calls to GCM_UPDATE to finalize the GCM.
 *  It performs the final GHASH to produce the resulting authentication TAG.
 *
 ******************************************************************************/
int gcm_finish(gcm_context *ctx,    // pointer to user-provided GCM context
               unsigned char *tag,  // pointer to buffer which receives the tag
               size_t tag_len)  // length, in bytes, of the tag-receiving buf
{
  unsigned char work_buf[16];
  uint64_t orig_len = ctx->len * 8;
  uint64_t orig_add_len = ctx->add_len * 8;
  size_t i;

  if (tag_len > sizeof(work_buf)) return -1;
  if (tag_len != 0) memcpy(tag, ctx->base_ectr, tag_len);

  if (orig_len || orig_add_len) {
    memset(work_buf, 0x00, 16);

    PUT_UINT32_BE((orig_add_len >> 32), work_buf, 0);
    PUT_UINT32_BE((orig_add_len), work_buf, 4);
    PUT_UINT32_BE((orig_len >> 32), work_buf, 8);
    PUT_UINT32_BE((orig_len), work_buf, 12);

    for (i = 0; i < 16; i++) ctx->buf[i] ^= work_buf[i];
    gcm_mult(ctx, ctx->buf, ctx->buf);
    for (i = 0; i < tag_len; i++) tag[i] ^= ctx->buf[i];
  }
  return (0);
}

/******************************************************************************
 *
 *  GCM_CRYPT_AND_TAG
 *
 *  This either encrypts or decrypts the user-provided data and, either
 *  way, generates an authentication tag of the requested length. It must be
 *  called with a GCM context whose key has already been set with GCM_SETKEY.
 *
 *  The user would typically call this explicitly to ENCRYPT a buffer of data
 *  and optional associated data, and produce its an authentication tag.
 *
 *  To reverse the process the user would typically call the companion
 *  GCM_AUTH_DECRYPT function to decrypt data and verify a user-provided
 *  authentication tag.  The GCM_AUTH_DECRYPT function calls this function
 *  to perform its decryption and tag generation, which it then compares.
 *
 ******************************************************************************/
int gcm_crypt_and_tag(
    gcm_context *ctx,            // gcm context with key already setup
    int mode,                    // cipher direction: GCM_ENCRYPT or GCM_DECRYPT
    const unsigned char *iv,     // pointer to the 12-byte initialization vector
    size_t iv_len,               // byte length if the IV. should always be 12
    const unsigned char *add,    // pointer to the non-ciphered additional data
    size_t add_len,              // byte length of the additional AEAD data
    const unsigned char *input,  // pointer to the cipher data source
    unsigned char *output,       // pointer to the cipher data destination
    size_t length,               // byte length of the cipher data
    unsigned char *tag,          // pointer to the tag to be generated
    size_t tag_len)              // byte length of the tag to be generated
{                                /*
                                    assuming that the caller has already invoked gcm_setkey to
                                    prepare the gcm context with the keying material, we simply
                                    invoke each of the three GCM sub-functions in turn...
                                 */
  if (gcm_start(ctx, mode, iv, iv_len, add, add_len) != 0) return -1;
  if (gcm_update(ctx, length, input, output) != 0) return -1;
  return gcm_finish(ctx, tag, tag_len);
}

/******************************************************************************
 *
 *  GCM_ZERO_CTX
 *
 *  The GCM context contains both the GCM context and the AES context.
 *  This includes keying and key-related material which is security-
 *  sensitive, so it MUST be zeroed after use. This function does that.
 *
 ******************************************************************************/
void gcm_zero_ctx(gcm_context *ctx) {
  // zero the context originally provided to us
  memset(ctx, 0, sizeof(gcm_context));
}
//
//  aes-gcm.c
//  Pods
//
//  Created by Markus Kosmal on 20/11/14.
//
//

int mg_aes_gcm_encrypt(unsigned char *output,  //
                       const unsigned char *input, size_t input_length,
                       const unsigned char *key, const size_t key_len,
                       const unsigned char *iv, const size_t iv_len,
                       unsigned char *aead, size_t aead_len, unsigned char *tag,
                       const size_t tag_len) {
  int ret = 0;      // our return value
  gcm_context ctx;  // includes the AES context structure

  gcm_setkey(&ctx, key, (unsigned int) key_len);

  ret = gcm_crypt_and_tag(&ctx, MG_ENCRYPT, iv, iv_len, aead, aead_len, input,
                          output, input_length, tag, tag_len);

  gcm_zero_ctx(&ctx);

  return (ret);
}

int mg_aes_gcm_decrypt(unsigned char *output, const unsigned char *input,
                       size_t input_length, const unsigned char *key,
                       const size_t key_len, const unsigned char *iv,
                       const size_t iv_len, unsigned char *aead,
                       size_t aead_len, const unsigned char *tag,
                       const size_t tag_len) {
  int ret = 0;      // our return value
  gcm_context ctx;  // includes the AES context structure
  unsigned char computed_tag[16];

  if (tag_len > sizeof(computed_tag)) return -1;

  gcm_setkey(&ctx, key, (unsigned int) key_len);

  ret = gcm_crypt_and_tag(&ctx, MG_DECRYPT, iv, iv_len, aead, aead_len, input,
                          output, input_length, computed_tag, tag_len);

  gcm_zero_ctx(&ctx);

  if (!mg_memeq(computed_tag, tag, tag_len)) ret = -1;

  return (ret);
}
#endif
// End of aes128 PD

#ifdef MG_ENABLE_LINES
#line 1 "src/tls_builtin.c"
#endif












#if MG_TLS == MG_TLS_BUILTIN

#ifndef MG_MAX_TLSOID_DEPTH
#define MG_MAX_TLSOID_DEPTH 15
#endif

// PKCS#8 algorithm OIDs
static const uint8_t mg_rsa_oid[] = {
    0x2a, 0x86, 0x48, 0x86, 0xf7,
    0x0d, 0x01, 0x01, 0x01  // 1.2.840.113549.1.1.1 rsaEncryption
};
static const uint8_t mg_ec_public_key_oid[] = {
    0x2a, 0x86, 0x48, 0xce, 0x3d, 0x02, 0x01  // 1.2.840.10045.2.1 ecPublicKey
};
static const uint8_t mg_secp256r1_oid[] = {
    0x2a, 0x86, 0x48, 0xce,
    0x3d, 0x03, 0x01, 0x07  // 1.2.840.10045.3.1.7 secp256r1
};

/* TLS 1.3 Record Content Type (RFC8446 B.1) */
#define MG_TLS_CHANGE_CIPHER 20
#define MG_TLS_ALERT 21
#define MG_TLS_HANDSHAKE 22
#define MG_TLS_APP_DATA 23
#define MG_TLS_HEARTBEAT 24

/* TLS 1.3 Handshake Message Type (RFC8446 B.3) */
#define MG_TLS_CLIENT_HELLO 1
#define MG_TLS_SERVER_HELLO 2
#define MG_TLS_ENCRYPTED_EXTENSIONS 8
#define MG_TLS_CERTIFICATE 11
#define MG_TLS_CERTIFICATE_REQUEST 13
#define MG_TLS_CERTIFICATE_VERIFY 15
#define MG_TLS_FINISHED 20

#define MG_TLS_RSA_USE_CRT 1  // CRT instead of naive RSA

// handshake is re-entrant, so we need to keep track of its state state names
// refer to RFC8446#A.1
enum mg_tls_hs_state {
  // Client state machine:
  MG_TLS_STATE_CLIENT_START,        // Send ClientHello
  MG_TLS_STATE_CLIENT_WAIT_SH,      // Wait for ServerHello
  MG_TLS_STATE_CLIENT_WAIT_EE,      // Wait for EncryptedExtensions
  MG_TLS_STATE_CLIENT_WAIT_CERT,    // Wait for Certificate
  MG_TLS_STATE_CLIENT_WAIT_CV,      // Wait for CertificateVerify
  MG_TLS_STATE_CLIENT_WAIT_FINISH,  // Wait for Finish
  MG_TLS_STATE_CLIENT_CONNECTED,    // Done

  // Server state machine:
  MG_TLS_STATE_SERVER_START,       // Wait for ClientHello
  MG_TLS_STATE_SERVER_RETRY,       // Wait for ClientHello after HelloRetry
  MG_TLS_STATE_SERVER_WAIT_CERT,   // Wait for Certificate
  MG_TLS_STATE_SERVER_WAIT_CV,     // Wait for CertificateVerify
  MG_TLS_STATE_SERVER_NEGOTIATED,  // Wait for Finish
  MG_TLS_STATE_SERVER_CONNECTED    // Done
};

// encryption keys for a TLS connection
struct tls_enc {
  uint32_t sseq;  // server sequence number, used in encryption
  uint32_t cseq;  // client sequence number, used in decryption
  // keys for AES encryption or ChaCha20
  uint8_t handshake_secret[32];
  uint8_t server_write_key[32];
  uint8_t server_write_iv[12];
  uint8_t server_finished_key[32];
  uint8_t client_write_key[32];
  uint8_t client_write_iv[12];
  uint8_t client_finished_key[32];
};

struct mg_rsa_key {
  struct mg_str n;     // modulus
  struct mg_str e;     // public exponent
  struct mg_str d;     // private exponent
  struct mg_str p;     // prime1
  struct mg_str q;     // prime2
  struct mg_str dP;    // exponent1 (d mod (p-1))
  struct mg_str dQ;    // exponent2 (d mod (q-1))
  struct mg_str qInv;  // coefficient ((inverse of q) mod p)
};

// per-connection TLS data
struct tls_data {
  enum mg_tls_hs_state state;  // keep track of connection handshake progress

  struct mg_iobuf send;  // For the receive path, we're reusing c->rtls
  size_t recv_offset;    // While c->rtls contains full records, reuse that
  size_t recv_len;       // buffer but point at individual decrypted messages

  uint8_t content_type;  // Last received record content type

  mg_sha256_ctx sha256;  // incremental SHA-256 hash for TLS handshake

  uint8_t random[32];      // client random from ClientHello
  uint8_t session_id[32];  // client session ID between the handshake states
  uint8_t p256_cli[32];    // client P-256 key between the handshake states
  uint8_t p256_peer[64];   // server: client P-256 public key X || Y
  uint8_t ecdhe_sec[32];   // P-256 ECDHE shared secret between the states

  bool skip_verification;    // do not perform checks on server certificate
  bool cert_requested;       // client received a CertificateRequest
  bool is_twoway;            // server is configured to authenticate clients
  bool is_sntp_pending;      // TLS handshake is waiting for wall-clock time
  struct mg_connection *timec;  // SNTP connection for wall-clock time
  struct mg_str cert_der;    // certificate in DER format
  struct mg_str ca_der;      // current CA certificate
  struct mg_str *ca_bundle_der; // bundle of CA certificates
  size_t ca_bundle_len;         // number of certificates in bundle
  struct mg_str *chain_der;  // certificate chain (intermediate certs)
  size_t chain_len;          // number of certificates in chain
  uint8_t ec_key[32];        // EC private key
  struct mg_rsa_key rsa;
  struct mg_str rsa_key_der;  // RSA private key in DER format
  char hostname[254];         // matching hostname

  bool is_ec_pubkey;         // EC or RSA. TODO(): currently unused
  uint8_t pubkey[512 + 16];  // server EC (64) or RSA (512+exp) public key to
                             // verify cert
  size_t pubkeysz;           // size of the server public key
  uint8_t sighash[32];       // calculated signature verification hash
  uint8_t sighash384[48];    // same, SHA-384, for ecdsa_secp384r1_sha384
  uint64_t hs_start;         // mg_millis() when the handshake started
  uint64_t hs_proc;          // ms spent processing handshake messages

  struct tls_enc enc;       // actual keys in use at this time
  struct tls_enc app_keys;  // storage during two-way auth handshake
};

#define TLS_RECHDR_SIZE 5  // 1 byte type, 2 bytes version, 2 bytes length
#define TLS_MSGHDR_SIZE 4  // 1 byte type, 3 bytes length

#ifdef MG_TLS_SSLKEYLOGFILE
#include <stdio.h>
static void mg_ssl_key_log(const char *label, uint8_t client_random[32],
                           uint8_t *secret, size_t secretsz) {
  FILE *f;
  char *keylogfile = getenv("SSLKEYLOGFILE");
  if (keylogfile == NULL) return;
  MG_DEBUG(("Dumping key log into %s", keylogfile));
  f = fopen(keylogfile, "a");
  if (f != NULL) {
    size_t i;
    fprintf(f, "%s ", label);
    for (i = 0; i < 32; i++) {
      fprintf(f, "%02x", client_random[i]);
    }
    fprintf(f, " ");
    for (i = 0; i < secretsz; i++) {
      fprintf(f, "%02x", secret[i]);
    }
    fprintf(f, "\n");
    fclose(f);
  } else {
    MG_ERROR(("Cannot open %s", keylogfile));
  }
}
#endif

// for derived tls keys we need SHA256([0]*32)
static uint8_t zeros[32] = {0};
static uint8_t zeros_sha256_digest[32] = {
    0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4,
    0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b,
    0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55};

// helper to hexdump buffers inline
static void mg_tls_hexdump(const char *msg, uint8_t *buf, size_t bufsz) {
  MG_VERBOSE(("%s: %M", msg, mg_print_hex, bufsz, buf));
}

// helper utilities to parse ASN.1 DER
struct mg_der_tlv {
  uint8_t type;
  uint32_t len;
  uint8_t *value;
};

// parse DER into a TLV record
static int mg_der_to_tlv(uint8_t *der, size_t dersz, struct mg_der_tlv *tlv) {
  uint32_t n = 0;
  if (dersz < 2) return -1;
  tlv->type = der[0];
  tlv->len = der[1];
  tlv->value = der + 2;
  if (tlv->len > 0x7f) {  // long-form length
    uint32_t i;
    n = tlv->len - 0x80;
    if (n == 0 || n > 4 || dersz < (2 + n)) return -1;
    tlv->len = 0;
    for (i = 0; i < n; i++) {
      tlv->len = (tlv->len << 8) | der[2 + i];
    }
    tlv->value += n;
  }
  if (tlv->len > dersz - (size_t) (tlv->value - der)) return -1;
  return (int) n;
}

static int mg_der_parse(uint8_t *der, size_t dersz, struct mg_der_tlv *tlv) {
  int n = mg_der_to_tlv(der, dersz, tlv);
  if (n < 0) return -1;
  if (tlv->len >= (uint32_t)((unsigned) -6)) return -1; // avoid overflow
  return 2 + n + (int) tlv->len; // 2: type, len; n = long form len bytes
}

static int mg_der_next(struct mg_der_tlv *parent, struct mg_der_tlv *child) {
  int consumed;
  if (parent->len == 0) return 0;
  consumed = mg_der_parse(parent->value, parent->len, child);
  if (consumed < 0) return -1;
  parent->value += consumed;
  parent->len -= (uint32_t) consumed;
  return 1;
}

static int der_find_oid(struct mg_der_tlv *tlv, const uint8_t *oid, size_t oid_len, struct mg_der_tlv *found, int depth) {
  struct mg_der_tlv parent, child;
  int r;
  parent = *tlv;
  while ((r = mg_der_next(&parent, &child)) > 0) {
    if (child.type == 0x06 && child.len == oid_len &&
        memcmp(child.value, oid, oid_len) == 0) {
      return mg_der_next(&parent, found);
    } else if (child.type & 0x20) {
      struct mg_der_tlv sub_parent = child;
      if (depth >= MG_MAX_TLSOID_DEPTH) {
        MG_ERROR(("too nested der"));
        return -1;
      }
      if ((r = der_find_oid(&sub_parent, oid, oid_len, found, depth + 1)) > 0)
        return 1;
      if (r < 0) return -1; // exit on error; r = 0 => not found, keep parsing
    }
  }
  if (r < 0) return -1;
  return 0;
}

static int mg_der_find_oid(struct mg_der_tlv *tlv, const uint8_t *oid,
                           size_t oid_len, struct mg_der_tlv *found) {
  return der_find_oid(tlv, oid, oid_len, found, 0);
}

#if 0
static void mg_der_debug(struct mg_der_tlv *tlv, int depth) {
  MG_DEBUG(("> %.*sd=%d Type: 0x%02X, Length: %u\n", depth * 4, " ", depth,
            tlv->type, tlv->len));

  if (tlv->type & 0x20) {  // Constructed: recurse into children
    struct mg_der_tlv child;
    struct mg_der_tlv parent = *tlv;
    while (mg_der_next(&parent, &child) > 0) {
      mg_der_debug(&child, depth + 1);
    }
  }
}
#endif

// Did we receive a full TLS record in the c->rtls buffer?
static bool mg_tls_got_record(struct mg_connection *c) {
  uint16_t rl;
  if (c->rtls.len < (size_t) TLS_RECHDR_SIZE) return false;
  rl = MG_LOAD_BE16(c->rtls.buf + 3);
  if (rl > 16384 + 256) {  // TLS 1.3 record max length (RFC-9846 5.1)
    mg_error(c, "TLS record too large");
    return false;
  }
  return c->rtls.len >= (size_t) (TLS_RECHDR_SIZE + rl);
}

// Remove a single TLS record from the recv buffer
static void mg_tls_drop_record(struct mg_connection *c) {
  struct mg_iobuf *rio = &c->rtls;
  uint16_t n = MG_LOAD_BE16(rio->buf + 3) + TLS_RECHDR_SIZE;
  mg_iobuf_del(rio, 0, n);
}

// Remove a single TLS message from decrypted buffer, remove the wrapping
// record if it was the last message within a record
static void mg_tls_drop_message(struct mg_connection *c) {
  uint32_t len;
  struct tls_data *tls = (struct tls_data *) c->tls;
  unsigned char *recv_buf = &c->rtls.buf[tls->recv_offset];
  if (tls->recv_len == 0) return;
  len = MG_LOAD_BE24(recv_buf + 1) + TLS_MSGHDR_SIZE;
  if (tls->recv_len < len) {
    mg_error(c, "wrong size");
    return;
  }
  mg_sha256_update(&tls->sha256, recv_buf, len);
  tls->recv_offset += len;
  tls->recv_len -= len;
  if (tls->recv_len == 0) {
    mg_tls_drop_record(c);
  }
}

// TLS1.3 secret derivation based on the key label
static void mg_tls_derive_secret(const char *label, uint8_t *key, size_t keysz,
                                 uint8_t *data, size_t datasz, uint8_t *hash,
                                 size_t hashsz) {
  size_t labelsz = strlen(label);
  uint8_t secret[32];
  uint8_t packed[256] = {0, (uint8_t) hashsz, (uint8_t) labelsz};
  // TODO: assert lengths of label, key, data and hash
  if (labelsz > 0) memmove(packed + 3, label, labelsz);
  packed[3 + labelsz] = (uint8_t) datasz;
  if (datasz > 0) memmove(packed + labelsz + 4, data, datasz);
  packed[4 + labelsz + datasz] = 1;

  mg_hmac_sha256(secret, key, keysz, packed, 5 + labelsz + datasz);
  memmove(hash, secret, hashsz);
}

// at this point we have ECDHE shared secret, we can generate a set of derived
// handshake encryption keys
static void mg_tls_generate_handshake_keys(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;

  mg_sha256_ctx sha256;
  uint8_t early_secret[32];
  uint8_t pre_extract_secret[32];
  uint8_t hello_hash[32];
  uint8_t server_hs_secret[32];
  uint8_t client_hs_secret[32];
#if MG_ENABLE_CHACHA20
  const size_t keysz = 32;
#else
  const size_t keysz = 16;
#endif

  mg_hmac_sha256(early_secret, NULL, 0, zeros, sizeof(zeros));
  mg_tls_derive_secret("tls13 derived", early_secret, 32, zeros_sha256_digest,
                       32, pre_extract_secret, 32);
  mg_hmac_sha256(tls->enc.handshake_secret, pre_extract_secret,
                 sizeof(pre_extract_secret), tls->ecdhe_sec,
                 sizeof(tls->ecdhe_sec));
  // ECDHE keys and secret are not needed anymore, wipe for forward secrecy
  mg_bzero(tls->p256_cli, sizeof(tls->p256_cli));
  mg_bzero(tls->ecdhe_sec, sizeof(tls->ecdhe_sec));
  mg_tls_hexdump("hs secret", tls->enc.handshake_secret, 32);

  // mg_sha256_final is not idempotent, need to copy sha256 context to calculate
  // the digest
  memmove(&sha256, &tls->sha256, sizeof(mg_sha256_ctx));
  mg_sha256_final(hello_hash, &sha256);

  mg_tls_hexdump("hello hash", hello_hash, 32);
  // derive keys needed for the rest of the handshake
  mg_tls_derive_secret("tls13 s hs traffic", tls->enc.handshake_secret, 32,
                       hello_hash, 32, server_hs_secret, 32);
  mg_tls_derive_secret("tls13 c hs traffic", tls->enc.handshake_secret, 32,
                       hello_hash, 32, client_hs_secret, 32);

  mg_tls_derive_secret("tls13 key", server_hs_secret, 32, NULL, 0,
                       tls->enc.server_write_key, keysz);
  mg_tls_derive_secret("tls13 iv", server_hs_secret, 32, NULL, 0,
                       tls->enc.server_write_iv, 12);
  mg_tls_derive_secret("tls13 finished", server_hs_secret, 32, NULL, 0,
                       tls->enc.server_finished_key, 32);

  mg_tls_derive_secret("tls13 key", client_hs_secret, 32, NULL, 0,
                       tls->enc.client_write_key, keysz);
  mg_tls_derive_secret("tls13 iv", client_hs_secret, 32, NULL, 0,
                       tls->enc.client_write_iv, 12);
  mg_tls_derive_secret("tls13 finished", client_hs_secret, 32, NULL, 0,
                       tls->enc.client_finished_key, 32);

  mg_tls_hexdump("s hs traffic", server_hs_secret, 32);
  mg_tls_hexdump("s key", tls->enc.server_write_key, keysz);
  mg_tls_hexdump("s iv", tls->enc.server_write_iv, 12);
  mg_tls_hexdump("s finished", tls->enc.server_finished_key, 32);
  mg_tls_hexdump("c hs traffic", client_hs_secret, 32);
  mg_tls_hexdump("c key", tls->enc.client_write_key, keysz);
  mg_tls_hexdump("c iv", tls->enc.client_write_iv, 12);
  mg_tls_hexdump("c finished", tls->enc.client_finished_key, 32);

#ifdef MG_TLS_SSLKEYLOGFILE
  mg_ssl_key_log("SERVER_HANDSHAKE_TRAFFIC_SECRET", tls->random,
                 server_hs_secret, 32);
  mg_ssl_key_log("CLIENT_HANDSHAKE_TRAFFIC_SECRET", tls->random,
                 client_hs_secret, 32);
#endif
}

static void mg_tls_generate_application_keys(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  uint8_t hash[32];
  uint8_t premaster_secret[32];
  uint8_t master_secret[32];
  uint8_t server_secret[32];
  uint8_t client_secret[32];
#if MG_ENABLE_CHACHA20
  const size_t keysz = 32;
#else
  const size_t keysz = 16;
#endif

  mg_sha256_ctx sha256;
  memmove(&sha256, &tls->sha256, sizeof(mg_sha256_ctx));
  mg_sha256_final(hash, &sha256);

  mg_tls_derive_secret("tls13 derived", tls->enc.handshake_secret, 32,
                       zeros_sha256_digest, 32, premaster_secret, 32);
  mg_hmac_sha256(master_secret, premaster_secret, 32, zeros, 32);

  mg_tls_derive_secret("tls13 s ap traffic", master_secret, 32, hash, 32,
                       server_secret, 32);
  mg_tls_derive_secret("tls13 key", server_secret, 32, NULL, 0,
                       tls->enc.server_write_key, keysz);
  mg_tls_derive_secret("tls13 iv", server_secret, 32, NULL, 0,
                       tls->enc.server_write_iv, 12);
  mg_tls_derive_secret("tls13 c ap traffic", master_secret, 32, hash, 32,
                       client_secret, 32);
  mg_tls_derive_secret("tls13 key", client_secret, 32, NULL, 0,
                       tls->enc.client_write_key, keysz);
  mg_tls_derive_secret("tls13 iv", client_secret, 32, NULL, 0,
                       tls->enc.client_write_iv, 12);

  mg_tls_hexdump("s ap traffic", server_secret, 32);
  mg_tls_hexdump("s key", tls->enc.server_write_key, keysz);
  mg_tls_hexdump("s iv", tls->enc.server_write_iv, 12);
  mg_tls_hexdump("s finished", tls->enc.server_finished_key, 32);
  mg_tls_hexdump("c ap traffic", client_secret, 32);
  mg_tls_hexdump("c key", tls->enc.client_write_key, keysz);
  mg_tls_hexdump("c iv", tls->enc.client_write_iv, 12);
  mg_tls_hexdump("c finished", tls->enc.client_finished_key, 32);
  tls->enc.sseq = tls->enc.cseq = 0;

#ifdef MG_TLS_SSLKEYLOGFILE
  mg_ssl_key_log("SERVER_TRAFFIC_SECRET_0", tls->random, server_secret, 32);
  mg_ssl_key_log("CLIENT_TRAFFIC_SECRET_0", tls->random, client_secret, 32);
#endif
}

// AES GCM encryption of the message + put encoded data into the write buffer
static bool mg_tls_encrypt(struct mg_connection *c, const uint8_t *msg,
                           size_t msgsz, uint8_t msgtype) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  struct mg_iobuf *wio = &tls->send;
  uint8_t *outmsg;
  uint8_t *tag;
  size_t encsz = msgsz + 16 + 1;
  uint8_t hdr[5] = {MG_TLS_APP_DATA, 0x03, 0x03,
                    (uint8_t) ((encsz >> 8) & 0xff), (uint8_t) (encsz & 0xff)};
  uint8_t associated_data[5] = {MG_TLS_APP_DATA, 0x03, 0x03,
                                (uint8_t) ((encsz >> 8) & 0xff),
                                (uint8_t) (encsz & 0xff)};
  uint8_t nonce[12];

  uint32_t seq = c->is_client ? tls->enc.cseq : tls->enc.sseq;
  uint8_t *key =
      c->is_client ? tls->enc.client_write_key : tls->enc.server_write_key;
  uint8_t *iv =
      c->is_client ? tls->enc.client_write_iv : tls->enc.server_write_iv;

  if (msgsz > 16384) {
    MG_ERROR(("msg longer than recordsz"));
    return false;
  }

#if MG_ENABLE_CHACHA20
#else
  mg_gcm_initialize();
#endif

  memmove(nonce, iv, sizeof(nonce));
  nonce[8] ^= (uint8_t) ((seq >> 24) & 255U);
  nonce[9] ^= (uint8_t) ((seq >> 16) & 255U);
  nonce[10] ^= (uint8_t) ((seq >> 8) & 255U);
  nonce[11] ^= (uint8_t) ((seq) & 255U);

  if (mg_iobuf_add(wio, wio->len, hdr, sizeof(hdr)) == 0 ||
      !mg_iobuf_resize(wio, wio->len + encsz))
    return false;
  outmsg = wio->buf + wio->len;
  tag = wio->buf + wio->len + msgsz + 1;
  memmove(outmsg, msg, msgsz);
  outmsg[msgsz] = msgtype;
#if MG_ENABLE_CHACHA20
  (void) tag;  // tag is only used in aes gcm
  {
    size_t n;
    uint8_t *enc = (uint8_t *) mg_calloc(1, msgsz + 256 + 1);
    if (enc == NULL) return false;
    n = mg_chacha20_poly1305_encrypt(enc, key, nonce, associated_data,
                                     sizeof(associated_data), outmsg,
                                     msgsz + 1);
    memmove(outmsg, enc, n);
    mg_free(enc);
  }
#else
  mg_aes_gcm_encrypt(outmsg, outmsg, msgsz + 1, key, 16, nonce, sizeof(nonce),
                     associated_data, sizeof(associated_data), tag, 16);
#endif
  c->is_client ? tls->enc.cseq++ : tls->enc.sseq++;
  wio->len += encsz;
  return true;
}

static void verbose_alert(uint8_t *buf) {
  uint8_t level = buf[0], desc = buf[1];
  MG_LOG(desc == 0 ? MG_LL_VERBOSE : MG_LL_INFO,  // close_notify is normal
         ("TLS ALERT received: level=%d, desc=%d (%s)", level, desc,
          desc == 0    ? "close_notify"
          : desc == 10 ? "unexpected_message"
          : desc == 20 ? "bad_record_mac"
          : desc == 21 ? "decryption_failed"
          : desc == 40 ? "handshake_failure"
          : desc == 42 ? "bad_certificate"
          : desc == 43 ? "unsupported_certificate"
          : desc == 44 ? "certificate_revoked"
          : desc == 45 ? "certificate_expired"
          : desc == 46 ? "certificate_unknown"
          : desc == 48 ? "unknown_ca"
                       : "unknown"));
}

// read an encrypted record, decrypt it in place
static int mg_tls_recv_record(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  struct mg_iobuf *rio = &c->rtls;
  uint16_t msgsz;
  uint8_t *msg;
  uint8_t nonce[12];
  int r;

  uint32_t seq = c->is_client ? tls->enc.sseq : tls->enc.cseq;
  uint8_t *key =
      c->is_client ? tls->enc.server_write_key : tls->enc.client_write_key;
  uint8_t *iv =
      c->is_client ? tls->enc.server_write_iv : tls->enc.client_write_iv;

  if (tls->recv_len > 0) {
    return 0; /* some data from previous record is still present */
  }
  for (;;) {
    if (!mg_tls_got_record(c)) {
      return MG_IO_WAIT;
    }
    if (rio->buf[0] == MG_TLS_APP_DATA) {
      break;
    } else if (rio->buf[0] == MG_TLS_CHANGE_CIPHER) {  // skip CCS
      mg_tls_drop_record(c);
    } else if (rio->buf[0] == MG_TLS_ALERT) {  // Skip Alerts
      if (rio->len >= 7) {
        verbose_alert(&rio->buf[5]);
      } else {
        MG_INFO(("TLS ALERT packet received (short)"));
      }
      mg_tls_drop_record(c);
    } else {
      mg_error(c, "unexpected packet");
      return -1;
    }
  }

  msgsz = MG_LOAD_BE16(rio->buf + 3);
  msg = rio->buf + 5;
  if (msgsz < 17) {
    mg_error(c, "wrong size");
    return -1;
  }

  memmove(nonce, iv, sizeof(nonce));
  nonce[8] ^= (uint8_t) ((seq >> 24) & 255U);
  nonce[9] ^= (uint8_t) ((seq >> 16) & 255U);
  nonce[10] ^= (uint8_t) ((seq >> 8) & 255U);
  nonce[11] ^= (uint8_t) ((seq) & 255U);
#if MG_ENABLE_CHACHA20
  {
    uint8_t associated_data[5] = {MG_TLS_APP_DATA, 0x03, 0x03,
                                  (uint8_t) ((msgsz >> 8) & 0xff),
                                  (uint8_t) (msgsz & 0xff)};
    uint8_t *dec = (uint8_t *) mg_calloc(1, msgsz);
    size_t n;
    if (dec == NULL) {
      mg_error(c, "TLS OOM");
      return -1;
    }
    n = mg_chacha20_poly1305_decrypt(dec, key, nonce, associated_data,
                                     sizeof(associated_data), msg, msgsz);
    if (n == (size_t) -1) {
      mg_free(dec);
      mg_error(c, "decryption error");
      return -1;
    }
    memmove(msg, dec, n);
    mg_free(dec);
  }
#else
  {
    uint8_t associated_data[5] = {MG_TLS_APP_DATA, 0x03, 0x03,
                                  (uint8_t) ((msgsz >> 8) & 0xff),
                                  (uint8_t) (msgsz & 0xff)};
    mg_gcm_initialize();
    if (mg_aes_gcm_decrypt(msg, msg, msgsz - 16, key, 16, nonce, sizeof(nonce),
                           associated_data, sizeof(associated_data),
                           msg + msgsz - 16, 16) != 0) {
      mg_error(c, "GCM tag verify failed");
      return -1;
    }
  }
#endif

  r = msgsz - 16 - 1;
  tls->content_type = msg[msgsz - 16 - 1];
  if (tls->content_type == MG_TLS_ALERT) {  // Process Alerts
    verbose_alert(msg);
    if (msg[0] == 2) {
      mg_error(c, "TLS Fatal alert");
      return -1;
    } else {
      mg_tls_drop_record(c);
      return MG_IO_WAIT;
    }
  }
  tls->recv_offset = (size_t) msg - (size_t) rio->buf;
  tls->recv_len = (size_t) msgsz - 16 - 1;
  c->is_client ? tls->enc.sseq++ : tls->enc.cseq++;
  return r;
}

// Hash the CertificateVerify content: SHA-256 into hash, and if hash384 is not
// NULL, SHA-384 into hash384 for ecdsa_secp384r1_sha384
static void mg_tls_calc_cert_verify_hash(struct mg_connection *c,
                                         uint8_t hash[32], uint8_t *hash384,
                                         bool is_client) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  uint8_t sig_content[130];
  mg_sha256_ctx sha256;

  memset(sig_content, 0x20, 64);
  if (is_client) {
    uint8_t client_context[34] = "TLS 1.3, client CertificateVerify";
    memcpy(sig_content + 64, client_context, sizeof(client_context));
  } else {
    uint8_t server_context[34] = "TLS 1.3, server CertificateVerify";
    memcpy(sig_content + 64, server_context, sizeof(server_context));
  }

  memmove(&sha256, &tls->sha256, sizeof(mg_sha256_ctx));
  mg_sha256_final(sig_content + 98, &sha256);

  mg_sha256_init(&sha256);
  mg_sha256_update(&sha256, sig_content, sizeof(sig_content));
  mg_sha256_final(hash, &sha256);
  if (hash384 != NULL) mg_sha384(hash384, sig_content, sizeof(sig_content));
}

// read and parse ClientHello record. Return 0 if it has a P-256 key share,
// 1 if it does not and HelloRetryRequest must be sent, negative otherwise
static int mg_tls_server_recv_hello(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  struct mg_iobuf *rio = &c->rtls;
  uint8_t session_id_len;
  uint16_t j;
  uint16_t cipher_suites_len;
  uint16_t ext_len;
  uint8_t *ext;
  uint16_t msgsz;
  bool have_p256 = false;
  bool is_retry = tls->state == MG_TLS_STATE_SERVER_RETRY;

  for (;;) {
    if (!mg_tls_got_record(c)) return MG_IO_WAIT;
    if (!is_retry || rio->buf[0] != MG_TLS_CHANGE_CIPHER) break;
    mg_tls_drop_record(c);  // compatibility mode CCS before 2nd ClientHello
  }
  if (rio->buf[0] != MG_TLS_HANDSHAKE || rio->buf[5] != MG_TLS_CLIENT_HELLO) {
    mg_error(c, "not a client hello packet");
    return -1;
  }
  if (rio->len < 50) goto fail;
  msgsz = MG_LOAD_BE16(rio->buf + 3);
  if (((uint32_t) msgsz + 4) > rio->len) goto fail;
  mg_sha256_update(&tls->sha256, rio->buf + 5, msgsz);
  // store client random
  memmove(tls->random, rio->buf + 11, sizeof(tls->random));
  // store session_id
  session_id_len = rio->buf[43];
  if (session_id_len == sizeof(tls->session_id)) {
    if (rio->len < (size_t)(46 + session_id_len)) goto fail; // 2: ciphers len
    memmove(tls->session_id, rio->buf + 44, session_id_len);
  } else if (session_id_len != 0) {
    MG_ERROR(("bad session id len"));
    goto fail;
  } // session_id_len is either sizeof(tls->session_id) or 0
  if (((uint32_t) 44 + 2 + session_id_len) > rio->len) goto fail;
  cipher_suites_len = MG_LOAD_BE16(rio->buf + 44 + session_id_len);
  if (((uint32_t) cipher_suites_len + 46 + 2 + 2 + session_id_len) > rio->len)
    goto fail;
  ext_len = MG_LOAD_BE16(rio->buf + 48 + session_id_len + cipher_suites_len);
  ext = rio->buf + 50 + session_id_len + cipher_suites_len;
  if (((unsigned char *) ext + ext_len) > (rio->buf + rio->len)) goto fail;
  for (j = 0; j < ext_len;) {
    uint16_t k;
    uint16_t key_exchange_len;
    uint8_t *key_exchange;
    uint16_t n = MG_LOAD_BE16(ext + j + 2);
    if (((uint32_t) n + j + 4) > ext_len) goto fail;
    if (MG_LOAD_BE16(ext + j) != 0x0033) {  // not a key share extension, ignore
      j += (uint16_t) (n + 4);
      continue;
    }
    key_exchange_len = MG_LOAD_BE16(ext + j + 4);
    key_exchange = ext + j + 6;
    if ((key_exchange + key_exchange_len) > ((uint8_t *) rio->buf + rio->len))
      goto fail;
    for (k = 0; k < key_exchange_len;) {
      uint16_t m = MG_LOAD_BE16(key_exchange + k + 2);
      uint16_t group = MG_LOAD_BE16(key_exchange + k);
      if (((uint32_t) m + k + 4) > key_exchange_len) goto fail;
      // secp256r1, uncompressed point 0x04 || X || Y
      if (group == 0x0017 && m == 65 && key_exchange[k + 4] == 0x04) {
        memmove(tls->p256_peer, key_exchange + k + 5, 64);
        have_p256 = true;
      }
      k += (uint16_t) (m + 4);
    }
    j += (uint16_t) (n + 4);
  }
  if (have_p256 || !is_retry) {  // No P-256 share: ask for it, but only once
    mg_tls_drop_record(c);
    return have_p256 ? 0 : 1;
  }
fail:
  mg_error(c, "bad client hello");
  return -1;
}

#define PLACEHOLDER_8B 'X', 'X', 'X', 'X', 'X', 'X', 'X', 'X'
#define PLACEHOLDER_16B PLACEHOLDER_8B, PLACEHOLDER_8B
#define PLACEHOLDER_32B PLACEHOLDER_16B, PLACEHOLDER_16B

// put ServerHello record into wio buffer
static bool mg_tls_server_send_hello(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  struct mg_iobuf *wio = &tls->send;
  uint8_t hello[155], pub[65], prv[32], hdr[5] = {0x16, 0x03, 0x03, 0, 0};
  size_t n = sizeof(hello);  // ServerHello length, including 4-byte header

  // calculate P-256 keyshare and the shared secret
  pub[0] = 0x04;  // uncompressed point 0x04 || X || Y
  if (!mg_p256_make_key(pub + 1, prv) ||
      !mg_p256_shared_secret(tls->p256_peer, prv, tls->ecdhe_sec)) {
    mg_bzero(prv, sizeof(prv));
    mg_error(c, "bad key");
    return false;
  }
  mg_bzero(prv, sizeof(prv));
  mg_tls_hexdump("s ecdhe sec", tls->ecdhe_sec, sizeof(tls->ecdhe_sec));

  hello[0] = 0x02;  // server hello, tls 1.2
  MG_STORE_BE24(hello + 1, n - 4);
  hello[4] = 0x03, hello[5] = 0x03;
  if (!mg_random(hello + 6, 32)) {  // server random, RFC 8446 4.1.3
    mg_error(c, "RNG");
    return false;
  }
  hello[38] = 0x20;  // session ID length + session ID (32 bytes)
  memmove(hello + 39, tls->session_id, sizeof(tls->session_id));
#if MG_ENABLE_CHACHA20
  hello[71] = 0x13, hello[72] = 0x03;  // TLS_CHACHA20_POLY1305_SHA256
#else
  hello[71] = 0x13, hello[72] = 0x01;  // TLS_AES_128_GCM_SHA256
#endif
  hello[73] = 0x00;                      // no compression
  MG_STORE_BE16(hello + 74, 65 + 14);    // extensions length
  MG_STORE_BE16(hello + 76, 0x0033);     // key share
  MG_STORE_BE16(hello + 78, 65 + 4);
  MG_STORE_BE16(hello + 80, 0x0017);     // secp256r1
  MG_STORE_BE16(hello + 82, 65);
  memmove(hello + 84, pub, sizeof(pub));
  memmove(hello + 149, "\x00\x2b\x00\x02\x03\x04", 6);  // tls1.3 == 0x304
  MG_STORE_BE16(hdr + 3, n);

  // server hello message
  if (mg_iobuf_add(wio, wio->len, hdr, sizeof(hdr)) == 0 ||
      mg_iobuf_add(wio, wio->len, hello, n) == 0)
    return false;
  mg_sha256_update(&tls->sha256, hello, n);

  // change cipher message, unless it was already sent after HelloRetryRequest
  if (tls->state != MG_TLS_STATE_SERVER_RETRY &&
      mg_iobuf_add(wio, wio->len, "\x14\x03\x03\x00\x01\x01", 6) == 0)
    return false;
  return true;
}

// HelloRetryRequest random, SHA-256 of "HelloRetryRequest", RFC 8446 4.1.3
static const uint8_t hrr_random[32] = {
    0xcf, 0x21, 0xad, 0x74, 0xe5, 0x9a, 0x61, 0x11, 0xbe, 0x1d, 0x8c,
    0x02, 0x1e, 0x65, 0xb8, 0x91, 0xc2, 0xa2, 0x11, 0x16, 0x7a, 0xbb,
    0x8c, 0x5e, 0x07, 0x9e, 0x09, 0xe2, 0xc8, 0xa8, 0x33, 0x9c};

// put HelloRetryRequest record into wio buffer, asking for a P-256 key share.
// Replace ClientHello in the transcript with message_hash, RFC 8446 4.4.1
static bool mg_tls_server_send_hrr(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  struct mg_iobuf *wio = &tls->send;
  uint8_t hrr[88], hdr[5] = {0x16, 0x03, 0x03, 0, sizeof(hrr)};
  uint8_t mh[4 + 32] = {0xfe, 0, 0, 32};  // message_hash(ClientHello1)

  hrr[0] = 0x02;  // server hello, tls 1.2
  MG_STORE_BE24(hrr + 1, sizeof(hrr) - 4);
  hrr[4] = 0x03, hrr[5] = 0x03;
  memmove(hrr + 6, hrr_random, sizeof(hrr_random));
  hrr[38] = 0x20;  // session ID length + session ID (32 bytes)
  memmove(hrr + 39, tls->session_id, sizeof(tls->session_id));
#if MG_ENABLE_CHACHA20
  hrr[71] = 0x13, hrr[72] = 0x03;  // TLS_CHACHA20_POLY1305_SHA256
#else
  hrr[71] = 0x13, hrr[72] = 0x01;  // TLS_AES_128_GCM_SHA256
#endif
  hrr[73] = 0x00;  // no compression
  // extensions: supported versions tls1.3, key share secp256r1 (group only)
  memmove(hrr + 74, "\x00\x0c\x00\x2b\x00\x02\x03\x04\x00\x33\x00\x02\x00\x17",
          14);

  mg_sha256_final(mh + 4, &tls->sha256);
  mg_sha256_init(&tls->sha256);
  mg_sha256_update(&tls->sha256, mh, sizeof(mh));
  mg_sha256_update(&tls->sha256, hrr, sizeof(hrr));

  // HelloRetryRequest message, followed by change cipher message
  return mg_iobuf_add(wio, wio->len, hdr, sizeof(hdr)) != 0 &&
         mg_iobuf_add(wio, wio->len, hrr, sizeof(hrr)) != 0 &&
         mg_iobuf_add(wio, wio->len, "\x14\x03\x03\x00\x01\x01", 6) != 0;
}

static bool mg_tls_server_send_ext(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  // server extensions
  uint8_t ext[6] = {0x08, 0, 0, 2, 0, 0};
  mg_sha256_update(&tls->sha256, ext, sizeof(ext));
  return mg_tls_encrypt(c, ext, sizeof(ext), MG_TLS_HANDSHAKE);
}

// signature algorithms we actually support: ecdsa_secp256r1_sha256,
// ecdsa_secp384r1_sha384, rsa_pss_rsae_sha256, rsa_pkcs1_sha256
static const uint8_t secp256r1_sig_algs[] = {
    0x00, 0x0d, 0x00, 0x0a, 0x00, 0x08, 0x04, 0x03,
    0x05, 0x03, 0x08, 0x04, 0x04, 0x01};

static bool mg_tls_server_send_cert_request(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  uint8_t req[13 + sizeof(secp256r1_sig_algs)];
  req[0] = MG_TLS_CERTIFICATE_REQUEST;  // handshake header
  MG_STORE_BE24(req + 1, 9 + sizeof(secp256r1_sig_algs));
  req[4] = 0;                                              // context length
  MG_STORE_BE16(req + 5, 6 + sizeof(secp256r1_sig_algs));  // extensions length
  MG_STORE_BE16(req + 7, 13);  // "signature algorithms"
  MG_STORE_BE16(req + 9, 2 + sizeof(secp256r1_sig_algs));  // length
  MG_STORE_BE16(
      req + 11,
      sizeof(secp256r1_sig_algs));  // signature hash algorithms length
  memcpy(req + 13, (uint8_t *) secp256r1_sig_algs, sizeof(secp256r1_sig_algs));
  mg_sha256_update(&tls->sha256, req, sizeof(req));
  return mg_tls_encrypt(c, req, sizeof(req), MG_TLS_HANDSHAKE);
}

static bool mg_tls_send_cert(struct mg_connection *c, bool is_client) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  int send_ca = !is_client && tls->ca_der.len > 0;
  // DER certificate + CA (server optional)
  size_t i, offset, total_size = tls->cert_der.len + 5;
  uint8_t *cert;
  bool res = false;
  for (i = 1; i < tls->chain_len; i++) {
    total_size += tls->chain_der[i].len + 5;
  }
  if (send_ca) {
    total_size += tls->ca_der.len + 5;
  }
  cert = (uint8_t *) mg_calloc(1, 13 + total_size);
  if (cert == NULL) return res;
  cert[0] = MG_TLS_CERTIFICATE;  // handshake header
  if (is_client && tls->cert_der.len == 0) total_size = 0; // empty list
  MG_STORE_BE24(cert + 1, total_size + 4);
  cert[4] = 0;                          // request context
  MG_STORE_BE24(cert + 5, total_size);  // 3 bytes: cert (s) length
  offset = 8;
  if (total_size > 0) { // handle empty list, RFC-8446 4.4.2
    MG_STORE_BE24(cert + offset, tls->cert_der.len);  // 3 bytes: 1st cert len
    offset += 3;
    // bytes 11+ are certificate in DER format
    memmove(cert + offset, tls->cert_der.buf, tls->cert_der.len);
    offset += tls->cert_der.len;
    MG_STORE_BE16(cert + offset, 0);  // certificate extensions (none)
    offset += 2;
    for (i = 1; i < tls->chain_len; i++) {
      MG_STORE_BE24(cert + offset, tls->chain_der[i].len);
      offset += 3;
      memmove(cert + offset, tls->chain_der[i].buf, tls->chain_der[i].len);
      offset += tls->chain_der[i].len;
      MG_STORE_BE16(cert + offset, 0);  // certificate extensions (none)
      offset += 2;
    }
    if (send_ca) {
      MG_STORE_BE24(cert + offset, tls->ca_der.len);  // 3 bytes: CA cert length
      offset += 3;
      memmove(cert + offset, tls->ca_der.buf,
              tls->ca_der.len);  // CA cert data
      offset += tls->ca_der.len;
      MG_STORE_BE16(cert + offset, 0);  // certificate extensions (none)
      offset += 2;
    }
  }
  mg_sha256_update(&tls->sha256, cert, offset);
  res = mg_tls_encrypt(c, cert, offset, MG_TLS_HANDSHAKE);
  mg_free(cert);
  return res;
}

static void mg_tls_mgf1(uint8_t *mask, size_t mask_len, const uint8_t *seed,
                        size_t seed_len) {
  uint32_t counter = 0;
  size_t chunk, chunk_len, generated = 0;
  while (generated < mask_len) {
    mg_sha256_ctx ctx;
    uint8_t digest[32];
    uint8_t ctr[4];
    ctr[0] = (uint8_t) (counter >> 24);
    ctr[1] = (uint8_t) (counter >> 16);
    ctr[2] = (uint8_t) (counter >> 8);
    ctr[3] = (uint8_t) counter;
    mg_sha256_init(&ctx);
    mg_sha256_update(&ctx, seed, seed_len);
    mg_sha256_update(&ctx, ctr, sizeof(ctr));
    mg_sha256_final(digest, &ctx);
    chunk_len = mask_len - generated;
    chunk = (chunk_len < sizeof(digest) ? chunk_len : sizeof(digest));
    memmove(mask + generated, digest, chunk);
    generated += chunk;
    counter++;
  }
}

static unsigned int mg_tls_rsa_bits(const struct mg_str *n) {
  size_t i = 0;
  unsigned int bits = 0;
  while (i < n->len && n->buf[i] == 0) i++;
  if (i == n->len) return 0;
  bits = (unsigned int) ((n->len - i) * 8);
  {
    uint8_t byte = (uint8_t) n->buf[i];
    while ((byte & 0x80U) == 0) {
      bits--;
      byte = (uint8_t) (byte << 1);
    }
  }
  return bits;
}

static bool mg_tls_pss_encode(const uint8_t *hash, size_t hashlen,
                              const struct mg_str *n, uint8_t *em) {
  size_t emlen = n->len, saltlen = hashlen, dblen, pslen, i;
  uint8_t salt[64];   // Max salt size for any reasonable hash
  uint8_t m[136];     // 8 + max hash (64) + max salt (64) = 136
  uint8_t H[64];      // Max hash size
  uint8_t DB[512];    // Max for 4096-bit RSA
  uint8_t mask[512];  // Max for 4096-bit RSA
  mg_sha256_ctx ctx;

  // Check bounds
  if (saltlen > sizeof(salt) || (8 + hashlen + saltlen) > sizeof(m) ||
      hashlen > sizeof(H) || emlen > sizeof(DB)) {
    MG_ERROR(("RSA key too large for static buffers"));
    return false;
  }
  if (emlen < hashlen + saltlen + 2) {
    return false;
  }
  if (!mg_random(salt, saltlen)) {
    return false;
  }
  MG_VERBOSE(("PSS salt: %M", mg_print_hex, saltlen, salt));

  // Build m = 8 zero bytes || hash || salt
  memset(m, 0, 8);
  memcpy(m + 8, hash, hashlen);
  memcpy(m + 8 + hashlen, salt, saltlen);
  mg_sha256_init(&ctx);
  mg_sha256_update(&ctx, m, 8 + hashlen + saltlen);
  mg_sha256_final(H, &ctx);
  MG_VERBOSE(("PSS H: %M", mg_print_hex, hashlen, H));

  dblen = emlen - hashlen - 1;
  pslen = emlen - hashlen - saltlen - 2;

  // Build DB = PS || 0x01 || salt
  memset(DB, 0, pslen);
  DB[pslen] = 0x01;
  memcpy(DB + pslen + 1, salt, saltlen);

  // Generate mask and apply to DB
  mg_tls_mgf1(mask, dblen, H, hashlen);
  for (i = 0; i < dblen; i++) DB[i] ^= mask[i];

  {
    // PSS standard: emBits = modulus_bit_length - 1
    unsigned rsa_bits = mg_tls_rsa_bits(n);
    unsigned embits = rsa_bits - 1;
    unsigned unused = (unsigned) (8 * emlen - embits);
    MG_VERBOSE(("RSA modulus bits: %u, emBits: %u, emlen: %zu, unused: %u",
                rsa_bits, embits, emlen, unused));
    if (unused > 0 && unused < 8) {
      uint8_t mask_byte = (uint8_t) (0xff >> unused);
      MG_VERBOSE(("Applying mask 0x%02x to first byte (was 0x%02x)", mask_byte,
                  DB[0]));
      DB[0] &= mask_byte;
      MG_VERBOSE(("First byte after mask: 0x%02x", DB[0]));
    }
  }

  // Build final em = maskedDB || H || 0xbc
  memcpy(em, DB, dblen);
  memcpy(em + dblen, H, hashlen);
  em[emlen - 1] = 0xbc;
  return true;
}

static bool mg_tls_rsa_sign(struct tls_data *tls, const uint8_t *em,
                            size_t emlen, uint8_t *sig) {
  const uint8_t *n = (const uint8_t *) tls->rsa.n.buf;
  size_t nlen = tls->rsa.n.len;
  int crt_result;
  while (nlen > 0 && *n == 0) n++, nlen--;
  if (emlen != nlen) return false;
#if MG_TLS_RSA_USE_CRT
  // RSA CRT (Chinese Remainder Theorem) optimization:
  // s1 = em^dP mod p
  // s2 = em^dQ mod q
  // h = qInv * (s1 - s2) mod p
  // s = s2 + h * q

  if (tls->rsa.p.len == 0 || tls->rsa.q.len == 0 || tls->rsa.dP.len == 0 ||
      tls->rsa.dQ.len == 0 || tls->rsa.qInv.len == 0) {
    MG_ERROR(("CRT parameters missing, cannot use CRT optimization"));
    return false;
  }

  MG_VERBOSE(("Using RSA-CRT optimization"));

  crt_result = mg_rsa_crt_sign(
      em, emlen, (const uint8_t *) tls->rsa.dP.buf, tls->rsa.dP.len,
      (const uint8_t *) tls->rsa.dQ.buf, tls->rsa.dQ.len,
      (const uint8_t *) tls->rsa.p.buf, tls->rsa.p.len,
      (const uint8_t *) tls->rsa.q.buf, tls->rsa.q.len,
      (const uint8_t *) tls->rsa.qInv.buf, tls->rsa.qInv.len, sig, nlen);

  if (crt_result == 0) {
    uint8_t check[512];
    bool verified;
    if (nlen > sizeof(check) ||
        mg_rsa_mod_pow(n, nlen, (const uint8_t *) tls->rsa.e.buf,
                       tls->rsa.e.len, sig, nlen, check, nlen) != 0) {
      MG_ERROR(("CRT signature verification failed"));
      return false;
    }
    verified = memcmp(check, em, emlen) == 0;
    mg_bzero(check, sizeof(check));
    if (!verified) {
      MG_ERROR(("CRT signature verification failed"));
      return false;
    }
    MG_VERBOSE(("CRT signature successful (first 4 bytes): %02x %02x %02x %02x",
                sig[0], sig[1], sig[2], sig[3]));
    MG_VERBOSE(("CRT signature successful (last 4 bytes): %02x %02x %02x %02x",
                sig[nlen - 4], sig[nlen - 3], sig[nlen - 2], sig[nlen - 1]));
    return true;
  } else {
    MG_ERROR(("CRT signing failed"));
    return false;
  }
#else
  int ret;
  // Standard RSA: s = em^d mod n
  memset(sig, 0, nlen);
  ret = mg_rsa_mod_pow(n, nlen, (const uint8_t *) tls->rsa.d.buf,
                       tls->rsa.d.len, em, emlen, sig, nlen);
  if (ret == 0) {
    MG_VERBOSE(("RSA signature first 4 bytes: %02x %02x %02x %02x", sig[0],
                sig[1], sig[2], sig[3]));
    MG_VERBOSE(("RSA signature last 4 bytes: %02x %02x %02x %02x",
                sig[nlen - 4], sig[nlen - 3], sig[nlen - 2], sig[nlen - 1]));
  }
  return ret == 0;
#endif
}

static size_t mg_rsa_trim_len(const uint8_t **p, size_t n) {
  while (n > 0 && **p == 0) (*p)++, n--;
  return n;
}

static bool mg_tls_send_cert_verify(struct mg_connection *c, bool is_client) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  uint8_t hash[32] = {0};

  mg_tls_calc_cert_verify_hash(c, (uint8_t *) hash, NULL, is_client);

  if (tls->rsa.n.len > 0 && tls->rsa.d.len > 0) {
    // RSA certificate verify packet
    const uint8_t *n = (const uint8_t *) tls->rsa.n.buf;
    size_t nlen = tls->rsa.n.len;
    struct mg_str rn;
    size_t emlen, verifysz;
    uint8_t em[512];      // Max for 4096-bit RSA
    uint8_t verify[520];  // 8 + 512 max
    nlen = mg_rsa_trim_len(&n, nlen);
    rn = mg_str_n((char *) n, nlen);
    emlen = nlen;
    verifysz = 8U + emlen;

    // Check bounds
    if (emlen > sizeof(em) || verifysz > sizeof(verify)) {
      MG_ERROR(("RSA key too large for static buffers"));
      return false;
    }

    if (!mg_tls_pss_encode(hash, sizeof(hash), &rn, em)) {
      MG_ERROR(("Failed PSS encode"));
      return false;
    }

    // Validate PSS encoded message format
    if (em[emlen - 1] != 0xbc) {
      MG_ERROR(("Invalid PSS encoding: last byte is 0x%02x, expected 0xbc",
                em[emlen - 1]));
      return false;
    }

    // Build verify packet header, then sign directly into the packet
    verify[0] = 0x0f;
    MG_STORE_BE24(verify + 1, emlen + 4);
    MG_STORE_BE16(verify + 4, 0x0804);
    MG_STORE_BE16(verify + 6, emlen);

    // Sign directly into the verify buffer (verify + 8 = signature location)
    memset(verify + 8, 0, emlen);  // Initialize signature area
    if (!mg_tls_rsa_sign(tls, em, emlen, verify + 8)) {
      MG_ERROR(("Failed RSA sign"));
      return false;
    }

    MG_VERBOSE(
        ("PSS EM first 4: %02x %02x %02x %02x", em[0], em[1], em[2], em[3]));
    MG_VERBOSE(("PSS EM last 4: %02x %02x %02x %02x", em[emlen - 4],
                em[emlen - 3], em[emlen - 2], em[emlen - 1]));

    mg_sha256_update(&tls->sha256, verify, verifysz);
    return mg_tls_encrypt(c, verify, verifysz, MG_TLS_HANDSHAKE);
  } else {
    // EC certificate verify packet
    uint8_t verify[82] = {0x0f, 0x00, 0x00, 0x00, 0x04, 0x03, 0x00, 0x00};
    size_t sigsz, verifysz = 0;
    int neg1, neg2;
    uint8_t sig[64] = {0};
    if (!mg_p256_sign(tls->ec_key, hash, sizeof(hash), sig)) {
      mg_error(c, "P-256 sign");
      return false;
    }

    neg1 = !!(sig[0] & 0x80);
    neg2 = !!(sig[32] & 0x80);
    verify[8] = 0x30;  // ASN.1 SEQUENCE
    verify[9] = (uint8_t) (68 + neg1 + neg2);
    verify[10] = 0x02;  // ASN.1 INTEGER
    verify[11] = (uint8_t) (32 + neg1);
    memmove(verify + 12 + neg1, sig, 32);
    verify[12 + 32 + neg1] = 0x02;  // ASN.1 INTEGER
    verify[13 + 32 + neg1] = (uint8_t) (32 + neg2);
    memmove(verify + 14 + 32 + neg1 + neg2, sig + 32, 32);

    sigsz = (size_t) (70 + neg1 + neg2);
    verifysz = 8U + sigsz;
    verify[3] = (uint8_t) (sigsz + 4);
    verify[7] = (uint8_t) sigsz;

    mg_sha256_update(&tls->sha256, verify, verifysz);
    return mg_tls_encrypt(c, verify, verifysz, MG_TLS_HANDSHAKE);
  }
}

static bool mg_tls_server_send_finish(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  mg_sha256_ctx sha256;
  uint8_t hash[32];
  uint8_t finish[36] = {0x14, 0, 0, 32};
  memmove(&sha256, &tls->sha256, sizeof(mg_sha256_ctx));
  mg_sha256_final(hash, &sha256);
  mg_hmac_sha256(finish + 4, tls->enc.server_finished_key, 32, hash, 32);
  if (!mg_tls_encrypt(c, finish, sizeof(finish), MG_TLS_HANDSHAKE))
    return false;
  mg_sha256_update(&tls->sha256, finish, sizeof(finish));
  return true;
}

static int mg_tls_server_recv_finish(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  unsigned char *recv_buf;
  // we have to backup sha256 value to restore it later, since Finished record
  // is exceptional and is not supposed to be added to the rolling hash
  // calculation.
  mg_sha256_ctx sha256 = tls->sha256;
  if (mg_tls_recv_record(c) < 0) {
    return -1;
  }
  recv_buf = &c->rtls.buf[tls->recv_offset];
  if (recv_buf[0] != MG_TLS_FINISHED) {
    mg_error(c, "expected Finish but got msg 0x%02x", recv_buf[0]);
    return -1;
  }
  mg_tls_drop_message(c);

  // restore hash
  tls->sha256 = sha256;
  return 0;
}

static bool mg_tls_client_send_hello(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  struct mg_iobuf *wio = &tls->send;

  uint8_t p256_pub[64];  // X || Y, sent as an uncompressed point 0x04 || X || Y

  // - "signature algorithms we actually support", see above
  //   uint8_t secp256r1_sig_algs[]
  // - all popular signature algorithms (if we don't care about verification)
  uint8_t all_sig_algs[34] = {
      0x00, 0x0d, 0x00, 0x1e, 0x00, 0x1c, 0x04, 0x03, 0x05, 0x03, 0x06, 0x03,
      0x08, 0x07, 0x08, 0x08, 0x08, 0x09, 0x08, 0x0a, 0x08, 0x0b, 0x08, 0x04,
      0x08, 0x05, 0x08, 0x06, 0x04, 0x01, 0x05, 0x01, 0x06, 0x01};
  uint8_t server_name_ext[9] = {0x00, 0x00, 0x00, 0xfe, 0x00,
                                0xfe, 0x00, 0x00, 0xfe};

  // clang-format off
  uint8_t msg_client_hello[178] = {
      // TLS Client Hello header reported as TLS1.2 (5)
      0x16, 0x03, 0x03, 0x00, 0xd8,
      // client hello, tls 1.2 (6)
      0x01, 0x00, 0x00, 0xd4, 0x03, 0x03,
      // random (32 bytes)
      PLACEHOLDER_32B,
      // session ID length + session ID (32 bytes)
      0x20, PLACEHOLDER_32B, 0x00,
      0x02,  // size = 2 bytes
#if MG_ENABLE_CHACHA20
      // TLS_CHACHA20_POLY1305_SHA256
      0x13, 0x03,
#else
      // TLS_AES_128_GCM_SHA256
      0x13, 0x01,
#endif
      // no compression
      0x01, 0x00,
      // extensions + keyshare
      0x00, 0xfe,
      // keyshare: secp256r1, uncompressed point
      0x00, 0x33, 0x00, 0x47, 0x00, 0x45,
      0x00, 0x17, 0x00, 0x41, 0x04, PLACEHOLDER_32B, PLACEHOLDER_32B,
      // supported groups (secp256r1)
      0x00, 0x0a, 0x00, 0x04, 0x00, 0x02, 0x00, 0x17,
      // supported versions (tls1.3 == 0x304)
      0x00, 0x2b, 0x00, 0x03, 0x02, 0x03, 0x04,
      // session ticket (none)
      0x00, 0x23, 0x00, 0x00, // 177 bytes till here
	};
  // clang-format on
  const char *hostname = tls->hostname;
  size_t hostnamesz = strlen(tls->hostname);
  size_t hostname_extsz = hostnamesz ? hostnamesz + 9 : 0;
  uint8_t *sig_alg =
      tls->skip_verification ? all_sig_algs : (uint8_t *) secp256r1_sig_algs;
  size_t sig_alg_sz = tls->skip_verification ? sizeof(all_sig_algs)
                                             : sizeof(secp256r1_sig_algs);

  // patch ClientHello with correct hostname ext length (if any)
  MG_STORE_BE16(msg_client_hello + 3,
                hostname_extsz + 216 - 9 - 34 + sig_alg_sz);
  MG_STORE_BE16(msg_client_hello + 7,
                hostname_extsz + 212 - 9 - 34 + sig_alg_sz);
  MG_STORE_BE16(msg_client_hello + 82,
                hostname_extsz + 137 - 9 - 34 + sig_alg_sz);

  if (hostnamesz > 0) {
    MG_STORE_BE16(server_name_ext + 2, hostnamesz + 5);
    MG_STORE_BE16(server_name_ext + 4, hostnamesz + 3);
    MG_STORE_BE16(server_name_ext + 7, hostnamesz);
  }

  // calculate keyshare
  if (!mg_p256_make_key(p256_pub, tls->p256_cli)) {
    mg_error(c, "P-256 key");
    return false;  // Do not send uninitialised p256_pub
  }

  // fill in the gaps: random + session ID + keyshare
  if (!mg_random(tls->session_id, sizeof(tls->session_id))) mg_error(c, "RNG");
  if (!mg_random(tls->random, sizeof(tls->random))) mg_error(c, "RNG");
  memmove(msg_client_hello + 11, tls->random, sizeof(tls->random));
  memmove(msg_client_hello + 44, tls->session_id, sizeof(tls->session_id));
  memmove(msg_client_hello + 95, p256_pub, sizeof(p256_pub));

  // client hello message
  if (mg_iobuf_add(wio, wio->len, msg_client_hello, sizeof(msg_client_hello)) ==
      0)
    return false;
  mg_sha256_update(&tls->sha256, msg_client_hello + 5,
                   sizeof(msg_client_hello) - 5);
  if (mg_iobuf_add(wio, wio->len, sig_alg, sig_alg_sz) == 0) return false;
  mg_sha256_update(&tls->sha256, sig_alg, sig_alg_sz);
  if (hostnamesz > 0) {
    if (mg_iobuf_add(wio, wio->len, server_name_ext, sizeof(server_name_ext)) ==
            0 ||
        mg_iobuf_add(wio, wio->len, hostname, hostnamesz) == 0)
      return false;
    mg_sha256_update(&tls->sha256, server_name_ext, sizeof(server_name_ext));
    mg_sha256_update(&tls->sha256, (uint8_t *) hostname, hostnamesz);
  }

  // change cipher message
  if (mg_iobuf_add(wio, wio->len, (const char *) "\x14\x03\x03\x00\x01\x01",
                   6) == 0)
    return false;
  return true;
}

static int mg_tls_client_recv_hello(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  struct mg_iobuf *rio = &c->rtls;
  uint16_t msgsz;
  uint8_t *ext;
  uint16_t ext_len;
  int j;

  if (!mg_tls_got_record(c)) {
    return MG_IO_WAIT;
  }
  if (rio->buf[0] != MG_TLS_HANDSHAKE || rio->len < 6 || rio->buf[5] != MG_TLS_SERVER_HELLO) {
    if (rio->buf[0] == MG_TLS_ALERT && rio->len >= 7) {
      verbose_alert(&rio->buf[5]);
      mg_error(c, "TLS error alert");
      return -1;
    }
    if (rio->len >= 6) MG_ERROR(("got packet type 0x%02x/0x%02x", rio->buf[0], rio->buf[5]));
    mg_error(c, "not a server hello packet");
    return -1;
  }

  if (rio->len < 5 + 39 + 32 + 3 + 2) goto fail;
  msgsz = MG_LOAD_BE16(rio->buf + 3);
  if (msgsz > rio->len - 5) goto fail;
  mg_sha256_update(&tls->sha256, rio->buf + 5, msgsz);

  ext_len = MG_LOAD_BE16(rio->buf + 5 + 39 + 32 + 3);
  ext = rio->buf + 5 + 39 + 32 + 3 + 2;
  if (ext_len > (rio->len - (5 + 39 + 32 + 3 + 2))) goto fail;

  for (j = 0; j < ext_len;) {
    uint16_t ext_type, ext_len2, group, key_exchange_len;
    uint8_t *key_exchange;
    if ((ext_len - j) < 4) goto fail;
    ext_type = MG_LOAD_BE16(ext + j);
    ext_len2 = MG_LOAD_BE16(ext + j + 2);
    if (ext_len2 > (ext_len - j - 4)) goto fail;
    if (ext_type != 0x0033) {  // not a key share extension, ignore
      j += (uint16_t) (ext_len2 + 4);
      continue;
    }
    if (ext_len2 < (2 + 2 + 32)) goto fail;
    group = MG_LOAD_BE16(ext + j + 4);
    key_exchange_len = MG_LOAD_BE16(ext + j + 6);
    key_exchange = ext + j + 8;
    if (key_exchange_len > ext_len2 - 4) goto fail;
    if (group != 0x0017) {  // secp256r1, uncompressed point 0x04 || X || Y
      mg_error(c, "bad key exchange group");
      return -1;
    }
    if (key_exchange_len != 65 || key_exchange[0] != 0x04 ||
        !mg_p256_shared_secret(key_exchange + 1, tls->p256_cli,
                               tls->ecdhe_sec)) {  // Validates the point
      mg_error(c, "bad key");
      return -1;
    }
    // mg_tls_hexdump("c ecdhe sec", tls->ecdhe_sec, 32);
    mg_tls_drop_record(c);
    /* generate handshake keys */
    mg_tls_generate_handshake_keys(c);
    return 0;
  }
fail:
  mg_error(c, "bad server hello");
  return -1;
}

static int mg_tls_client_recv_ext(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  unsigned char *recv_buf;
  if (mg_tls_recv_record(c) < 0) {
    return -1;
  }
  recv_buf = &c->rtls.buf[tls->recv_offset];
  if (recv_buf[0] != MG_TLS_ENCRYPTED_EXTENSIONS) {
    mg_error(c, "expected server extensions but got msg 0x%02x", recv_buf[0]);
    return -1;
  }
  mg_tls_drop_message(c);
  return 0;
}

struct mg_tls_cert {
  bool is_ec_pubkey;
  bool is_ca;
  struct mg_str sn;
  struct mg_str pubkey;
  struct mg_der_tlv issuer;
  struct mg_der_tlv subj;
  struct mg_str sig;    // signature
  uint8_t tbshash[48];  // 32B for sha256/secp256, 48B for sha384/secp384
  size_t tbshashsz;     // actual TBS hash size
};

static void mg_der_debug_cert_name(const char *name, struct mg_der_tlv *tlv) {
  struct mg_der_tlv v;
  struct mg_str cn, c, o, ou;
  if (mg_log_level < MG_LL_VERBOSE) return; // skip recursive computations
  cn = c = o = ou = mg_str("");
  if (mg_der_find_oid(tlv, (uint8_t *) "\x55\x04\x03", 3, &v) > 0)
    cn = mg_str_n((const char *) v.value, v.len);
  if (mg_der_find_oid(tlv, (uint8_t *) "\x55\x04\x06", 3, &v) > 0)
    c = mg_str_n((const char *) v.value, v.len);
  if (mg_der_find_oid(tlv, (uint8_t *) "\x55\x04\x0a", 3, &v) > 0)
    o = mg_str_n((const char *) v.value, v.len);
  if (mg_der_find_oid(tlv, (uint8_t *) "\x55\x04\x0b", 3, &v) > 0)
    ou = mg_str_n((const char *) v.value, v.len);
  MG_VERBOSE(("%s: CN=%.*s, C=%.*s, O=%.*s, OU=%.*s", name, cn.len, cn.buf,
              c.len, c.buf, o.len, o.buf, ou.len, ou.buf));
}

static uint64_t asnt2t(uint8_t *v, uint8_t type) {
  unsigned int y, mo, d, h, mi, ss;
  y = 10U * (*v++ - '0'), y += (*v++ - '0');
  if (type == 0x17) {       // UTCTime, RFC-5280 4.1.2.5.1 YYMMDDHHMMSSZ
    if (y >= 50) return (uint64_t) 0;  // 19YY is in the past
    y += 2000;
  } else {  // GeneralizedTime, RFC-5280 4.1.2.5.2 YYYYMMDDHHMMSSZ
    y *= 100U, y += 10U * (*v++ - '0'), y += (*v++ - '0');
  }
  mo = 10U * (*v++ - '0'), mo += (*v++ - '0');
  d = 10U * (*v++ - '0'), d += (*v++ - '0');
  h = 10U * (*v++ - '0'), h += (*v++ - '0');
  mi = 10U * (*v++ - '0'), mi += (*v++ - '0');
  ss = 10U * (*v++ - '0'), ss += (*v++ - '0');
  if (*v != 'Z') return 0; // invalid
  return mg_timegm(y, mo, d, h, mi, ss);
}

static int mg_tls_parse_cert_der(void *buf, size_t dersz,
                                 struct mg_tls_cert *cert) {
  uint8_t *tbs, *der = (uint8_t *) buf;
  size_t tbssz;
  struct mg_der_tlv root, tbs_cert, field, algo;  // pubkey, signature;
  struct mg_der_tlv pki, pki_algo, pki_key, pki_curve, raw_sig;

  // Parse outermost SEQUENCE
  if (mg_der_parse(der, dersz, &root) <= 0 || root.type != 0x30) return -1;

  // Parse TBSCertificate SEQUENCE
  tbs = root.value;
  if (mg_der_next(&root, &tbs_cert) <= 0 || tbs_cert.type != 0x30) return -1;
  tbssz = (size_t) (tbs_cert.value + tbs_cert.len - tbs);

  // Parse Version (optional field)
  if (mg_der_next(&tbs_cert, &field) <= 0) return -1;
  if (field.type == 0xa0) {  // v3
    if (mg_der_parse(field.value, field.len, &field) <= 0 || field.len != 1 ||
        field.value[0] != 2)
      return -1;
    if (mg_der_next(&tbs_cert, &field) <= 0) return -1;
  }

  // Parse Serial Number
  if (field.type != 2) return -1;
  cert->sn = mg_str_n((char *) field.value, field.len);
  MG_VERBOSE(("cert s/n: %M", mg_print_hex, cert->sn.len, cert->sn.buf));

  // Parse signature algorithm (first occurrence)
  if (mg_der_next(&tbs_cert, &field) <= 0 || field.type != 0x30) return -1;
  if (mg_der_next(&field, &algo) <= 0 || algo.type != 0x06) return -1;

  MG_VERBOSE(("sig algo (oid): %M", mg_print_hex, algo.len, algo.value));
  // Signature algorithm OID mapping
  if (algo.len == 8 &&
      memcmp(algo.value, "\x2A\x86\x48\xCE\x3D\x04\x03\x02", 8) == 0) {
    MG_VERBOSE(("sig algo: ECDSA with SHA256"));
    mg_sha256(cert->tbshash, tbs, tbssz);
    cert->tbshashsz = 32;
  } else if (algo.len == 9 &&
             memcmp(algo.value, "\x2A\x86\x48\x86\xF7\x0D\x01\x01\x0B", 9) ==
                 0) {
    MG_VERBOSE(("sig algo: RSA with SHA256"));
    mg_sha256(cert->tbshash, tbs, tbssz);
    cert->tbshashsz = 32;
  } else if (algo.len == 8 &&
             memcmp(algo.value, "\x2A\x86\x48\xCE\x3D\x04\x03\x03", 8) == 0) {
    MG_VERBOSE(("sig algo: ECDSA with SHA384"));
    mg_sha384(cert->tbshash, tbs, tbssz);
    cert->tbshashsz = 48;
  } else if (algo.len == 9 &&
             memcmp(algo.value, "\x2A\x86\x48\x86\xF7\x0D\x01\x01\x0C", 9) ==
                 0) {
    MG_VERBOSE(("sig algo: RSA with SHA384"));
    mg_sha384(cert->tbshash, tbs, tbssz);
    cert->tbshashsz = 48;
  } else {
    MG_ERROR(
        ("sig algo: unsupported OID: %M", mg_print_hex, algo.len, algo.value));
    return -1;
  }
  MG_VERBOSE(("tbs hash: %M", mg_print_hex, cert->tbshashsz, cert->tbshash));

  // issuer
  if (mg_der_next(&tbs_cert, &field) <= 0 || field.type != 0x30) return -1;
  cert->issuer = field;
  mg_der_debug_cert_name("issuer", &field);

  // validity dates (before/after)
  if (mg_der_next(&tbs_cert, &field) <= 0 || field.type != 0x30) return -1;
  {
    struct mg_der_tlv before, after;
    uint64_t now = mg_now() / 1000U, t;
    if (mg_der_next(&field, &before) <= 0 || ((before.type != 0x17 || before.len != 13) && (before.type != 0x18 || before.len != 15))) return -1;
    if (now < (t = asnt2t(before.value, before.type))) {
      MG_ERROR(("cert is not yet valid: before=%.*s (%lu), now=%lu", before.len, before.value, t, now));
      return -1;
    }
    if (mg_der_next(&field, &after) <= 0 || ((after.type != 0x17 || after.len != 13) && (after.type != 0x18 || after.len != 15))) return -1;
    if (memcmp(after.value, "99991231235959Z", 15) == 0) { // RFC-5280 4.1.2.5
      MG_ERROR(("No well-defined expiration date"));
      return -1;
    }
    if (now > (t = asnt2t(after.value, after.type))) {
      MG_ERROR(("cert is no longer valid: after=%.*s (%lu), now=%lu", after.len, after.value, t, now));
      return -1;
    }
  }

  // subject
  if (mg_der_next(&tbs_cert, &field) <= 0 || field.type != 0x30) return -1;
  cert->subj = field;
  mg_der_debug_cert_name("subject", &field);

  // subject public key info
  if (mg_der_next(&tbs_cert, &field) <= 0 || field.type != 0x30) return -1;

  if (mg_der_next(&field, &pki) <= 0 || pki.type != 0x30) return -1;
  if (mg_der_next(&pki, &pki_algo) <= 0 || pki_algo.type != 0x06) return -1;

  // public key algorithm
  MG_VERBOSE(("pk algo (oid): %M", mg_print_hex, pki_algo.len, pki_algo.value));
  if (pki_algo.len == 8 &&
      memcmp(pki_algo.value, "\x2A\x86\x48\xCE\x3D\x03\x01\x07", 8) == 0) {
    cert->is_ec_pubkey = true;
    MG_VERBOSE(("pk algo: ECDSA secp256r1"));
  } else if (pki_algo.len == 8 &&
             memcmp(pki_algo.value, "\x2A\x86\x48\xCE\x3D\x03\x01\x08", 8) ==
                 0) {
    cert->is_ec_pubkey = true;
    MG_VERBOSE(("pk algo: ECDSA secp384r1"));
  } else if (pki_algo.len == 7 &&
             memcmp(pki_algo.value, "\x2A\x86\x48\xCE\x3D\x02\x01", 7) == 0) {
    cert->is_ec_pubkey = true;
    MG_VERBOSE(("pk algo: EC public key"));
  } else if (pki_algo.len == 9 &&
             memcmp(pki_algo.value, "\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01",
                    9) == 0) {
    cert->is_ec_pubkey = false;
    MG_VERBOSE(("pk algo: RSA"));
  } else {
    MG_ERROR(("unsupported pk algo: %M", mg_print_hex, pki_algo.len,
              pki_algo.value));
    return -1;
  }

  // Parse public key
  if (cert->is_ec_pubkey) {
    if (mg_der_next(&pki, &pki_curve) <= 0 || pki_curve.type != 0x06) return -1;
  }
  if (mg_der_next(&field, &pki_key) <= 0 || pki_key.type != 0x03) return -1;

  if (cert->is_ec_pubkey) {  // Skip leading 0x00 and 0x04 (=uncompressed)
    if (pki_key.len < 2) return -1;
    cert->pubkey = mg_str_n((char *) pki_key.value + 2, pki_key.len - 2);
  } else {  // Skip leading 0x00 byte
    if (pki_key.len < 1) return -1;
    cert->pubkey = mg_str_n((char *) pki_key.value + 1, pki_key.len - 1);
  }

  { // parse optional fields
    int r; // unique ids
    while ((r = mg_der_next(&tbs_cert, &field)) > 0 && field.type == 0xa1);
    if (r > 0 && field.type == 0xa3) { // extensions
      bool ca = false, certsign = true;
      struct mg_der_tlv ext, e, i;  // ext[e(i, ...), ...]
      if (mg_der_next(&field, &ext) <= 0 || ext.type != 0x30) return -1;
      while (mg_der_next(&ext, &e) > 0) {
        if (mg_der_next(&e, &i) <= 0 || i.type != 0x06) return -1;
        if (i.len == 3 && memcmp(i.value, (uint8_t *) "\x55\x1d\x13", 3) == 0) {
          struct mg_der_tlv s, v; // basicConstraints
          MG_VERBOSE(("basicConstraints"));
          if (mg_der_next(&e, &i) <= 0 || i.type != 0x01 || i.len != 1 || *i.value != 0xff) return -1;
          if (mg_der_next(&e, &i) <= 0 || i.type != 0x04) return -1;
          if (mg_der_next(&i, &s) <= 0 || s.type != 0x30) return -1;
          if (s.len == 0) break;
          if (mg_der_next(&s, &v) <= 0) return -1;
          if (v.type == 0x01 && v.len == 1 && *v.value == 0xff) ca = true;
        } else if (i.len == 3 && memcmp(i.value, (uint8_t *) "\x55\x1d\x0f", 3) == 0) {
          struct mg_der_tlv b; // keyUsage, enforce when present
          MG_VERBOSE(("keyUsage"));
          if (mg_der_next(&e, &i) <= 0) return -1;
          if (i.type == 0x01) { // SHOULD, defaults to false
            if (i.len != 1) return -1;
            if (mg_der_next(&e, &i) <= 0) return -1;
          }
          if (i.type != 0x04) return -1;
          if (mg_der_next(&i, &b) <= 0 || b.type != 0x03) return -1;
          if (!(b.value[1] & MG_BIT(2))) certsign = false;
        }
      }
      if (ca && certsign) cert->is_ca = true;
    }
  }
  // Parse signature
  if (mg_der_next(&root, &field) <= 0 || field.type != 0x30) return -1;
  if (mg_der_next(&root, &raw_sig) <= 0 || raw_sig.type != 0x03) return -1;
  if (raw_sig.len < 1 || raw_sig.value[0] != 0x00) return -1;

  cert->sig = mg_str_n((char *) raw_sig.value + 1, raw_sig.len - 1);
  MG_VERBOSE(("sig: %M", mg_print_hex, cert->sig.len, cert->sig.buf));

  return 0;
}

static int countdots(struct mg_str s) {
  int count = 0;
  size_t len = s.len;
  char *p = s.buf;
  while (len--) {
    if (*(p++) == '.') ++count;
  }
  return count;
}

static int mg_tls_verify_cert_san(const uint8_t *der, size_t dersz,
                                  const char *server_name,
                                  struct mg_addr *ref_ip) {
  struct mg_der_tlv root, field, name;
  bool is_addr = ref_ip != NULL;
  if (mg_der_parse((uint8_t *) der, dersz, &root) < 0) {
    MG_ERROR(("failed to parse certificate"));
    return -1;
  }
  if (mg_der_find_oid(&root, (uint8_t *) "\x55\x1d\x11", 3, &field) <= 0) {
    MG_ERROR(("failed to extract SAN"));
    return -1;
  }
  if (mg_der_parse(field.value, field.len, &field) < 0) {
    MG_ERROR(("SAN is not a constructed object"));
    return -1;
  }
  while (mg_der_next(&field, &name) > 0) {
    if (is_addr) {  // match ref IP to IP and name to DNS name (RFC-9525 6.2)
      if (name.type == 0x87 && name.len == 4) {  // this is an IPv4 address
        uint32_t cert_ip;
        MG_VERBOSE(("Found SAN, IP: %M", mg_print_ip4, name.value));
        memcpy(&cert_ip, name.value, 4);
        if (!ref_ip->is_ip6 && cert_ip == ref_ip->addr.ip4)  // RFC-9525 6.4
          return 1;  // and byte-matches the one we wanted to connected to
      }
#if MG_ENABLE_IPV6
      else if (name.type == 0x87 && name.len == 16) {  // is an IPv6 address
        MG_VERBOSE(("Found SAN, IPv6: %M", mg_print_ip6, name.value));
        if (ref_ip->is_ip6 && memcmp(name.value, ref_ip->addr.ip6, 16) == 0)
          return 1;  // and byte-matches the one we wanted to connected to
      }
#endif
    } else if (name.type == 0x82) {  // this is a DNS name
      struct mg_str sn, tn;
      MG_VERBOSE(("Found SAN, DNS: %.*s", name.len, name.value));
      sn = mg_str(server_name), tn = mg_str_n((char *) name.value, name.len);
      if (countdots(sn) == countdots(tn) && mg_match(sn, tn, NULL))
        return 1;  // and matches the host name
    }
  }
  return -1;
}

static int mg_tls_verify_cert_signature(const struct mg_tls_cert *cert,
                                        const struct mg_tls_cert *issuer) {
  if (issuer->is_ec_pubkey) {
    uint8_t sig[128];
    struct mg_der_tlv seq = {0, 0, 0}, a = {0, 0, 0}, b = {0, 0, 0};
    mg_der_parse((uint8_t *) cert->sig.buf, cert->sig.len, &seq);
    mg_der_next(&seq, &a);
    mg_der_next(&seq, &b);
    if (a.len == 0 || b.len == 0) {
      MG_ERROR(("cert verification error"));
      return 0;
    }
    if (issuer->pubkey.len == 64) {
      const uint32_t N = 32;
      if (a.len > N) a.value += (a.len - N), a.len = N; // padding
      if (b.len > N) b.value += (b.len - N), b.len = N;
      memset(sig, 0, N - a.len); // short encoding
      memmove(sig + (N - a.len), a.value, a.len);
      memset(sig + N, 0, N - b.len);
      memmove(sig + N + (N - b.len), b.value, b.len);
      return mg_p256_verify((uint8_t *) issuer->pubkey.buf, cert->tbshash,
                            cert->tbshashsz, sig);
    } else if (issuer->pubkey.len == 96) {
      const uint32_t N = 48;
      if (a.len > N) a.value += (a.len - N), a.len = N;  // padding
      if (b.len > N) b.value += (b.len - N), b.len = N;
      memset(sig, 0, N - a.len);  // short encoding
      memmove(sig + (N - a.len), a.value, a.len);
      memset(sig + N, 0, N - b.len);
      memmove(sig + N + (N - b.len), b.value, b.len);
      return mg_p384_verify((uint8_t *) issuer->pubkey.buf, cert->tbshash,
                            cert->tbshashsz, sig);
    } else {
      MG_ERROR(("unsupported public key length: %d", issuer->pubkey.len));
      return 0;
    }
  } else {
    const uint8_t *n;
    size_t nlen;
    uint8_t sig2[512];  // 4096 bits
    struct mg_der_tlv seq, modulus, exponent;
    if (mg_der_parse((uint8_t *) issuer->pubkey.buf, issuer->pubkey.len,
                     &seq) <= 0 ||
        mg_der_next(&seq, &modulus) <= 0 || modulus.type != 2 ||
        modulus.len == 0 || mg_der_next(&seq, &exponent) <= 0 ||
        exponent.type != 2 || exponent.len == 0) {
      return -1;
    }
    n = modulus.value, nlen = mg_rsa_trim_len(&n, modulus.len);
    if (nlen > sizeof(sig2) || cert->tbshashsz > nlen ||
        mg_rsa_mod_pow(modulus.value, modulus.len, exponent.value,
                       exponent.len, (uint8_t *) cert->sig.buf, cert->sig.len,
                       sig2, nlen) != 0) {
      return 0;
    }

    return mg_rsa_pkcs_verify(sig2, nlen, cert->tbshash, cert->tbshashsz);
  }
}

static int mg_tls_verify_cert_cn(struct mg_der_tlv *subj, const char *host) {
  struct mg_der_tlv v;
  int matched = 0;
  if (mg_der_find_oid(subj, (uint8_t *) "\x55\x04\x03", 3, &v) > 0) {
    struct mg_str hn, cn;
    MG_VERBOSE(("using CN: %.*s <-> %s", v.len, v.value, host));
    hn = mg_str(host), cn = mg_str_n((char *) v.value, v.len);
    matched = (int) (countdots(hn) == countdots(cn) && mg_match(hn, cn, NULL));
  }
  return matched;
}

// 1: found, 0: not found, < 0: parse error: -1: bundle (unused), -2: name
static int tls_bundle_find(struct tls_data *tls, struct mg_der_tlv *name,
                           struct mg_tls_cert *cert) {
  size_t i;
  struct mg_der_tlv v;
  struct mg_str *p = tls->ca_bundle_der, tgt;
  if (mg_der_find_oid(name, (uint8_t *) "\x55\x04\x03", 3, &v) <= 0) return -2;
  tgt = mg_str_n((const char *) v.value, v.len);
  for (i = 0; i < tls->ca_bundle_len; i++, p++) {
    struct mg_str subj;
    if (mg_tls_parse_cert_der(p->buf, p->len, cert) < 0 ||
        mg_der_find_oid(&cert->subj, (uint8_t *) "\x55\x04\x03", 3, &v) <= 0) {
      MG_ERROR(("failed to parse certificate #%u in bundle", i + 1));
      continue; // skip this certificate but don't halt the process
    }
    subj = mg_str_n((const char *) v.value, v.len);
    MG_VERBOSE(("#%u: %.*s (%.*s)", i + 1, subj.len, subj.buf, tgt.len, tgt.buf));
    if (mg_strcasecmp(subj, tgt) == 0) return 1;
  }
  return 0;
}

bool mg_aton_(struct mg_str str, struct mg_addr *addr);

static int mg_tls_recv_cert(struct mg_connection *c, bool is_client) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  unsigned char *recv_buf;

  if (mg_tls_recv_record(c) < 0) {
    return -1;
  }

  recv_buf = &c->rtls.buf[tls->recv_offset];

  if (recv_buf[0] == MG_TLS_CERTIFICATE_REQUEST) {
    MG_VERBOSE(("got certificate request"));
    mg_tls_drop_message(c);
    tls->cert_requested = 1;
    return -1;
  }

  if (recv_buf[0] != MG_TLS_CERTIFICATE) {
    mg_error(c, "expected %s certificate but got msg 0x%02x",
             is_client ? "server" : "client", recv_buf[0]);
    return -1;
  }

  if (tls->recv_len < 13) { // 8 + 3 + 2, chain len + cert len + ext len
    mg_error(c, "certificate list too short");
    return -1;
  }

  {
    // Normally, there are 2-3 certs in a chain (when is_client)
    struct mg_tls_cert certs[8];
    int certnum = 0;
    uint32_t full_cert_chain_len = MG_LOAD_BE24(recv_buf + 1);
    uint32_t cert_chain_len = MG_LOAD_BE24(recv_buf + 5);
    uint8_t *p = recv_buf + 8;
    uint8_t *endp = recv_buf + cert_chain_len + 8;
    bool found_ca = false;
    struct mg_tls_cert ca;

    if (cert_chain_len != full_cert_chain_len - 4 || cert_chain_len > (tls->recv_len - 8)) {
      MG_ERROR(("full chain length: %d, chain length: %d, msg length: %d", full_cert_chain_len, cert_chain_len, tls->recv_len));
      mg_error(c, "invalid certificate chain length");
      return -1;
    }

    memset(certs, 0, sizeof(certs));
    memset(&ca, 0, sizeof(ca));

    if (tls->ca_der.len > 0) { // single cert or server
      if (mg_tls_parse_cert_der(tls->ca_der.buf, tls->ca_der.len, &ca) < 0) {
        mg_error(c, "failed to parse CA certificate");
        return -1;
      }
      MG_VERBOSE(("CA serial: %M", mg_print_hex, ca.sn.len, ca.sn.buf));
    }

    while (p < endp) {
      struct mg_tls_cert *ci = &certs[certnum++];
      uint32_t certsz;
      uint16_t certext;
      uint8_t *cert = p + 3;

      if ((endp - p) < 5) { // 3 + 2, cert len + ext len fields
        mg_error(c, "truncated certificate in chain");
        return -1;
      }
      certsz = MG_LOAD_BE24(p);
      p = cert + certsz + 2; // skip cert extensions (size only, not supported)
      if (p > endp) {
        mg_error(c, "invalid certificate length");
        return -1;
      }
      certext = MG_LOAD_BE16(cert + certsz);
      if (certext != 0) {
        mg_error(c, "certificate extensions are not supported");
        return -1;
      }

      if (mg_tls_parse_cert_der(cert, certsz, ci) < 0) {
        mg_error(c, "failed to parse certificate");
        return -1;
      }

      if (ci == certs) {
        // First certificate in the chain is peer cert, check SAN if requested,
        // and store public key for further CertVerify step
        if (tls->hostname[0] != '\0') {
          struct mg_addr addr;  // use aton_ so localhost evals as a name
          bool is_addr = mg_aton_(mg_str_s(tls->hostname), &addr);
          if (mg_tls_verify_cert_san(cert, certsz, tls->hostname,
                                     is_addr ? &addr : NULL) <= 0 &&
              (!is_addr ||  // DNS CN fallback
               mg_tls_verify_cert_cn(&ci->subj, tls->hostname) <= 0)) {
            mg_error(c, "failed to verify hostname");
            return -1;
          }
        }
        if (ci->pubkey.len > sizeof(tls->pubkey)) {
          mg_error(c, "peer public key too large");
          return -1;
        }
        memmove(tls->pubkey, ci->pubkey.buf, ci->pubkey.len);
        tls->pubkeysz = ci->pubkey.len;
      } else {
        if (!ci->is_ca || !mg_tls_verify_cert_signature(ci - 1, ci)) {
          mg_error(c, "failed to verify certificate chain %s", ci->is_ca ? "" : "(not a true CA)");
          return -1;
        }
      }

      if (tls->ca_bundle_len > 0) {  // bundle, find subject and compare keys
        int r;
        MG_VERBOSE(("Search current cert in bundle"));
        r = tls_bundle_find(tls, &ci->subj, &ca);  // find current cert ci
        if (r == -1) {
          mg_error(c, "failed to parse CA bundle");
          return -1;
        } else if (r > 0 && ca.pubkey.len == ci->pubkey.len &&
                   memcmp(ca.pubkey.buf, ci->pubkey.buf, ca.pubkey.len) == 0) {
          found_ca = true;
          MG_VERBOSE(("CA serial: %M", mg_print_hex, ca.sn.len, ca.sn.buf));
          break;
        }  // else r = 0 or -2 => subj not found or pubkey not matching
      }

      // Single CA: accept the chain if the CA itself is in it, e.g. a
      // cross-signed copy of the root. Same subject and same public key
      // means the same trusted key, and the links below ci are verified
      if (tls->ca_der.len > 0 && ca.subj.len == ci->subj.len &&
          memcmp(ca.subj.value, ci->subj.value, ca.subj.len) == 0 &&
          ca.pubkey.len == ci->pubkey.len &&
          memcmp(ca.pubkey.buf, ci->pubkey.buf, ca.pubkey.len) == 0) {
        found_ca = true;
        MG_VERBOSE(("CA found in chain at depth %d", certnum - 1));
        break;
      }

      if (certnum == sizeof(certs) / sizeof(certs[0]) - 1) {
        mg_error(c, "too many certificates in the chain");
        return -1;
      }
    }

    if (!found_ca && certnum > 0 && tls->ca_bundle_len > 0) {  // bundle
      int r;
      MG_VERBOSE(("Search bundle for issuer of last cert in chain"));
      r = tls_bundle_find(tls, &certs[certnum - 1].issuer, &ca);
      if (r <= 0) {
        mg_error(c, "failed to %s",
                 r == 0    ? "find issuing CA in bundle"
                 : r == -1 ? "parse CA bundle"
                           : "parse name OID");
        return -1;
      } else if (!ca.is_ca ||  // candidate issuer found
                 !mg_tls_verify_cert_signature(&certs[certnum - 1], &ca)) {
        mg_error(c, "failed to verify CA");
        return -1;
      }  // issuer verified. NOTE(): may fail if two diff CAs share a CN
      MG_VERBOSE(("CA serial: %M", mg_print_hex, ca.sn.len, ca.sn.buf));
    }
    if (!found_ca && tls->ca_der.len > 0) {  // single cert or server
      if (certnum < 1 ||
          !mg_tls_verify_cert_signature(&certs[certnum - 1], &ca)) {
        mg_error(c, "failed to verify CA");
        return -1;
      } else if (is_client) {
        MG_VERBOSE(("no CA in chain; verification with builtin CA passed"));
      }
    }
  }
  mg_tls_drop_message(c);
  mg_tls_calc_cert_verify_hash(c, tls->sighash, tls->sighash384, !is_client);
  return 0;
}

static int mg_tls_recv_cert_verify(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  unsigned char *recv_buf;
  if (mg_tls_recv_record(c) < 0) {
    return -1;
  }
  recv_buf = &c->rtls.buf[tls->recv_offset];
  if (recv_buf[0] != MG_TLS_CERTIFICATE_VERIFY) {
    mg_error(c, "expected %s certificate verify but got msg 0x%02x",
             c->is_client ? "server" : "client", recv_buf[0]);
    return -1;
  }
  if (tls->recv_len < 8) {
    mg_error(c, "server certificate verify is too short: %d bytes",
             tls->recv_len);
    return -1;
  }

  // Ignore CertificateVerify if strict checks are not required
  if (tls->skip_verification) {
    mg_tls_drop_message(c);
    return 0;
  }

  {
    uint16_t sigalg = MG_LOAD_BE16(recv_buf + 4);
    uint16_t siglen = MG_LOAD_BE16(recv_buf + 6);
    uint8_t *sigbuf = recv_buf + 8;
    if (siglen > tls->recv_len - 8) {
      mg_error(c, "invalid certverify signature length: %d, expected %d",
               siglen, tls->recv_len - 8);
      return -1;
    }
    MG_VERBOSE(
        ("certificate verification, algo=%04x, siglen=%d", sigalg, siglen));

    if (sigalg == 0x0804) {  // rsa_pss_rsae_sha256
      uint8_t sig2[512];     // 2048 or 4096 bits
      const uint8_t *n;
      size_t nlen;
      struct mg_der_tlv seq, modulus, exponent;

      if (mg_der_parse(tls->pubkey, tls->pubkeysz, &seq) <= 0 ||
          mg_der_next(&seq, &modulus) <= 0 || modulus.type != 2 ||
          modulus.len == 0 || mg_der_next(&seq, &exponent) <= 0 ||
          exponent.type != 2 || exponent.len == 0) {
        mg_error(c, "invalid public key");
        return -1;
      }

      n = modulus.value, nlen = mg_rsa_trim_len(&n, modulus.len);
      if (nlen > sizeof(sig2) ||
          mg_rsa_mod_pow(modulus.value, modulus.len, exponent.value,
                         exponent.len, sigbuf, siglen, sig2, nlen) != 0 ||
          !mg_rsa_verify(sig2, nlen, tls->sighash)) {
        mg_error(c, "failed to verify RSA certificate (certverify)");
        return -1;
      }
      MG_VERBOSE(("certificate verification successful (RSA)"));
    } else if (sigalg == 0x0403 || sigalg == 0x0503) {
      // ecdsa_secp256r1_sha256 or ecdsa_secp384r1_sha384. Extract certificate
      // signature and verify it using pubkey and sighash
      bool is384 = sigalg == 0x0503;
      uint32_t N = is384 ? 48 : 32;  // curve size: r and s are N bytes each
      int ok;
      uint8_t sig[96];
      struct mg_der_tlv seq, r, s;
      memset(sig, 0, sizeof(sig));
      if (tls->pubkeysz != 2 * N) {
        mg_error(c, "certverify scheme %04x does not match the key", sigalg);
        return -1;
      }
      if (mg_der_to_tlv(sigbuf, siglen, &seq) < 0) {
        mg_error(c, "verification message is not an ASN.1 DER sequence");
        return -1;
      }
      if (mg_der_to_tlv(seq.value, seq.len, &r) < 0) {
        mg_error(c, "missing first part of the signature");
        return -1;
      }
      if (mg_der_to_tlv(r.value + r.len, seq.len - r.len, &s) < 0) {
        mg_error(c, "missing second part of the signature");
        return -1;
      }
      // Integers may be padded with zeroes
      if (r.len > N) r.value = r.value + (r.len - N), r.len = N;
      if (s.len > N) s.value = s.value + (s.len - N), s.len = N;

      // r or s may be shorter than N bytes, "right-justify" (network order)
      memmove(sig + (N - r.len), r.value, r.len);
      memmove(sig + N + (N - s.len), s.value, s.len);

      if (is384) {
        ok = mg_p384_verify(tls->pubkey, tls->sighash384,
                            sizeof(tls->sighash384), sig);
      } else {
        ok = mg_p256_verify(tls->pubkey, tls->sighash, sizeof(tls->sighash),
                            sig);
      }
      if (ok != 1) {
        mg_error(c, "failed to verify EC certificate (certverify)");
        return -1;
      }
      MG_VERBOSE(("certificate verification successful (EC)"));
    } else {
      // From
      // https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml:
      //   0805 = rsa_pss_rsae_sha384
      //   0806 = rsa_pss_rsae_sha512
      //   0807 = ed25519
      //   0808 = ed448
      //   0809 = rsa_pss_pss_sha256
      //   080A = rsa_pss_pss_sha384
      //   080B = rsa_pss_pss_sha512
      MG_ERROR(("unsupported certverify signature scheme: %x of %d bytes",
                sigalg, siglen));
      return -1;
    }
  }
  mg_tls_drop_message(c);
  return 0;
}

static int mg_tls_client_recv_finish(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  unsigned char *recv_buf;
  if (mg_tls_recv_record(c) < 0) {
    return -1;
  }
  recv_buf = &c->rtls.buf[tls->recv_offset];
  if (recv_buf[0] != MG_TLS_FINISHED) {
    mg_error(c, "expected server finished but got msg 0x%02x", recv_buf[0]);
    return -1;
  }
  mg_tls_drop_message(c);
  return 0;
}

static bool mg_tls_client_send_finish(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  mg_sha256_ctx sha256;
  uint8_t hash[32];
  uint8_t finish[36] = {0x14, 0, 0, 32};
  memmove(&sha256, &tls->sha256, sizeof(mg_sha256_ctx));
  mg_sha256_final(hash, &sha256);
  mg_hmac_sha256(finish + 4, tls->enc.client_finished_key, 32, hash, 32);
  return mg_tls_encrypt(c, finish, sizeof(finish), MG_TLS_HANDSHAKE);
}

static bool mg_tls_client_handshake(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  switch (tls->state) {
    case MG_TLS_STATE_CLIENT_START:
      if (!mg_tls_client_send_hello(c)) return false;
      tls->state = MG_TLS_STATE_CLIENT_WAIT_SH;
      // Fallthrough
    case MG_TLS_STATE_CLIENT_WAIT_SH:
      if (mg_tls_client_recv_hello(c) < 0) break;
      tls->state = MG_TLS_STATE_CLIENT_WAIT_EE;
      // Fallthrough
    case MG_TLS_STATE_CLIENT_WAIT_EE:
      if (mg_tls_client_recv_ext(c) < 0) break;
      tls->state = MG_TLS_STATE_CLIENT_WAIT_CERT;
      // Fallthrough
    case MG_TLS_STATE_CLIENT_WAIT_CERT:
      if (mg_tls_recv_cert(c, true) < 0) break;
      tls->state = MG_TLS_STATE_CLIENT_WAIT_CV;
      // Fallthrough
    case MG_TLS_STATE_CLIENT_WAIT_CV:
      if (mg_tls_recv_cert_verify(c) < 0) break;
      tls->state = MG_TLS_STATE_CLIENT_WAIT_FINISH;
      // Fallthrough
    case MG_TLS_STATE_CLIENT_WAIT_FINISH:
      if (mg_tls_client_recv_finish(c) < 0) break;
      if (tls->cert_requested && tls->cert_der.len > 0) {  // two-way auth
        // generate application keys at this point, keep using handshake keys
        struct tls_enc hs_keys = tls->enc;
        mg_tls_generate_application_keys(c);
        tls->app_keys = tls->enc;
        tls->enc = hs_keys;
        if (!mg_tls_send_cert(c, true) || !mg_tls_send_cert_verify(c, true) ||
            !mg_tls_client_send_finish(c))
          return false;
        tls->enc = tls->app_keys;
      } else {
        if (!mg_tls_client_send_finish(c)) return false;
        mg_tls_generate_application_keys(c);
      }
      tls->state = MG_TLS_STATE_CLIENT_CONNECTED;
      c->is_tls_hs = 0;
      mg_call(c, MG_EV_TLS_HS, NULL);
      break;
    default:
      mg_error(c, "unexpected client state: %d", tls->state);
      break;
  }
  return true;
}

static bool mg_tls_server_handshake(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  int r;
  switch (tls->state) {
    case MG_TLS_STATE_SERVER_START:
    case MG_TLS_STATE_SERVER_RETRY:
      if ((r = mg_tls_server_recv_hello(c)) < 0) break;
      if (r == 1) {  // No P-256 key share, ask for it
        if (!mg_tls_server_send_hrr(c)) return false;
        tls->state = MG_TLS_STATE_SERVER_RETRY;
        break;
      }
      if (!mg_tls_server_send_hello(c)) return false;
      mg_tls_generate_handshake_keys(c);
      if (!mg_tls_server_send_ext(c)) return false;
      if (tls->is_twoway && !mg_tls_server_send_cert_request(c)) return false;
      if (!mg_tls_send_cert(c, false)) return false;
      if (!mg_tls_send_cert_verify(c, false)) return false;
      if (!mg_tls_server_send_finish(c)) return false;
      if (tls->is_twoway) {
        // generate application keys at this point, keep using handshake keys
        struct tls_enc hs_keys = tls->enc;
        mg_tls_generate_application_keys(c);
        tls->app_keys = tls->enc;
        tls->enc = hs_keys;
        tls->state = MG_TLS_STATE_SERVER_WAIT_CERT;
        break;
      }
      tls->state = MG_TLS_STATE_SERVER_NEGOTIATED;
      // fallthrough
    case MG_TLS_STATE_SERVER_NEGOTIATED:
      if (mg_tls_server_recv_finish(c) < 0) break;
      if (tls->is_twoway) {  // use previously generated keys
        tls->enc = tls->app_keys;
      } else {  // generate keys now
        mg_tls_generate_application_keys(c);
      }
      tls->state = MG_TLS_STATE_SERVER_CONNECTED;
      c->is_tls_hs = 0;
      break;
    case MG_TLS_STATE_SERVER_WAIT_CERT:
      if (mg_tls_recv_cert(c, false) < 0) break;
      tls->state = MG_TLS_STATE_SERVER_WAIT_CV;
      // Fallthrough
    case MG_TLS_STATE_SERVER_WAIT_CV:
      if (mg_tls_recv_cert_verify(c) < 0) break;
      tls->state = MG_TLS_STATE_SERVER_NEGOTIATED;
      break;
    default:
      mg_error(c, "unexpected server state: %d", tls->state);
      break;
  }
  return true;
}

void mg_tls_handshake(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  long n;
  bool res;
  uint64_t t;
  if (c->is_closing) return;  // we don't clear rx buf, so ignore what's left
  if (tls->is_sntp_pending) return;
  t = mg_millis();
  if (c->is_client) {
    // will clear is_hs when sending last chunk
    res = mg_tls_client_handshake(c);
  } else {
    res = mg_tls_server_handshake(c);
  }
  tls->hs_proc += mg_millis() - t;
  if (res && !c->is_tls_hs) {
    // CPU: key exchange, signing, certificate verification. Wait: the rest of
    // the total, waiting for the peer and the network
    unsigned long total = (unsigned long) (mg_millis() - tls->hs_start);
    unsigned long cpu = (unsigned long) tls->hs_proc;
    MG_DEBUG(("%lu TLS handshake done in %lu ms: %lu ms CPU, %lu ms peer wait",
              c->id, total, cpu, total > cpu ? total - cpu : 0));
  }
  if (!res) {
    mg_error(c, "TLS OOM");
    return;
  }
  while (tls->send.len > 0 &&
         (n = mg_io_send(c, tls->send.buf, tls->send.len)) > 0) {
    mg_iobuf_del(&tls->send, 0, (size_t) n);
  }  // if last chunk fails to be sent, it will be sent with first app data,
     // otherwise, it needs to be flushed
}

static int mg_rsa_parse_der_int(const uint8_t **p, const uint8_t *end,
                                struct mg_str *out) {
  const uint8_t *start = *p, *value_start, *value_end;
  uint8_t i;
  uint32_t len;

  if (end - start < 2) {
    MG_VERBOSE(("DER INT: not enough bytes (%d < 2)", (int) (end - start)));
    return -1;
  }
  if (start[0] != 0x02) {
    MG_VERBOSE(("DER INT: expected 0x02, got 0x%02x", start[0]));
    return -1;
  }

  len = start[1];
  *p = start + 2;

  if (len > 0x7F) {
    // Long form length
    uint8_t len_bytes = len & 0x7F;
    MG_VERBOSE(("DER INT: long form, %d length bytes", len_bytes));
    if (len_bytes == 0 || len_bytes > 4 || (size_t) (end - *p) < len_bytes) {
      MG_VERBOSE(("DER INT: not enough bytes for length"));
      return -1;
    }
    len = 0;
    for (i = 0; i < len_bytes; i++) {
      len = (len << 8) | (*p)[i];
    }
    *p += len_bytes;
  }

  MG_VERBOSE(("DER INT: length=%u, remaining=%d", len, (int) (end - *p)));

  if ((size_t) (end - *p) < len) {
    MG_VERBOSE(("DER INT: length exceeds remaining bytes"));
    return -1;
  }

  // The encoded length tells us how many bytes to consume from the stream
  value_start = *p;
  value_end = *p + len;

  // Skip leading zero byte if present (for positive numbers)
  // This doesn't change how many bytes we consume, just what we expose
  if (len > 0 && (*p)[0] == 0x00) {
    (*p)++;
    len--;
  }

  out->buf = (char *) *p;
  out->len = len;

  // Advance pointer by the ORIGINAL encoded length, not the adjusted length
  *p = value_end;

  MG_VERBOSE(("DER INT: parsed %u bytes (skipped zero=%d)", len,
              (size_t) (value_end - value_start) != (size_t) len ? 1 : 0));
  return 0;
}

// RFC 5915 ECPrivateKey ::= SEQUENCE {
//   version INTEGER { ecPrivkeyVer1(1) },
//   privateKey OCTET STRING,
//   parameters [0] ECParameters {{ NamedCurve }} OPTIONAL,
//   publicKey [1] BIT STRING OPTIONAL
// }
static int mg_parse_ec_private_key(const uint8_t *der, size_t dersz,
                                   uint8_t *ec_key) {
  struct mg_der_tlv root, version, private_key_octets;

  if (mg_der_parse((uint8_t *) der, dersz, &root) < 0 || root.type != 0x30) {
    MG_ERROR(("EC private key: invalid SEQUENCE"));
    return -1;
  }

  if (mg_der_next(&root, &version) <= 0 || version.type != 0x02) {
    MG_ERROR(("EC private key: invalid version"));
    return -1;
  }

  if (mg_der_next(&root, &private_key_octets) <= 0 ||
      private_key_octets.type != 0x04) {
    MG_ERROR(("EC private key: invalid privateKey OCTET STRING"));
    return -1;
  }

  if (private_key_octets.len != 32) {
    MG_ERROR(
        ("EC private key: expected 32 bytes, got %u", private_key_octets.len));
    return -1;
  }

  memcpy(ec_key, private_key_octets.value, 32);
  return 0;
}

// Parse RSA private key from DER format
// RSAPrivateKey ::= SEQUENCE {
//   version           INTEGER (0),
//   modulus           INTEGER,  -- n
//   publicExponent    INTEGER,  -- e
//   privateExponent   INTEGER,  -- d
//   prime1            INTEGER,  -- p
//   prime2            INTEGER,  -- q
//   exponent1         INTEGER,  -- dP = d mod (p-1)
//   exponent2         INTEGER,  -- dQ = d mod (q-1)
//   coefficient       INTEGER,  -- qInv = (inverse of q) mod p
// }
static int mg_rsa_parse_key(const uint8_t *der, size_t dersz,
                            struct mg_rsa_key *key) {
  const uint8_t *p = der;
  const uint8_t *end = der + dersz;
  uint32_t seq_len;
  struct mg_str version;

  memset(key, 0, sizeof(*key));

  // Parse outer SEQUENCE
  if (end - p < 2) {
    MG_ERROR(("RSA key too short for SEQUENCE header"));
    return -1;
  }
  if (p[0] != 0x30) {
    MG_ERROR(("RSA key: expected SEQUENCE (0x30), got 0x%02x", p[0]));
    return -1;
  }

  seq_len = p[1];
  p += 2;

  if (seq_len > 0x7F) {
    // Long form length
    uint8_t i, len_bytes = seq_len & 0x7F;
    MG_VERBOSE(("Long form length: %d bytes", len_bytes));
    if (len_bytes == 0 || len_bytes > 4 || (size_t) (end - p) < len_bytes) {
      MG_ERROR(("Not enough bytes for long form length"));
      return -1;
    }
    seq_len = 0;
    for (i = 0; i < len_bytes; i++) {
      seq_len = (seq_len << 8) | p[i];
    }
    p += len_bytes;
  }

  MG_VERBOSE(
      ("SEQUENCE length: %u, total DER size: %u", seq_len, (unsigned) dersz));

  if ((size_t) (end - p) < seq_len) {
    MG_ERROR(("SEQUENCE length exceeds buffer"));
    return -1;
  }
  end = p + seq_len;  // Adjust end to sequence boundary

  // Parse version (should be 0)
  MG_VERBOSE(("Before version: offset=%d, bytes: %02x %02x %02x %02x",
              (int) (p - der), p < end ? p[0] : 0xFF,
              p + 1 < end ? p[1] : 0xFF, p + 2 < end ? p[2] : 0xFF,
              p + 3 < end ? p[3] : 0xFF));
  if (mg_rsa_parse_der_int(&p, end, &version) < 0) {
    MG_ERROR(("Failed to parse version"));
    return -1;
  }
  MG_DEBUG(("Version: %d byte(s), value=%d, offset now=%d", (int) version.len,
            version.len > 0 ? (int) (unsigned char) version.buf[0] : -1,
            (int) (p - der)));

  // Parse the 8 components: n, e, d, p, q, dP, dQ, qInv
  MG_VERBOSE(("Before n: offset=%d, bytes: %02x %02x %02x %02x %02x %02x",
              (int) (p - der), p < end ? p[0] : 0xFF,
              p + 1 < end ? p[1] : 0xFF, p + 2 < end ? p[2] : 0xFF,
              p + 3 < end ? p[3] : 0xFF, p + 4 < end ? p[4] : 0xFF,
              p + 5 < end ? p[5] : 0xFF));
  if (mg_rsa_parse_der_int(&p, end, &key->n) < 0) {
    MG_ERROR(("Failed to parse n (modulus)"));
    return -1;
  }
  MG_VERBOSE(("Parsed n: %d bytes, offset now=%d, consumed=%d bytes total",
              (int) key->n.len, (int) (p - der), (int) (p - der)));
  MG_VERBOSE(("  First 8 bytes of n: %02x %02x %02x %02x %02x %02x %02x %02x",
              key->n.len > 0 ? (unsigned char) key->n.buf[0] : 0xFF,
              key->n.len > 1 ? (unsigned char) key->n.buf[1] : 0xFF,
              key->n.len > 2 ? (unsigned char) key->n.buf[2] : 0xFF,
              key->n.len > 3 ? (unsigned char) key->n.buf[3] : 0xFF,
              key->n.len > 4 ? (unsigned char) key->n.buf[4] : 0xFF,
              key->n.len > 5 ? (unsigned char) key->n.buf[5] : 0xFF,
              key->n.len > 6 ? (unsigned char) key->n.buf[6] : 0xFF,
              key->n.len > 7 ? (unsigned char) key->n.buf[7] : 0xFF));
  MG_VERBOSE(("  Next bytes after n: %02x %02x %02x %02x %02x %02x",
              p < end ? p[0] : 0xFF, p + 1 < end ? p[1] : 0xFF,
              p + 2 < end ? p[2] : 0xFF, p + 3 < end ? p[3] : 0xFF,
              p + 4 < end ? p[4] : 0xFF, p + 5 < end ? p[5] : 0xFF));

  if (mg_rsa_parse_der_int(&p, end, &key->e) < 0) {
    MG_ERROR(("Failed to parse e (public exponent), bytes remaining: %d",
              (int) (end - p)));
    if (end - p >= 4) {
      MG_ERROR(("  Next 4 bytes: %02x %02x %02x %02x", p[0], p[1], p[2], p[3]));
    }
    return -1;
  }
  if (mg_rsa_parse_der_int(&p, end, &key->d) < 0) {
    MG_ERROR(("Failed to parse d (private exponent)"));
    return -1;
  }
  if (mg_rsa_parse_der_int(&p, end, &key->p) < 0) {
    MG_ERROR(("Failed to parse p (prime1)"));
    return -1;
  }
  if (mg_rsa_parse_der_int(&p, end, &key->q) < 0) {
    MG_ERROR(("Failed to parse q (prime2)"));
    return -1;
  }
  if (mg_rsa_parse_der_int(&p, end, &key->dP) < 0) {
    MG_ERROR(("Failed to parse dP (exponent1)"));
    return -1;
  }
  if (mg_rsa_parse_der_int(&p, end, &key->dQ) < 0) {
    MG_ERROR(("Failed to parse dQ (exponent2)"));
    return -1;
  }
  if (mg_rsa_parse_der_int(&p, end, &key->qInv) < 0) {
    MG_ERROR(("Failed to parse qInv (coefficient)"));
    return -1;
  }

  MG_VERBOSE(("Successfully parsed RSA key"));
  return 0;
}

// PKCS#8 PrivateKeyInfo ::= SEQUENCE {
//   version INTEGER,
//   privateKeyAlgorithm AlgorithmIdentifier,
//   privateKey OCTET STRING,
//   attributes [0] Attributes OPTIONAL
// }
// AlgorithmIdentifier ::= SEQUENCE {
//   algorithm OBJECT IDENTIFIER,
//   parameters ANY OPTIONAL
// }
static int mg_parse_pkcs8_key(const uint8_t *der, size_t dersz,
                              struct mg_rsa_key *rsa, uint8_t *ec_key) {
  struct mg_der_tlv root, version, alg_id, private_key_octets;
  struct mg_der_tlv alg_oid, alg_params;

  if (mg_der_parse((uint8_t *) der, dersz, &root) < 0 || root.type != 0x30) {
    MG_ERROR(("PKCS#8: invalid PrivateKeyInfo SEQUENCE"));
    return -1;
  }

  if (mg_der_next(&root, &version) <= 0 || version.type != 0x02) {
    MG_ERROR(("PKCS#8: invalid version"));
    return -1;
  }

  if (mg_der_next(&root, &alg_id) <= 0 || alg_id.type != 0x30) {
    MG_ERROR(("PKCS#8: invalid AlgorithmIdentifier SEQUENCE"));
    return -1;
  }

  if (mg_der_next(&alg_id, &alg_oid) <= 0 || alg_oid.type != 0x06) {
    MG_ERROR(("PKCS#8: invalid algorithm OID"));
    return -1;
  }

  if (mg_der_next(&root, &private_key_octets) <= 0 ||
      private_key_octets.type != 0x04) {
    MG_ERROR(("PKCS#8: invalid privateKey OCTET STRING"));
    return -1;
  }

  if (alg_oid.len == sizeof(mg_rsa_oid) &&
      memcmp(alg_oid.value, mg_rsa_oid, sizeof(mg_rsa_oid)) == 0) {
    struct mg_rsa_key rsa_key;
    if (rsa == NULL) return -1;
    if (mg_rsa_parse_key(private_key_octets.value, private_key_octets.len,
                         &rsa_key) < 0) {
      MG_ERROR(("PKCS#8: failed to parse inner RSA key"));
      return -1;
    }
    *rsa = rsa_key;
    return 0;

  } else if (alg_oid.len == sizeof(mg_ec_public_key_oid) &&
             memcmp(alg_oid.value, mg_ec_public_key_oid,
                    sizeof(mg_ec_public_key_oid)) == 0) {
    if (ec_key == NULL) return -1;
    if (mg_der_next(&alg_id, &alg_params) <= 0 || alg_params.type != 0x06) {
      MG_ERROR(("PKCS#8: invalid EC parameters OID"));
      return -1;
    }

    if (alg_params.len != sizeof(mg_secp256r1_oid) ||
        memcmp(alg_params.value, mg_secp256r1_oid, sizeof(mg_secp256r1_oid)) !=
            0) {
      MG_ERROR(("PKCS#8: unsupported EC curve (only secp256r1 supported)"));
      return -1;
    }

    return mg_parse_ec_private_key(private_key_octets.value,
                                   private_key_octets.len, ec_key);

  } else {
    MG_ERROR(("PKCS#8: unsupported algorithm"));
    return -1;
  }
}

static int mg_parse_pem(const struct mg_str pem, const struct mg_str label,
                        struct mg_str *der) {
  size_t n = 0, m = 0;
  char *s;
  const char *c;
  struct mg_str caps[6];  // number of wildcards + 1
  if (!mg_match(pem, mg_str("#-----BEGIN #-----#-----END #-----#"), caps)) {
    *der = mg_strdup(pem);
    return 0;
  }
  if (mg_strcmp(caps[1], label) != 0 || mg_strcmp(caps[3], label) != 0) {
    return -1;  // bad label
  }
  if ((s = (char *) mg_calloc(1, caps[2].len)) == NULL) {
    return -1;
  }

  for (c = caps[2].buf; c < caps[2].buf + caps[2].len; c++) {
    if (*c == ' ' || *c == '\n' || *c == '\r' || *c == '\t') {
      continue;
    }
    s[n++] = *c;
  }
  m = mg_base64_decode(s, n, s, n);
  if (m == 0) {
    mg_free(s);
    return -1;
  }
  der->buf = s;
  der->len = m;
  return 0;
}

static int mg_parse_pem_certs(const struct mg_str pem, struct mg_str **ders) {
  int count = 0;
  struct mg_str *certs = NULL;
  const char *p = pem.buf;
  const char *end = pem.buf + pem.len;
  const char *begin_marker = "-----BEGIN CERTIFICATE-----";
  const char *end_marker = "-----END CERTIFICATE-----";
  size_t begin_len = strlen(begin_marker);
  size_t end_len = strlen(end_marker);

  while (p < end) {
    const char *s, *begin = NULL, *finish = NULL;
    struct mg_str cert_pem, cert_der, *new_certs;
    int i;

    for (s = p; s <= end - (int) begin_len; s++) {
      if (memcmp(s, begin_marker, begin_len) == 0) {
        begin = s;
        break;
      }
    }
    if (begin == NULL) break;

    for (s = begin + begin_len; s <= end - (int) end_len; s++) {
      if (memcmp(s, end_marker, end_len) == 0) {
        finish = s + end_len;
        break;
      }
    }
    if (finish == NULL) {
      for (i = 0; i < count; i++) mg_free((void *) certs[i].buf);
      mg_free(certs);
      return -1;
    }

    cert_pem = mg_str_n(begin, (size_t) (finish - begin));
    if (mg_parse_pem(cert_pem, mg_str_s("CERTIFICATE"), &cert_der) < 0) {
      for (i = 0; i < count; i++) mg_free((void *) certs[i].buf);
      mg_free(certs);
      return -1;
    }

    new_certs =
        (struct mg_str *) mg_calloc((size_t) count + 1, sizeof(*new_certs));
    if (new_certs == NULL) {
      mg_free((void *) cert_der.buf);
      for (i = 0; i < count; i++) mg_free((void *) certs[i].buf);
      mg_free(certs);
      return -1;
    }
    if (count > 0) {
      memmove(new_certs, certs, (size_t) count * sizeof(struct mg_str));
      mg_free(certs);
    }
    certs = new_certs;

    certs[count++] = cert_der;
    p = finish;
  }

  *ders = certs;
  return count;
}

size_t mg_p256_parse_private_key(struct mg_str key, uint8_t *buf, size_t len) {
  struct mg_str der = mg_str_n(NULL, 0);
  size_t n = 0;
  if (buf == NULL || len < 32) return 0;
  // current mg_parse_ec_private_key() only handles 32-byte keys
  if (mg_parse_pem(key, mg_str_s("EC PRIVATE KEY"), &der) == 0) {
    if (mg_parse_ec_private_key((uint8_t *) der.buf, der.len, buf) == 0) n = 32;
  } else if (mg_parse_pem(key, mg_str_s("PRIVATE KEY"), &der) == 0) {
    if (mg_parse_pkcs8_key((uint8_t *) der.buf, der.len, NULL, buf) == 0) n = 32;
  }
  mg_free((void *) der.buf);
  return n;
}

static void sync_time_cancel(struct mg_connection *c) {
  c->fn = NULL;
  c->is_closing = 1;
}

static void sync_time_cb(struct mg_connection *c, int ev, void *ev_data) {
  if (ev == MG_EV_SNTP_TIME || (ev == MG_EV_CLOSE)) {
    struct mg_connection *tlsc = (struct mg_connection *) c->fn_data;
    struct tls_data *tls = (struct tls_data *) tlsc->tls;
    sync_time_cancel(c);  // c = tls->timec
    tls->timec = NULL;
    tls->is_sntp_pending = false;
    if (ev == MG_EV_SNTP_TIME) mg_tls_handshake(tlsc);
    if (ev == MG_EV_CLOSE) mg_error(tlsc, "time sync failed");
  }
  (void) ev_data;
}

void mg_tls_init(struct mg_connection *c, const struct mg_tls_opts *opts) {
  struct mg_str key;
  struct tls_data *tls =
      (struct tls_data *) mg_calloc(1, sizeof(struct tls_data));
  if (tls == NULL) {
    mg_error(c, "tls oom");
    return;
  }
  tls->hs_start = mg_millis();

  tls->state =
      c->is_client ? MG_TLS_STATE_CLIENT_START : MG_TLS_STATE_SERVER_START;

  tls->skip_verification = opts->skip_verification;
  // tls->send.align = MG_IO_SIZE;

  c->tls = tls;
  c->is_tls = c->is_tls_hs = 1;
  mg_sha256_init(&tls->sha256);

  // save hostname (client extension)
  if (opts->name.len > 0) {
    if (opts->name.len >= sizeof(tls->hostname) - 1) {
      mg_error(c, "hostname too long");
      return;
    }
    strncpy((char *) tls->hostname, opts->name.buf, sizeof(tls->hostname) - 1);
    tls->hostname[opts->name.len] = 0;
  }

  // server CA certificate; parse PEM [bundle] or DER
  if (opts->ca.len > 0)  {
    struct mg_str *all_certs = NULL;
    int cert_count = mg_parse_pem_certs(opts->ca, &all_certs);

    if (cert_count > 1 && c->is_client) { // use bundle for clients only
      tls->ca_bundle_len = (size_t) cert_count;
      tls->ca_bundle_der = all_certs;
      MG_VERBOSE(("%d-cert bundle", cert_count));
    } else if (cert_count > 0) {
      tls->ca_der.buf = all_certs[0].buf;
      tls->ca_der.len = all_certs[0].len;
      mg_free(all_certs);
    } else { // parse again for a possible DER (or a truncated begin string)
      if (mg_parse_pem(opts->ca, mg_str_s("CERTIFICATE"), &tls->ca_der) < 0) {
        MG_ERROR(("Failed to load CA certificate"));
        goto xit;
      }
    } // ca_bundle_len != 0 && ca_der.len = 0 => bundle
    if (!c->is_client) tls->is_twoway = true;  // server + CA: two-way auth
  }

  if (opts->cert.buf == NULL || opts->cert.len == 0) {
    MG_VERBOSE(("No certificate provided"));
    goto xit;
  }

  // parse PEM or DER certificate
  {
    struct mg_str *all_certs = NULL;
    int cert_count = mg_parse_pem_certs(opts->cert, &all_certs);

    if (cert_count > 0) {
      tls->cert_der.buf = all_certs[0].buf;
      tls->cert_der.len = all_certs[0].len;
      if (cert_count > 1) {
        tls->chain_len = (size_t) cert_count;
        tls->chain_der = all_certs;
      } else {
        mg_free(all_certs);
      }
    } else { // parse again for a possible DER (or a truncated begin string)
      if (mg_parse_pem(opts->cert, mg_str_s("CERTIFICATE"), &tls->cert_der) <
          0) {
        MG_ERROR(("Failed to load certificate"));
        return;
      }
    }
  }

  // parse PEM or DER EC key
  if (opts->key.buf == NULL || opts->key.len == 0) {
    mg_error(c, "Certificate provided without a private key");
    return;
  }

  if (mg_parse_pem(opts->key, mg_str_s("EC PRIVATE KEY"), &key) == 0) {
    // expect ASN.1 SEQUENCE=[INTEGER=1, BITSTRING of 32 bytes, ...]
    // 30 nn 02 01 01 04 20 [key] ...
    if (key.len < (2 + 5 + 32) || key.buf[0] != 0x30 ||
        (key.buf[1] & 0x80) != 0 ||
        memcmp(key.buf + 2, "\x02\x01\x01\x04\x20", 5) != 0) {
      mg_error(c, "EC private key: invalid ASN.1");
    } else {
      memmove(tls->ec_key, key.buf + 7, 32);
    }
    mg_free((void *) key.buf);
  } else if (mg_parse_pem(opts->key, mg_str_s("RSA PRIVATE KEY"), &key) == 0) {
    struct mg_rsa_key rsa_key;
    // RSA private key found, store it for later use
    tls->rsa_key_der = key;

    // parse and validate the key structure
    // we keep the DER buffer, rsa_key just points into it
    if (mg_rsa_parse_key((const uint8_t *) key.buf, key.len, &rsa_key) < 0) {
      MG_ERROR(("Failed to parse RSA private key structure"));
      mg_free((void *) key.buf);
      tls->rsa_key_der = mg_str_n(NULL, 0);
      mg_error(c, "Invalid RSA private key format");
      return;
    }

    MG_VERBOSE(("RSA key components:"));
    MG_VERBOSE(("  n (modulus):  %d bytes", (int) rsa_key.n.len));
    MG_VERBOSE(("  e (pubexp):   %d bytes", (int) rsa_key.e.len));
    MG_VERBOSE(("  d (privexp):  %d bytes", (int) rsa_key.d.len));
    MG_VERBOSE(("  p (prime1):   %d bytes", (int) rsa_key.p.len));
    MG_VERBOSE(("  q (prime2):   %d bytes", (int) rsa_key.q.len));
    MG_VERBOSE(("  dP:           %d bytes", (int) rsa_key.dP.len));
    MG_VERBOSE(("  dQ:           %d bytes", (int) rsa_key.dQ.len));
    MG_VERBOSE(("  qInv:         %d bytes", (int) rsa_key.qInv.len));

    // Copy parsed RSA key components to tls->rsa for signing operations
    tls->rsa = rsa_key;
  } else if (mg_parse_pem(opts->key, mg_str_s("PRIVATE KEY"), &key) == 0) {
    if (mg_parse_pkcs8_key((const uint8_t *) key.buf, key.len, &tls->rsa,
                           tls->ec_key) == 0) {
      if (tls->rsa.n.len > 0) {
        tls->rsa_key_der = key;
        MG_INFO(("Parsed PKCS#8 RSA private key: %d bytes", (int) key.len));
      } else {
        mg_free((void *) key.buf);
        MG_VERBOSE(("Parsed PKCS#8 EC private key"));
      }
    } else {
      mg_free((void *) key.buf);
      mg_error(c, "Unsupported PKCS#8 private key format, algorithm, or curve");
      return;
    }
  } else {
    mg_error(
        c, "Expected EC PRIVATE KEY, RSA PRIVATE KEY, or PRIVATE KEY (PKCS#8)");
    return;
  }
xit:
  if (mg_boot_timestamp_ms == 0 && !tls->skip_verification &&
      (tls->ca_bundle_len > 0 || tls->ca_der.len > 0)) {
    struct mg_connection *timec;
    timec = mg_sntp_connect(c->mgr, NULL, sync_time_cb, c);
    if (timec == NULL) {
      mg_error(c, "time sync failed");
      return;
    }
    tls->is_sntp_pending = true;
    tls->timec = timec;
  }
}

void mg_tls_free(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  size_t i;
  if (tls != NULL) {
    if (tls->timec != NULL) {
      sync_time_cancel(tls->timec);
      tls->timec = NULL;
    }
    mg_iobuf_free(&tls->send);
    if (tls->chain_der != NULL) {
      for (i = 0; i < tls->chain_len; i++) {
        mg_free((void *) tls->chain_der[i].buf);
      }
      mg_free(tls->chain_der);
    } else {
      mg_free((void *) tls->cert_der.buf);
    }
    if (tls->ca_bundle_der != NULL) {
      for (i = 0; i < tls->ca_bundle_len; i++) {
        mg_free((void *) tls->ca_bundle_der[i].buf);
      }
      mg_free(tls->ca_bundle_der);
    }
    mg_free((void *) tls->ca_der.buf);
    mg_free((void *) tls->rsa_key_der.buf);
    mg_bzero((unsigned char *) tls, sizeof(*tls));  // Wipe keys and secrets
  }
  mg_free(c->tls);
  c->tls = NULL;
}

long mg_tls_send(struct mg_connection *c, const void *buf, size_t len) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  long n = MG_IO_WAIT;
  bool was_throttled = c->is_tls_throttled;  // see #3074
  if (!was_throttled) {                      // encrypt new data
    size_t i, rs = MG_TLS_RECORD_SIZE > 16384 ? 16384 : MG_TLS_RECORD_SIZE;
    if (len > MG_IO_SIZE) len = MG_IO_SIZE;
    for (i = 0; i < len; i += rs) {  // Split into records, send them at once
      size_t m = len - i < rs ? len - i : rs;
      if (!mg_tls_encrypt(c, (const uint8_t *) buf + i, m, MG_TLS_APP_DATA))
        return 0;  // returning 0 means an OOM condition (iobuf couldn't
                   // resize), yet this is so far recoverable, let the caller
                   // decide
    }
  }  // else, resend outstanding encrypted data in tls->send
  while (tls->send.len > 0 &&
         (n = mg_io_send(c, tls->send.buf, tls->send.len)) > 0) {
    mg_iobuf_del(&tls->send, 0, (size_t) n);
  }  // if last chunk fails to be sent, it needs to be flushed
  c->is_tls_throttled = (tls->send.len > 0 && n == MG_IO_WAIT);
  MG_VERBOSE(("%lu %ld %ld %ld %c %c", c->id, (long) len, (long) tls->send.len,
              n, was_throttled ? 'T' : 't', c->is_tls_throttled ? 'T' : 't'));
  if (n == MG_IO_ERR) return MG_IO_ERR;
  if (was_throttled) return MG_IO_WAIT;  // sent throttled data instead
  return (long) len;  // return len even when throttled, already encripted that
}

long mg_tls_recv(struct mg_connection *c, void *buf, size_t len) {
  int r = 0;
  struct tls_data *tls = (struct tls_data *) c->tls;
  unsigned char *recv_buf;
  size_t minlen;

  for (;;) {
    r = mg_tls_recv_record(c);
    if (r < 0) return r;
    if (tls->content_type == MG_TLS_APP_DATA) break;
    tls->recv_len = 0;
    mg_tls_drop_record(c);
  }

  if (buf == NULL || len == 0) return 0L;
  recv_buf = &c->rtls.buf[tls->recv_offset];
  minlen = len < tls->recv_len ? len : tls->recv_len;
  memmove(buf, recv_buf, minlen);
  tls->recv_offset += minlen;
  tls->recv_len -= minlen;
  if (tls->recv_len == 0) mg_tls_drop_record(c);
  return (long) minlen;
}

size_t mg_tls_pending(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  return tls != NULL ? tls->recv_len : 0;
}

long mg_tls_flush(struct mg_connection *c) {
  struct tls_data *tls = (struct tls_data *) c->tls;
  long n = 0;
  while (tls->send.len > 0 &&
         (n = mg_io_send(c, tls->send.buf, tls->send.len)) > 0) {
    mg_iobuf_del(&tls->send, 0, (size_t) n);
  }
  if (n == MG_IO_ERR) return n;
  c->is_tls_throttled = tls->send.len > 0;
  return c->is_tls_throttled ? MG_IO_WAIT : 0;
}

void mg_tls_ctx_init(struct mg_mgr *mgr) {
  (void) mgr;
}

void mg_tls_ctx_free(struct mg_mgr *mgr) {
  (void) mgr;
}
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/tls_chacha20.c"
#endif
// portable8439 v1.0.1
// Source: https://github.com/DavyLandman/portable8439
// Licensed under CC0-1.0
// Contains poly1305-donna e6ad6e091d30d7f4ec2d4f978be1fcfcbce72781 (Public
// Domain)





#if MG_TLS == MG_TLS_BUILTIN && MG_ENABLE_CHACHA20
// ******* BEGIN: chacha-portable/chacha-portable.h ********

#if !defined(__cplusplus) && !defined(_MSC_VER) && \
    (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L)
#error "C99 or newer required"
#endif

#define CHACHA20_KEY_SIZE (32)
#define CHACHA20_NONCE_SIZE (12)

#if defined(_MSC_VER) || defined(__cplusplus)
// add restrict support
#if ((defined(_MSC_VER) && _MSC_VER >= 1900) && !defined(__cplusplus)) || \
    defined(__clang__) || defined(__GNUC__)
#define restrict __restrict
#else
#define restrict
#endif
#endif

// xor data with a ChaCha20 keystream as per RFC8439
static PORTABLE_8439_DECL void chacha20_xor_stream(
    uint8_t *restrict dest, const uint8_t *restrict source, size_t length,
    const uint8_t key[CHACHA20_KEY_SIZE],
    const uint8_t nonce[CHACHA20_NONCE_SIZE], uint32_t counter);

static PORTABLE_8439_DECL void rfc8439_keygen(
    uint8_t poly_key[32], const uint8_t key[CHACHA20_KEY_SIZE],
    const uint8_t nonce[CHACHA20_NONCE_SIZE]);

// ******* END:   chacha-portable/chacha-portable.h ********
// ******* BEGIN: poly1305-donna/poly1305-donna.h ********

#include <stddef.h>

typedef struct poly1305_context {
  size_t aligner;
  unsigned char opaque[136];
} poly1305_context;

static PORTABLE_8439_DECL void poly1305_init(poly1305_context *ctx,
                                             const unsigned char key[32]);
static PORTABLE_8439_DECL void poly1305_update(poly1305_context *ctx,
                                               const unsigned char *m,
                                               size_t bytes);
static PORTABLE_8439_DECL void poly1305_finish(poly1305_context *ctx,
                                               unsigned char mac[16]);

// ******* END:   poly1305-donna/poly1305-donna.h ********
// ******* BEGIN: chacha-portable.c ********

// this is a fresh implementation of chacha20, based on the description in
// rfc8349 it's such a nice compact algorithm that it is easy to do. In
// relationship to other c implementation this implementation:
//  - pure c99
//  - big & little endian support
//  - safe for architectures that don't support unaligned reads
//
// Next to this, we try to be fast as possible without resorting inline
// assembly.

// based on https://sourceforge.net/p/predef/wiki/Endianness/
#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \
    __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define __HAVE_LITTLE_ENDIAN 1
#elif defined(__LITTLE_ENDIAN__) || defined(__ARMEL__) ||                 \
    defined(__THUMBEL__) || defined(__AARCH64EL__) || defined(_MIPSEL) || \
    defined(__MIPSEL) || defined(__MIPSEL__) || defined(__XTENSA_EL__) || \
    defined(__AVR__)
#define __HAVE_LITTLE_ENDIAN 1
#endif
// DO NOT test for LITTLE_ENDIAN, as it is defined as 1234 when including
// sys/types.h in GCC

#ifndef TEST_SLOW_PATH
#if defined(__HAVE_LITTLE_ENDIAN)
#define FAST_PATH
#endif
#endif

#define CHACHA20_STATE_WORDS (16)
#define CHACHA20_BLOCK_SIZE (CHACHA20_STATE_WORDS * sizeof(uint32_t))

#ifdef FAST_PATH
#define store_32_le(target, source) memcpy(&(target), source, sizeof(uint32_t))
#else
#define store_32_le(target, source)                                 \
  target = (uint32_t) (source)[0] | ((uint32_t) (source)[1]) << 8 | \
           ((uint32_t) (source)[2]) << 16 | ((uint32_t) (source)[3]) << 24
#endif

static void initialize_state(uint32_t state[CHACHA20_STATE_WORDS],
                             const uint8_t key[CHACHA20_KEY_SIZE],
                             const uint8_t nonce[CHACHA20_NONCE_SIZE],
                             uint32_t counter) {
#if 0
#ifdef static_assert
  static_assert(sizeof(uint32_t) == 4,
                "We don't support systems that do not conform to standard of "
                "uint32_t being exact 32bit wide");
#endif
#endif
  state[0] = 0x61707865;
  state[1] = 0x3320646e;
  state[2] = 0x79622d32;
  state[3] = 0x6b206574;
  store_32_le(state[4], key);
  store_32_le(state[5], key + 4);
  store_32_le(state[6], key + 8);
  store_32_le(state[7], key + 12);
  store_32_le(state[8], key + 16);
  store_32_le(state[9], key + 20);
  store_32_le(state[10], key + 24);
  store_32_le(state[11], key + 28);
  state[12] = counter;
  store_32_le(state[13], nonce);
  store_32_le(state[14], nonce + 4);
  store_32_le(state[15], nonce + 8);
}

#define increment_counter(state) (state)[12]++

// source: http://blog.regehr.org/archives/1063
#define rotl32a(x, n) ((x) << (n)) | ((x) >> (32 - (n)))

#define Qround(a, b, c, d) \
  a += b;                  \
  d ^= a;                  \
  d = rotl32a(d, 16);      \
  c += d;                  \
  b ^= c;                  \
  b = rotl32a(b, 12);      \
  a += b;                  \
  d ^= a;                  \
  d = rotl32a(d, 8);       \
  c += d;                  \
  b ^= c;                  \
  b = rotl32a(b, 7);

#define TIMES16(x)                                                          \
  x(0) x(1) x(2) x(3) x(4) x(5) x(6) x(7) x(8) x(9) x(10) x(11) x(12) x(13) \
      x(14) x(15)

static void core_block(const uint32_t *restrict start,
                       uint32_t *restrict output) {
  int i;
// instead of working on the output array,
// we let the compiler allocate 16 local variables on the stack
#define __LV(i) uint32_t __t##i = start[i];
  TIMES16(__LV)

#define __Q(a, b, c, d) Qround(__t##a, __t##b, __t##c, __t##d)

  for (i = 0; i < 10; i++) {
    __Q(0, 4, 8, 12);
    __Q(1, 5, 9, 13);
    __Q(2, 6, 10, 14);
    __Q(3, 7, 11, 15);
    __Q(0, 5, 10, 15);
    __Q(1, 6, 11, 12);
    __Q(2, 7, 8, 13);
    __Q(3, 4, 9, 14);
  }

#define __FIN(i) output[i] = start[i] + __t##i;
  TIMES16(__FIN)
}

#define U8(x) ((uint8_t) ((x) & 0xFF))

#ifdef FAST_PATH
#define xor32_le(dst, src, pad)            \
  uint32_t __value;                        \
  memcpy(&__value, src, sizeof(uint32_t)); \
  __value ^= *(pad);                       \
  memcpy(dst, &__value, sizeof(uint32_t));
#else
#define xor32_le(dst, src, pad)           \
  (dst)[0] = (src)[0] ^ U8(*(pad));       \
  (dst)[1] = (src)[1] ^ U8(*(pad) >> 8);  \
  (dst)[2] = (src)[2] ^ U8(*(pad) >> 16); \
  (dst)[3] = (src)[3] ^ U8(*(pad) >> 24);
#endif

#define index8_32(a, ix) ((a) + ((ix) * sizeof(uint32_t)))

#define xor32_blocks(dest, source, pad, words)                    \
  for (i = 0; i < words; i++) {                                   \
    xor32_le(index8_32(dest, i), index8_32(source, i), (pad) + i) \
  }

static void xor_block(uint8_t *restrict dest, const uint8_t *restrict source,
                      const uint32_t *restrict pad, unsigned int chunk_size) {
  unsigned int i, full_blocks = chunk_size / (unsigned int) sizeof(uint32_t);
  // have to be carefull, we are going back from uint32 to uint8, so endianness
  // matters again
  xor32_blocks(dest, source, pad, full_blocks)

      dest += full_blocks * sizeof(uint32_t);
  source += full_blocks * sizeof(uint32_t);
  pad += full_blocks;

  switch (chunk_size % sizeof(uint32_t)) {
    case 1:
      dest[0] = source[0] ^ U8(*pad);
      break;
    case 2:
      dest[0] = source[0] ^ U8(*pad);
      dest[1] = source[1] ^ U8(*pad >> 8);
      break;
    case 3:
      dest[0] = source[0] ^ U8(*pad);
      dest[1] = source[1] ^ U8(*pad >> 8);
      dest[2] = source[2] ^ U8(*pad >> 16);
      break;
  }
}

static void chacha20_xor_stream(uint8_t *restrict dest,
                                const uint8_t *restrict source, size_t length,
                                const uint8_t key[CHACHA20_KEY_SIZE],
                                const uint8_t nonce[CHACHA20_NONCE_SIZE],
                                uint32_t counter) {
  uint32_t state[CHACHA20_STATE_WORDS];
  uint32_t pad[CHACHA20_STATE_WORDS];
  size_t i, b, last_block, full_blocks = length / CHACHA20_BLOCK_SIZE;
  initialize_state(state, key, nonce, counter);
  for (b = 0; b < full_blocks; b++) {
    core_block(state, pad);
    increment_counter(state);
    xor32_blocks(dest, source, pad, CHACHA20_STATE_WORDS) dest +=
        CHACHA20_BLOCK_SIZE;
    source += CHACHA20_BLOCK_SIZE;
  }
  last_block = length % CHACHA20_BLOCK_SIZE;
  if (last_block > 0) {
    core_block(state, pad);
    xor_block(dest, source, pad, (unsigned int) last_block);
  }
}

#ifdef FAST_PATH
#define serialize(poly_key, result) memcpy(poly_key, result, 32)
#else
#define store32_le(target, source)   \
  (target)[0] = U8(*(source));       \
  (target)[1] = U8(*(source) >> 8);  \
  (target)[2] = U8(*(source) >> 16); \
  (target)[3] = U8(*(source) >> 24);

#define serialize(poly_key, result)                 \
  for (i = 0; i < 32 / sizeof(uint32_t); i++) {     \
    store32_le(index8_32(poly_key, i), result + i); \
  }
#endif

static void rfc8439_keygen(uint8_t poly_key[32],
                           const uint8_t key[CHACHA20_KEY_SIZE],
                           const uint8_t nonce[CHACHA20_NONCE_SIZE]) {
  uint32_t state[CHACHA20_STATE_WORDS];
  uint32_t result[CHACHA20_STATE_WORDS];
  size_t i;
  initialize_state(state, key, nonce, 0);
  core_block(state, result);
  serialize(poly_key, result);
  (void) i;
}
// ******* END: chacha-portable.c ********
// ******* BEGIN: poly1305-donna.c ********

/* auto detect between 32bit / 64bit */
#if /* uint128 available on 64bit system*/                              \
    (defined(__SIZEOF_INT128__) &&                                      \
     defined(__LP64__))                       /* MSVC 64bit compiler */ \
    || (defined(_MSC_VER) && defined(_M_X64)) /* gcc >= 4.4 64bit */    \
    || (defined(__GNUC__) && defined(__LP64__) &&                       \
        ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ >= 4))))
#define __GUESS64
#else
#define __GUESS32
#endif

#if defined(POLY1305_8BIT)
/*
        poly1305 implementation using 8 bit * 8 bit = 16 bit multiplication and
32 bit addition

        based on the public domain reference version in supercop by djb
static */

#if defined(_MSC_VER) && _MSC_VER < 1700
#define POLY1305_NOINLINE
#elif defined(_MSC_VER)
#define POLY1305_NOINLINE __declspec(noinline)
#elif defined(__GNUC__)
#define POLY1305_NOINLINE __attribute__((noinline))
#else
#define POLY1305_NOINLINE
#endif

#define poly1305_block_size 16

/* 17 + sizeof(size_t) + 51*sizeof(unsigned char) */
typedef struct poly1305_state_internal_t {
  unsigned char buffer[poly1305_block_size];
  size_t leftover;
  unsigned char h[17];
  unsigned char r[17];
  unsigned char pad[17];
  unsigned char final;
} poly1305_state_internal_t;

static void poly1305_init(poly1305_context *ctx, const unsigned char key[32]) {
  poly1305_state_internal_t *st = (poly1305_state_internal_t *) ctx;
  size_t i;

  st->leftover = 0;

  /* h = 0 */
  for (i = 0; i < 17; i++) st->h[i] = 0;

  /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
  st->r[0] = key[0] & 0xff;
  st->r[1] = key[1] & 0xff;
  st->r[2] = key[2] & 0xff;
  st->r[3] = key[3] & 0x0f;
  st->r[4] = key[4] & 0xfc;
  st->r[5] = key[5] & 0xff;
  st->r[6] = key[6] & 0xff;
  st->r[7] = key[7] & 0x0f;
  st->r[8] = key[8] & 0xfc;
  st->r[9] = key[9] & 0xff;
  st->r[10] = key[10] & 0xff;
  st->r[11] = key[11] & 0x0f;
  st->r[12] = key[12] & 0xfc;
  st->r[13] = key[13] & 0xff;
  st->r[14] = key[14] & 0xff;
  st->r[15] = key[15] & 0x0f;
  st->r[16] = 0;

  /* save pad for later */
  for (i = 0; i < 16; i++) st->pad[i] = key[i + 16];
  st->pad[16] = 0;

  st->final = 0;
}

static void poly1305_add(unsigned char h[17], const unsigned char c[17]) {
  unsigned short u;
  unsigned int i;
  for (u = 0, i = 0; i < 17; i++) {
    u += (unsigned short) h[i] + (unsigned short) c[i];
    h[i] = (unsigned char) u & 0xff;
    u >>= 8;
  }
}

static void poly1305_squeeze(unsigned char h[17], unsigned long hr[17]) {
  unsigned long u;
  unsigned int i;
  u = 0;
  for (i = 0; i < 16; i++) {
    u += hr[i];
    h[i] = (unsigned char) u & 0xff;
    u >>= 8;
  }
  u += hr[16];
  h[16] = (unsigned char) u & 0x03;
  u >>= 2;
  u += (u << 2); /* u *= 5; */
  for (i = 0; i < 16; i++) {
    u += h[i];
    h[i] = (unsigned char) u & 0xff;
    u >>= 8;
  }
  h[16] += (unsigned char) u;
}

static void poly1305_freeze(unsigned char h[17]) {
  const unsigned char minusp[17] = {0x05, 0x00, 0x00, 0x00, 0x00, 0x00,
                                    0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
                                    0x00, 0x00, 0x00, 0x00, 0xfc};
  unsigned char horig[17], negative;
  unsigned int i;

  /* compute h + -p */
  for (i = 0; i < 17; i++) horig[i] = h[i];
  poly1305_add(h, minusp);

  /* select h if h < p, or h + -p if h >= p */
  negative = -(h[16] >> 7);
  for (i = 0; i < 17; i++) h[i] ^= negative & (horig[i] ^ h[i]);
}

static void poly1305_blocks(poly1305_state_internal_t *st,
                            const unsigned char *m, size_t bytes) {
  const unsigned char hibit = st->final ^ 1; /* 1 << 128 */

  while (bytes >= poly1305_block_size) {
    unsigned long hr[17], u;
    unsigned char c[17];
    unsigned int i, j;

    /* h += m */
    for (i = 0; i < 16; i++) c[i] = m[i];
    c[16] = hibit;
    poly1305_add(st->h, c);

    /* h *= r */
    for (i = 0; i < 17; i++) {
      u = 0;
      for (j = 0; j <= i; j++) {
        u += (unsigned short) st->h[j] * st->r[i - j];
      }
      for (j = i + 1; j < 17; j++) {
        unsigned long v = (unsigned short) st->h[j] * st->r[i + 17 - j];
        v = ((v << 8) + (v << 6)); /* v *= (5 << 6); */
        u += v;
      }
      hr[i] = u;
    }

    /* (partial) h %= p */
    poly1305_squeeze(st->h, hr);

    m += poly1305_block_size;
    bytes -= poly1305_block_size;
  }
}

static POLY1305_NOINLINE void poly1305_finish(poly1305_context *ctx,
                                              unsigned char mac[16]) {
  poly1305_state_internal_t *st = (poly1305_state_internal_t *) ctx;
  size_t i;

  /* process the remaining block */
  if (st->leftover) {
    size_t i = st->leftover;
    st->buffer[i++] = 1;
    for (; i < poly1305_block_size; i++) st->buffer[i] = 0;
    st->final = 1;
    poly1305_blocks(st, st->buffer, poly1305_block_size);
  }

  /* fully reduce h */
  poly1305_freeze(st->h);

  /* h = (h + pad) % (1 << 128) */
  poly1305_add(st->h, st->pad);
  for (i = 0; i < 16; i++) mac[i] = st->h[i];

  /* zero out the state */
  for (i = 0; i < 17; i++) st->h[i] = 0;
  for (i = 0; i < 17; i++) st->r[i] = 0;
  for (i = 0; i < 17; i++) st->pad[i] = 0;
}
#elif defined(POLY1305_16BIT)
/*
        poly1305 implementation using 16 bit * 16 bit = 32 bit multiplication
and 32 bit addition static */

#if defined(_MSC_VER) && _MSC_VER < 1700
#define POLY1305_NOINLINE
#elif defined(_MSC_VER)
#define POLY1305_NOINLINE __declspec(noinline)
#elif defined(__GNUC__)
#define POLY1305_NOINLINE __attribute__((noinline))
#else
#define POLY1305_NOINLINE
#endif

#define poly1305_block_size 16

/* 17 + sizeof(size_t) + 18*sizeof(unsigned short) */
typedef struct poly1305_state_internal_t {
  unsigned char buffer[poly1305_block_size];
  size_t leftover;
  unsigned short r[10];
  unsigned short h[10];
  unsigned short pad[8];
  unsigned char final;
} poly1305_state_internal_t;

/* interpret two 8 bit unsigned integers as a 16 bit unsigned integer in little
 * endian */
static unsigned short U8TO16(const unsigned char *p) {
  return (((unsigned short) (p[0] & 0xff)) |
          ((unsigned short) (p[1] & 0xff) << 8));
}

/* store a 16 bit unsigned integer as two 8 bit unsigned integers in little
 * endian */
static void U16TO8(unsigned char *p, unsigned short v) {
  p[0] = (v) & 0xff;
  p[1] = (v >> 8) & 0xff;
}

static void poly1305_init(poly1305_context *ctx, const unsigned char key[32]) {
  poly1305_state_internal_t *st = (poly1305_state_internal_t *) ctx;
  unsigned short t0, t1, t2, t3, t4, t5, t6, t7;
  size_t i;

  /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
  t0 = U8TO16(&key[0]);
  st->r[0] = (t0) & 0x1fff;
  t1 = U8TO16(&key[2]);
  st->r[1] = ((t0 >> 13) | (t1 << 3)) & 0x1fff;
  t2 = U8TO16(&key[4]);
  st->r[2] = ((t1 >> 10) | (t2 << 6)) & 0x1f03;
  t3 = U8TO16(&key[6]);
  st->r[3] = ((t2 >> 7) | (t3 << 9)) & 0x1fff;
  t4 = U8TO16(&key[8]);
  st->r[4] = ((t3 >> 4) | (t4 << 12)) & 0x00ff;
  st->r[5] = ((t4 >> 1)) & 0x1ffe;
  t5 = U8TO16(&key[10]);
  st->r[6] = ((t4 >> 14) | (t5 << 2)) & 0x1fff;
  t6 = U8TO16(&key[12]);
  st->r[7] = ((t5 >> 11) | (t6 << 5)) & 0x1f81;
  t7 = U8TO16(&key[14]);
  st->r[8] = ((t6 >> 8) | (t7 << 8)) & 0x1fff;
  st->r[9] = ((t7 >> 5)) & 0x007f;

  /* h = 0 */
  for (i = 0; i < 10; i++) st->h[i] = 0;

  /* save pad for later */
  for (i = 0; i < 8; i++) st->pad[i] = U8TO16(&key[16 + (2 * i)]);

  st->leftover = 0;
  st->final = 0;
}

static void poly1305_blocks(poly1305_state_internal_t *st,
                            const unsigned char *m, size_t bytes) {
  const unsigned short hibit = (st->final) ? 0 : (1 << 11); /* 1 << 128 */
  unsigned short t0, t1, t2, t3, t4, t5, t6, t7;
  unsigned long d[10];
  unsigned long c;

  while (bytes >= poly1305_block_size) {
    size_t i, j;

    /* h += m[i] */
    t0 = U8TO16(&m[0]);
    st->h[0] += (t0) & 0x1fff;
    t1 = U8TO16(&m[2]);
    st->h[1] += ((t0 >> 13) | (t1 << 3)) & 0x1fff;
    t2 = U8TO16(&m[4]);
    st->h[2] += ((t1 >> 10) | (t2 << 6)) & 0x1fff;
    t3 = U8TO16(&m[6]);
    st->h[3] += ((t2 >> 7) | (t3 << 9)) & 0x1fff;
    t4 = U8TO16(&m[8]);
    st->h[4] += ((t3 >> 4) | (t4 << 12)) & 0x1fff;
    st->h[5] += ((t4 >> 1)) & 0x1fff;
    t5 = U8TO16(&m[10]);
    st->h[6] += ((t4 >> 14) | (t5 << 2)) & 0x1fff;
    t6 = U8TO16(&m[12]);
    st->h[7] += ((t5 >> 11) | (t6 << 5)) & 0x1fff;
    t7 = U8TO16(&m[14]);
    st->h[8] += ((t6 >> 8) | (t7 << 8)) & 0x1fff;
    st->h[9] += ((t7 >> 5)) | hibit;

    /* h *= r, (partial) h %= p */
    for (i = 0, c = 0; i < 10; i++) {
      d[i] = c;
      for (j = 0; j < 10; j++) {
        d[i] += (unsigned long) st->h[j] *
                ((j <= i) ? st->r[i - j] : (5 * st->r[i + 10 - j]));
        /* Sum(h[i] * r[i] * 5) will overflow slightly above 6 products with an
         * unclamped r, so carry at 5 */
        if (j == 4) {
          c = (d[i] >> 13);
          d[i] &= 0x1fff;
        }
      }
      c += (d[i] >> 13);
      d[i] &= 0x1fff;
    }
    c = ((c << 2) + c); /* c *= 5 */
    c += d[0];
    d[0] = ((unsigned short) c & 0x1fff);
    c = (c >> 13);
    d[1] += c;

    for (i = 0; i < 10; i++) st->h[i] = (unsigned short) d[i];

    m += poly1305_block_size;
    bytes -= poly1305_block_size;
  }
}

static POLY1305_NOINLINE void poly1305_finish(poly1305_context *ctx,
                                              unsigned char mac[16]) {
  poly1305_state_internal_t *st = (poly1305_state_internal_t *) ctx;
  unsigned short c;
  unsigned short g[10];
  unsigned short mask;
  unsigned long f;
  size_t i;

  /* process the remaining block */
  if (st->leftover) {
    size_t i = st->leftover;
    st->buffer[i++] = 1;
    for (; i < poly1305_block_size; i++) st->buffer[i] = 0;
    st->final = 1;
    poly1305_blocks(st, st->buffer, poly1305_block_size);
  }

  /* fully carry h */
  c = st->h[1] >> 13;
  st->h[1] &= 0x1fff;
  for (i = 2; i < 10; i++) {
    st->h[i] += c;
    c = st->h[i] >> 13;
    st->h[i] &= 0x1fff;
  }
  st->h[0] += (c * 5);
  c = st->h[0] >> 13;
  st->h[0] &= 0x1fff;
  st->h[1] += c;
  c = st->h[1] >> 13;
  st->h[1] &= 0x1fff;
  st->h[2] += c;

  /* compute h + -p */
  g[0] = st->h[0] + 5;
  c = g[0] >> 13;
  g[0] &= 0x1fff;
  for (i = 1; i < 10; i++) {
    g[i] = st->h[i] + c;
    c = g[i] >> 13;
    g[i] &= 0x1fff;
  }

  /* select h if h < p, or h + -p if h >= p */
  mask = (c ^ 1) - 1;
  for (i = 0; i < 10; i++) g[i] &= mask;
  mask = ~mask;
  for (i = 0; i < 10; i++) st->h[i] = (st->h[i] & mask) | g[i];

  /* h = h % (2^128) */
  st->h[0] = ((st->h[0]) | (st->h[1] << 13)) & 0xffff;
  st->h[1] = ((st->h[1] >> 3) | (st->h[2] << 10)) & 0xffff;
  st->h[2] = ((st->h[2] >> 6) | (st->h[3] << 7)) & 0xffff;
  st->h[3] = ((st->h[3] >> 9) | (st->h[4] << 4)) & 0xffff;
  st->h[4] = ((st->h[4] >> 12) | (st->h[5] << 1) | (st->h[6] << 14)) & 0xffff;
  st->h[5] = ((st->h[6] >> 2) | (st->h[7] << 11)) & 0xffff;
  st->h[6] = ((st->h[7] >> 5) | (st->h[8] << 8)) & 0xffff;
  st->h[7] = ((st->h[8] >> 8) | (st->h[9] << 5)) & 0xffff;

  /* mac = (h + pad) % (2^128) */
  f = (unsigned long) st->h[0] + st->pad[0];
  st->h[0] = (unsigned short) f;
  for (i = 1; i < 8; i++) {
    f = (unsigned long) st->h[i] + st->pad[i] + (f >> 16);
    st->h[i] = (unsigned short) f;
  }

  for (i = 0; i < 8; i++) U16TO8(mac + (i * 2), st->h[i]);

  /* zero out the state */
  for (i = 0; i < 10; i++) st->h[i] = 0;
  for (i = 0; i < 10; i++) st->r[i] = 0;
  for (i = 0; i < 8; i++) st->pad[i] = 0;
}
#elif defined(POLY1305_32BIT) || \
    (!defined(POLY1305_64BIT) && defined(__GUESS32))
/*
        poly1305 implementation using 32 bit * 32 bit = 64 bit multiplication
and 64 bit addition static */

#if defined(_MSC_VER) && _MSC_VER < 1700
#define POLY1305_NOINLINE
#elif defined(_MSC_VER)
#define POLY1305_NOINLINE __declspec(noinline)
#elif defined(__GNUC__)
#define POLY1305_NOINLINE __attribute__((noinline))
#else
#define POLY1305_NOINLINE
#endif

#define poly1305_block_size 16

/* 17 + sizeof(size_t) + 14*sizeof(unsigned long) */
typedef struct poly1305_state_internal_t {
  unsigned long r[5];
  unsigned long h[5];
  unsigned long pad[4];
  size_t leftover;
  unsigned char buffer[poly1305_block_size];
  unsigned char final;
} poly1305_state_internal_t;

/* interpret four 8 bit unsigned integers as a 32 bit unsigned integer in little
 * endian */
static unsigned long U8TO32(const unsigned char *p) {
  return (((unsigned long) (p[0] & 0xff)) |
          ((unsigned long) (p[1] & 0xff) << 8) |
          ((unsigned long) (p[2] & 0xff) << 16) |
          ((unsigned long) (p[3] & 0xff) << 24));
}

/* store a 32 bit unsigned integer as four 8 bit unsigned integers in little
 * endian */
static void U32TO8(unsigned char *p, unsigned long v) {
  p[0] = (unsigned char) ((v) & 0xff);
  p[1] = (unsigned char) ((v >> 8) & 0xff);
  p[2] = (unsigned char) ((v >> 16) & 0xff);
  p[3] = (unsigned char) ((v >> 24) & 0xff);
}

static void poly1305_init(poly1305_context *ctx, const unsigned char key[32]) {
  poly1305_state_internal_t *st = (poly1305_state_internal_t *) ctx;

  /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
  st->r[0] = (U8TO32(&key[0])) & 0x3ffffff;
  st->r[1] = (U8TO32(&key[3]) >> 2) & 0x3ffff03;
  st->r[2] = (U8TO32(&key[6]) >> 4) & 0x3ffc0ff;
  st->r[3] = (U8TO32(&key[9]) >> 6) & 0x3f03fff;
  st->r[4] = (U8TO32(&key[12]) >> 8) & 0x00fffff;

  /* h = 0 */
  st->h[0] = 0;
  st->h[1] = 0;
  st->h[2] = 0;
  st->h[3] = 0;
  st->h[4] = 0;

  /* save pad for later */
  st->pad[0] = U8TO32(&key[16]);
  st->pad[1] = U8TO32(&key[20]);
  st->pad[2] = U8TO32(&key[24]);
  st->pad[3] = U8TO32(&key[28]);

  st->leftover = 0;
  st->final = 0;
}

static void poly1305_blocks(poly1305_state_internal_t *st,
                            const unsigned char *m, size_t bytes) {
  const unsigned long hibit = (st->final) ? 0 : (1UL << 24); /* 1 << 128 */
  unsigned long r0, r1, r2, r3, r4;
  unsigned long s1, s2, s3, s4;
  unsigned long h0, h1, h2, h3, h4;
  uint64_t d0, d1, d2, d3, d4;
  unsigned long c;

  r0 = st->r[0];
  r1 = st->r[1];
  r2 = st->r[2];
  r3 = st->r[3];
  r4 = st->r[4];

  s1 = r1 * 5;
  s2 = r2 * 5;
  s3 = r3 * 5;
  s4 = r4 * 5;

  h0 = st->h[0];
  h1 = st->h[1];
  h2 = st->h[2];
  h3 = st->h[3];
  h4 = st->h[4];

  while (bytes >= poly1305_block_size) {
    /* h += m[i] */
    h0 += (U8TO32(m + 0)) & 0x3ffffff;
    h1 += (U8TO32(m + 3) >> 2) & 0x3ffffff;
    h2 += (U8TO32(m + 6) >> 4) & 0x3ffffff;
    h3 += (U8TO32(m + 9) >> 6) & 0x3ffffff;
    h4 += (U8TO32(m + 12) >> 8) | hibit;

    /* h *= r */
    d0 = ((uint64_t) h0 * r0) + ((uint64_t) h1 * s4) + ((uint64_t) h2 * s3) +
         ((uint64_t) h3 * s2) + ((uint64_t) h4 * s1);
    d1 = ((uint64_t) h0 * r1) + ((uint64_t) h1 * r0) + ((uint64_t) h2 * s4) +
         ((uint64_t) h3 * s3) + ((uint64_t) h4 * s2);
    d2 = ((uint64_t) h0 * r2) + ((uint64_t) h1 * r1) + ((uint64_t) h2 * r0) +
         ((uint64_t) h3 * s4) + ((uint64_t) h4 * s3);
    d3 = ((uint64_t) h0 * r3) + ((uint64_t) h1 * r2) + ((uint64_t) h2 * r1) +
         ((uint64_t) h3 * r0) + ((uint64_t) h4 * s4);
    d4 = ((uint64_t) h0 * r4) + ((uint64_t) h1 * r3) + ((uint64_t) h2 * r2) +
         ((uint64_t) h3 * r1) + ((uint64_t) h4 * r0);

    /* (partial) h %= p */
    c = (unsigned long) (d0 >> 26);
    h0 = (unsigned long) d0 & 0x3ffffff;
    d1 += c;
    c = (unsigned long) (d1 >> 26);
    h1 = (unsigned long) d1 & 0x3ffffff;
    d2 += c;
    c = (unsigned long) (d2 >> 26);
    h2 = (unsigned long) d2 & 0x3ffffff;
    d3 += c;
    c = (unsigned long) (d3 >> 26);
    h3 = (unsigned long) d3 & 0x3ffffff;
    d4 += c;
    c = (unsigned long) (d4 >> 26);
    h4 = (unsigned long) d4 & 0x3ffffff;
    h0 += c * 5;
    c = (h0 >> 26);
    h0 = h0 & 0x3ffffff;
    h1 += c;

    m += poly1305_block_size;
    bytes -= poly1305_block_size;
  }

  st->h[0] = h0;
  st->h[1] = h1;
  st->h[2] = h2;
  st->h[3] = h3;
  st->h[4] = h4;
}

static POLY1305_NOINLINE void poly1305_finish(poly1305_context *ctx,
                                              unsigned char mac[16]) {
  poly1305_state_internal_t *st = (poly1305_state_internal_t *) ctx;
  unsigned long h0, h1, h2, h3, h4, c;
  unsigned long g0, g1, g2, g3, g4;
  uint64_t f;
  unsigned long mask;

  /* process the remaining block */
  if (st->leftover) {
    size_t i = st->leftover;
    st->buffer[i++] = 1;
    for (; i < poly1305_block_size; i++) st->buffer[i] = 0;
    st->final = 1;
    poly1305_blocks(st, st->buffer, poly1305_block_size);
  }

  /* fully carry h */
  h0 = st->h[0];
  h1 = st->h[1];
  h2 = st->h[2];
  h3 = st->h[3];
  h4 = st->h[4];

  c = h1 >> 26;
  h1 = h1 & 0x3ffffff;
  h2 += c;
  c = h2 >> 26;
  h2 = h2 & 0x3ffffff;
  h3 += c;
  c = h3 >> 26;
  h3 = h3 & 0x3ffffff;
  h4 += c;
  c = h4 >> 26;
  h4 = h4 & 0x3ffffff;
  h0 += c * 5;
  c = h0 >> 26;
  h0 = h0 & 0x3ffffff;
  h1 += c;

  /* compute h + -p */
  g0 = h0 + 5;
  c = g0 >> 26;
  g0 &= 0x3ffffff;
  g1 = h1 + c;
  c = g1 >> 26;
  g1 &= 0x3ffffff;
  g2 = h2 + c;
  c = g2 >> 26;
  g2 &= 0x3ffffff;
  g3 = h3 + c;
  c = g3 >> 26;
  g3 &= 0x3ffffff;
  g4 = h4 + c - (1UL << 26);

  /* select h if h < p, or h + -p if h >= p */
  mask = (g4 >> ((sizeof(unsigned long) * 8) - 1)) - 1;
  g0 &= mask;
  g1 &= mask;
  g2 &= mask;
  g3 &= mask;
  g4 &= mask;
  mask = ~mask;
  h0 = (h0 & mask) | g0;
  h1 = (h1 & mask) | g1;
  h2 = (h2 & mask) | g2;
  h3 = (h3 & mask) | g3;
  h4 = (h4 & mask) | g4;

  /* h = h % (2^128) */
  h0 = ((h0) | (h1 << 26)) & 0xffffffff;
  h1 = ((h1 >> 6) | (h2 << 20)) & 0xffffffff;
  h2 = ((h2 >> 12) | (h3 << 14)) & 0xffffffff;
  h3 = ((h3 >> 18) | (h4 << 8)) & 0xffffffff;

  /* mac = (h + pad) % (2^128) */
  f = (uint64_t) h0 + st->pad[0];
  h0 = (unsigned long) f;
  f = (uint64_t) h1 + st->pad[1] + (f >> 32);
  h1 = (unsigned long) f;
  f = (uint64_t) h2 + st->pad[2] + (f >> 32);
  h2 = (unsigned long) f;
  f = (uint64_t) h3 + st->pad[3] + (f >> 32);
  h3 = (unsigned long) f;

  U32TO8(mac + 0, h0);
  U32TO8(mac + 4, h1);
  U32TO8(mac + 8, h2);
  U32TO8(mac + 12, h3);

  /* zero out the state */
  st->h[0] = 0;
  st->h[1] = 0;
  st->h[2] = 0;
  st->h[3] = 0;
  st->h[4] = 0;
  st->r[0] = 0;
  st->r[1] = 0;
  st->r[2] = 0;
  st->r[3] = 0;
  st->r[4] = 0;
  st->pad[0] = 0;
  st->pad[1] = 0;
  st->pad[2] = 0;
  st->pad[3] = 0;
}

#else
/*
        poly1305 implementation using 64 bit * 64 bit = 128 bit multiplication
and 128 bit addition static */

#if defined(_MSC_VER)

typedef struct uint128_t {
  uint64_t lo;
  uint64_t hi;
} uint128_t;

#define MUL128(out, x, y) out.lo = _umul128((x), (y), &out.hi)
#define ADD(out, in)                \
  {                                 \
    uint64_t t = out.lo;            \
    out.lo += in.lo;                \
    out.hi += (out.lo < t) + in.hi; \
  }
#define ADDLO(out, in)      \
  {                         \
    uint64_t t = out.lo;    \
    out.lo += in;           \
    out.hi += (out.lo < t); \
  }
#define SHR(in, shift) (__shiftright128(in.lo, in.hi, (shift)))
#define LO(in) (in.lo)

#if defined(_MSC_VER) && _MSC_VER < 1700
#define POLY1305_NOINLINE
#else
#define POLY1305_NOINLINE __declspec(noinline)
#endif
#elif defined(__GNUC__)
#if defined(__SIZEOF_INT128__)
// Get rid of GCC warning "ISO C does not support '__int128' types"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpedantic"
typedef unsigned __int128 uint128_t;
#pragma GCC diagnostic pop
#else
typedef unsigned uint128_t __attribute__((mode(TI)));
#endif

#define MUL128(out, x, y) out = ((uint128_t) x * y)
#define ADD(out, in) out += in
#define ADDLO(out, in) out += in
#define SHR(in, shift) (uint64_t) (in >> (shift))
#define LO(in) (uint64_t) (in)

#define POLY1305_NOINLINE __attribute__((noinline))
#endif

#define poly1305_block_size 16

/* 17 + sizeof(size_t) + 8*sizeof(uint64_t) */
typedef struct poly1305_state_internal_t {
  uint64_t r[3];
  uint64_t h[3];
  uint64_t pad[2];
  size_t leftover;
  unsigned char buffer[poly1305_block_size];
  unsigned char final;
} poly1305_state_internal_t;

/* interpret eight 8 bit unsigned integers as a 64 bit unsigned integer in
 * little endian */
static uint64_t U8TO64(const unsigned char *p) {
  return (((uint64_t) (p[0] & 0xff)) | ((uint64_t) (p[1] & 0xff) << 8) |
          ((uint64_t) (p[2] & 0xff) << 16) | ((uint64_t) (p[3] & 0xff) << 24) |
          ((uint64_t) (p[4] & 0xff) << 32) | ((uint64_t) (p[5] & 0xff) << 40) |
          ((uint64_t) (p[6] & 0xff) << 48) | ((uint64_t) (p[7] & 0xff) << 56));
}

/* store a 64 bit unsigned integer as eight 8 bit unsigned integers in little
 * endian */
static void U64TO8(unsigned char *p, uint64_t v) {
  p[0] = (unsigned char) ((v) & 0xff);
  p[1] = (unsigned char) ((v >> 8) & 0xff);
  p[2] = (unsigned char) ((v >> 16) & 0xff);
  p[3] = (unsigned char) ((v >> 24) & 0xff);
  p[4] = (unsigned char) ((v >> 32) & 0xff);
  p[5] = (unsigned char) ((v >> 40) & 0xff);
  p[6] = (unsigned char) ((v >> 48) & 0xff);
  p[7] = (unsigned char) ((v >> 56) & 0xff);
}

static void poly1305_init(poly1305_context *ctx, const unsigned char key[32]) {
  poly1305_state_internal_t *st = (poly1305_state_internal_t *) ctx;
  uint64_t t0, t1;

  /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
  t0 = U8TO64(&key[0]);
  t1 = U8TO64(&key[8]);

  st->r[0] = (t0) & 0xffc0fffffff;
  st->r[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffff;
  st->r[2] = ((t1 >> 24)) & 0x00ffffffc0f;

  /* h = 0 */
  st->h[0] = 0;
  st->h[1] = 0;
  st->h[2] = 0;

  /* save pad for later */
  st->pad[0] = U8TO64(&key[16]);
  st->pad[1] = U8TO64(&key[24]);

  st->leftover = 0;
  st->final = 0;
}

static void poly1305_blocks(poly1305_state_internal_t *st,
                            const unsigned char *m, size_t bytes) {
  const uint64_t hibit = (st->final) ? 0 : ((uint64_t) 1 << 40); /* 1 << 128 */
  uint64_t r0, r1, r2;
  uint64_t s1, s2;
  uint64_t h0, h1, h2;
  uint64_t c;
  uint128_t d0, d1, d2, d;

  r0 = st->r[0];
  r1 = st->r[1];
  r2 = st->r[2];

  h0 = st->h[0];
  h1 = st->h[1];
  h2 = st->h[2];

  s1 = r1 * (5 << 2);
  s2 = r2 * (5 << 2);

  while (bytes >= poly1305_block_size) {
    uint64_t t0, t1;

    /* h += m[i] */
    t0 = U8TO64(&m[0]);
    t1 = U8TO64(&m[8]);

    h0 += ((t0) & 0xfffffffffff);
    h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffff);
    h2 += (((t1 >> 24)) & 0x3ffffffffff) | hibit;

    /* h *= r */
    MUL128(d0, h0, r0);
    MUL128(d, h1, s2);
    ADD(d0, d);
    MUL128(d, h2, s1);
    ADD(d0, d);
    MUL128(d1, h0, r1);
    MUL128(d, h1, r0);
    ADD(d1, d);
    MUL128(d, h2, s2);
    ADD(d1, d);
    MUL128(d2, h0, r2);
    MUL128(d, h1, r1);
    ADD(d2, d);
    MUL128(d, h2, r0);
    ADD(d2, d);

    /* (partial) h %= p */
    c = SHR(d0, 44);
    h0 = LO(d0) & 0xfffffffffff;
    ADDLO(d1, c);
    c = SHR(d1, 44);
    h1 = LO(d1) & 0xfffffffffff;
    ADDLO(d2, c);
    c = SHR(d2, 42);
    h2 = LO(d2) & 0x3ffffffffff;
    h0 += c * 5;
    c = (h0 >> 44);
    h0 = h0 & 0xfffffffffff;
    h1 += c;

    m += poly1305_block_size;
    bytes -= poly1305_block_size;
  }

  st->h[0] = h0;
  st->h[1] = h1;
  st->h[2] = h2;
}

static POLY1305_NOINLINE void poly1305_finish(poly1305_context *ctx,
                                              unsigned char mac[16]) {
  poly1305_state_internal_t *st = (poly1305_state_internal_t *) ctx;
  uint64_t h0, h1, h2, c;
  uint64_t g0, g1, g2;
  uint64_t t0, t1;

  /* process the remaining block */
  if (st->leftover) {
    size_t i = st->leftover;
    st->buffer[i] = 1;
    for (i = i + 1; i < poly1305_block_size; i++) st->buffer[i] = 0;
    st->final = 1;
    poly1305_blocks(st, st->buffer, poly1305_block_size);
  }

  /* fully carry h */
  h0 = st->h[0];
  h1 = st->h[1];
  h2 = st->h[2];

  c = (h1 >> 44);
  h1 &= 0xfffffffffff;
  h2 += c;
  c = (h2 >> 42);
  h2 &= 0x3ffffffffff;
  h0 += c * 5;
  c = (h0 >> 44);
  h0 &= 0xfffffffffff;
  h1 += c;
  c = (h1 >> 44);
  h1 &= 0xfffffffffff;
  h2 += c;
  c = (h2 >> 42);
  h2 &= 0x3ffffffffff;
  h0 += c * 5;
  c = (h0 >> 44);
  h0 &= 0xfffffffffff;
  h1 += c;

  /* compute h + -p */
  g0 = h0 + 5;
  c = (g0 >> 44);
  g0 &= 0xfffffffffff;
  g1 = h1 + c;
  c = (g1 >> 44);
  g1 &= 0xfffffffffff;
  g2 = h2 + c - ((uint64_t) 1 << 42);

  /* select h if h < p, or h + -p if h >= p */
  c = (g2 >> ((sizeof(uint64_t) * 8) - 1)) - 1;
  g0 &= c;
  g1 &= c;
  g2 &= c;
  c = ~c;
  h0 = (h0 & c) | g0;
  h1 = (h1 & c) | g1;
  h2 = (h2 & c) | g2;

  /* h = (h + pad) */
  t0 = st->pad[0];
  t1 = st->pad[1];

  h0 += ((t0) & 0xfffffffffff);
  c = (h0 >> 44);
  h0 &= 0xfffffffffff;
  h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffff) + c;
  c = (h1 >> 44);
  h1 &= 0xfffffffffff;
  h2 += (((t1 >> 24)) & 0x3ffffffffff) + c;
  h2 &= 0x3ffffffffff;

  /* mac = h % (2^128) */
  h0 = ((h0) | (h1 << 44));
  h1 = ((h1 >> 20) | (h2 << 24));

  U64TO8(&mac[0], h0);
  U64TO8(&mac[8], h1);

  /* zero out the state */
  st->h[0] = 0;
  st->h[1] = 0;
  st->h[2] = 0;
  st->r[0] = 0;
  st->r[1] = 0;
  st->r[2] = 0;
  st->pad[0] = 0;
  st->pad[1] = 0;
}

#endif

static void poly1305_update(poly1305_context *ctx, const unsigned char *m,
                            size_t bytes) {
  poly1305_state_internal_t *st = (poly1305_state_internal_t *) ctx;
  size_t i;

  /* handle leftover */
  if (st->leftover) {
    size_t want = (poly1305_block_size - st->leftover);
    if (want > bytes) want = bytes;
    for (i = 0; i < want; i++) st->buffer[st->leftover + i] = m[i];
    bytes -= want;
    m += want;
    st->leftover += want;
    if (st->leftover < poly1305_block_size) return;
    poly1305_blocks(st, st->buffer, poly1305_block_size);
    st->leftover = 0;
  }

  /* process full blocks */
  if (bytes >= poly1305_block_size) {
    size_t want = (bytes & (size_t) ~(poly1305_block_size - 1));
    poly1305_blocks(st, m, want);
    m += want;
    bytes -= want;
  }

  /* store leftover */
  if (bytes) {
    for (i = 0; i < bytes; i++) st->buffer[st->leftover + i] = m[i];
    st->leftover += bytes;
  }
}

// ******* END: poly1305-donna.c ********
// ******* BEGIN: portable8439.c ********

#define __CHACHA20_BLOCK_SIZE (64)
#define __POLY1305_KEY_SIZE (32)

static PORTABLE_8439_DECL uint8_t __ZEROES[16] = {0};
static PORTABLE_8439_DECL void pad_if_needed(poly1305_context *ctx,
                                             size_t size) {
  size_t padding = size % 16;
  if (padding != 0) {
    poly1305_update(ctx, __ZEROES, 16 - padding);
  }
}

#define __u8(v) ((uint8_t) ((v) & 0xFF))

// TODO: make this depending on the unaligned/native read size possible
static PORTABLE_8439_DECL void write_64bit_int(poly1305_context *ctx,
                                               uint64_t value) {
  uint8_t result[8];
  result[0] = __u8(value);
  result[1] = __u8(value >> 8);
  result[2] = __u8(value >> 16);
  result[3] = __u8(value >> 24);
  result[4] = __u8(value >> 32);
  result[5] = __u8(value >> 40);
  result[6] = __u8(value >> 48);
  result[7] = __u8(value >> 56);
  poly1305_update(ctx, result, 8);
}

static PORTABLE_8439_DECL void poly1305_calculate_mac(
    uint8_t *mac, const uint8_t *cipher_text, size_t cipher_text_size,
    const uint8_t key[RFC_8439_KEY_SIZE],
    const uint8_t nonce[RFC_8439_NONCE_SIZE], const uint8_t *ad,
    size_t ad_size) {
  // init poly key (section 2.6)
  uint8_t poly_key[__POLY1305_KEY_SIZE] = {0};
  poly1305_context poly_ctx;
  rfc8439_keygen(poly_key, key, nonce);
  // start poly1305 mac
  poly1305_init(&poly_ctx, poly_key);

  if (ad != NULL && ad_size > 0) {
    // write AD if present
    poly1305_update(&poly_ctx, ad, ad_size);
    pad_if_needed(&poly_ctx, ad_size);
  }

  // now write the cipher text
  poly1305_update(&poly_ctx, cipher_text, cipher_text_size);
  pad_if_needed(&poly_ctx, cipher_text_size);

  // write sizes
  write_64bit_int(&poly_ctx, ad_size);
  write_64bit_int(&poly_ctx, cipher_text_size);

  // calculate MAC
  poly1305_finish(&poly_ctx, mac);
}

#define MG_PM(p) ((size_t) (p))

// pointers overlap if the smaller either ahead of the end,
// or its end is before the start of the other
//
// s_size should be smaller or equal to b_size
#define MG_OVERLAPPING(s, s_size, b, b_size) \
  (MG_PM(s) < MG_PM((b) + (b_size))) && (MG_PM(b) < MG_PM((s) + (s_size)))

PORTABLE_8439_DECL size_t mg_chacha20_poly1305_encrypt(
    uint8_t *restrict cipher_text, const uint8_t key[RFC_8439_KEY_SIZE],
    const uint8_t nonce[RFC_8439_NONCE_SIZE], const uint8_t *restrict ad,
    size_t ad_size, const uint8_t *restrict plain_text,
    size_t plain_text_size) {
  size_t new_size = plain_text_size + RFC_8439_TAG_SIZE;
  if (MG_OVERLAPPING(plain_text, plain_text_size, cipher_text, new_size)) {
    return (size_t) -1;
  }
  chacha20_xor_stream(cipher_text, plain_text, plain_text_size, key, nonce, 1);
  poly1305_calculate_mac(cipher_text + plain_text_size, cipher_text,
                         plain_text_size, key, nonce, ad, ad_size);
  return new_size;
}

PORTABLE_8439_DECL size_t mg_chacha20_poly1305_decrypt(
    uint8_t *restrict plain_text, const uint8_t key[RFC_8439_KEY_SIZE],
    const uint8_t nonce[RFC_8439_NONCE_SIZE], const uint8_t *restrict ad,
    size_t ad_size, const uint8_t *restrict cipher_text,
    size_t cipher_text_size) {
  // first we calculate the mac and see if it lines up, only then do we decrypt
  size_t actual_size = cipher_text_size - RFC_8439_TAG_SIZE;
  uint8_t computed_mac[RFC_8439_TAG_SIZE];
  if (MG_OVERLAPPING(plain_text, actual_size, cipher_text, cipher_text_size)) {
    return (size_t) -1;
  }

  poly1305_calculate_mac(computed_mac, cipher_text, actual_size, key, nonce, ad,
                         ad_size);

  if (!mg_memeq(computed_mac, cipher_text + actual_size, RFC_8439_TAG_SIZE))
    return (size_t) -1;

  chacha20_xor_stream(plain_text, cipher_text, actual_size, key, nonce, 1);
  return actual_size;
}
// ******* END:   portable8439.c ********
#endif  // MG_TLS == MG_TLS_BUILTIN

#ifdef MG_ENABLE_LINES
#line 1 "src/tls_dummy.c"
#endif


#if MG_TLS == MG_TLS_NONE
void mg_tls_init(struct mg_connection *c, const struct mg_tls_opts *opts) {
  (void) opts;
  mg_error(c, "TLS is not enabled");
}
void mg_tls_handshake(struct mg_connection *c) {
  (void) c;
}
void mg_tls_free(struct mg_connection *c) {
  (void) c;
}
long mg_tls_recv(struct mg_connection *c, void *buf, size_t len) {
  return c == NULL || buf == NULL || len == 0 ? 0 : -1;
}
long mg_tls_send(struct mg_connection *c, const void *buf, size_t len) {
  return c == NULL || buf == NULL || len == 0 ? 0 : -1;
}
size_t mg_tls_pending(struct mg_connection *c) {
  (void) c;
  return 0;
}
long mg_tls_flush(struct mg_connection *c) {
  (void) c;
  return 0;
}
void mg_tls_ctx_init(struct mg_mgr *mgr) {
  (void) mgr;
}
void mg_tls_ctx_free(struct mg_mgr *mgr) {
  (void) mgr;
}
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/tls_mbed.c"
#endif






#if MG_TLS == MG_TLS_MBED

#if defined(MBEDTLS_VERSION_NUMBER) && MBEDTLS_VERSION_NUMBER >= 0x03000000 && \
    MBEDTLS_VERSION_NUMBER < 0x04000000
#define MG_MBEDTLS_RNG_GET , mg_mbed_rng, NULL
#else
#define MG_MBEDTLS_RNG_GET
#endif

static int mg_tls_err(struct mg_connection *c, int rc) {
  char s[80];
  mbedtls_strerror(rc, s, sizeof(s));
  MG_ERROR(("%lu %s", ((struct mg_connection *) c)->id, s));
  return rc;
}

#if defined(MBEDTLS_VERSION_NUMBER) && MBEDTLS_VERSION_NUMBER >= 0x04000000
#else
static int mg_mbed_rng(void *ctx, unsigned char *buf, size_t len) {
  (void) ctx;
  return mg_random(buf, len) ? 0 : -1;
}
#endif

static bool mg_load_cert(struct mg_str str, mbedtls_x509_crt *p) {
  int rc;
  if (str.buf == NULL || str.buf[0] == '\0' || str.buf[0] == '*') return true;
  if (!MG_IS_DER(str.buf)) str.len++;  // PEM, include trailing NUL
  if ((rc = mbedtls_x509_crt_parse(p, (uint8_t *) str.buf, str.len)) != 0) {
    MG_ERROR(("cert err %#x", -rc));
    return false;
  }
  return true;
}

static bool mg_load_key(struct mg_str str, mbedtls_pk_context *p) {
  int rc;
  if (str.buf == NULL || str.buf[0] == '\0' || str.buf[0] == '*') return true;
  if (!MG_IS_DER(str.buf)) str.len++;  // PEM, include trailing NUL
  if ((rc = mbedtls_pk_parse_key(p, (uint8_t *) str.buf, str.len, NULL,
                                 0 MG_MBEDTLS_RNG_GET)) != 0) {
    MG_ERROR(("key err %#x", -rc));
    return false;
  }
  return true;
}

void mg_tls_free(struct mg_connection *c) {
  struct mg_tls *tls = (struct mg_tls *) c->tls;
  if (tls != NULL) {
    mbedtls_ssl_free(&tls->ssl);
    mbedtls_pk_free(&tls->pk);
    mbedtls_x509_crt_free(&tls->ca);
    mbedtls_x509_crt_free(&tls->cert);
    mbedtls_ssl_config_free(&tls->conf);
#ifdef MBEDTLS_SSL_SESSION_TICKETS
    mbedtls_ssl_ticket_free(&tls->ticket);
#endif
    // PSA has global data. Do not call mbedtls_psa_crypto_free() here,
    // it will free all global resources. Call it when actually freeing all
    // application resources (main() exits)
    mg_free(tls);
    c->tls = NULL;
  }
}

static int mg_net_send(void *ctx, const unsigned char *buf, size_t len) {
  long n = mg_io_send((struct mg_connection *) ctx, buf, len);
  MG_VERBOSE(("%lu n=%ld e=%d", ((struct mg_connection *) ctx)->id, n, errno));
  if (n == MG_IO_WAIT) return MBEDTLS_ERR_SSL_WANT_WRITE;
  // if (n == MG_IO_RESET) return MBEDTLS_ERR_NET_CONN_RESET;
  if (n == MG_IO_ERR) return MBEDTLS_ERR_NET_SEND_FAILED;
  return (int) n;
}

static int mg_net_recv(void *ctx, unsigned char *buf, size_t len) {
  long n = mg_io_recv((struct mg_connection *) ctx, buf, len);
  MG_VERBOSE(("%lu n=%ld", ((struct mg_connection *) ctx)->id, n));
  if (n == MG_IO_WAIT) return MBEDTLS_ERR_SSL_WANT_READ;
  // if (n == MG_IO_RESET) return MBEDTLS_ERR_NET_CONN_RESET;
  if (n == MG_IO_ERR) return MBEDTLS_ERR_NET_RECV_FAILED;
  return (int) n;
}

void mg_tls_handshake(struct mg_connection *c) {
  struct mg_tls *tls = (struct mg_tls *) c->tls;
  int rc = mbedtls_ssl_handshake(&tls->ssl);
  if (rc == 0) {  // Success
    if (tls->check_name && (mbedtls_ssl_get_verify_result(&tls->ssl) &
                            MBEDTLS_X509_BADCERT_CN_MISMATCH)) {
      mg_error(c, "failed to verify hostname");
      return;
    } // ignore MBEDTLS_X509_BADCERT_NOT_TRUSTED, no CA cert given
    MG_DEBUG(("%lu success", c->id));
    c->is_tls_hs = 0;
    mg_call(c, MG_EV_TLS_HS, NULL);
  } else if (rc == MBEDTLS_ERR_SSL_WANT_READ ||
             rc == MBEDTLS_ERR_SSL_WANT_WRITE) {  // Still pending
    MG_VERBOSE(("%lu pending, %d%d %d (-%#x)", c->id, c->is_connecting,
                c->is_tls_hs, rc, -rc));
  } else {
    mg_error(c, "TLS handshake: -%#x", -mg_tls_err(c, rc));  // Error
  }
}

static void debug_cb(void *c, int lev, const char *s, int n, const char *s2) {
  n = (int) strlen(s2) - 1;
  MG_INFO(("%lu %d %.*s", ((struct mg_connection *) c)->id, lev, n, s2));
  (void) s;
}

void mg_tls_init(struct mg_connection *c, const struct mg_tls_opts *opts) {
  struct mg_tls *tls = (struct mg_tls *) mg_calloc(1, sizeof(*tls));
  int rc = 0;
  bool check_name = false;
  c->tls = tls;
  if (c->tls == NULL) {
    mg_error(c, "TLS OOM");
    goto fail;
  }
  if (c->is_listening) goto fail;
  MG_DEBUG(("%lu Setting TLS", c->id));
  MG_PROF_ADD(c, "mbedtls_init_start");
  mbedtls_ssl_init(&tls->ssl);
  mbedtls_ssl_config_init(&tls->conf);
  mbedtls_x509_crt_init(&tls->ca);
  mbedtls_x509_crt_init(&tls->cert);
  mbedtls_pk_init(&tls->pk);
  mbedtls_ssl_conf_dbg(&tls->conf, debug_cb, c);
#if defined(MG_MBEDTLS_DEBUG_LEVEL)
  mbedtls_debug_set_threshold(MG_MBEDTLS_DEBUG_LEVEL);
#endif
  if ((rc = mbedtls_ssl_config_defaults(
           &tls->conf,
           c->is_client ? MBEDTLS_SSL_IS_CLIENT : MBEDTLS_SSL_IS_SERVER,
           MBEDTLS_SSL_TRANSPORT_STREAM, MBEDTLS_SSL_PRESET_DEFAULT)) != 0) {
    mg_error(c, "tls defaults %#x", -mg_tls_err(c, rc));
    goto fail;
  }
#if defined(MBEDTLS_VERSION_NUMBER) && MBEDTLS_VERSION_NUMBER >= 0x04000000
  MG_INFO(("PSA is in control of random number generation"));
#else
  mbedtls_ssl_conf_rng(&tls->conf, mg_mbed_rng, c);
#endif

  if (c->is_client && opts->name.buf != NULL && opts->name.len > 0 &&
      opts->name.buf[0] != '\0') {
    char *host = mg_mprintf("%.*s", opts->name.len, opts->name.buf);
    mbedtls_ssl_set_hostname(&tls->ssl, host);
#if !defined(MBEDTLS_VERSION_NUMBER) || MBEDTLS_VERSION_NUMBER < 0x03030000 || \
    MBEDTLS_VERSION_NUMBER >= 0x04000000
    // NOTE: MBEDTLS_SSL_VERIFY_OPTIONAL is not supported for TLS1.3 on client
    // side See https://github.com/Mbed-TLS/mbedtls/issues/7075
    check_name = true;
#endif
    MG_DEBUG(("%lu hostname verification: %s", c->id, host));
    mg_free(host);
  } else {
    MG_DEBUG(("%lu skipping hostname verification", c->id));
    mbedtls_ssl_set_hostname(&tls->ssl, NULL);
  }
  if (opts->ca.len == 0 || mg_strcmp(opts->ca, mg_str("*")) == 0) {
    mbedtls_ssl_conf_authmode(&tls->conf, check_name  // see set comment above
                                              ? MBEDTLS_SSL_VERIFY_OPTIONAL
                                              : MBEDTLS_SSL_VERIFY_NONE);
    tls->check_name = check_name; // host name set but no CA cert given
  } else {
    if (mg_load_cert(opts->ca, &tls->ca) == false) goto fail;
    mbedtls_ssl_conf_ca_chain(&tls->conf, &tls->ca, NULL);
    mbedtls_ssl_conf_authmode(&tls->conf, MBEDTLS_SSL_VERIFY_REQUIRED);
  }
  if (!mg_load_cert(opts->cert, &tls->cert)) goto fail;
  if (!mg_load_key(opts->key, &tls->pk)) goto fail;
  if (tls->cert.version &&
      (rc = mbedtls_ssl_conf_own_cert(&tls->conf, &tls->cert, &tls->pk)) != 0) {
    mg_error(c, "own cert %#x", -mg_tls_err(c, rc));
    goto fail;
  }

#ifdef MBEDTLS_SSL_SESSION_TICKETS
  if (!c->is_client && c->mgr->tls_ctx != NULL) {
    mbedtls_ssl_conf_session_tickets_cb(
        &tls->conf, mbedtls_ssl_ticket_write, mbedtls_ssl_ticket_parse,
        &((struct mg_tls_ctx *) c->mgr->tls_ctx)->tickets);
  }
#endif

  if ((rc = mbedtls_ssl_setup(&tls->ssl, &tls->conf)) != 0) {
    mg_error(c, "setup err %#x", -mg_tls_err(c, rc));
    goto fail;
  }
  c->is_tls = 1;
  c->is_tls_hs = 1;
  mbedtls_ssl_set_bio(&tls->ssl, c, mg_net_send, mg_net_recv, 0);
  MG_PROF_ADD(c, "mbedtls_init_end");
  return;
fail:
  mg_tls_free(c);
}

size_t mg_tls_pending(struct mg_connection *c) {
  struct mg_tls *tls = (struct mg_tls *) c->tls;
  return tls == NULL ? 0 : mbedtls_ssl_get_bytes_avail(&tls->ssl);
}

long mg_tls_recv(struct mg_connection *c, void *buf, size_t len) {
  struct mg_tls *tls = (struct mg_tls *) c->tls;
  long n = mbedtls_ssl_read(&tls->ssl, (unsigned char *) buf, len);
  if (!c->is_tls_hs && (buf == NULL || len == 0) && n == 0) return 0;  // MIP
  if (n == MBEDTLS_ERR_SSL_WANT_READ || n == MBEDTLS_ERR_SSL_WANT_WRITE)
    return MG_IO_WAIT;
#if defined(MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET)
  if (n == MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET) {
    return MG_IO_WAIT;
  }
#endif
  if (n <= 0) return MG_IO_ERR;
  return n;
}

long mg_tls_send(struct mg_connection *c, const void *buf, size_t len) {
  struct mg_tls *tls = (struct mg_tls *) c->tls;
  long n;
  bool was_throttled = c->is_tls_throttled;  // see #3074
  n = was_throttled ? mbedtls_ssl_write(&tls->ssl, tls->throttled_buf,
                                        tls->throttled_len) /* flush old data */
                    : mbedtls_ssl_write(&tls->ssl, (unsigned char *) buf,
                                        len);  // encrypt current data
#if defined(MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET)
  if (n == MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET)
    return MG_IO_WAIT;
#endif
  c->is_tls_throttled =
      (n == MBEDTLS_ERR_SSL_WANT_READ || n == MBEDTLS_ERR_SSL_WANT_WRITE);
  if (was_throttled && c->is_tls_throttled) return MG_IO_WAIT;  // no changes
  if (c->is_tls_throttled) {
    tls->throttled_buf =
        (unsigned char *) buf;  // MbedTLS code actually ignores
    tls->throttled_len = len;   // these, but let's play API rules
    return MG_IO_WAIT;
  }
  if (n <= 0) return MG_IO_ERR;
  return n;
}

long mg_tls_flush(struct mg_connection *c) {
  struct mg_tls *tls = (struct mg_tls *) c->tls;
  long n = 0;
  if (c->is_tls_throttled && c->is_draining) {
    n = mbedtls_ssl_write(&tls->ssl, tls->throttled_buf, tls->throttled_len);
#if defined(MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET)
    if (n == MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET) return MG_IO_WAIT;
#endif
    c->is_tls_throttled =
        (n == MBEDTLS_ERR_SSL_WANT_READ || n == MBEDTLS_ERR_SSL_WANT_WRITE);
    if (c->is_tls_throttled) return MG_IO_WAIT;
    if (n <= 0) return MG_IO_ERR;
  }
  return 0;
}

void mg_tls_ctx_init(struct mg_mgr *mgr) {
#if defined(MBEDTLS_VERSION_NUMBER) && MBEDTLS_VERSION_NUMBER >= 0x03000000 && \
    defined(MBEDTLS_PSA_CRYPTO_C)
  psa_crypto_init();  // Initializes global PSA resources, no-op if already done
#endif
#ifdef MBEDTLS_SSL_SESSION_TICKETS
  {
    struct mg_tls_ctx *ctx = (struct mg_tls_ctx *) mg_calloc(1, sizeof(*ctx));
    if (ctx == NULL) {
      MG_ERROR(("TLS context init OOM"));
    } else {
      int rc;
      mbedtls_ssl_ticket_init(&ctx->tickets);
#if defined(MBEDTLS_VERSION_NUMBER) && MBEDTLS_VERSION_NUMBER >= 0x04000000
      rc = mbedtls_ssl_ticket_setup(&ctx->tickets, PSA_ALG_GCM,
                                    PSA_KEY_TYPE_AES, 128, 86400);
#else
      rc = mbedtls_ssl_ticket_setup(&ctx->tickets, mg_mbed_rng, NULL,
                                         MBEDTLS_CIPHER_AES_128_GCM, 86400);
#endif
      if (rc != 0) {
        MG_ERROR((" mbedtls_ssl_ticket_setup %#x", -rc));
        mbedtls_ssl_ticket_free(&ctx->tickets);
        mg_free(ctx);
      } else {
        mgr->tls_ctx = ctx;
      }
    }
  }
#else
  (void) mgr;
#endif
}

void mg_tls_ctx_free(struct mg_mgr *mgr) {
  struct mg_tls_ctx *ctx = (struct mg_tls_ctx *) mgr->tls_ctx;
  if (ctx != NULL) {
#ifdef MBEDTLS_SSL_SESSION_TICKETS
    mbedtls_ssl_ticket_free(&ctx->tickets);
#endif
    mg_free(ctx);
    mgr->tls_ctx = NULL;
  }
}
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/tls_openssl.c"
#endif




#if MG_TLS == MG_TLS_OPENSSL || MG_TLS == MG_TLS_WOLFSSL

static int tls_err_cb(const char *s, size_t len, void *c) {
  int n = (int) len - 1;
  MG_ERROR(("%lu %.*s", ((struct mg_connection *) c)->id, n, s));
  return 0;  // undocumented
}

static int mg_tls_err(struct mg_connection *c, struct mg_tls *tls, int res) {
  int err = SSL_get_error(tls->ssl, res);
  // We've just fetched the last error from the queue.
  // Now we need to clear the error queue. If we do not, then the following
  // can happen (actually reported):
  //  - A new connection is accept()-ed with cert error (e.g. self-signed cert)
  //  - Since all accept()-ed connections share listener's context,
  //  - *ALL* SSL accepted connection report read error on the next poll cycle.
  //    Thus a single errored connection can close all the rest, unrelated ones.
  // Clearing the error keeps the shared SSL_CTX in an OK state.

  if (err != 0) ERR_print_errors_cb(tls_err_cb, c);
  ERR_clear_error();
  if (err == SSL_ERROR_WANT_READ) return 0;
  if (err == SSL_ERROR_WANT_WRITE) return 0;
  return err;
}

#if MG_TLS != MG_TLS_WOLFSSL
static STACK_OF(X509_INFO) * load_ca_certs(struct mg_str ca) {
  BIO *bio = BIO_new_mem_buf(ca.buf, (int) ca.len);
  STACK_OF(X509_INFO) *certs =
      bio ? PEM_X509_INFO_read_bio(bio, NULL, NULL, NULL) : NULL;
  if (bio) BIO_free(bio);
  return certs;
}

static bool add_ca_certs(SSL_CTX *ctx, STACK_OF(X509_INFO) * certs) {
  int i;
  X509_STORE *cert_store = SSL_CTX_get_cert_store(ctx);
  if (cert_store == NULL) return false;
  for (i = 0; i < sk_X509_INFO_num(certs); i++) {
    X509_INFO *cert_info = sk_X509_INFO_value(certs, i);
    if (cert_info->x509 && !X509_STORE_add_cert(cert_store, cert_info->x509))
      return false;
  }
  return true;
}
#endif

static EVP_PKEY *load_key(struct mg_str s) {
  BIO *bio = BIO_new_mem_buf(s.buf, (int) (long) s.len);
  EVP_PKEY *key = bio ? PEM_read_bio_PrivateKey(bio, NULL, 0, NULL) : NULL;
  if (bio) BIO_free(bio);
  return key;
}

static int load_cert(SSL *ssl, struct mg_str s) {
  BIO *bio = BIO_new_mem_buf(s.buf, (int) (long) s.len);
  X509 *cert = NULL;
  int rc = 0;
  if (bio == NULL) return 0;
  if (MG_IS_DER(s.buf)) {
    cert = d2i_X509_bio(bio, NULL);
    rc = cert == NULL ? 0 : SSL_use_certificate(ssl, cert);
  } else {
    cert = PEM_read_bio_X509(bio, NULL, NULL, NULL);
    rc = cert == NULL ? 0 : SSL_use_certificate(ssl, cert);
#if MG_TLS != MG_TLS_WOLFSSL
    X509_free(cert);
    while (rc == 1) {
      cert = PEM_read_bio_X509(bio, NULL, NULL, NULL);
      if (cert == NULL) {
        ERR_clear_error();  // PEM_read_bio_X509 sets an error on EOF
        break;
      }
      rc = (int) SSL_add1_chain_cert(ssl, cert);
      X509_free(cert);
    }
    cert = NULL;
#endif
  }
  X509_free(cert);
  if (bio) BIO_free(bio);
  return rc;
}

static long mg_bio_ctrl(BIO *b, int cmd, long larg, void *pargs) {
  long ret = 0;
  if (cmd == BIO_CTRL_PUSH) ret = 1;
  if (cmd == BIO_CTRL_POP) ret = 1;
  if (cmd == BIO_CTRL_FLUSH) ret = 1;
#if MG_TLS == MG_TLS_OPENSSL
  if (cmd == BIO_C_SET_NBIO) ret = 1;
#endif
  // MG_DEBUG(("%d -> %ld", cmd, ret));
  (void) b, (void) cmd, (void) larg, (void) pargs;
  return ret;
}

static int mg_bio_read(BIO *bio, char *buf, int len) {
  struct mg_connection *c = (struct mg_connection *) BIO_get_data(bio);
  long res = mg_io_recv(c, buf, (size_t) len);
  // MG_DEBUG(("%p %d %ld", buf, len, res));
  len = res > 0 ? (int) res : -1;
  if (res == MG_IO_WAIT) BIO_set_retry_read(bio);
  return len;
}

static int mg_bio_write(BIO *bio, const char *buf, int len) {
  struct mg_connection *c = (struct mg_connection *) BIO_get_data(bio);
  long res = mg_io_send(c, buf, (size_t) len);
  // MG_DEBUG(("%p %d %ld", buf, len, res));
  len = res > 0 ? (int) res : -1;
  if (res == MG_IO_WAIT) BIO_set_retry_write(bio);
  return len;
}

#ifdef MG_TLS_SSLKEYLOGFILE
static void ssl_keylog_cb(const SSL *ssl, const char *line) {
  FILE *f;
  char *keylogfile = getenv("SSLKEYLOGFILE");
  if (keylogfile == NULL) return;
  f = fopen(keylogfile, "a");
  if (f != NULL) {
    fprintf(f, "%s\n", line);
    fflush(f);
    fclose(f);
  } else {
    MG_ERROR(("Cannot open %s", keylogfile));
  }
  (void) ssl;
}
#endif

void mg_tls_free(struct mg_connection *c) {
  struct mg_tls *tls = (struct mg_tls *) c->tls;
  if (tls == NULL) return;
  SSL_free(tls->ssl);
  SSL_CTX_free(tls->ctx);
  BIO_meth_free(tls->bm);
  mg_free(tls->name);
  mg_free(tls);
  c->tls = NULL;
}

void mg_tls_init(struct mg_connection *c, const struct mg_tls_opts *opts) {
  struct mg_tls *tls = (struct mg_tls *) mg_calloc(1, sizeof(*tls));
  const char *id = "mongoose";
  bool check_name = false;
  static unsigned char s_initialised = 0;
  BIO *bio = NULL;
  int rc;
  c->tls = tls;
  if (tls == NULL) {
    mg_error(c, "TLS OOM");
    goto fail;
  }

  if (!s_initialised) {
    SSL_library_init();
    s_initialised++;
  }
  MG_DEBUG(("%lu Setting TLS", c->id));
  tls->ctx = c->is_client ? SSL_CTX_new(TLS_client_method())
                          : SSL_CTX_new(TLS_server_method());
  if (tls->ctx == NULL) {
    mg_error(c, "SSL_CTX_new");
    goto fail;
  }
#ifdef MG_TLS_SSLKEYLOGFILE
  SSL_CTX_set_keylog_callback(tls->ctx, ssl_keylog_cb);
#endif
  if ((tls->ssl = SSL_new(tls->ctx)) == NULL) {
    mg_error(c, "SSL_new");
    goto fail;
  }
  SSL_set_session_id_context(tls->ssl, (const uint8_t *) id,
                             (unsigned) strlen(id));
  // Disable deprecated protocols
  SSL_set_options(tls->ssl, SSL_OP_NO_SSLv2);
  SSL_set_options(tls->ssl, SSL_OP_NO_SSLv3);
  SSL_set_options(tls->ssl, SSL_OP_NO_TLSv1);
  SSL_set_options(tls->ssl, SSL_OP_NO_TLSv1_1);
#ifdef MG_ENABLE_OPENSSL_NO_COMPRESSION
  SSL_set_options(tls->ssl, SSL_OP_NO_COMPRESSION);
#endif
#ifdef MG_ENABLE_OPENSSL_CIPHER_SERVER_PREFERENCE
  SSL_set_options(tls->ssl, SSL_OP_CIPHER_SERVER_PREFERENCE);
#endif

if (c->is_client && opts->name.buf != NULL && opts->name.len > 0 && opts->name.buf[0] != '\0') {
  tls->name = mg_mprintf("%.*s", (int) opts->name.len, opts->name.buf);
  if (tls->name == NULL) {
    mg_error(c, "TLS OOM");
    goto fail;
  }
  check_name = true;
}
#if MG_TLS == MG_TLS_WOLFSSL && !defined(OPENSSL_COMPATIBLE_DEFAULTS)
  if (opts->ca.len == 0 || mg_strcmp(opts->ca, mg_str("*")) == 0) {
    // Older versions require that either the CA is loaded or SSL_VERIFY_NONE
    // explicitly set
    SSL_set_verify(tls->ssl, SSL_VERIFY_NONE, NULL);
  }
#endif

  if (opts->ca.buf != NULL && opts->ca.len > 0 && opts->ca.buf[0] != '\0') {
    SSL_set_verify(tls->ssl, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT,
                   NULL);
#if MG_TLS == MG_TLS_WOLFSSL
    extern int wolfSSL_CTX_load_verify_buffer(SSL_CTX *, const unsigned char *,
                                              long, int);
    rc = wolfSSL_CTX_load_verify_buffer(tls->ctx,
                                        (const unsigned char *) opts->ca.buf,
                                        (long) opts->ca.len, SSL_FILETYPE_PEM);
    if (rc != 1) {
      mg_error(c, "CA err");
      goto fail;
    }
#else
    STACK_OF(X509_INFO) *certs = load_ca_certs(opts->ca);
    rc = add_ca_certs(tls->ctx, certs);
    sk_X509_INFO_pop_free(certs, X509_INFO_free);
    if (!rc) {
      mg_error(c, "CA err");
      goto fail;
    }
#endif
  } else {
    tls->check_name = check_name; // host name set but no CA cert given
  }

  if (opts->cert.buf != NULL && opts->cert.buf[0] != '\0') {
    rc = load_cert(tls->ssl, opts->cert);
    if (rc != 1) {
      mg_error(c, "CERT err %d", mg_tls_err(c, tls, rc));
      goto fail;
    }
  }
  if (opts->key.buf != NULL && opts->key.buf[0] != '\0') {
    EVP_PKEY *key = load_key(opts->key);
    rc = key == NULL ? 0 : SSL_use_PrivateKey(tls->ssl, key);
    EVP_PKEY_free(key);
    if (key == NULL || rc != 1) {
      mg_error(c, "KEY err %d", mg_tls_err(c, tls, rc));
      goto fail;
    }
  }

  SSL_set_mode(tls->ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
#if MG_TLS == MG_TLS_OPENSSL && OPENSSL_VERSION_NUMBER > 0x10002000L
  (void) SSL_set_ecdh_auto(tls->ssl, 1);
#endif
#if OPENSSL_VERSION_NUMBER >= 0x10100000L
  if (tls->name != NULL) {
#if MG_TLS != MG_TLS_WOLFSSL || LIBWOLFSSL_VERSION_HEX >= 0x05005002
    SSL_set1_host(tls->ssl, tls->name);
#else
    X509_VERIFY_PARAM_set1_host(SSL_get0_param(tls->ssl), tls->name, 0);
#endif
    SSL_set_tlsext_host_name(tls->ssl, tls->name);
  }
#endif
#if MG_TLS == MG_TLS_WOLFSSL
  tls->bm = BIO_meth_new(0, "bio_mg");
#else
  tls->bm = BIO_meth_new(BIO_get_new_index() | BIO_TYPE_SOURCE_SINK, "bio_mg");
#endif
  BIO_meth_set_write(tls->bm, mg_bio_write);
  BIO_meth_set_read(tls->bm, mg_bio_read);
  BIO_meth_set_ctrl(tls->bm, mg_bio_ctrl);

  bio = BIO_new(tls->bm);
  BIO_set_data(bio, c);
  SSL_set_bio(tls->ssl, bio, bio);

  c->is_tls = 1;
  c->is_tls_hs = 1;
  MG_DEBUG(("%lu SSL %s OK", c->id, c->is_accepted ? "accept" : "client"));
  return;
fail:
  mg_tls_free(c);
}

void mg_tls_handshake(struct mg_connection *c) {
  struct mg_tls *tls = (struct mg_tls *) c->tls;
  int rc = c->is_client ? SSL_connect(tls->ssl) : SSL_accept(tls->ssl);
  if (rc == 1) {
    if (tls->check_name) {  // host name set but no CA cert given
      X509 *cert = SSL_get_peer_certificate(tls->ssl);
      bool ok =
          cert != NULL && X509_check_host(cert, tls->name, 0, 0, NULL) == 1;
      X509_free(cert);
      if (!ok) {
        mg_error(c, "failed to verify hostname");
        return;
      }
    }
    MG_DEBUG(("%lu success", c->id));
    c->is_tls_hs = 0;
    mg_call(c, MG_EV_TLS_HS, NULL);
  } else {
    int code = mg_tls_err(c, tls, rc);
    if (code != 0) mg_error(c, "tls hs: rc %d, err %d", rc, code);
  }
}

size_t mg_tls_pending(struct mg_connection *c) {
  struct mg_tls *tls = (struct mg_tls *) c->tls;
  return tls == NULL ? 0 : (size_t) SSL_pending(tls->ssl);
}

long mg_tls_recv(struct mg_connection *c, void *buf, size_t len) {
  struct mg_tls *tls = (struct mg_tls *) c->tls;
  int n;
#if MG_TLS == MG_TLS_WOLFSSL
  uint8_t dummy; // WolfSSL requires destination != NULL
  if (buf == NULL && len == 0) buf = &dummy;
#endif
  n = SSL_read(tls->ssl, buf, (int) len);
  if (!c->is_tls_hs && (buf == NULL || len == 0) && n == 0) return 0;  // MIP
  if (n < 0 && mg_tls_err(c, tls, n) == 0) return MG_IO_WAIT;
  if (n <= 0) return MG_IO_ERR;
  return n;
}

long mg_tls_send(struct mg_connection *c, const void *buf, size_t len) {
  struct mg_tls *tls = (struct mg_tls *) c->tls;
  int n = SSL_write(tls->ssl, buf, (int) len);
  if (n < 0 && mg_tls_err(c, tls, n) == 0) return MG_IO_WAIT;
  if (n <= 0) return MG_IO_ERR;
  return n;
}

long mg_tls_flush(struct mg_connection *c) {
  (void) c;
  return 0;
}

void mg_tls_ctx_init(struct mg_mgr *mgr) {
  (void) mgr;
}

void mg_tls_ctx_free(struct mg_mgr *mgr) {
  (void) mgr;
}
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/tls_p256.c"
#endif
// P-256 API, mg_p256_*(), used by built-in TLS, JWT and OTA. Implemented by
// BearSSL code below (MG_P256=MG_P256_BEAR) or tls_p256_cortex.c (LENNGREN)
// BearSSL code: https://bearssl.org commit 7bea48e. Shared bignum and ECDSA
// code is always compiled, it is also used by P-384, see tls_p384.c
//
// Copyright (c) 2016 Thomas Pornin <pornin@bolet.org>
// Copyright (c) 2017 Thomas Pornin <pornin@bolet.org>
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// Mongoose changes: br_ / BR_ names are prefixed with mg_ / MG_, as are the
// constant-time helpers from inner.h (NOT, MUX, EQ, ...). Structs are not
// typedef-ed. ECDSA signing takes the caller's k instead of HMAC-DRBG, k is
// computed by mg_p256_sign() (RFC 6979). ECDSA functions take the curve id and
// public key directly




#if MG_TLS == MG_TLS_BUILTIN || defined(MG_OTA_PUBLIC_KEY)

// From BearSSL src/codec/ccopy.c
void
mg_br_ccopy(uint32_t ctl, void *dst, const void *src, size_t len)
{
	unsigned char *d;
	const unsigned char *s;

	d = (unsigned char *)dst;
	s = (const unsigned char *)src;
	while (len -- > 0) {
		uint32_t x, y;

		x = *s ++;
		y = *d;
		*d = (unsigned char)MG_BR_MUX(ctl, x, y);
		d ++;
	}
}

// From BearSSL src/int/i32_div32.c
uint32_t
mg_br_divrem(uint32_t hi, uint32_t lo, uint32_t d, uint32_t *r)
{
	/* TODO: optimize this */
	uint32_t q;
	uint32_t ch, cf;
	int k;

	q = 0;
	ch = MG_BR_EQ(hi, d);
	hi = MG_BR_MUX(ch, 0, hi);
	for (k = 31; k > 0; k --) {
		int j;
		uint32_t w, ctl, hi2, lo2;

		j = 32 - k;
		w = (hi << j) | (lo >> k);
		ctl = MG_BR_GE(w, d) | (hi >> k);
		hi2 = (w - d) >> j;
		lo2 = lo - (d << k);
		hi = MG_BR_MUX(ctl, hi2, hi);
		lo = MG_BR_MUX(ctl, lo2, lo);
		q |= ctl << k;
	}
	cf = MG_BR_GE(lo, d) | hi;
	q |= cf;
	*r = MG_BR_MUX(cf, lo - d, lo);
	return q;
}

// From BearSSL src/int/i31_add.c
uint32_t
mg_br_i31_add(uint32_t *a, const uint32_t *b, uint32_t ctl)
{
	uint32_t cc;
	size_t u, m;

	cc = 0;
	m = (a[0] + 63) >> 5;
	for (u = 1; u < m; u ++) {
		uint32_t aw, bw, naw;

		aw = a[u];
		bw = b[u];
		naw = aw + bw + cc;
		cc = naw >> 31;
		a[u] = MG_BR_MUX(ctl, naw & (uint32_t)0x7FFFFFFF, aw);
	}
	return cc;
}

// From BearSSL src/int/i31_bitlen.c
uint32_t
mg_br_i31_bit_length(uint32_t *x, size_t xlen)
{
	uint32_t tw, twk;

	tw = 0;
	twk = 0;
	while (xlen -- > 0) {
		uint32_t w, c;

		c = MG_BR_EQ(tw, 0);
		w = x[xlen];
		tw = MG_BR_MUX(c, w, tw);
		twk = MG_BR_MUX(c, (uint32_t)xlen, twk);
	}
	return (twk << 5) + MG_BR_BIT_LENGTH(tw);
}

// From BearSSL src/int/i31_decmod.c
uint32_t
mg_br_i31_decode_mod(uint32_t *x, const void *src, size_t len, const uint32_t *m)
{
	/*
	 * Two-pass algorithm: in the first pass, we determine whether the
	 * value fits; in the second pass, we do the actual write.
	 *
	 * During the first pass, 'r' contains the comparison result so
	 * far:
	 *  0x00000000   value is equal to the modulus
	 *  0x00000001   value is greater than the modulus
	 *  0xFFFFFFFF   value is lower than the modulus
	 *
	 * Since we iterate starting with the least significant bytes (at
	 * the end of src[]), each new comparison overrides the previous
	 * except when the comparison yields 0 (equal).
	 *
	 * During the second pass, 'r' is either 0xFFFFFFFF (value fits)
	 * or 0x00000000 (value does not fit).
	 *
	 * We must iterate over all bytes of the source, _and_ possibly
	 * some extra virtual bytes (with value 0) so as to cover the
	 * complete modulus as well. We also add 4 such extra bytes beyond
	 * the modulus length because it then guarantees that no accumulated
	 * partial word remains to be processed.
	 */
	const unsigned char *buf;
	size_t mlen, tlen;
	int pass;
	uint32_t r;

	buf = (const unsigned char *)src;
	mlen = (m[0] + 31) >> 5;
	tlen = (mlen << 2);
	if (tlen < len) {
		tlen = len;
	}
	tlen += 4;
	r = 0;
	for (pass = 0; pass < 2; pass ++) {
		size_t u, v;
		uint32_t acc;
		int acc_len;

		v = 1;
		acc = 0;
		acc_len = 0;
		for (u = 0; u < tlen; u ++) {
			uint32_t b;

			if (u < len) {
				b = buf[len - 1 - u];
			} else {
				b = 0;
			}
			acc |= (b << acc_len);
			acc_len += 8;
			if (acc_len >= 31) {
				uint32_t xw;

				xw = acc & (uint32_t)0x7FFFFFFF;
				acc_len -= 31;
				acc = b >> (8 - acc_len);
				if (v <= mlen) {
					if (pass) {
						x[v] = r & xw;
					} else {
						uint32_t cc;

						cc = (uint32_t)MG_BR_CMP(xw, m[v]);
						r = MG_BR_MUX(MG_BR_EQ(cc, 0), r, cc);
					}
				} else {
					if (!pass) {
						r = MG_BR_MUX(MG_BR_EQ(xw, 0), r, 1);
					}
				}
				v ++;
			}
		}

		/*
		 * When we reach this point at the end of the first pass:
		 * r is either 0, 1 or -1; we want to set r to 0 if it
		 * is equal to 0 or 1, and leave it to -1 otherwise.
		 *
		 * When we reach this point at the end of the second pass:
		 * r is either 0 or -1; we want to leave that value
		 * untouched. This is a subcase of the previous.
		 */
		r >>= 1;
		r |= (r << 1);
	}

	x[0] = m[0];
	return r & (uint32_t)1;
}

// From BearSSL src/int/i31_decode.c
void
mg_br_i31_decode(uint32_t *x, const void *src, size_t len)
{
	const unsigned char *buf;
	size_t u, v;
	uint32_t acc;
	int acc_len;

	buf = (const unsigned char *)src;
	u = len;
	v = 1;
	acc = 0;
	acc_len = 0;
	while (u -- > 0) {
		uint32_t b;

		b = buf[u];
		acc |= (b << acc_len);
		acc_len += 8;
		if (acc_len >= 31) {
			x[v ++] = acc & (uint32_t)0x7FFFFFFF;
			acc_len -= 31;
			acc = b >> (8 - acc_len);
		}
	}
	if (acc_len != 0) {
		x[v ++] = acc;
	}
	x[0] = mg_br_i31_bit_length(x + 1, v - 1);
}

// From BearSSL src/int/i31_encode.c
void
mg_br_i31_encode(void *dst, size_t len, const uint32_t *x)
{
	unsigned char *buf;
	size_t k, xlen;
	uint32_t acc;
	int acc_len;

	xlen = (x[0] + 31) >> 5;
	if (xlen == 0) {
		memset(dst, 0, len);
		return;
	}
	buf = (unsigned char *)dst + len;
	k = 1;
	acc = 0;
	acc_len = 0;
	while (len != 0) {
		uint32_t w;

		w = (k <= xlen) ? x[k] : 0;
		k ++;
		if (acc_len == 0) {
			acc = w;
			acc_len = 31;
		} else {
			uint32_t z;

			z = acc | (w << acc_len);
			acc_len --;
			acc = w >> (31 - acc_len);
			if (len >= 4) {
				buf -= 4;
				len -= 4;
				mg_br_enc32be(buf, z);
			} else {
				switch (len) {
				case 3:
					buf[-3] = (unsigned char)(z >> 16);
					/* fall through */
				case 2:
					buf[-2] = (unsigned char)(z >> 8);
					/* fall through */
				case 1:
					buf[-1] = (unsigned char)z;
					break;
				}
				return;
			}
		}
	}
}

// From BearSSL src/int/i31_fmont.c
void
mg_br_i31_from_monty(uint32_t *x, const uint32_t *m, uint32_t m0i)
{
	size_t len, u, v;

	len = (m[0] + 31) >> 5;
	for (u = 0; u < len; u ++) {
		uint32_t f;
		uint64_t cc;

		f = MG_BR_MUL31_lo(x[1], m0i);
		cc = 0;
		for (v = 0; v < len; v ++) {
			uint64_t z;

			z = (uint64_t)x[v + 1] + MG_BR_MUL31(f, m[v + 1]) + cc;
			cc = z >> 31;
			if (v != 0) {
				x[v] = (uint32_t)z & 0x7FFFFFFF;
			}
		}
		x[len] = (uint32_t)cc;
	}

	/*
	 * We may have to do an extra subtraction, but only if the
	 * value in x[] is indeed greater than or equal to that of m[],
	 * which is why we must do two calls (first call computes the
	 * carry, second call performs the subtraction only if the carry
	 * is 0).
	 */
	mg_br_i31_sub(x, m, MG_BR_NOT(mg_br_i31_sub(x, m, 0)));
}

// From BearSSL src/int/i31_iszero.c
uint32_t
mg_br_i31_iszero(const uint32_t *x)
{
	uint32_t z;
	size_t u;

	z = 0;
	for (u = (x[0] + 31) >> 5; u > 0; u --) {
		z |= x[u];
	}
	return ~(z | -z) >> 31;
}

// From BearSSL src/int/i31_modpow.c
void
mg_br_i31_modpow(uint32_t *x,
	const unsigned char *e, size_t elen,
	const uint32_t *m, uint32_t m0i, uint32_t *t1, uint32_t *t2)
{
	size_t mlen;
	uint32_t k;

	/*
	 * 'mlen' is the length of m[] expressed in bytes (including
	 * the "bit length" first field).
	 */
	mlen = ((m[0] + 63) >> 5) * sizeof m[0];

	/*
	 * Throughout the algorithm:
	 * -- t1[] is in Montgomery representation; it contains x, x^2,
	 * x^4, x^8...
	 * -- The result is accumulated, in normal representation, in
	 * the x[] array.
	 * -- t2[] is used as destination buffer for each multiplication.
	 *
	 * Note that there is no need to call mg_br_i32_from_monty().
	 */
	memcpy(t1, x, mlen);
	mg_br_i31_to_monty(t1, m);
	mg_br_i31_zero(x, m[0]);
	x[1] = 1;
	for (k = 0; k < ((uint32_t)elen << 3); k ++) {
		uint32_t ctl;

		ctl = (e[elen - 1 - (k >> 3)] >> (k & 7)) & 1;
		mg_br_i31_montymul(t2, x, t1, m, m0i);
		MG_BR_CCOPY(ctl, x, t2, mlen);
		mg_br_i31_montymul(t2, t1, t1, m, m0i);
		memcpy(t1, t2, mlen);
	}
}

// From BearSSL src/int/i31_montmul.c
void
mg_br_i31_montymul(uint32_t *d, const uint32_t *x, const uint32_t *y,
	const uint32_t *m, uint32_t m0i)
{
	/*
	 * Each outer loop iteration computes:
	 *   d <- (d + xu*y + f*m) / 2^31
	 * We have xu <= 2^31-1 and f <= 2^31-1.
	 * Thus, if d <= 2*m-1 on input, then:
	 *   2*m-1 + 2*(2^31-1)*m <= (2^32)*m-1
	 * and the new d value is less than 2*m.
	 *
	 * We represent d over 31-bit words, with an extra word 'dh'
	 * which can thus be only 0 or 1.
	 */
	size_t len, len4, u, v;
	uint32_t dh;

	len = (m[0] + 31) >> 5;
	len4 = len & ~(size_t)3;
	mg_br_i31_zero(d, m[0]);
	dh = 0;
	for (u = 0; u < len; u ++) {
		/*
		 * The carry for each operation fits on 32 bits:
		 *   d[v+1] <= 2^31-1
		 *   xu*y[v+1] <= (2^31-1)*(2^31-1)
		 *   f*m[v+1] <= (2^31-1)*(2^31-1)
		 *   r <= 2^32-1
		 *   (2^31-1) + 2*(2^31-1)*(2^31-1) + (2^32-1) = 2^63 - 2^31
		 * After division by 2^31, the new r is then at most 2^32-1
		 *
		 * Using a 32-bit carry has performance benefits on 32-bit
		 * systems; however, on 64-bit architectures, we prefer to
		 * keep the carry (r) in a 64-bit register, thus avoiding some
		 * "clear high bits" operations.
		 */
		uint32_t f, xu;
		uint32_t r;  /* Mongoose: 32-bit carry only */

		xu = x[u + 1];
		f = MG_BR_MUL31_lo((d[1] + MG_BR_MUL31_lo(x[u + 1], y[1])), m0i);

		r = 0;
		for (v = 0; v < len4; v += 4) {
			uint64_t z;

			z = (uint64_t)d[v + 1] + MG_BR_MUL31(xu, y[v + 1])
				+ MG_BR_MUL31(f, m[v + 1]) + r;
			r = (uint32_t)(z >> 31);
			d[v + 0] = (uint32_t)z & 0x7FFFFFFF;
			z = (uint64_t)d[v + 2] + MG_BR_MUL31(xu, y[v + 2])
				+ MG_BR_MUL31(f, m[v + 2]) + r;
			r = (uint32_t)(z >> 31);
			d[v + 1] = (uint32_t)z & 0x7FFFFFFF;
			z = (uint64_t)d[v + 3] + MG_BR_MUL31(xu, y[v + 3])
				+ MG_BR_MUL31(f, m[v + 3]) + r;
			r = (uint32_t)(z >> 31);
			d[v + 2] = (uint32_t)z & 0x7FFFFFFF;
			z = (uint64_t)d[v + 4] + MG_BR_MUL31(xu, y[v + 4])
				+ MG_BR_MUL31(f, m[v + 4]) + r;
			r = (uint32_t)(z >> 31);
			d[v + 3] = (uint32_t)z & 0x7FFFFFFF;
		}
		for (; v < len; v ++) {
			uint64_t z;

			z = (uint64_t)d[v + 1] + MG_BR_MUL31(xu, y[v + 1])
				+ MG_BR_MUL31(f, m[v + 1]) + r;
			r = (uint32_t)(z >> 31);
			d[v] = (uint32_t)z & 0x7FFFFFFF;
		}

		/*
		 * Since the new dh can only be 0 or 1, the addition of
		 * the old dh with the carry MUST fit on 32 bits, and
		 * thus can be done into dh itself.
		 */
		dh += r;
		d[len] = dh & 0x7FFFFFFF;
		dh >>= 31;
	}

	/*
	 * We must write back the bit length because it was overwritten in
	 * the loop (not overwriting it would require a test in the loop,
	 * which would yield bigger and slower code).
	 */
	d[0] = m[0];

	/*
	 * d[] may still be greater than m[] at that point; notably, the
	 * 'dh' word may be non-zero.
	 */
	mg_br_i31_sub(d, m, MG_BR_NEQ(dh, 0) | MG_BR_NOT(mg_br_i31_sub(d, m, 0)));
}

// From BearSSL src/int/i31_muladd.c
void
mg_br_i31_muladd_small(uint32_t *x, uint32_t z, const uint32_t *m)
{
	uint32_t m_bitlen;
	unsigned mblr;
	size_t u, mlen;
	uint32_t a0, a1, b0, hi, g, q, tb;
	uint32_t under, over;
	uint32_t cc;

	/*
	 * We can test on the modulus bit length since we accept to
	 * leak that length.
	 */
	m_bitlen = m[0];
	if (m_bitlen == 0) {
		return;
	}
	if (m_bitlen <= 31) {
		uint32_t lo;

		hi = x[1] >> 1;
		lo = (x[1] << 31) | z;
		x[1] = mg_br_rem(hi, lo, m[1]);
		return;
	}
	mlen = (m_bitlen + 31) >> 5;
	mblr = (unsigned)m_bitlen & 31;

	/*
	 * Principle: we estimate the quotient (x*2^31+z)/m by
	 * doing a 64/32 division with the high words.
	 *
	 * Let:
	 *   w = 2^31
	 *   a = (w*a0 + a1) * w^N + a2
	 *   b = b0 * w^N + b2
	 * such that:
	 *   0 <= a0 < w
	 *   0 <= a1 < w
	 *   0 <= a2 < w^N
	 *   w/2 <= b0 < w
	 *   0 <= b2 < w^N
	 *   a < w*b
	 * I.e. the two top words of a are a0:a1, the top word of b is
	 * b0, we ensured that b0 is "full" (high bit set), and a is
	 * such that the quotient q = a/b fits on one word (0 <= q < w).
	 *
	 * If a = b*q + r (with 0 <= r < q), we can estimate q by
	 * doing an Euclidean division on the top words:
	 *   a0*w+a1 = b0*u + v  (with 0 <= v < b0)
	 * Then the following holds:
	 *   0 <= u <= w
	 *   u-2 <= q <= u
	 */
	hi = x[mlen];
	if (mblr == 0) {
		a0 = x[mlen];
		memmove(x + 2, x + 1, (mlen - 1) * sizeof *x);
		x[1] = z;
		a1 = x[mlen];
		b0 = m[mlen];
	} else {
		a0 = ((x[mlen] << (31 - mblr)) | (x[mlen - 1] >> mblr))
			& 0x7FFFFFFF;
		memmove(x + 2, x + 1, (mlen - 1) * sizeof *x);
		x[1] = z;
		a1 = ((x[mlen] << (31 - mblr)) | (x[mlen - 1] >> mblr))
			& 0x7FFFFFFF;
		b0 = ((m[mlen] << (31 - mblr)) | (m[mlen - 1] >> mblr))
			& 0x7FFFFFFF;
	}

	/*
	 * We estimate a divisor q. If the quotient returned by mg_br_div()
	 * is g:
	 * -- If a0 == b0 then g == 0; we want q = 0x7FFFFFFF.
	 * -- Otherwise:
	 *    -- if g == 0 then we set q = 0;
	 *    -- otherwise, we set q = g - 1.
	 * The properties described above then ensure that the true
	 * quotient is q-1, q or q+1.
	 *
	 * Take care that a0, a1 and b0 are 31-bit words, not 32-bit. We
	 * must adjust the parameters to mg_br_div() accordingly.
	 */
	g = mg_br_div(a0 >> 1, a1 | (a0 << 31), b0);
	q = MG_BR_MUX(MG_BR_EQ(a0, b0), 0x7FFFFFFF, MG_BR_MUX(MG_BR_EQ(g, 0), 0, g - 1));

	/*
	 * We subtract q*m from x (with the extra high word of value 'hi').
	 * Since q may be off by 1 (in either direction), we may have to
	 * add or subtract m afterwards.
	 *
	 * The 'tb' flag will be true (1) at the end of the loop if the
	 * result is greater than or equal to the modulus (not counting
	 * 'hi' or the carry).
	 */
	cc = 0;
	tb = 1;
	for (u = 1; u <= mlen; u ++) {
		uint32_t mw, zw, xw, nxw;
		uint64_t zl;

		mw = m[u];
		zl = MG_BR_MUL31(mw, q) + cc;
		cc = (uint32_t)(zl >> 31);
		zw = (uint32_t)zl & (uint32_t)0x7FFFFFFF;
		xw = x[u];
		nxw = xw - zw;
		cc += nxw >> 31;
		nxw &= 0x7FFFFFFF;
		x[u] = nxw;
		tb = MG_BR_MUX(MG_BR_EQ(nxw, mw), tb, MG_BR_GT(nxw, mw));
	}

	/*
	 * If we underestimated q, then either cc < hi (one extra bit
	 * beyond the top array word), or cc == hi and tb is true (no
	 * extra bit, but the result is not lower than the modulus). In
	 * these cases we must subtract m once.
	 *
	 * Otherwise, we may have overestimated, which will show as
	 * cc > hi (thus a negative result). Correction is adding m once.
	 */
	over = MG_BR_GT(cc, hi);
	under = ~over & (tb | MG_BR_LT(cc, hi));
	mg_br_i31_add(x, m, over);
	mg_br_i31_sub(x, m, under);
}

// From BearSSL src/int/i31_ninv31.c
uint32_t
mg_br_i31_ninv31(uint32_t x)
{
	uint32_t y;

	y = 2 - x;
	y *= 2 - y * x;
	y *= 2 - y * x;
	y *= 2 - y * x;
	y *= 2 - y * x;
	return MG_BR_MUX(x & 1, -y, 0) & 0x7FFFFFFF;
}

// From BearSSL src/int/i31_rshift.c
void
mg_br_i31_rshift(uint32_t *x, int count)
{
	size_t u, len;
	uint32_t r;

	len = (x[0] + 31) >> 5;
	if (len == 0) {
		return;
	}
	r = x[1] >> count;
	for (u = 2; u <= len; u ++) {
		uint32_t w;

		w = x[u];
		x[u - 1] = ((w << (31 - count)) | r) & 0x7FFFFFFF;
		r = w >> count;
	}
	x[len] = r;
}

// From BearSSL src/int/i31_sub.c
uint32_t
mg_br_i31_sub(uint32_t *a, const uint32_t *b, uint32_t ctl)
{
	uint32_t cc;
	size_t u, m;

	cc = 0;
	m = (a[0] + 63) >> 5;
	for (u = 1; u < m; u ++) {
		uint32_t aw, bw, naw;

		aw = a[u];
		bw = b[u];
		naw = aw - bw - cc;
		cc = naw >> 31;
		a[u] = MG_BR_MUX(ctl, naw & 0x7FFFFFFF, aw);
	}
	return cc;
}

// From BearSSL src/int/i31_tmont.c
void
mg_br_i31_to_monty(uint32_t *x, const uint32_t *m)
{
	uint32_t k;

	for (k = (m[0] + 31) >> 5; k > 0; k --) {
		mg_br_i31_muladd_small(x, 0, m);
	}
}

// From BearSSL src/ec/ecdsa_i31_bits.c
void
mg_br_ecdsa_i31_bits2int(uint32_t *x,
	const void *src, size_t len, uint32_t ebitlen)
{
	uint32_t bitlen, hbitlen;
	int sc;

	bitlen = ebitlen - (ebitlen >> 5);
	hbitlen = (uint32_t)len << 3;
	if (hbitlen > bitlen) {
		len = (bitlen + 7) >> 3;
		sc = (int)((hbitlen - bitlen) & 7);
	} else {
		sc = 0;
	}
	mg_br_i31_zero(x, ebitlen);
	mg_br_i31_decode(x, src, len);
	mg_br_i31_rshift(x, sc);
	x[0] = ebitlen;
}

// From BearSSL src/ec/ecdsa_i31_vrfy_raw.c
#define I31_LEN     ((MG_BR_MAX_EC_SIZE + 61) / 31)
#define POINT_LEN   (1 + (((MG_BR_MAX_EC_SIZE + 7) >> 3) << 1))

uint32_t
mg_br_ecdsa_i31_vrfy_raw(const struct mg_br_ec_impl *impl, int curve,
	const void *hash, size_t hash_len,
	const unsigned char *q, size_t qlen,
	const void *sig, size_t sig_len)
{
	/*
	 * IMPORTANT: this code is fit only for curves with a prime
	 * order. This is needed so that modular reduction of the X
	 * coordinate of a point can be done with a simple subtraction.
	 */
	const unsigned char *order;
	uint32_t n[I31_LEN], r[I31_LEN], s[I31_LEN], t1[I31_LEN], t2[I31_LEN];
	unsigned char tx[(MG_BR_MAX_EC_SIZE + 7) >> 3];
	unsigned char ty[(MG_BR_MAX_EC_SIZE + 7) >> 3];
	unsigned char eU[POINT_LEN];
	size_t nlen, rlen, ulen;
	uint32_t n0i, res;

	/*
	 * If the curve is not supported, then report an error.
	 */
	if (((impl->supported_curves >> curve) & 1) == 0) {
		return 0;
	}

	/*
	 * Get the curve parameters (generator and order).
	 */
	order = impl->order(curve, &nlen);
	impl->generator(curve, &ulen);

	/*
	 * Signature length must be even.
	 */
	if (sig_len & 1) {
		return 0;
	}
	rlen = sig_len >> 1;

	/*
	 * Public key point must have the proper size for this curve.
	 */
	if (qlen != ulen) {
		return 0;
	}

	/*
	 * Get modulus; then decode the r and s values. They must be
	 * lower than the modulus, and s must not be null.
	 */
	mg_br_i31_decode(n, order, nlen);
	n0i = mg_br_i31_ninv31(n[1]);
	if (!mg_br_i31_decode_mod(r, sig, rlen, n)) {
		return 0;
	}
	if (!mg_br_i31_decode_mod(s, (const unsigned char *)sig + rlen, rlen, n)) {
		return 0;
	}
	if (mg_br_i31_iszero(s)) {
		return 0;
	}

	/*
	 * Invert s. We do that with a modular exponentiation; we use
	 * the fact that for all the curves we support, the least
	 * significant byte is not 0 or 1, so we can subtract 2 without
	 * any carry to process.
	 * We also want 1/s in Montgomery representation, which can be
	 * done by converting _from_ Montgomery representation before
	 * the inversion (because (1/s)*R = 1/(s/R)).
	 */
	mg_br_i31_from_monty(s, n, n0i);
	memcpy(tx, order, nlen);
	tx[nlen - 1] -= 2;
	mg_br_i31_modpow(s, tx, nlen, n, n0i, t1, t2);

	/*
	 * Truncate the hash to the modulus length (in bits) and reduce
	 * it modulo the curve order. The modular reduction can be done
	 * with a subtraction since the truncation already reduced the
	 * value to the modulus bit length.
	 */
	mg_br_ecdsa_i31_bits2int(t1, hash, hash_len, n[0]);
	mg_br_i31_sub(t1, n, mg_br_i31_sub(t1, n, 0) ^ 1);

	/*
	 * Multiply the (truncated, reduced) hash value with 1/s, result in
	 * t2, encoded in ty.
	 */
	mg_br_i31_montymul(t2, t1, s, n, n0i);
	mg_br_i31_encode(ty, nlen, t2);

	/*
	 * Multiply r with 1/s, result in t1, encoded in tx.
	 */
	mg_br_i31_montymul(t1, r, s, n, n0i);
	mg_br_i31_encode(tx, nlen, t1);

	/*
	 * Compute the point x*Q + y*G.
	 */
	memcpy(eU, q, ulen);
	res = impl->muladd(eU, NULL, ulen,
		tx, nlen, ty, nlen, curve);

	/*
	 * Get the X coordinate, reduce modulo the curve order, and
	 * compare with the 'r' value.
	 *
	 * The modular reduction can be done with subtractions because
	 * we work with curves of prime order, so the curve order is
	 * close to the field order (Hasse's theorem).
	 */
	mg_br_i31_zero(t1, n[0]);
	mg_br_i31_decode(t1, &eU[1], ulen >> 1);
	t1[0] = n[0];
	mg_br_i31_sub(t1, n, mg_br_i31_sub(t1, n, 0) ^ 1);
	res &= ~mg_br_i31_sub(t1, r, 1);
	res &= mg_br_i31_iszero(t1);
	return res;
}

// From BearSSL src/ec/ecdsa_i31_sign_raw.c, k is given by the caller
#define ORDER_LEN   ((MG_BR_MAX_EC_SIZE + 7) >> 3)

size_t
mg_br_ecdsa_i31_sign_raw(const struct mg_br_ec_impl *impl, int curve,
	const void *hash_value, size_t hash_len,
	const unsigned char *xb, size_t xlen,
	const unsigned char *kb, void *sig)
{
	/*
	 * IMPORTANT: this code is fit only for curves with a prime
	 * order. This is needed so that modular reduction of the X
	 * coordinate of a point can be done with a simple subtraction.
	 * We also rely on the last byte of the curve order to be distinct
	 * from 0 and 1.
	 */
	const unsigned char *order;
	uint32_t n[I31_LEN], r[I31_LEN], s[I31_LEN], x[I31_LEN];
	uint32_t m[I31_LEN], k[I31_LEN], t1[I31_LEN], t2[I31_LEN];
	unsigned char tt[ORDER_LEN << 1];
	unsigned char eU[POINT_LEN];
	size_t nlen, ulen;
	uint32_t n0i, ctl;

	/*
	 * If the curve is not supported, then exit with an error.
	 */
	if (((impl->supported_curves >> curve) & 1) == 0) {
		return 0;
	}

	/*
	 * Get modulus.
	 */
	order = impl->order(curve, &nlen);
	mg_br_i31_decode(n, order, nlen);
	n0i = mg_br_i31_ninv31(n[1]);

	/*
	 * Get private key as an i31 integer. This also checks that the
	 * private key is well-defined (not zero, and less than the
	 * curve order).
	 */
	if (!mg_br_i31_decode_mod(x, xb, xlen, n)) {
		return 0;
	}
	if (mg_br_i31_iszero(x)) {
		return 0;
	}

	/*
	 * Truncate and reduce the hash value modulo the curve order.
	 */
	mg_br_ecdsa_i31_bits2int(m, hash_value, hash_len, n[0]);
	mg_br_i31_sub(m, n, mg_br_i31_sub(m, n, 0) ^ 1);

	/*
	 * Mongoose: the "k" value is given by the caller (e.g. RFC 6979,
	 * computed by uECC). It must be in [1, n-1].
	 */
	if (!mg_br_i31_decode_mod(k, kb, nlen, n) || mg_br_i31_iszero(k)) {
		return 0;
	}

	/*
	 * Compute k*G and extract the X coordinate, then reduce it
	 * modulo the curve order. Since we support only curves with
	 * prime order, that reduction is only a matter of computing
	 * a subtraction.
	 */
	mg_br_i31_encode(tt, nlen, k);
	ulen = impl->mulgen(eU, tt, nlen, curve);
	mg_br_i31_zero(r, n[0]);
	mg_br_i31_decode(r, &eU[1], ulen >> 1);
	r[0] = n[0];
	mg_br_i31_sub(r, n, mg_br_i31_sub(r, n, 0) ^ 1);

	/*
	 * Compute 1/k in double-Montgomery representation. We do so by
	 * first converting _from_ Montgomery representation (twice),
	 * then using a modular exponentiation.
	 */
	mg_br_i31_from_monty(k, n, n0i);
	mg_br_i31_from_monty(k, n, n0i);
	memcpy(tt, order, nlen);
	tt[nlen - 1] -= 2;
	mg_br_i31_modpow(k, tt, nlen, n, n0i, t1, t2);

	/*
	 * Compute s = (m+xr)/k (mod n).
	 * The k[] array contains R^2/k (double-Montgomery representation);
	 * we thus can use direct Montgomery multiplications and conversions
	 * from Montgomery, avoiding any call to mg_br_i31_to_monty() (which
	 * is slower).
	 */
	mg_br_i31_from_monty(m, n, n0i);
	mg_br_i31_montymul(t1, x, r, n, n0i);
	ctl = mg_br_i31_add(t1, m, 1);
	ctl |= mg_br_i31_sub(t1, n, 0) ^ 1;
	mg_br_i31_sub(t1, n, ctl);
	mg_br_i31_montymul(s, t1, k, n, n0i);

	/*
	 * Encode r and s in the signature.
	 */
	mg_br_i31_encode(sig, nlen, r);
	mg_br_i31_encode((unsigned char *)sig + nlen, nlen, s);
	return nlen << 1;
}

#if MG_P256 == MG_P256_BEAR
// From BearSSL src/ec/ec_p256_m31.c
/*
 * If MG_BR_NO_ARITH_SHIFT is undefined, or defined to 0, then we _assume_
 * that right-shifting a signed negative integer copies the sign bit
 * (arithmetic right-shift). This is "implementation-defined behaviour",
 * i.e. it is not undefined, but it may differ between compilers. Each
 * compiler is supposed to document its behaviour in that respect. GCC
 * explicitly defines that an arithmetic right shift is used. We expect
 * all other compilers to do the same, because underlying CPU offer an
 * arithmetic right shift opcode that could not be used otherwise.
 */
#define MG_BR_NO_ARITH_SHIFT 0
#if MG_BR_NO_ARITH_SHIFT
#define ARSH(x, n)    (((uint32_t)(x) >> (n)) \
                      | ((-((uint32_t)(x) >> 31)) << (32 - (n))))
#define ARSHW(x, n)   (((uint64_t)(x) >> (n)) \
                      | ((-((uint64_t)(x) >> 63)) << (64 - (n))))
#else
#define ARSH(x, n)    ((uint32_t)((*(int32_t *)&(x)) >> (n)))
#define ARSHW(x, n)   ((uint64_t)((*(int64_t *)&(x)) >> (n)))
#endif

/*
 * Convert an integer from unsigned big-endian encoding to a sequence of
 * 30-bit words in little-endian order. The final "partial" word is
 * returned.
 */
static uint32_t
be8_to_le30(uint32_t *dst, const unsigned char *src, size_t len)
{
	uint32_t acc;
	int acc_len;

	acc = 0;
	acc_len = 0;
	while (len -- > 0) {
		uint32_t b;

		b = src[len];
		if (acc_len < 22) {
			acc |= b << acc_len;
			acc_len += 8;
		} else {
			*dst ++ = (acc | (b << acc_len)) & 0x3FFFFFFF;
			acc = b >> (30 - acc_len);
			acc_len -= 22;
		}
	}
	return acc;
}

/*
 * Convert an integer (30-bit words, little-endian) to unsigned
 * big-endian encoding. The total encoding length is provided; all
 * the destination bytes will be filled.
 */
static void
le30_to_be8(unsigned char *dst, size_t len, const uint32_t *src)
{
	uint32_t acc;
	int acc_len;

	acc = 0;
	acc_len = 0;
	while (len -- > 0) {
		if (acc_len < 8) {
			uint32_t w;

			w = *src ++;
			dst[len] = (unsigned char)(acc | (w << acc_len));
			acc = w >> (8 - acc_len);
			acc_len += 22;
		} else {
			dst[len] = (unsigned char)acc;
			acc >>= 8;
			acc_len -= 8;
		}
	}
}

/*
 * Multiply two integers. Source integers are represented as arrays of
 * nine 30-bit words, for values up to 2^270-1. Result is encoded over
 * 18 words of 30 bits each.
 */
static void
mul9(uint32_t *d, const uint32_t *a, const uint32_t *b)
{
	/*
	 * Maximum intermediate result is no more than
	 * 10376293531797946367, which fits in 64 bits. Reason:
	 *
	 *   10376293531797946367 = 9 * (2^30-1)^2 + 9663676406
	 *   10376293531797946367 < 9663676407 * 2^30
	 *
	 * Thus, adding together 9 products of 30-bit integers, with
	 * a carry of at most 9663676406, yields an integer that fits
	 * on 64 bits and generates a carry of at most 9663676406.
	 */
	uint64_t t[17];
	uint64_t cc;
	int i;

	t[ 0] = MG_BR_MUL31(a[0], b[0]);
	t[ 1] = MG_BR_MUL31(a[0], b[1])
		+ MG_BR_MUL31(a[1], b[0]);
	t[ 2] = MG_BR_MUL31(a[0], b[2])
		+ MG_BR_MUL31(a[1], b[1])
		+ MG_BR_MUL31(a[2], b[0]);
	t[ 3] = MG_BR_MUL31(a[0], b[3])
		+ MG_BR_MUL31(a[1], b[2])
		+ MG_BR_MUL31(a[2], b[1])
		+ MG_BR_MUL31(a[3], b[0]);
	t[ 4] = MG_BR_MUL31(a[0], b[4])
		+ MG_BR_MUL31(a[1], b[3])
		+ MG_BR_MUL31(a[2], b[2])
		+ MG_BR_MUL31(a[3], b[1])
		+ MG_BR_MUL31(a[4], b[0]);
	t[ 5] = MG_BR_MUL31(a[0], b[5])
		+ MG_BR_MUL31(a[1], b[4])
		+ MG_BR_MUL31(a[2], b[3])
		+ MG_BR_MUL31(a[3], b[2])
		+ MG_BR_MUL31(a[4], b[1])
		+ MG_BR_MUL31(a[5], b[0]);
	t[ 6] = MG_BR_MUL31(a[0], b[6])
		+ MG_BR_MUL31(a[1], b[5])
		+ MG_BR_MUL31(a[2], b[4])
		+ MG_BR_MUL31(a[3], b[3])
		+ MG_BR_MUL31(a[4], b[2])
		+ MG_BR_MUL31(a[5], b[1])
		+ MG_BR_MUL31(a[6], b[0]);
	t[ 7] = MG_BR_MUL31(a[0], b[7])
		+ MG_BR_MUL31(a[1], b[6])
		+ MG_BR_MUL31(a[2], b[5])
		+ MG_BR_MUL31(a[3], b[4])
		+ MG_BR_MUL31(a[4], b[3])
		+ MG_BR_MUL31(a[5], b[2])
		+ MG_BR_MUL31(a[6], b[1])
		+ MG_BR_MUL31(a[7], b[0]);
	t[ 8] = MG_BR_MUL31(a[0], b[8])
		+ MG_BR_MUL31(a[1], b[7])
		+ MG_BR_MUL31(a[2], b[6])
		+ MG_BR_MUL31(a[3], b[5])
		+ MG_BR_MUL31(a[4], b[4])
		+ MG_BR_MUL31(a[5], b[3])
		+ MG_BR_MUL31(a[6], b[2])
		+ MG_BR_MUL31(a[7], b[1])
		+ MG_BR_MUL31(a[8], b[0]);
	t[ 9] = MG_BR_MUL31(a[1], b[8])
		+ MG_BR_MUL31(a[2], b[7])
		+ MG_BR_MUL31(a[3], b[6])
		+ MG_BR_MUL31(a[4], b[5])
		+ MG_BR_MUL31(a[5], b[4])
		+ MG_BR_MUL31(a[6], b[3])
		+ MG_BR_MUL31(a[7], b[2])
		+ MG_BR_MUL31(a[8], b[1]);
	t[10] = MG_BR_MUL31(a[2], b[8])
		+ MG_BR_MUL31(a[3], b[7])
		+ MG_BR_MUL31(a[4], b[6])
		+ MG_BR_MUL31(a[5], b[5])
		+ MG_BR_MUL31(a[6], b[4])
		+ MG_BR_MUL31(a[7], b[3])
		+ MG_BR_MUL31(a[8], b[2]);
	t[11] = MG_BR_MUL31(a[3], b[8])
		+ MG_BR_MUL31(a[4], b[7])
		+ MG_BR_MUL31(a[5], b[6])
		+ MG_BR_MUL31(a[6], b[5])
		+ MG_BR_MUL31(a[7], b[4])
		+ MG_BR_MUL31(a[8], b[3]);
	t[12] = MG_BR_MUL31(a[4], b[8])
		+ MG_BR_MUL31(a[5], b[7])
		+ MG_BR_MUL31(a[6], b[6])
		+ MG_BR_MUL31(a[7], b[5])
		+ MG_BR_MUL31(a[8], b[4]);
	t[13] = MG_BR_MUL31(a[5], b[8])
		+ MG_BR_MUL31(a[6], b[7])
		+ MG_BR_MUL31(a[7], b[6])
		+ MG_BR_MUL31(a[8], b[5]);
	t[14] = MG_BR_MUL31(a[6], b[8])
		+ MG_BR_MUL31(a[7], b[7])
		+ MG_BR_MUL31(a[8], b[6]);
	t[15] = MG_BR_MUL31(a[7], b[8])
		+ MG_BR_MUL31(a[8], b[7]);
	t[16] = MG_BR_MUL31(a[8], b[8]);

	/*
	 * Propagate carries.
	 */
	cc = 0;
	for (i = 0; i < 17; i ++) {
		uint64_t w;

		w = t[i] + cc;
		d[i] = (uint32_t)w & 0x3FFFFFFF;
		cc = w >> 30;
	}
	d[17] = (uint32_t)cc;
}

/*
 * Square a 270-bit integer, represented as an array of nine 30-bit words.
 * Result uses 18 words of 30 bits each.
 */
static void
square9(uint32_t *d, const uint32_t *a)
{
	uint64_t t[17];
	uint64_t cc;
	int i;

	t[ 0] = MG_BR_MUL31(a[0], a[0]);
	t[ 1] = ((MG_BR_MUL31(a[0], a[1])) << 1);
	t[ 2] = MG_BR_MUL31(a[1], a[1])
		+ ((MG_BR_MUL31(a[0], a[2])) << 1);
	t[ 3] = ((MG_BR_MUL31(a[0], a[3])
		+ MG_BR_MUL31(a[1], a[2])) << 1);
	t[ 4] = MG_BR_MUL31(a[2], a[2])
		+ ((MG_BR_MUL31(a[0], a[4])
		+ MG_BR_MUL31(a[1], a[3])) << 1);
	t[ 5] = ((MG_BR_MUL31(a[0], a[5])
		+ MG_BR_MUL31(a[1], a[4])
		+ MG_BR_MUL31(a[2], a[3])) << 1);
	t[ 6] = MG_BR_MUL31(a[3], a[3])
		+ ((MG_BR_MUL31(a[0], a[6])
		+ MG_BR_MUL31(a[1], a[5])
		+ MG_BR_MUL31(a[2], a[4])) << 1);
	t[ 7] = ((MG_BR_MUL31(a[0], a[7])
		+ MG_BR_MUL31(a[1], a[6])
		+ MG_BR_MUL31(a[2], a[5])
		+ MG_BR_MUL31(a[3], a[4])) << 1);
	t[ 8] = MG_BR_MUL31(a[4], a[4])
		+ ((MG_BR_MUL31(a[0], a[8])
		+ MG_BR_MUL31(a[1], a[7])
		+ MG_BR_MUL31(a[2], a[6])
		+ MG_BR_MUL31(a[3], a[5])) << 1);
	t[ 9] = ((MG_BR_MUL31(a[1], a[8])
		+ MG_BR_MUL31(a[2], a[7])
		+ MG_BR_MUL31(a[3], a[6])
		+ MG_BR_MUL31(a[4], a[5])) << 1);
	t[10] = MG_BR_MUL31(a[5], a[5])
		+ ((MG_BR_MUL31(a[2], a[8])
		+ MG_BR_MUL31(a[3], a[7])
		+ MG_BR_MUL31(a[4], a[6])) << 1);
	t[11] = ((MG_BR_MUL31(a[3], a[8])
		+ MG_BR_MUL31(a[4], a[7])
		+ MG_BR_MUL31(a[5], a[6])) << 1);
	t[12] = MG_BR_MUL31(a[6], a[6])
		+ ((MG_BR_MUL31(a[4], a[8])
		+ MG_BR_MUL31(a[5], a[7])) << 1);
	t[13] = ((MG_BR_MUL31(a[5], a[8])
		+ MG_BR_MUL31(a[6], a[7])) << 1);
	t[14] = MG_BR_MUL31(a[7], a[7])
		+ ((MG_BR_MUL31(a[6], a[8])) << 1);
	t[15] = ((MG_BR_MUL31(a[7], a[8])) << 1);
	t[16] = MG_BR_MUL31(a[8], a[8]);

	/*
	 * Propagate carries.
	 */
	cc = 0;
	for (i = 0; i < 17; i ++) {
		uint64_t w;

		w = t[i] + cc;
		d[i] = (uint32_t)w & 0x3FFFFFFF;
		cc = w >> 30;
	}
	d[17] = (uint32_t)cc;
}

/*
 * Base field modulus for P-256.
 */
static const uint32_t F256[] = {

	0x3FFFFFFF, 0x3FFFFFFF, 0x3FFFFFFF, 0x0000003F, 0x00000000,
	0x00000000, 0x00001000, 0x3FFFC000, 0x0000FFFF
};

/*
 * The 'b' curve equation coefficient for P-256.
 */
static const uint32_t P256_B[] = {

	0x27D2604B, 0x2F38F0F8, 0x053B0F63, 0x0741AC33, 0x1886BC65,
	0x2EF555DA, 0x293E7B3E, 0x0D762A8E, 0x00005AC6
};

/*
 * Addition in the field. Source operands shall fit on 257 bits; output
 * will be lower than twice the modulus.
 */
static void
add_f256(uint32_t *d, const uint32_t *a, const uint32_t *b)
{
	uint32_t w, cc;
	int i;

	cc = 0;
	for (i = 0; i < 9; i ++) {
		w = a[i] + b[i] + cc;
		d[i] = w & 0x3FFFFFFF;
		cc = w >> 30;
	}
	w >>= 16;
	d[8] &= 0xFFFF;
	d[3] -= w << 6;
	d[6] -= w << 12;
	d[7] += w << 14;
	cc = w;
	for (i = 0; i < 9; i ++) {
		w = d[i] + cc;
		d[i] = w & 0x3FFFFFFF;
		cc = ARSH(w, 30);
	}
}

/*
 * Subtraction in the field. Source operands shall be smaller than twice
 * the modulus; the result will fulfil the same property.
 */
static void
sub_f256(uint32_t *d, const uint32_t *a, const uint32_t *b)
{
	uint32_t w, cc;
	int i;

	/*
	 * We really compute a - b + 2*p to make sure that the result is
	 * positive.
	 */
	w = a[0] - b[0] - 0x00002;
	d[0] = w & 0x3FFFFFFF;
	w = a[1] - b[1] + ARSH(w, 30);
	d[1] = w & 0x3FFFFFFF;
	w = a[2] - b[2] + ARSH(w, 30);
	d[2] = w & 0x3FFFFFFF;
	w = a[3] - b[3] + ARSH(w, 30) + 0x00080;
	d[3] = w & 0x3FFFFFFF;
	w = a[4] - b[4] + ARSH(w, 30);
	d[4] = w & 0x3FFFFFFF;
	w = a[5] - b[5] + ARSH(w, 30);
	d[5] = w & 0x3FFFFFFF;
	w = a[6] - b[6] + ARSH(w, 30) + 0x02000;
	d[6] = w & 0x3FFFFFFF;
	w = a[7] - b[7] + ARSH(w, 30) - 0x08000;
	d[7] = w & 0x3FFFFFFF;
	w = a[8] - b[8] + ARSH(w, 30) + 0x20000;
	d[8] = w & 0xFFFF;
	w >>= 16;
	d[8] &= 0xFFFF;
	d[3] -= w << 6;
	d[6] -= w << 12;
	d[7] += w << 14;
	cc = w;
	for (i = 0; i < 9; i ++) {
		w = d[i] + cc;
		d[i] = w & 0x3FFFFFFF;
		cc = ARSH(w, 30);
	}
}

/*
 * Compute a multiplication in F256. Source operands shall be less than
 * twice the modulus.
 */
static void
mul_f256(uint32_t *d, const uint32_t *a, const uint32_t *b)
{
	uint32_t t[18];
	uint64_t s[18];
	uint64_t cc, x;
	uint32_t z, c;
	int i;

	mul9(t, a, b);

	/*
	 * Modular reduction: each high word in added/subtracted where
	 * necessary.
	 *
	 * The modulus is:
	 *    p = 2^256 - 2^224 + 2^192 + 2^96 - 1
	 * Therefore:
	 *    2^256 = 2^224 - 2^192 - 2^96 + 1 mod p
	 *
	 * For a word x at bit offset n (n >= 256), we have:
	 *    x*2^n = x*2^(n-32) - x*2^(n-64)
	 *            - x*2^(n - 160) + x*2^(n-256) mod p
	 *
	 * Thus, we can nullify the high word if we reinject it at some
	 * proper emplacements.
	 *
	 * We use 64-bit intermediate words to allow for carries to
	 * accumulate easily, before performing the final propagation.
	 */
	for (i = 0; i < 18; i ++) {
		s[i] = t[i];
	}

	for (i = 17; i >= 9; i --) {
		uint64_t y;

		y = s[i];
		s[i - 1] += ARSHW(y, 2);
		s[i - 2] += (y << 28) & 0x3FFFFFFF;
		s[i - 2] -= ARSHW(y, 4);
		s[i - 3] -= (y << 26) & 0x3FFFFFFF;
		s[i - 5] -= ARSHW(y, 10);
		s[i - 6] -= (y << 20) & 0x3FFFFFFF;
		s[i - 8] += ARSHW(y, 16);
		s[i - 9] += (y << 14) & 0x3FFFFFFF;
	}

	/*
	 * Carry propagation must be signed. Moreover, we may have overdone
	 * it a bit, and obtain a negative result.
	 *
	 * The loop above ran 9 times; each time, each word was augmented
	 * by at most one extra word (in absolute value). Thus, the top
	 * word must in fine fit in 39 bits, so the carry below will fit
	 * on 9 bits.
	 */
	cc = 0;
	for (i = 0; i < 9; i ++) {
		x = s[i] + cc;
		d[i] = (uint32_t)x & 0x3FFFFFFF;
		cc = ARSHW(x, 30);
	}

	/*
	 * All nine words fit on 30 bits, but there may be an extra
	 * carry for a few bits (at most 9), and that carry may be
	 * negative. Moreover, we want the result to fit on 257 bits.
	 * The two lines below ensure that the word in d[] has length
	 * 256 bits, and the (signed) carry (beyond 2^256) is in cc. The
	 * significant length of cc is less than 24 bits, so we will be
	 * able to switch to 32-bit operations.
	 */
	cc = ARSHW(x, 16);
	d[8] &= 0xFFFF;

	/*
	 * One extra round of reduction, for cc*2^256, which means
	 * adding cc*(2^224-2^192-2^96+1) to a 256-bit (nonnegative)
	 * value. If cc is negative, then it may happen (rarely, but
	 * not neglectibly so) that the result would be negative. In
	 * order to avoid that, if cc is negative, then we add the
	 * modulus once. Note that if cc is negative, then propagating
	 * that carry must yield a value lower than the modulus, so
	 * adding the modulus once will keep the final result under
	 * twice the modulus.
	 */
	z = (uint32_t)cc;
	d[3] -= z << 6;
	d[6] -= (z << 12) & 0x3FFFFFFF;
	d[7] -= ARSH(z, 18);
	d[7] += (z << 14) & 0x3FFFFFFF;
	d[8] += ARSH(z, 16);
	c = z >> 31;
	d[0] -= c;
	d[3] += c << 6;
	d[6] += c << 12;
	d[7] -= c << 14;
	d[8] += c << 16;
	for (i = 0; i < 9; i ++) {
		uint32_t w;

		w = d[i] + z;
		d[i] = w & 0x3FFFFFFF;
		z = ARSH(w, 30);
	}
}

/*
 * Compute a square in F256. Source operand shall be less than
 * twice the modulus.
 */
static void
square_f256(uint32_t *d, const uint32_t *a)
{
	uint32_t t[18];
	uint64_t s[18];
	uint64_t cc, x;
	uint32_t z, c;
	int i;

	square9(t, a);

	/*
	 * Modular reduction: each high word in added/subtracted where
	 * necessary.
	 *
	 * The modulus is:
	 *    p = 2^256 - 2^224 + 2^192 + 2^96 - 1
	 * Therefore:
	 *    2^256 = 2^224 - 2^192 - 2^96 + 1 mod p
	 *
	 * For a word x at bit offset n (n >= 256), we have:
	 *    x*2^n = x*2^(n-32) - x*2^(n-64)
	 *            - x*2^(n - 160) + x*2^(n-256) mod p
	 *
	 * Thus, we can nullify the high word if we reinject it at some
	 * proper emplacements.
	 *
	 * We use 64-bit intermediate words to allow for carries to
	 * accumulate easily, before performing the final propagation.
	 */
	for (i = 0; i < 18; i ++) {
		s[i] = t[i];
	}

	for (i = 17; i >= 9; i --) {
		uint64_t y;

		y = s[i];
		s[i - 1] += ARSHW(y, 2);
		s[i - 2] += (y << 28) & 0x3FFFFFFF;
		s[i - 2] -= ARSHW(y, 4);
		s[i - 3] -= (y << 26) & 0x3FFFFFFF;
		s[i - 5] -= ARSHW(y, 10);
		s[i - 6] -= (y << 20) & 0x3FFFFFFF;
		s[i - 8] += ARSHW(y, 16);
		s[i - 9] += (y << 14) & 0x3FFFFFFF;
	}

	/*
	 * Carry propagation must be signed. Moreover, we may have overdone
	 * it a bit, and obtain a negative result.
	 *
	 * The loop above ran 9 times; each time, each word was augmented
	 * by at most one extra word (in absolute value). Thus, the top
	 * word must in fine fit in 39 bits, so the carry below will fit
	 * on 9 bits.
	 */
	cc = 0;
	for (i = 0; i < 9; i ++) {
		x = s[i] + cc;
		d[i] = (uint32_t)x & 0x3FFFFFFF;
		cc = ARSHW(x, 30);
	}

	/*
	 * All nine words fit on 30 bits, but there may be an extra
	 * carry for a few bits (at most 9), and that carry may be
	 * negative. Moreover, we want the result to fit on 257 bits.
	 * The two lines below ensure that the word in d[] has length
	 * 256 bits, and the (signed) carry (beyond 2^256) is in cc. The
	 * significant length of cc is less than 24 bits, so we will be
	 * able to switch to 32-bit operations.
	 */
	cc = ARSHW(x, 16);
	d[8] &= 0xFFFF;

	/*
	 * One extra round of reduction, for cc*2^256, which means
	 * adding cc*(2^224-2^192-2^96+1) to a 256-bit (nonnegative)
	 * value. If cc is negative, then it may happen (rarely, but
	 * not neglectibly so) that the result would be negative. In
	 * order to avoid that, if cc is negative, then we add the
	 * modulus once. Note that if cc is negative, then propagating
	 * that carry must yield a value lower than the modulus, so
	 * adding the modulus once will keep the final result under
	 * twice the modulus.
	 */
	z = (uint32_t)cc;
	d[3] -= z << 6;
	d[6] -= (z << 12) & 0x3FFFFFFF;
	d[7] -= ARSH(z, 18);
	d[7] += (z << 14) & 0x3FFFFFFF;
	d[8] += ARSH(z, 16);
	c = z >> 31;
	d[0] -= c;
	d[3] += c << 6;
	d[6] += c << 12;
	d[7] -= c << 14;
	d[8] += c << 16;
	for (i = 0; i < 9; i ++) {
		uint32_t w;

		w = d[i] + z;
		d[i] = w & 0x3FFFFFFF;
		z = ARSH(w, 30);
	}
}

/*
 * Perform a "final reduction" in field F256 (field for curve P-256).
 * The source value must be less than twice the modulus. If the value
 * is not lower than the modulus, then the modulus is subtracted and
 * this function returns 1; otherwise, it leaves it untouched and it
 * returns 0.
 */
static uint32_t
reduce_final_f256(uint32_t *d)
{
	uint32_t t[9];
	uint32_t cc;
	int i;

	cc = 0;
	for (i = 0; i < 9; i ++) {
		uint32_t w;

		w = d[i] - F256[i] - cc;
		cc = w >> 31;
		t[i] = w & 0x3FFFFFFF;
	}
	cc ^= 1;
	MG_BR_CCOPY(cc, d, t, sizeof t);
	return cc;
}

/*
 * Jacobian coordinates for a point in P-256: affine coordinates (X,Y)
 * are such that:
 *   X = x / z^2
 *   Y = y / z^3
 * For the point at infinity, z = 0.
 * Each point thus admits many possible representations.
 *
 * Coordinates are represented in arrays of 32-bit integers, each holding
 * 30 bits of data. Values may also be slightly greater than the modulus,
 * but they will always be lower than twice the modulus.
 */
struct p256_jacobian {
	uint32_t x[9];
	uint32_t y[9];
	uint32_t z[9];
};

/*
 * Convert a point to affine coordinates:
 *  - If the point is the point at infinity, then all three coordinates
 *    are set to 0.
 *  - Otherwise, the 'z' coordinate is set to 1, and the 'x' and 'y'
 *    coordinates are the 'X' and 'Y' affine coordinates.
 * The coordinates are guaranteed to be lower than the modulus.
 */
static void
p256_to_affine(struct p256_jacobian *P)
{
	uint32_t t1[9], t2[9];
	int i;

	/*
	 * Invert z with a modular exponentiation: the modulus is
	 * p = 2^256 - 2^224 + 2^192 + 2^96 - 1, and the exponent is
	 * p-2. Exponent bit pattern (from high to low) is:
	 *  - 32 bits of value 1
	 *  - 31 bits of value 0
	 *  - 1 bit of value 1
	 *  - 96 bits of value 0
	 *  - 94 bits of value 1
	 *  - 1 bit of value 0
	 *  - 1 bit of value 1
	 * Thus, we precompute z^(2^31-1) to speed things up.
	 *
	 * If z = 0 (point at infinity) then the modular exponentiation
	 * will yield 0, which leads to the expected result (all three
	 * coordinates set to 0).
	 */

	/*
	 * A simple square-and-multiply for z^(2^31-1). We could save about
	 * two dozen multiplications here with an addition chain, but
	 * this would require a bit more code, and extra stack buffers.
	 */
	memcpy(t1, P->z, sizeof P->z);
	for (i = 0; i < 30; i ++) {
		square_f256(t1, t1);
		mul_f256(t1, t1, P->z);
	}

	/*
	 * Square-and-multiply. Apart from the squarings, we have a few
	 * multiplications to set bits to 1; we multiply by the original z
	 * for setting 1 bit, and by t1 for setting 31 bits.
	 */
	memcpy(t2, P->z, sizeof P->z);
	for (i = 1; i < 256; i ++) {
		square_f256(t2, t2);
		switch (i) {
		case 31:
		case 190:
		case 221:
		case 252:
			mul_f256(t2, t2, t1);
			break;
		case 63:
		case 253:
		case 255:
			mul_f256(t2, t2, P->z);
			break;
		}
	}

	/*
	 * Now that we have 1/z, multiply x by 1/z^2 and y by 1/z^3.
	 */
	mul_f256(t1, t2, t2);
	mul_f256(P->x, t1, P->x);
	mul_f256(t1, t1, t2);
	mul_f256(P->y, t1, P->y);
	reduce_final_f256(P->x);
	reduce_final_f256(P->y);

	/*
	 * Multiply z by 1/z. If z = 0, then this will yield 0, otherwise
	 * this will set z to 1.
	 */
	mul_f256(P->z, P->z, t2);
	reduce_final_f256(P->z);
}

/*
 * Double a point in P-256. This function works for all valid points,
 * including the point at infinity.
 */
static void
p256_double(struct p256_jacobian *Q)
{
	/*
	 * Doubling formulas are:
	 *
	 *   s = 4*x*y^2
	 *   m = 3*(x + z^2)*(x - z^2)
	 *   x' = m^2 - 2*s
	 *   y' = m*(s - x') - 8*y^4
	 *   z' = 2*y*z
	 *
	 * These formulas work for all points, including points of order 2
	 * and points at infinity:
	 *   - If y = 0 then z' = 0. But there is no such point in P-256
	 *     anyway.
	 *   - If z = 0 then z' = 0.
	 */
	uint32_t t1[9], t2[9], t3[9], t4[9];

	/*
	 * Compute z^2 in t1.
	 */
	square_f256(t1, Q->z);

	/*
	 * Compute x-z^2 in t2 and x+z^2 in t1.
	 */
	add_f256(t2, Q->x, t1);
	sub_f256(t1, Q->x, t1);

	/*
	 * Compute 3*(x+z^2)*(x-z^2) in t1.
	 */
	mul_f256(t3, t1, t2);
	add_f256(t1, t3, t3);
	add_f256(t1, t3, t1);

	/*
	 * Compute 4*x*y^2 (in t2) and 2*y^2 (in t3).
	 */
	square_f256(t3, Q->y);
	add_f256(t3, t3, t3);
	mul_f256(t2, Q->x, t3);
	add_f256(t2, t2, t2);

	/*
	 * Compute x' = m^2 - 2*s.
	 */
	square_f256(Q->x, t1);
	sub_f256(Q->x, Q->x, t2);
	sub_f256(Q->x, Q->x, t2);

	/*
	 * Compute z' = 2*y*z.
	 */
	mul_f256(t4, Q->y, Q->z);
	add_f256(Q->z, t4, t4);

	/*
	 * Compute y' = m*(s - x') - 8*y^4. Note that we already have
	 * 2*y^2 in t3.
	 */
	sub_f256(t2, t2, Q->x);
	mul_f256(Q->y, t1, t2);
	square_f256(t4, t3);
	add_f256(t4, t4, t4);
	sub_f256(Q->y, Q->y, t4);
}

/*
 * Add point P2 to point P1.
 *
 * This function computes the wrong result in the following cases:
 *
 *   - If P1 == 0 but P2 != 0
 *   - If P1 != 0 but P2 == 0
 *   - If P1 == P2
 *
 * In all three cases, P1 is set to the point at infinity.
 *
 * Returned value is 0 if one of the following occurs:
 *
 *   - P1 and P2 have the same Y coordinate
 *   - P1 == 0 and P2 == 0
 *   - The Y coordinate of one of the points is 0 and the other point is
 *     the point at infinity.
 *
 * The third case cannot actually happen with valid points, since a point
 * with Y == 0 is a point of order 2, and there is no point of order 2 on
 * curve P-256.
 *
 * Therefore, assuming that P1 != 0 and P2 != 0 on input, then the caller
 * can apply the following:
 *
 *   - If the result is not the point at infinity, then it is correct.
 *   - Otherwise, if the returned value is 1, then this is a case of
 *     P1+P2 == 0, so the result is indeed the point at infinity.
 *   - Otherwise, P1 == P2, so a "double" operation should have been
 *     performed.
 */
static uint32_t
p256_add(struct p256_jacobian *P1, const struct p256_jacobian *P2)
{
	/*
	 * Addtions formulas are:
	 *
	 *   u1 = x1 * z2^2
	 *   u2 = x2 * z1^2
	 *   s1 = y1 * z2^3
	 *   s2 = y2 * z1^3
	 *   h = u2 - u1
	 *   r = s2 - s1
	 *   x3 = r^2 - h^3 - 2 * u1 * h^2
	 *   y3 = r * (u1 * h^2 - x3) - s1 * h^3
	 *   z3 = h * z1 * z2
	 */
	uint32_t t1[9], t2[9], t3[9], t4[9], t5[9], t6[9], t7[9];
	uint32_t ret;
	int i;

	/*
	 * Compute u1 = x1*z2^2 (in t1) and s1 = y1*z2^3 (in t3).
	 */
	square_f256(t3, P2->z);
	mul_f256(t1, P1->x, t3);
	mul_f256(t4, P2->z, t3);
	mul_f256(t3, P1->y, t4);

	/*
	 * Compute u2 = x2*z1^2 (in t2) and s2 = y2*z1^3 (in t4).
	 */
	square_f256(t4, P1->z);
	mul_f256(t2, P2->x, t4);
	mul_f256(t5, P1->z, t4);
	mul_f256(t4, P2->y, t5);

	/*
	 * Compute h = h2 - u1 (in t2) and r = s2 - s1 (in t4).
	 * We need to test whether r is zero, so we will do some extra
	 * reduce.
	 */
	sub_f256(t2, t2, t1);
	sub_f256(t4, t4, t3);
	reduce_final_f256(t4);
	ret = 0;
	for (i = 0; i < 9; i ++) {
		ret |= t4[i];
	}
	ret = (ret | -ret) >> 31;

	/*
	 * Compute u1*h^2 (in t6) and h^3 (in t5);
	 */
	square_f256(t7, t2);
	mul_f256(t6, t1, t7);
	mul_f256(t5, t7, t2);

	/*
	 * Compute x3 = r^2 - h^3 - 2*u1*h^2.
	 */
	square_f256(P1->x, t4);
	sub_f256(P1->x, P1->x, t5);
	sub_f256(P1->x, P1->x, t6);
	sub_f256(P1->x, P1->x, t6);

	/*
	 * Compute y3 = r*(u1*h^2 - x3) - s1*h^3.
	 */
	sub_f256(t6, t6, P1->x);
	mul_f256(P1->y, t4, t6);
	mul_f256(t1, t5, t3);
	sub_f256(P1->y, P1->y, t1);

	/*
	 * Compute z3 = h*z1*z2.
	 */
	mul_f256(t1, P1->z, P2->z);
	mul_f256(P1->z, t1, t2);

	return ret;
}

/*
 * Add point P2 to point P1. This is a specialised function for the
 * case when P2 is a non-zero point in affine coordinate.
 *
 * This function computes the wrong result in the following cases:
 *
 *   - If P1 == 0
 *   - If P1 == P2
 *
 * In both cases, P1 is set to the point at infinity.
 *
 * Returned value is 0 if one of the following occurs:
 *
 *   - P1 and P2 have the same Y coordinate
 *   - The Y coordinate of P2 is 0 and P1 is the point at infinity.
 *
 * The second case cannot actually happen with valid points, since a point
 * with Y == 0 is a point of order 2, and there is no point of order 2 on
 * curve P-256.
 *
 * Therefore, assuming that P1 != 0 on input, then the caller
 * can apply the following:
 *
 *   - If the result is not the point at infinity, then it is correct.
 *   - Otherwise, if the returned value is 1, then this is a case of
 *     P1+P2 == 0, so the result is indeed the point at infinity.
 *   - Otherwise, P1 == P2, so a "double" operation should have been
 *     performed.
 */
static uint32_t
p256_add_mixed(struct p256_jacobian *P1, const struct p256_jacobian *P2)
{
	/*
	 * Addtions formulas are:
	 *
	 *   u1 = x1
	 *   u2 = x2 * z1^2
	 *   s1 = y1
	 *   s2 = y2 * z1^3
	 *   h = u2 - u1
	 *   r = s2 - s1
	 *   x3 = r^2 - h^3 - 2 * u1 * h^2
	 *   y3 = r * (u1 * h^2 - x3) - s1 * h^3
	 *   z3 = h * z1
	 */
	uint32_t t1[9], t2[9], t3[9], t4[9], t5[9], t6[9], t7[9];
	uint32_t ret;
	int i;

	/*
	 * Compute u1 = x1 (in t1) and s1 = y1 (in t3).
	 */
	memcpy(t1, P1->x, sizeof t1);
	memcpy(t3, P1->y, sizeof t3);

	/*
	 * Compute u2 = x2*z1^2 (in t2) and s2 = y2*z1^3 (in t4).
	 */
	square_f256(t4, P1->z);
	mul_f256(t2, P2->x, t4);
	mul_f256(t5, P1->z, t4);
	mul_f256(t4, P2->y, t5);

	/*
	 * Compute h = h2 - u1 (in t2) and r = s2 - s1 (in t4).
	 * We need to test whether r is zero, so we will do some extra
	 * reduce.
	 */
	sub_f256(t2, t2, t1);
	sub_f256(t4, t4, t3);
	reduce_final_f256(t4);
	ret = 0;
	for (i = 0; i < 9; i ++) {
		ret |= t4[i];
	}
	ret = (ret | -ret) >> 31;

	/*
	 * Compute u1*h^2 (in t6) and h^3 (in t5);
	 */
	square_f256(t7, t2);
	mul_f256(t6, t1, t7);
	mul_f256(t5, t7, t2);

	/*
	 * Compute x3 = r^2 - h^3 - 2*u1*h^2.
	 */
	square_f256(P1->x, t4);
	sub_f256(P1->x, P1->x, t5);
	sub_f256(P1->x, P1->x, t6);
	sub_f256(P1->x, P1->x, t6);

	/*
	 * Compute y3 = r*(u1*h^2 - x3) - s1*h^3.
	 */
	sub_f256(t6, t6, P1->x);
	mul_f256(P1->y, t4, t6);
	mul_f256(t1, t5, t3);
	sub_f256(P1->y, P1->y, t1);

	/*
	 * Compute z3 = h*z1*z2.
	 */
	mul_f256(P1->z, P1->z, t2);

	return ret;
}

/*
 * Decode a P-256 point. This function does not support the point at
 * infinity. Returned value is 0 if the point is invalid, 1 otherwise.
 */
static uint32_t
p256_decode(struct p256_jacobian *P, const void *src, size_t len)
{
	const unsigned char *buf;
	uint32_t tx[9], ty[9], t1[9], t2[9];
	uint32_t bad;
	int i;

	if (len != 65) {
		return 0;
	}
	buf = (const unsigned char *)src;

	/*
	 * First byte must be 0x04 (uncompressed format). We could support
	 * "hybrid format" (first byte is 0x06 or 0x07, and encodes the
	 * least significant bit of the Y coordinate), but it is explicitly
	 * forbidden by RFC 5480 (section 2.2).
	 */
	bad = MG_BR_NEQ(buf[0], 0x04);

	/*
	 * Decode the coordinates, and check that they are both lower
	 * than the modulus.
	 */
	tx[8] = be8_to_le30(tx, buf + 1, 32);
	ty[8] = be8_to_le30(ty, buf + 33, 32);
	bad |= reduce_final_f256(tx);
	bad |= reduce_final_f256(ty);

	/*
	 * Check curve equation.
	 */
	square_f256(t1, tx);
	mul_f256(t1, tx, t1);
	square_f256(t2, ty);
	sub_f256(t1, t1, tx);
	sub_f256(t1, t1, tx);
	sub_f256(t1, t1, tx);
	add_f256(t1, t1, P256_B);
	sub_f256(t1, t1, t2);
	reduce_final_f256(t1);
	for (i = 0; i < 9; i ++) {
		bad |= t1[i];
	}

	/*
	 * Copy coordinates to the point structure.
	 */
	memcpy(P->x, tx, sizeof tx);
	memcpy(P->y, ty, sizeof ty);
	memset(P->z, 0, sizeof P->z);
	P->z[0] = 1;
	return MG_BR_EQ(bad, 0);
}

/*
 * Encode a point into a buffer. This function assumes that the point is
 * valid, in affine coordinates, and not the point at infinity.
 */
static void
p256_encode(void *dst, const struct p256_jacobian *P)
{
	unsigned char *buf;

	buf = (unsigned char *)dst;
	buf[0] = 0x04;
	le30_to_be8(buf + 1, 32, P->x);
	le30_to_be8(buf + 33, 32, P->y);
}

/*
 * Multiply a curve point by an integer. The integer is assumed to be
 * lower than the curve order, and the base point must not be the point
 * at infinity.
 */
static void
p256_mul(struct p256_jacobian *P, const unsigned char *x, size_t xlen)
{
	/*
	 * qz is a flag that is initially 1, and remains equal to 1
	 * as long as the point is the point at infinity.
	 *
	 * We use a 2-bit window to handle multiplier bits by pairs.
	 * The precomputed window really is the points P2 and P3.
	 */
	uint32_t qz;
	struct p256_jacobian P2, P3, Q, T, U;

	/*
	 * Compute window values.
	 */
	P2 = *P;
	p256_double(&P2);
	P3 = *P;
	p256_add(&P3, &P2);

	/*
	 * We start with Q = 0. We process multiplier bits 2 by 2.
	 */
	memset(&Q, 0, sizeof Q);
	qz = 1;
	while (xlen -- > 0) {
		int k;

		for (k = 6; k >= 0; k -= 2) {
			uint32_t bits;
			uint32_t bnz;

			p256_double(&Q);
			p256_double(&Q);
			T = *P;
			U = Q;
			bits = (*x >> k) & (uint32_t)3;
			bnz = MG_BR_NEQ(bits, 0);
			MG_BR_CCOPY(MG_BR_EQ(bits, 2), &T, &P2, sizeof T);
			MG_BR_CCOPY(MG_BR_EQ(bits, 3), &T, &P3, sizeof T);
			p256_add(&U, &T);
			MG_BR_CCOPY(bnz & qz, &Q, &T, sizeof Q);
			MG_BR_CCOPY(bnz & ~qz, &Q, &U, sizeof Q);
			qz &= ~bnz;
		}
		x ++;
	}
	*P = Q;
}

/*
 * Precomputed window: k*G points, where G is the curve generator, and k
 * is an integer from 1 to 15 (inclusive). The X and Y coordinates of
 * the point are encoded as 9 words of 30 bits each (little-endian
 * order).
 */
static const uint32_t Gwin[15][18] = {

	{ 0x1898C296, 0x1284E517, 0x1EB33A0F, 0x00DF604B,
	  0x2440F277, 0x339B958E, 0x04247F8B, 0x347CB84B,
	  0x00006B17, 0x37BF51F5, 0x2ED901A0, 0x3315ECEC,
	  0x338CD5DA, 0x0F9E162B, 0x1FAD29F0, 0x27F9B8EE,
	  0x10B8BF86, 0x00004FE3 },

	{ 0x07669978, 0x182D23F1, 0x3F21B35A, 0x225A789D,
	  0x351AC3C0, 0x08E00C12, 0x34F7E8A5, 0x1EC62340,
	  0x00007CF2, 0x227873D1, 0x3812DE74, 0x0E982299,
	  0x1F6B798F, 0x3430DBBA, 0x366B1A7D, 0x2D040293,
	  0x154436E3, 0x00000777 },

	{ 0x06E7FD6C, 0x2D05986F, 0x3ADA985F, 0x31ADC87B,
	  0x0BF165E6, 0x1FBE5475, 0x30A44C8F, 0x3934698C,
	  0x00005ECB, 0x227D5032, 0x29E6C49E, 0x04FB83D9,
	  0x0AAC0D8E, 0x24A2ECD8, 0x2C1B3869, 0x0FF7E374,
	  0x19031266, 0x00008734 },

	{ 0x2B030852, 0x024C0911, 0x05596EF5, 0x07F8B6DE,
	  0x262BD003, 0x3779967B, 0x08FBBA02, 0x128D4CB4,
	  0x0000E253, 0x184ED8C6, 0x310B08FC, 0x30EE0055,
	  0x3F25B0FC, 0x062D764E, 0x3FB97F6A, 0x33CC719D,
	  0x15D69318, 0x0000E0F1 },

	{ 0x03D033ED, 0x05552837, 0x35BE5242, 0x2320BF47,
	  0x268FDFEF, 0x13215821, 0x140D2D78, 0x02DE9454,
	  0x00005159, 0x3DA16DA4, 0x0742ED13, 0x0D80888D,
	  0x004BC035, 0x0A79260D, 0x06FCDAFE, 0x2727D8AE,
	  0x1F6A2412, 0x0000E0C1 },

	{ 0x3C2291A9, 0x1AC2ABA4, 0x3B215B4C, 0x131D037A,
	  0x17DDE302, 0x0C90B2E2, 0x0602C92D, 0x05CA9DA9,
	  0x0000B01A, 0x0FC77FE2, 0x35F1214E, 0x07E16BDF,
	  0x003DDC07, 0x2703791C, 0x3038B7EE, 0x3DAD56FE,
	  0x041D0C8D, 0x0000E85C },

	{ 0x3187B2A3, 0x0018A1C0, 0x00FEF5B3, 0x3E7E2E2A,
	  0x01FB607E, 0x2CC199F0, 0x37B4625B, 0x0EDBE82F,
	  0x00008E53, 0x01F400B4, 0x15786A1B, 0x3041B21C,
	  0x31CD8CF2, 0x35900053, 0x1A7E0E9B, 0x318366D0,
	  0x076F780C, 0x000073EB },

	{ 0x1B6FB393, 0x13767707, 0x3CE97DBB, 0x348E2603,
	  0x354CADC1, 0x09D0B4EA, 0x1B053404, 0x1DE76FBA,
	  0x000062D9, 0x0F09957E, 0x295029A8, 0x3E76A78D,
	  0x3B547DAE, 0x27CEE0A2, 0x0575DC45, 0x1D8244FF,
	  0x332F647A, 0x0000AD5A },

	{ 0x10949EE0, 0x1E7A292E, 0x06DF8B3D, 0x02B2E30B,
	  0x31F8729E, 0x24E35475, 0x30B71878, 0x35EDBFB7,
	  0x0000EA68, 0x0DD048FA, 0x21688929, 0x0DE823FE,
	  0x1C53FAA9, 0x0EA0C84D, 0x052A592A, 0x1FCE7870,
	  0x11325CB2, 0x00002A27 },

	{ 0x04C5723F, 0x30D81A50, 0x048306E4, 0x329B11C7,
	  0x223FB545, 0x085347A8, 0x2993E591, 0x1B5ACA8E,
	  0x0000CEF6, 0x04AF0773, 0x28D2EEA9, 0x2751EEEC,
	  0x037B4A7F, 0x3B4C1059, 0x08F37674, 0x2AE906E1,
	  0x18A88A6A, 0x00008786 },

	{ 0x34BC21D1, 0x0CCE474D, 0x15048BF4, 0x1D0BB409,
	  0x021CDA16, 0x20DE76C3, 0x34C59063, 0x04EDE20E,
	  0x00003ED1, 0x282A3740, 0x0BE3BBF3, 0x29889DAE,
	  0x03413697, 0x34C68A09, 0x210EBE93, 0x0C8A224C,
	  0x0826B331, 0x00009099 },

	{ 0x0624E3C4, 0x140317BA, 0x2F82C99D, 0x260C0A2C,
	  0x25D55179, 0x194DCC83, 0x3D95E462, 0x356F6A05,
	  0x0000741D, 0x0D4481D3, 0x2657FC8B, 0x1BA5CA71,
	  0x3AE44B0D, 0x07B1548E, 0x0E0D5522, 0x05FDC567,
	  0x2D1AA70E, 0x00000770 },

	{ 0x06072C01, 0x23857675, 0x1EAD58A9, 0x0B8A12D9,
	  0x1EE2FC79, 0x0177CB61, 0x0495A618, 0x20DEB82B,
	  0x0000177C, 0x2FC7BFD8, 0x310EEF8B, 0x1FB4DF39,
	  0x3B8530E8, 0x0F4E7226, 0x0246B6D0, 0x2A558A24,
	  0x163353AF, 0x000063BB },

	{ 0x24D2920B, 0x1C249DCC, 0x2069C5E5, 0x09AB2F9E,
	  0x36DF3CF1, 0x1991FD0C, 0x062B97A7, 0x1E80070E,
	  0x000054E7, 0x20D0B375, 0x2E9F20BD, 0x35090081,
	  0x1C7A9DDC, 0x22E7C371, 0x087E3016, 0x03175421,
	  0x3C6ECA7D, 0x0000F599 },

	{ 0x259B9D5F, 0x0D9A318F, 0x23A0EF16, 0x00EBE4B7,
	  0x088265AE, 0x2CDE2666, 0x2BAE7ADF, 0x1371A5C6,
	  0x0000F045, 0x0D034F36, 0x1F967378, 0x1B5FA3F4,
	  0x0EC8739D, 0x1643E62A, 0x1653947E, 0x22D1F4E6,
	  0x0FB8D64B, 0x0000B5B9 }
};

/*
 * Lookup one of the Gwin[] values, by index. This is constant-time.
 */
static void
lookup_Gwin(struct p256_jacobian *T, uint32_t idx)
{
	uint32_t xy[18];
	uint32_t k;
	size_t u;

	memset(xy, 0, sizeof xy);
	for (k = 0; k < 15; k ++) {
		uint32_t m;

		m = -MG_BR_EQ(idx, k + 1);
		for (u = 0; u < 18; u ++) {
			xy[u] |= m & Gwin[k][u];
		}
	}
	memcpy(T->x, &xy[0], sizeof T->x);
	memcpy(T->y, &xy[9], sizeof T->y);
	memset(T->z, 0, sizeof T->z);
	T->z[0] = 1;
}

/*
 * Multiply the generator by an integer. The integer is assumed non-zero
 * and lower than the curve order.
 */
static void
p256_mulgen(struct p256_jacobian *P, const unsigned char *x, size_t xlen)
{
	/*
	 * qz is a flag that is initially 1, and remains equal to 1
	 * as long as the point is the point at infinity.
	 *
	 * We use a 4-bit window to handle multiplier bits by groups
	 * of 4. The precomputed window is constant static data, with
	 * points in affine coordinates; we use a constant-time lookup.
	 */
	struct p256_jacobian Q;
	uint32_t qz;

	memset(&Q, 0, sizeof Q);
	qz = 1;
	while (xlen -- > 0) {
		int k;
		unsigned bx;

		bx = *x ++;
		for (k = 0; k < 2; k ++) {
			uint32_t bits;
			uint32_t bnz;
			struct p256_jacobian T, U;

			p256_double(&Q);
			p256_double(&Q);
			p256_double(&Q);
			p256_double(&Q);
			bits = (bx >> 4) & 0x0F;
			bnz = MG_BR_NEQ(bits, 0);
			lookup_Gwin(&T, bits);
			U = Q;
			p256_add_mixed(&U, &T);
			MG_BR_CCOPY(bnz & qz, &Q, &T, sizeof Q);
			MG_BR_CCOPY(bnz & ~qz, &Q, &U, sizeof Q);
			qz &= ~bnz;
			bx <<= 4;
		}
	}
	*P = Q;
}

static const unsigned char P256_G[] = {
	0x04, 0x6B, 0x17, 0xD1, 0xF2, 0xE1, 0x2C, 0x42, 0x47, 0xF8,
	0xBC, 0xE6, 0xE5, 0x63, 0xA4, 0x40, 0xF2, 0x77, 0x03, 0x7D,
	0x81, 0x2D, 0xEB, 0x33, 0xA0, 0xF4, 0xA1, 0x39, 0x45, 0xD8,
	0x98, 0xC2, 0x96, 0x4F, 0xE3, 0x42, 0xE2, 0xFE, 0x1A, 0x7F,
	0x9B, 0x8E, 0xE7, 0xEB, 0x4A, 0x7C, 0x0F, 0x9E, 0x16, 0x2B,
	0xCE, 0x33, 0x57, 0x6B, 0x31, 0x5E, 0xCE, 0xCB, 0xB6, 0x40,
	0x68, 0x37, 0xBF, 0x51, 0xF5
};

static const unsigned char P256_N[] = {
	0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF,
	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xBC, 0xE6, 0xFA, 0xAD,
	0xA7, 0x17, 0x9E, 0x84, 0xF3, 0xB9, 0xCA, 0xC2, 0xFC, 0x63,
	0x25, 0x51
};

static const unsigned char *
p256_api_generator(int curve, size_t *len)
{
	(void)curve;
	*len = sizeof P256_G;
	return P256_G;
}

static const unsigned char *
p256_api_order(int curve, size_t *len)
{
	(void)curve;
	*len = sizeof P256_N;
	return P256_N;
}

static size_t
p256_api_xoff(int curve, size_t *len)
{
	(void)curve;
	*len = 32;
	return 1;
}

static uint32_t
p256_api_mul(unsigned char *G, size_t Glen,
	const unsigned char *x, size_t xlen, int curve)
{
	uint32_t r;
	struct p256_jacobian P;

	(void)curve;
	if (Glen != 65) {
		return 0;
	}
	r = p256_decode(&P, G, Glen);
	p256_mul(&P, x, xlen);
	p256_to_affine(&P);
	p256_encode(G, &P);
	return r;
}

static size_t
p256_api_mulgen(unsigned char *R,
	const unsigned char *x, size_t xlen, int curve)
{
	struct p256_jacobian P;

	(void)curve;
	p256_mulgen(&P, x, xlen);
	p256_to_affine(&P);
	p256_encode(R, &P);
	return 65;
}

static uint32_t
p256_api_muladd(unsigned char *A, const unsigned char *B, size_t len,
	const unsigned char *x, size_t xlen,
	const unsigned char *y, size_t ylen, int curve)
{
	struct p256_jacobian P, Q;
	uint32_t r, t, z;
	int i;

	(void)curve;
	if (len != 65) {
		return 0;
	}
	r = p256_decode(&P, A, len);
	p256_mul(&P, x, xlen);
	if (B == NULL) {
		p256_mulgen(&Q, y, ylen);
	} else {
		r &= p256_decode(&Q, B, len);
		p256_mul(&Q, y, ylen);
	}

	/*
	 * The final addition may fail in case both points are equal.
	 */
	t = p256_add(&P, &Q);
	reduce_final_f256(P.z);
	z = 0;
	for (i = 0; i < 9; i ++) {
		z |= P.z[i];
	}
	z = MG_BR_EQ(z, 0);
	p256_double(&Q);

	/*
	 * If z is 1 then either P+Q = 0 (t = 1) or P = Q (t = 0). So we
	 * have the following:
	 *
	 *   z = 0, t = 0   return P (normal addition)
	 *   z = 0, t = 1   return P (normal addition)
	 *   z = 1, t = 0   return Q (a 'double' case)
	 *   z = 1, t = 1   report an error (P+Q = 0)
	 */
	MG_BR_CCOPY(z & ~t, &P, &Q, sizeof Q);
	p256_to_affine(&P);
	p256_encode(A, &P);
	r &= ~(z & t);
	return r;
}

static const struct mg_br_ec_impl mg_br_ec_p256_m31 = {
	(uint32_t)0x00800000,
	&p256_api_generator,
	&p256_api_order,
	&p256_api_xoff,
	&p256_api_mul,
	&p256_api_mulgen,
	&p256_api_muladd
};

#undef ARSH
#undef ARSHW

// mg_p256_*() for MG_P256_BEAR, see tls_p256.h
int mg_p256_make_key(uint8_t public_key[64], uint8_t private_key[32]) {
  const unsigned char *n;
  unsigned char q[65];
  uint32_t x[I31_LEN], nn[I31_LEN];
  size_t nlen;
  int tries, ok = 0;
  n = mg_br_ec_p256_m31.order(MG_BR_EC_secp256r1, &nlen);
  mg_br_i31_decode(nn, n, nlen);
  for (tries = 0; tries < 64 && !ok; tries++) {  // Private key in [1, n-1]
    if (!mg_random(private_key, 32)) break;
    ok = mg_br_i31_decode_mod(x, private_key, 32, nn) && !mg_br_i31_iszero(x);
  }
  if (ok) {
    mg_br_ec_p256_m31.mulgen(q, private_key, 32, MG_BR_EC_secp256r1);
    memcpy(public_key, q + 1, 64);
  }
  mg_bzero((unsigned char *) x, sizeof(x));
  return ok;
}

int mg_p256_compute_public_key(const uint8_t private_key[32],
                               uint8_t public_key[64]) {
  const unsigned char *n;
  unsigned char q[65];
  uint32_t x[I31_LEN], nn[I31_LEN];
  size_t nlen;
  int ok;
  n = mg_br_ec_p256_m31.order(MG_BR_EC_secp256r1, &nlen);
  mg_br_i31_decode(nn, n, nlen);
  ok = mg_br_i31_decode_mod(x, private_key, 32, nn) && !mg_br_i31_iszero(x);
  if (ok) {
    mg_br_ec_p256_m31.mulgen(q, private_key, 32, MG_BR_EC_secp256r1);
    memcpy(public_key, q + 1, 64);
  }
  mg_bzero((unsigned char *) x, sizeof(x));
  return ok;
}

int mg_p256_shared_secret(const uint8_t public_key[64],
                          const uint8_t private_key[32], uint8_t secret[32]) {
  unsigned char q[65];
  uint32_t ok;
  q[0] = 0x04;
  memcpy(q + 1, public_key, 64);
  ok = mg_br_ec_p256_m31.mul(q, sizeof(q), private_key, 32,
                             MG_BR_EC_secp256r1);  // Validates the point
  memcpy(secret, q + 1, 32);
  mg_bzero(q, sizeof(q));
  return ok == 1;
}

int mg_p256_sign_with_k(const uint8_t private_key[32], const uint8_t *hash,
                        size_t hash_size, const uint8_t k[32],
                        uint8_t signature[64]) {
  return mg_br_ecdsa_i31_sign_raw(&mg_br_ec_p256_m31, MG_BR_EC_secp256r1,
                                  hash, hash_size, private_key, 32, k,
                                  signature) == 64;
}

int mg_p256_verify(const uint8_t public_key[64], const uint8_t *hash,
                   size_t hash_size, const uint8_t signature[64]) {
  unsigned char q[65];
  q[0] = 0x04;
  memcpy(q + 1, public_key, 64);
  return mg_br_ecdsa_i31_vrfy_raw(&mg_br_ec_p256_m31, MG_BR_EC_secp256r1, hash,
                                  hash_size, q, sizeof(q), signature, 64) == 1;
}
#endif  // MG_P256 == MG_P256_BEAR

// P-256 group order n, big endian
static const uint8_t s_p256_n[32] = {
    0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff,
    0xff, 0xff, 0xff, 0xff, 0xff, 0xbc, 0xe6, 0xfa, 0xad, 0xa7, 0x17,
    0x9e, 0x84, 0xf3, 0xb9, 0xca, 0xc2, 0xfc, 0x63, 0x25, 0x51};

// Compute d = a - n, return 1 if a < n (borrow). Constant time
static uint32_t p256_sub_n(uint8_t d[32], const uint8_t a[32]) {
  uint32_t borrow = 0, v;
  int i;
  for (i = 31; i >= 0; i--) {
    v = (uint32_t) a[i] - s_p256_n[i] - borrow;
    d[i] = (uint8_t) v;
    borrow = (v >> 8) & 1;
  }
  return borrow;
}

// ECDSA with deterministic k, RFC 6979 section 3.2, HMAC-SHA256. For P-256,
// qlen == hlen == 256 bits, so bits2int() is just the leftmost 32 bytes
int mg_p256_sign(const uint8_t private_key[32], const uint8_t *hash,
                 size_t hash_size, uint8_t signature[64]) {
  uint8_t v[32], k[32], buf[32 + 1 + 32 + 32], h1[32], t[32];
  uint32_t lt, nz;
  int i, tries, ok = 0;

  // h1 = bits2octets(hash): leftmost 256 bits, reduced modulo n
  memset(h1, 0, sizeof(h1));
  if (hash_size >= 32) {
    memcpy(h1, hash, 32);
  } else {
    memcpy(h1 + 32 - hash_size, hash, hash_size);
  }
  lt = p256_sub_n(t, h1);
  for (i = 0; i < 32; i++) h1[i] = (uint8_t) MG_BR_MUX(lt, h1[i], t[i]);

  // Steps b-g: V = 0x01..., K = 0x00..., two HMAC rounds with x and h1
  memset(v, 1, sizeof(v));
  memset(k, 0, sizeof(k));
  memcpy(buf + 33, private_key, 32);
  memcpy(buf + 65, h1, 32);
  for (i = 0; i < 2; i++) {
    memcpy(buf, v, 32);
    buf[32] = (uint8_t) i;
    mg_hmac_sha256(k, k, 32, buf, sizeof(buf));
    mg_hmac_sha256(v, k, 32, v, 32);
  }

  // Step h: candidate k = V, must be in [1, n-1]
  for (tries = 0; tries < 64 && !ok; tries++) {
    mg_hmac_sha256(v, k, 32, v, 32);
    for (nz = 0, i = 0; i < 32; i++) nz |= v[i];
    lt = p256_sub_n(t, v);
    if (nz != 0 && lt) {
      ok = mg_p256_sign_with_k(private_key, hash, hash_size, v, signature);
    }
    if (!ok) {  // K = HMAC_K(V || 0x00), V = HMAC_K(V)
      memcpy(buf, v, 32);
      buf[32] = 0;
      mg_hmac_sha256(k, k, 32, buf, 33);
      mg_hmac_sha256(v, k, 32, v, 32);
    }
  }
  mg_bzero(v, sizeof(v));
  mg_bzero(k, sizeof(k));
  mg_bzero(buf, sizeof(buf));
  mg_bzero(t, sizeof(t));
  return ok;
}

#undef I31_LEN
#undef POINT_LEN
#undef ORDER_LEN
#endif  // MG_TLS == MG_TLS_BUILTIN || defined(MG_OTA_PUBLIC_KEY)

#ifdef MG_ENABLE_LINES
#line 1 "src/tls_p256_cortex.c"
#endif
// P-256 for Cortex-M4/M7/M33, https://github.com/Emill/P256-Cortex-M4
// commit ecd3ec2
// Enabled by MG_P256=MG_P256_LENNGREN. Provides mg_p256_*(), see tls_p256.h
//
// Copyright (c) 2017-2021 Emil Lenngren
// Copyright (c) 2021 Shortcut Labs AB
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// Mongoose changes: the library's C code, header, config and GCC assembly are
// merged into one file, the assembly is a top-level __asm__ without .end.
// Trimmed with unifdef to keygen, ECDH, sign and verify. Uses the no-FPU field
// multiplication and the constant RAM access pattern table lookup (safe with a
// data cache). Internal C functions are static, casts added in slide_257().
// mg_p256_*() wrappers are at the end




#if MG_P256 == MG_P256_LENNGREN && \
    (MG_TLS == MG_TLS_BUILTIN || defined(MG_OTA_PUBLIC_KEY))

typedef const uint32_t (*constarr)[8];

struct FGInteger {
    // To get the value this struct represents,
    // interpret signed_value as a two's complement 288-bit little endian integer,
    // and negate if flip_sign is -1
    int flip_sign; // 0 or -1
    uint32_t signed_value[9]; // of 288 bits, 257 are useful (top 31 bits are sign-extended from bit 256)
};

struct SignPrecomp {  // p256_sign_step1() result
    uint32_t r[8];
    uint32_t k_inv[8];
};

struct XYInteger {
    // To get the value this struct represents,
    // interpret signed_value as an unsigned 288-bit little endian integer,
    // and negate if flip_sign is -1
    int flip_sign; // 0 or -1
    uint32_t value[8]; // unsigned value, 0 <= value < P256_order
};

int P256_divsteps2_31(int delta, uint32_t f, uint32_t g, uint32_t res_matrix[4]);
void P256_matrix_mul_fg_9(uint32_t a, uint32_t b, const struct FGInteger fg[2], struct FGInteger *res);
void P256_matrix_mul_mod_n(uint32_t a, uint32_t b, const struct XYInteger xy[2], struct XYInteger *res);

void P256_to_montgomery(uint32_t aR[8], const uint32_t a[8]);
void P256_from_montgomery(uint32_t a[8], const uint32_t aR[8]);
bool P256_check_range_p(const uint32_t a[8]);

bool P256_check_range_n(const uint32_t a[8]);
void P256_mul_mod_n(uint32_t res[8], const uint32_t a[8], const uint32_t b[8]);
void P256_add_mod_n(uint32_t res[8], const uint32_t a[8], const uint32_t b[8]);
void P256_reduce_mod_n_32bytes(uint32_t res[8], const uint32_t a[8]);

void P256_select_point(uint32_t (*output)[8], uint32_t* table, uint32_t num_coordinates, uint32_t index);

void P256_jacobian_to_affine(uint32_t affine_mont_x[8], uint32_t affine_mont_y[8], const uint32_t jacobian_mont[3][8]);
bool P256_point_is_on_curve(const uint32_t x_mont[8], const uint32_t y_mont[8]);
void P256_double_j(uint32_t jacobian_point_out[3][8], const uint32_t jacobian_point_in[3][8]);
void P256_add_sub_j(uint32_t jacobian_point1[3][8], const uint32_t (*point2)[8], bool is_sub, bool p2_is_affine);
bool P256_verify_last_step(const uint32_t r[8], const uint32_t jacobian_point[3][8]);

void P256_negate_mod_p_if(uint32_t out[8], const uint32_t in[8], uint32_t should_negate);
void P256_negate_mod_n_if(uint32_t out[8], const uint32_t in[8], uint32_t should_negate);

extern uint32_t P256_order[9];

static const uint32_t one_montgomery[8] = {1, 0, 0, 0xffffffff, 0xffffffff, 0xffffffff, 0xfffffffe, 0};

// This table contains 1G, 3G, 5G, ... 15G in affine coordinates in montgomery form
static const uint32_t p256_basepoint_precomp[8][2][8] = {
{{0x18a9143c, 0x79e730d4, 0x5fedb601, 0x75ba95fc, 0x77622510, 0x79fb732b, 0xa53755c6, 0x18905f76},
{0xce95560a, 0xddf25357, 0xba19e45c, 0x8b4ab8e4, 0xdd21f325, 0xd2e88688, 0x25885d85, 0x8571ff18}},
{{0x4eebc127, 0xffac3f90, 0x87d81fb, 0xb027f84a, 0x87cbbc98, 0x66ad77dd, 0xb6ff747e, 0x26936a3f},
{0xc983a7eb, 0xb04c5c1f, 0x861fe1a, 0x583e47ad, 0x1a2ee98e, 0x78820831, 0xe587cc07, 0xd5f06a29}},
{{0xc45c61f5, 0xbe1b8aae, 0x94b9537d, 0x90ec649a, 0xd076c20c, 0x941cb5aa, 0x890523c8, 0xc9079605},
{0xe7ba4f10, 0xeb309b4a, 0xe5eb882b, 0x73c568ef, 0x7e7a1f68, 0x3540a987, 0x2dd1e916, 0x73a076bb}},
{{0xa0173b4f, 0x746354e, 0xd23c00f7, 0x2bd20213, 0xc23bb08, 0xf43eaab5, 0xc3123e03, 0x13ba5119},
{0x3f5b9d4d, 0x2847d030, 0x5da67bdd, 0x6742f2f2, 0x77c94195, 0xef933bdc, 0x6e240867, 0xeaedd915}},
{{0x264e20e8, 0x75c96e8f, 0x59a7a841, 0xabe6bfed, 0x44c8eb00, 0x2cc09c04, 0xf0c4e16b, 0xe05b3080},
{0xa45f3314, 0x1eb7777a, 0xce5d45e3, 0x56af7bed, 0x88b12f1a, 0x2b6e019a, 0xfd835f9b, 0x86659cd}},
{{0x6245e404, 0xea7d260a, 0x6e7fdfe0, 0x9de40795, 0x8dac1ab5, 0x1ff3a415, 0x649c9073, 0x3e7090f1},
{0x2b944e88, 0x1a768561, 0xe57f61c8, 0x250f939e, 0x1ead643d, 0xc0daa89, 0xe125b88e, 0x68930023}},
{{0x4b2ed709, 0xccc42563, 0x856fd30d, 0xe356769, 0x559e9811, 0xbcbcd43f, 0x5395b759, 0x738477ac},
{0xc00ee17f, 0x35752b90, 0x742ed2e3, 0x68748390, 0xbd1f5bc1, 0x7cd06422, 0xc9e7b797, 0xfbc08769}},
{{0xbc60055b, 0x72bcd8b7, 0x56e27e4b, 0x3cc23ee, 0xe4819370, 0xee337424, 0xad3da09, 0xe2aa0e43},
{0x6383c45d, 0x40b8524f, 0x42a41b25, 0xd7663554, 0x778a4797, 0x64efa6de, 0x7079adf4, 0x2042170a}}
};

// This contains two tables, 8 points each in affine coordinates in montgomery form
// The first table contains these points:
// (2^192 - 2^128 - 2^64 - 1)G
// (2^192 - 2^128 - 2^64 + 1)G
// (2^192 - 2^128 + 2^64 - 1)G
// (2^192 - 2^128 + 2^64 + 1)G
// (2^192 + 2^128 - 2^64 - 1)G
// (2^192 + 2^128 - 2^64 + 1)G
// (2^192 + 2^128 + 2^64 - 1)G
// (2^192 + 2^128 + 2^64 + 1)G
// The second table contains the same points multiplied by 2^32
static const uint32_t p256_basepoint_precomp2[2][8][2][8] =
{
{
{{0x670844e0, 0x52d8a7c9, 0xef68a29d, 0xe33bdc, 0x4bdb7361, 0xf3d2848, 0x91c5304d, 0x5222c821},
{0xdf73fc25, 0xea6d2944, 0x255c81b, 0xa04c0f55, 0xefe488a8, 0x29acdc97, 0x80a560de, 0xbe2e158f}},
{{0x2b13e673, 0xfc8511ee, 0xd103ed24, 0xffc58dee, 0xea7e99b8, 0x1022523a, 0x4afc8a17, 0x8f43ea39},
{0xc5f33d0b, 0x8f4e2dbc, 0xd0aa1681, 0x3bc099fa, 0x79ff9df1, 0xffbb7b41, 0xd58b57c4, 0x180de09d}},
{{0x8bd1cda5, 0x56430752, 0x8e05eda5, 0x1807577f, 0x956896e9, 0x99c699b, 0xf1f0efb5, 0x83d6093d},
{0xed97061c, 0xef5af17e, 0x30d4c3c, 0x35b977b8, 0x49229439, 0x81fa75a2, 0xa0b6d35d, 0xf5a22070}},
{{0x74f81cf1, 0x814c5365, 0x120065b, 0xe30baff7, 0x15132621, 0x80ae1256, 0x36a80788, 0x16d2b8cb},
{0xecc50bca, 0x33d14697, 0x17aedd21, 0x19a9dfb0, 0xedc3f766, 0x523fbcc7, 0xb2cf5afd, 0x9c4de6dd}},
{{0xcf0d9f6d, 0x5305a9e6, 0x81a9b021, 0x5839172f, 0x75c687cf, 0xcca7a4dd, 0x844be22f, 0x36d59b3e},
{0x111a53e9, 0xcace7e62, 0xf063f3a1, 0x91c843d4, 0xda812da, 0xbf77e5f0, 0x437f3176, 0xe64af9c}},
{{0xcf07517d, 0xdbd568bb, 0xba6830b9, 0x2f1afba2, 0xe6c4c2a6, 0x15b6807c, 0xe4966aef, 0x91c7eabc},
{0xd6b2b6e6, 0x716dea1b, 0x19f85b4b, 0x248c43d1, 0x4a315e2a, 0x16dcfd60, 0xc72b3d0b, 0x15fdd303}},
{{0x42b7dfd5, 0xe40bf9f4, 0x2d934f2a, 0x673689f3, 0x30a6f50b, 0x8314beb4, 0x976ec64e, 0xd17af2bc},
{0x1ee7ddf1, 0x39f66c4f, 0x68ea373c, 0x7f68e18b, 0x53d0b186, 0x5166c1f2, 0x7be58f14, 0x95dda601}},
{{0x42913074, 0xd5ae356, 0x48a542b1, 0x55491b27, 0xb310732a, 0x469ca665, 0x5f1a4cc1, 0x29591d52},
{0xb84f983f, 0xe76f5b6b, 0x9f5f84e1, 0xbe7eef41, 0x80baa189, 0x1200d496, 0x18ef332c, 0x6376551f}}
},
{
{{0x7c4e54f5, 0xb9e5cbc0, 0xe1410e34, 0xc53a1a17, 0xec454425, 0x3e199130, 0x1700902e, 0xb029c97e},
{0x786423b6, 0x2de66e11, 0xb41a95be, 0x262dc914, 0x451b683, 0x51766abd, 0x85bb6fb1, 0x55ad5f34}},
{{0x9066cb79, 0x74f4f1c, 0x30c8b94e, 0x1ab31bd6, 0xd74275b3, 0x6d3f012f, 0x9ddcce40, 0xa214d0b1},
{0xd165050a, 0x24aedf74, 0xe0e5dc3e, 0x95f17ece, 0xd9224456, 0x6ada9cda, 0x2dd60eea, 0x1fadb2d1}},
{{0xe20cfb9b, 0xa3d83091, 0xba76e0cb, 0xae79c975, 0xc8858a6e, 0xa5f2a588, 0x874a3168, 0xe897a5f4},
{0x7d48f096, 0xf6c1ef40, 0xc35b132c, 0x1f9c516b, 0x53c479fd, 0xe1040f91, 0x9df06743, 0x60e881f}},
{{0x52a90e51, 0x9e0ad72, 0x38c50a96, 0xb7e66ea3, 0x7d997770, 0xab32ad05, 0x445671cb, 0xceaffe2},
{0x5d37cc99, 0xdfbe753c, 0xe0fea2d5, 0x95d068cc, 0x4dd77cb6, 0x1e37cdda, 0x55530688, 0x88c5a4bb}},
{{0xc7744f1, 0x3413f033, 0xbc816702, 0x23c05c89, 0x1192b5ac, 0x2322ee9a, 0x373180bb, 0xc1636a0},
{0xbdde0207, 0xfe2f3d4, 0xc23578d8, 0xe1a093a, 0xc888ead, 0x6e5f0d1, 0x52a2b660, 0x9ca285a5}},
{{0xce923964, 0xdae76995, 0xa34c7993, 0xcc96493a, 0xea73d9e7, 0xd19b5144, 0x311e6e34, 0x4a5c263},
{0xd9a2a443, 0x7db5b32b, 0x2cfd960c, 0x3754bd33, 0xa430f15, 0xc5bcc98, 0xd9a94574, 0x5651201f}},
{{0xfc0418fe, 0xebdd8921, 0x34e20036, 0x37015b39, 0xdf03a353, 0xcf4fcd8f, 0xf12cab16, 0xdc2de6e1},
{0xd071df14, 0x9c17cc1a, 0x63415530, 0xd7c5e6a3, 0x68f3fb1e, 0xb5301660, 0x18269301, 0xb5f70bc9}},
{{0x79ec1a0f, 0x2d8daefd, 0xceb39c97, 0x3bbcd6fd, 0x58f61a95, 0xf5575ffc, 0xadf7b420, 0xdbd986c4},
{0x15f39eb7, 0x81aa8814, 0xb98d976c, 0x6ee2fcf5, 0xcf2f717d, 0x5465475d, 0x6860bbd0, 0x8e24d3c4}}
}
};

// Takes the leftmost 256 bits in hash (treated as big endian),
// and converts to little endian integer z.
static void hash_to_z(uint32_t z[8], const uint8_t* hash, uint32_t hashlen) {
    if (hashlen > 32) {
        hashlen = 32;
    }
    for (uint32_t i = 0; i < hashlen; i++) {
        ((uint8_t*)z)[i] = hash[hashlen - 1 - i];
    }
    for (uint32_t i = hashlen; i < 32; i++) {
        ((uint8_t*)z)[i] = 0;
    }
}

// Creates a representation of a (little endian integer),
// so that r[0] + 2*r[1] + 2^2*r[2] + 2^3*r[3] + ... = a,
// where each r[i] is -15, -13, ..., 11, 13, 15 or 0.
// Only around 1/5.5 of the r[i] will be non-zero.
static void slide_257(signed char r[257], const uint8_t a[32]) {
    for (int i = 0; i < 256; ++i) {
        r[i] = 1 & (a[i >> 3] >> (i & 7));
    }
    r[256] = 0;

    for (int i = 0; i < 256; i++) {
        if (r[i] != 0) {
            for (int b = 1; b <= 4 && i + b < 256; b++) {
                if (r[i + b] != 0) {
                    if (r[i] + (r[i + b] << b) <= 15) {
                        r[i] = (signed char) (r[i] + (r[i + b] << b)); r[i + b] = 0;
                    } else if (r[i] - (r[i + b] << b) >= -15) {
                        r[i] = (signed char) (r[i] - (r[i + b] << b));
                        for (;;) {
                            r[i + b] = 0;
                            b++;
                            if (!r[i + b]) {
                                r[i + b] = 1;
                                b--; // Will be added back after loop footer b++
                                break;
                            }
                        }
                    } else {
                        break;
                    }
                }
            }
        }
    }
}

static void P256_mod_n_inv(uint32_t out[8], const uint32_t in[8]) {
    // This function follows the algorithm in section 12.1 of https://gcd.cr.yp.to/safegcd-20190413.pdf.
    // It has been altered in the following ways:
    //   1. Due to 32-bit cpu, we use 24 * 31 iterations instead of 12 * 62.
    //   2. P-256 modulus instead of 2^255-19.
    //      744 iterations are still enough and slightly more than the required 741 (floor((49*256+57)/17)).
    //   3. Step 5 has been corrected to go back to step 2 instead of step 3.
    //   4. The order of the matrix multiplications in step 6 has been changed to (T24*(T23*(T22*(...*(T1*[0, 1]))))),
    //      where [0, 1] is a column vector to make it possible to be able to extract the "top-right corner", v, of T24*T23*...*T1.
    //      The result v will then be contained in the first element of the resulting column vector.
    
    struct {
        struct FGInteger fg[2]; // f and g
        struct XYInteger xy[2]; // x and y
    } state[2]; // "current" and "next"
    
    state[0].fg[0].flip_sign = 0; // non-negative f
    memcpy(&state[0].fg[0].signed_value, P256_order, 36); // f
    state[0].fg[1].flip_sign = 0; // non-negative g
    memcpy(&state[0].fg[1].signed_value, in, 32); // g
    state[0].fg[1].signed_value[8] = 0; // upper bits of g are 0
    memset(&state[0].xy, 0, sizeof(state[0].xy));
    // We later need a factor 2^-744. The montgomery multiplication gives 2^(24*-32)=2^-768, so multiply the init value (1) by 2^24 here.
    state[0].xy[1].value[0] = 1U << 24;
    
    int delta = 1;
    for (int i = 0; i < 24; i++) {
        // Scaled translation matrix Ti
        uint32_t matrix[4]; // element range: [-2^30, 2^31] (negative numbers are stored in two's complement form)
        
        // Decode f and g into two's complement representation and use the lowest 32 bits in the P256_divsteps2_31 calculation
        uint32_t negate_f = state[i % 2].fg[0].flip_sign;
        uint32_t negate_g = state[i % 2].fg[1].flip_sign;
        delta = P256_divsteps2_31(delta, (state[i % 2].fg[0].signed_value[0] ^ negate_f) - negate_f, (state[i % 2].fg[1].signed_value[0] ^ negate_g) - negate_g, matrix);
        
        // "Jump step", calculates the new f and g values that applies after 31 divstep2 iterations
        P256_matrix_mul_fg_9(matrix[0], matrix[1], state[i % 2].fg, &state[(i + 1) % 2].fg[0]);
        P256_matrix_mul_fg_9(matrix[2], matrix[3], state[i % 2].fg, &state[(i + 1) % 2].fg[1]);
        
        // Iterate the result vector
        // Due to montgomery multiplication inside this function, each step also adds a 2^-32 factor
        P256_matrix_mul_mod_n(matrix[0], matrix[1], state[i % 2].xy, &state[(i + 1) % 2].xy[0]);
        P256_matrix_mul_mod_n(matrix[2], matrix[3], state[i % 2].xy, &state[(i + 1) % 2].xy[1]);
    }
    // Calculates val^-1 = sgn(f) * v * 2^-744, where v is the "top-right corner" of the resulting T24*T23*...*T1 matrix.
    // In this implementation, at this point x contains v * 2^-744.
    P256_negate_mod_n_if(out, &state[0].xy[0].value[0], (state[0].xy[0].flip_sign ^ state[0].fg[0].flip_sign ^ state[0].fg[0].signed_value[8]) & 1);
}

// Constant time abs
static inline uint32_t abs_int(int8_t a) {
    uint32_t a_u = (uint32_t)(int32_t)a;
    uint32_t mask = a_u >> 31;
    mask |= mask << 1;
    mask |= mask << 2;
    uint32_t result = (-a) & mask;
    result |= a & (mask ^ 0xf);
    return result;
}

// Calculates scalar*P in constant time (except for the scalars 2 and n-2, for which the results take a few extra cycles to compute)
static void scalarmult_variable_base(uint32_t output_mont_x[8], uint32_t output_mont_y[8], const uint32_t input_mont_x[8], const uint32_t input_mont_y[8], const uint32_t scalar[8]) {
    // Based on https://eprint.iacr.org/2014/130.pdf, Algorithm 1.
    
    uint32_t scalar2[8];
    int8_t e[64];
    
    // The algorithm used requires the scalar to be odd. If even, negate the scalar modulo p to make it odd, and later negate the end result.
    bool even = (scalar[0] & 1) ^ 1;
    P256_negate_mod_n_if(scalar2, scalar, even);
    
    // Rewrite the scalar as e[0] + 2^4*e[1] + 2^8*e[2] + ... + 2^252*e[63], where each e[i] is an odd number and -15 <= e[i] <= 15.
    e[0] = scalar2[0] & 0xf;
    for (int i = 1; i < 64; i++) {
        // Extract 4 bits
        e[i] = (scalar2[i / 8] >> ((i % 8) * 4)) & 0xf;
        // If even, subtract 2^4 from e[i - 1] and add 1 to e[i]
        e[i - 1] -= (int8_t) (((e[i] & 1) ^ 1) << 4);
        e[i] |= 1;
    }
    
    // Create a table of P, 3P, 5P, ... 15P.
    uint32_t table[8][3][8];
    memcpy(table[0][0], input_mont_x, 32);
    memcpy(table[0][1], input_mont_y, 32);
    memcpy(table[0][2], one_montgomery, 32);
    P256_double_j(table[7], (constarr)table[0]);
    for (int i = 1; i < 8; i++) {
        memcpy(table[i], table[7], 96);
        P256_add_sub_j(table[i], (constarr)table[i - 1], 0, 0);
    }
    
    // Calculate the result as (((((((((e[63]*G)*2^4)+e[62])*2^4)+e[61])*2^4)...)+e[1])*2^4)+e[0] = (2^252*e[63] + 2^248*e[62] + ... + e[0])*G.
    
    uint32_t current_point[3][8];
    
    // e[63] is never negative
    P256_select_point(current_point, (uint32_t*)table, 3, e[63] >> 1);
    
    for (uint32_t i = 63; i --> 0;) {
        for (int j = 3; j >= 0; j--) {
            P256_double_j(current_point, (constarr)current_point);
        }
        uint32_t selected_point[3][8];
        P256_select_point(selected_point, (uint32_t*)table, 3, abs_int(e[i]) >> 1);
        P256_negate_mod_p_if(selected_point[1], selected_point[1], (uint8_t)e[i] >> 7);
        
        // There is (only) one odd input scalar that leads to an exception when i == 0: n-2,
        // in that case current_point will be equal to selected_point and hence a doubling
        // will occur instead. We don't bother fixing the same constant time for that case since
        // the probability of that random value to be generated is around 1/2^255 and an
        // attacker could easily test this case anyway.
        P256_add_sub_j(current_point, (constarr)selected_point, false, false);
    }
    P256_jacobian_to_affine(output_mont_x, output_mont_y, (constarr)current_point);
    
    // If the scalar was initially even, we now negate the result to get the correct result, since -(scalar*G) = (-scalar*G).
    // This is done by negating y, since -(x,y) = (x,-y).
    P256_negate_mod_p_if(output_mont_y, output_mont_y, even);
}

#define get_bit(arr, i) ((arr[(i) / 32] >> ((i) % 32)) & 1)

// Calculates scalar*G in constant time
static void scalarmult_fixed_base(uint32_t output_mont_x[8], uint32_t output_mont_y[8], const uint32_t scalar[8]) {
    uint32_t scalar2[8];
    
    // Just as with the algorithm used in variable base scalar multiplication, this algorithm requires the scalar to be odd.
    bool even = (scalar[0] & 1) ^ 1;
    P256_negate_mod_n_if(scalar2, scalar, even);
    
    // This algorithm conceptually rewrites the odd scalar as s[0] + 2^1*s[1] + 2^2*s[2] + ... + 2^255*s[255], where each s[i] is -1 or 1.
    // By initially setting s[i] to the corresponding bit S[i] in the original odd scalar S, we go from lsb to msb, and whenever a value s[i] is 0,
    // increase s[i] by 1 and decrease s[i-1] by 2.
    // This will result in that s[i] = S[i+1] == 1 ? 1 : -1 for i < 255, and s[255] = 1.
    
    // We then form the scalars abs(s[j] + s[j+64]*2^64 + s[j+128]*2^128 + s[j+192]*2^192)*(2^32 * floor(j / 32)) for different 0 <= j < 64.
    // Each scalar times G has already been precomputed in p256_basepoint_precomp2.
    // That way we only need 31 point doublings and 63 point additions.
    
    uint32_t current_point[3][8];
    uint32_t selected_point[2][8];
    
    for (uint32_t i = 32; i --> 0;) {
        {
            uint32_t mask = get_bit(scalar2, i + 32 + 1) | (get_bit(scalar2, i + 64 + 32 + 1) << 1) | (get_bit(scalar2, i + 2 * 64 + 32 + 1) << 2);
            if (i == 31) {
                P256_select_point(current_point, (uint32_t*)p256_basepoint_precomp2[1], 2, mask);
                memcpy(current_point[2], one_montgomery, 32);
            } else {
                P256_double_j(current_point, (constarr)current_point);
                
                uint32_t sign = get_bit(scalar2, i + 3 * 64 + 32 + 1) - 1; // positive: 0, negative: -1
                mask = (mask ^ sign) & 7;
                P256_select_point(selected_point, (uint32_t*)p256_basepoint_precomp2[1], 2, mask);
                P256_negate_mod_p_if(selected_point[1], selected_point[1], sign & 1);
                P256_add_sub_j(current_point, (constarr)selected_point, false, true);
            }
        }
        {
            uint32_t mask = get_bit(scalar2, i + 1) | (get_bit(scalar2, i + 64 + 1) << 1) | (get_bit(scalar2, i + 2 * 64 + 1) << 2);
            uint32_t sign = get_bit(scalar2, i + 3 * 64 + 1) - 1; // positive: 0, negative: -1
            mask = (mask ^ sign) & 7;
            P256_select_point(selected_point, (uint32_t*)p256_basepoint_precomp2[0], 2, mask);
            P256_negate_mod_p_if(selected_point[1], selected_point[1], sign & 1);
            P256_add_sub_j(current_point, (constarr)selected_point, false, true);
        }
    }
    P256_jacobian_to_affine(output_mont_x, output_mont_y, (constarr)current_point);
    
    // Negate final result if the scalar was initially even.
    P256_negate_mod_p_if(output_mont_y, output_mont_y, even);
}

static void p256_convert_endianness(void* output, const void* input, size_t byte_len) {
    for (size_t i = 0; i < byte_len / 2; i++) {
        uint8_t t = ((uint8_t*)input)[byte_len - 1 - i];
        ((uint8_t*)output)[byte_len - 1 - i] = ((uint8_t*)input)[i];
        ((uint8_t*)output)[i] = t;
    }
}

static bool p256_verify(const uint32_t public_key_x[8], const uint32_t public_key_y[8], const uint8_t* hash, uint32_t hashlen_in_bytes, const uint32_t r[8], const uint32_t s[8]) {
    if (!P256_check_range_n(r) || !P256_check_range_n(s)) {
        return false;
    }
    
    if (!P256_check_range_p(public_key_x) || !P256_check_range_p(public_key_y)) {
        return false;
    }
    
    uint32_t pk_table[8][3][8];
    P256_to_montgomery(pk_table[0][0], public_key_x);
    P256_to_montgomery(pk_table[0][1], public_key_y);
    memcpy(pk_table[0][2], one_montgomery, 32);
    
    if (!P256_point_is_on_curve(pk_table[0][0], pk_table[0][1])) {
        return false;
    }
    
    // Create a table of P, 3P, 5P, ..., 15P, where P is the public key.
    P256_double_j(pk_table[7], (constarr)pk_table[0]);
    for (int i = 1; i < 8; i++) {
        memcpy(pk_table[i], pk_table[7], 96);
        P256_add_sub_j(pk_table[i], (constarr)pk_table[i - 1], 0, 0);
    }
    
    uint32_t z[8], w[8], u1[8], u2[8];
    
    hash_to_z(z, hash, hashlen_in_bytes);
    
    P256_mod_n_inv(w, s);
    
    P256_mul_mod_n(u1, z, w);
    P256_mul_mod_n(u2, r, w);
    
    // Each value in these arrays will be an odd integer v, so that -15 <= v <= 15.
    // Around 1/5.5 of them will be non-zero.
    signed char slide_bp[257], slide_pk[257];
    slide_257(slide_bp, (uint8_t*)u1);
    slide_257(slide_pk, (uint8_t*)u2);
    
    uint32_t cp[3][8] = {0};
    
    for (int i = 256; i >= 0; i--) {
        P256_double_j(cp, (constarr)cp);
        if (slide_bp[i] > 0) {
            P256_add_sub_j(cp, p256_basepoint_precomp[slide_bp[i]/2], 0, 1);
        } else if (slide_bp[i] < 0) {
            P256_add_sub_j(cp, p256_basepoint_precomp[(-slide_bp[i])/2], 1, 1);
        }
        if (slide_pk[i] > 0) {
            P256_add_sub_j(cp, (constarr)pk_table[slide_pk[i]/2], 0, 0);
        } else if (slide_pk[i] < 0) {
            P256_add_sub_j(cp, (constarr)pk_table[(-slide_pk[i])/2], 1, 0);
        }
    }
    
    return P256_verify_last_step(r, (constarr)cp);
}

static bool p256_sign_step1(struct SignPrecomp *result, const uint32_t k[8]) {
    do {
        uint32_t point_res[2][8];
        if (!P256_check_range_n(k)) {
            break;
        }
        scalarmult_fixed_base(point_res[0], point_res[1], k);
        P256_mod_n_inv(result->k_inv, k);
        P256_from_montgomery(result->r, point_res[0]);
        P256_reduce_mod_n_32bytes(result->r, result->r);
        
        uint32_t r_sum = 0;
        for (int i = 0; i < 8; i++) {
            r_sum |= result->r[i];
        }
        if (r_sum == 0) {
            break;
        }
        return true;
    } while (false);
    
    memset(result, 0, sizeof(struct SignPrecomp));
    return false;
}

static bool p256_sign_step2(uint32_t r[8], uint32_t s[8], const uint8_t* hash, uint32_t hashlen_in_bytes, const uint32_t private_key[8], struct SignPrecomp *sign_precomp) {
    do {
        if (!P256_check_range_n(sign_precomp->k_inv) || !P256_check_range_n(sign_precomp->r)) { // just make sure user did not input an obviously invalid precomp
            break;
        }
        uint32_t *const z = r;
        hash_to_z(z, hash, hashlen_in_bytes);
        P256_mul_mod_n(s, sign_precomp->r, private_key);
        P256_add_mod_n(s, z, s);
        P256_mul_mod_n(s, sign_precomp->k_inv, s);
        
        memcpy(r, sign_precomp->r, 32);
        
        uint32_t s_sum = 0;
        for (int i = 0; i < 8; i++) {
            s_sum |= s[i];
        }
        if (s_sum == 0) {
            break;
        }
        memset(sign_precomp, 0, sizeof(*sign_precomp));
        return true;
    } while (false);
    
    memset(r, 0, 32);
    memset(s, 0, 32);
    return false;
}

static bool p256_sign(uint32_t r[8], uint32_t s[8], const uint8_t* hash, uint32_t hashlen_in_bytes, const uint32_t private_key[8], const uint32_t k[8]) {
    struct SignPrecomp t;
    if (!p256_sign_step1(&t, k)) {
        memset(r, 0, 32);
        memset(s, 0, 32);
        return false;
    }
    return p256_sign_step2(r, s, hash, hashlen_in_bytes, private_key, &t);
}

static bool p256_scalarmult_base(uint32_t result_x[8], uint32_t result_y[8], const uint32_t scalar[8]) {
    if (!P256_check_range_n(scalar)) {
        return false;
    }
    scalarmult_fixed_base(result_x, result_y, scalar);
    P256_from_montgomery(result_x, result_x);
    P256_from_montgomery(result_y, result_y);
    return true;
    
}

static bool p256_keygen(uint32_t public_key_x[8], uint32_t public_key_y[8], const uint32_t private_key[8]) {
    return p256_scalarmult_base(public_key_x, public_key_y, private_key);
}


static bool p256_scalarmult_generic_no_scalar_check(uint32_t output_mont_x[8], uint32_t output_mont_y[8], const uint32_t scalar[8], const uint32_t in_x[8], const uint32_t in_y[8]) {
    if (!P256_check_range_p(in_x) || !P256_check_range_p(in_y)) {
        return false;
    }
    
    P256_to_montgomery(output_mont_x, in_x);
    P256_to_montgomery(output_mont_y, in_y);
    
    if (!P256_point_is_on_curve(output_mont_x, output_mont_y)) {
        return false;
    }
    
    scalarmult_variable_base(output_mont_x, output_mont_y, output_mont_x, output_mont_y, scalar);
    return true;
}

static bool p256_ecdh_calc_shared_secret(uint8_t shared_secret[32], const uint32_t private_key[8], const uint32_t others_public_key_x[8], const uint32_t others_public_key_y[8]) {
    uint32_t result_x[8], result_y[8];
    if (!p256_scalarmult_generic_no_scalar_check(result_x, result_y, private_key, others_public_key_x, others_public_key_y)) {
        return false;
    }
    P256_from_montgomery(result_x, result_x);
    p256_convert_endianness(shared_secret, result_x, 32);
    return true;
}

// Assembler, from p256-cortex-m4-asm-gcc.S, converted to a top-level asm
// block so it can live in the amalgamated mongoose.c
__asm__(
    ".pushsection .text.mg_p256,\"ax\",%progbits\n"
    "\t.syntax unified\n"
    "\t.thumb\n"
    "\t.align 2\n"
    "\t.type P256_select_point, %function\n"
    "P256_select_point:\n"
    "\t.global P256_select_point\n"
    "\tpush {r0,r2,r3,r4-r11,lr}\n"
    "\tsubs r2,#1\n"
    "\tlsls r2,#5\n"
    "0:\n"
    "\trsbs r3,#0\n"
    "\tsbcs r3,r3,r3\n"
    "\tmvns r3,r3\n"
    "\tldm r1!,{r6-r12,lr}\n"
    "\tands r6,r3\n"
    "\tands r7,r3\n"
    "\tand r8,r3\n"
    "\tand r9,r3\n"
    "\tand r10,r3\n"
    "\tand r11,r3\n"
    "\tand r12,r3\n"
    "\tand lr,r3\n"
    "\tadds r1,r2\n"
    "\tmovs r3,#1\n"
    "1:\n"
    "\tldr r0,[sp,#8]\n"
    "\teors r0,r3\n"
    "\tmrs r0,apsr\n"
    "\tlsrs r0,#30\n"
    "\tldm r1!,{r4,r5}\n"
    "\tumlal r6,r2,r0,r4\n"
    "\tumlal r7,r3,r0,r5\n"
    "\tldm r1!,{r4,r5}\n"
    "\tumlal r8,r2,r0,r4\n"
    "\tumlal r9,r3,r0,r5\n"
    "\tldm r1!,{r4,r5}\n"
    "\tumlal r10,r2,r0,r4\n"
    "\tumlal r11,r3,r0,r5\n"
    "\tldm r1!,{r4,r5}\n"
    "\tumlal r12,r2,r0,r4\n"
    "\tumlal lr,r3,r0,r5\n"
    "\tadds r1,r2\n"
    "\tadds r3,#1\n"
    "\tcmp r3,#8\n"
    "\tbne 1b\n"
    "\tldm sp,{r0,r4}\n"
    "\tstm r0!,{r6-r12,lr}\n"
    "\tstr r0,[sp]\n"
    "\tsub r1,r1,r2, lsl #3\n"
    "\tsubs r1,#224\n"
    "\tsubs r4,#1\n"
    "\tstr r4,[sp,#4]\n"
    "\tldr r3,[sp,#8]\n"
    "\tbne 0b\n"
    "\tadd sp,#12\n"
    "\tpop {r4-r11,pc}\n"
    "\t.size P256_select_point, .-P256_select_point\n"
    "\t.type mul288x288, %function\n"
    "mul288x288:\n"
    "\tpush {r4-r11,lr}\n"
    "\tmov r4,r0\n"
    "\tmov r5,r2\n"
    "\tmov r6,r1\n"
    "\tmovs r1,#72\n"
    "\tbl setzero\n"
    "\tldm r5,{r0-r2,r8-r12,lr}\n"
    "\tmovs r7,#9\n"
    "0:\n"
    "\tldm r6!,{r5}\n"
    "\tpush {r6,r7}\n"
    "\tmovs r3,#0\n"
    "\tldm r4,{r6,r7}\n"
    "\tumaal r6,r3,r5,r0\n"
    "\tumaal r7,r3,r5,r1\n"
    "\tstm r4!,{r6,r7}\n"
    "\tldm r4,{r6,r7}\n"
    "\tumaal r6,r3,r5,r2\n"
    "\tumaal r7,r3,r5,r8\n"
    "\tstm r4!,{r6,r7}\n"
    "\tldm r4,{r6,r7}\n"
    "\tumaal r6,r3,r5,r9\n"
    "\tumaal r7,r3,r5,r10\n"
    "\tstm r4!,{r6,r7}\n"
    "\tldm r4,{r6,r7}\n"
    "\tumaal r6,r3,r5,r11\n"
    "\tumaal r7,r3,r5,r12\n"
    "\tstm r4!,{r6,r7}\n"
    "\tldm r4,{r6}\n"
    "\tumaal r3,r6,r5,lr\n"
    "\tstm r4!,{r3,r6}\n"
    "\tsubs r4,r4,#36\n"
    "\tpop {r6,r7}\n"
    "\tsubs r7,r7,#1\n"
    "\tbne 0b\n"
    "\tpop {r4-r11,pc}\n"
    "\t.size mul288x288, .-mul288x288\n"
    "\t.type setzero, %function\n"
    "setzero:\n"
    "\tmovs r2,#0\n"
    "\tmovs r3,#0\n"
    "0:\n"
    "\tstm r0!,{r2,r3}\n"
    "\tsubs r1,r1,#8\n"
    "\tbne 0b\n"
    "\tbx lr\n"
    "\t.size setzero, .-setzero\n"
    "\t.type P256_mulmod, %function\n"
    "P256_mulmod:\n"
    "\tpush {r2,lr}\n"
    "\tsub sp,#28\n"
    "\tldm r2,{r2,r3,r4,r5}\n"
    "\tldm r1!,{r0,r10,lr}\n"
    "\tumull r6,r11,r2,r0\n"
    "\tumull r7,r12,r3,r0\n"
    "\tumaal r7,r11,r2,r10\n"
    "\tpush {r6,r7}\n"
    "\tumull r8,r6,r4,r0\n"
    "\tumaal r8,r11,r3,r10\n"
    "\tumull r9,r7,r5,r0\n"
    "\tumaal r9,r11,r4,r10\n"
    "\tumaal r11,r7,r5,r10\n"
    "\tumaal r8,r12,r2,lr\n"
    "\tumaal r9,r12,r3,lr\n"
    "\tumaal r11,r12,r4,lr\n"
    "\tumaal r12,r7,r5,lr\n"
    "\tldm r1!,{r0,r10,lr}\n"
    "\tumaal r9,r6,r2,r0\n"
    "\tumaal r11,r6,r3,r0\n"
    "\tumaal r12,r6,r4,r0\n"
    "\tumaal r6,r7,r5,r0\n"
    "\tstrd r8,r9,[sp,#8]\n"
    "\tmov r9,#0\n"
    "\tumaal r11,r9,r2,r10\n"
    "\tumaal r12,r9,r3,r10\n"
    "\tumaal r6,r9,r4,r10\n"
    "\tumaal r7,r9,r5,r10\n"
    "\tmov r10,#0\n"
    "\tumaal r12,r10,r2,lr\n"
    "\tumaal r6,r10,r3,lr\n"
    "\tumaal r7,r10,r4,lr\n"
    "\tumaal r9,r10,r5,lr\n"
    "\tldr r8,[r1],#4\n"
    "\tmov lr,#0\n"
    "\tumaal lr,r6,r2,r8\n"
    "\tumaal r7,r6,r3,r8\n"
    "\tumaal r9,r6,r4,r8\n"
    "\tumaal r10,r6,r5,r8\n"
    "\tldr r8,[r1],#-28\n"
    "\tmov r0,#0\n"
    "\tumaal r7,r0,r2,r8\n"
    "\tumaal r9,r0,r3,r8\n"
    "\tumaal r10,r0,r4,r8\n"
    "\tumaal r6,r0,r5,r8\n"
    "\tpush {r0}\n"
    "\tldr r2,[sp,#40]\n"
    "\tadds r2,r2,#16\n"
    "\tldm r2,{r2,r3,r4,r5}\n"
    "\tldr r8,[r1],#4\n"
    "\tmov r0,#0\n"
    "\tumaal r11,r0,r2,r8\n"
    "\tstr r11,[sp,#16+4]\n"
    "\tumaal r12,r0,r3,r8\n"
    "\tumaal lr,r0,r4,r8\n"
    "\tumaal r0,r7,r5,r8\n"
    "\tldr r8,[r1],#4\n"
    "\tmov r11,#0\n"
    "\tumaal r12,r11,r2,r8\n"
    "\tstr r12,[sp,#20+4]\n"
    "\tumaal lr,r11,r3,r8\n"
    "\tumaal r0,r11,r4,r8\n"
    "\tumaal r11,r7,r5,r8\n"
    "\tldr r8,[r1],#4\n"
    "\tmov r12,#0\n"
    "\tumaal lr,r12,r2,r8\n"
    "\tstr lr,[sp,#24+4]\n"
    "\tumaal r0,r12,r3,r8\n"
    "\tumaal r11,r12,r4,r8\n"
    "\tumaal r10,r12,r5,r8\n"
    "\tldr r8,[r1],#4\n"
    "\tmov lr,#0\n"
    "\tumaal r0,lr,r2,r8\n"
    "\tstr r0,[sp,#28+4]\n"
    "\tumaal r11,lr,r3,r8\n"
    "\tumaal r10,lr,r4,r8\n"
    "\tumaal r6,lr,r5,r8\n"
    "\tldm r1!,{r0,r8}\n"
    "\tumaal r11,r9,r2,r0\n"
    "\tstr r11,[sp,#32+4]\n"
    "\tumaal r9,r10,r3,r0\n"
    "\tumaal r10,r6,r4,r0\n"
    "\tpop {r11}\n"
    "\tumaal r11,r6,r5,r0\n"
    "\tumaal r9,r7,r2,r8\n"
    "\tumaal r10,r7,r3,r8\n"
    "\tumaal r11,r7,r4,r8\n"
    "\tumaal r6,r7,r5,r8\n"
    "\tldm r1!,{r0,r8}\n"
    "\tumaal r10,r12,r2,r0\n"
    "\tumaal r11,r12,r3,r0\n"
    "\tumaal r6,r12,r4,r0\n"
    "\tumaal r7,r12,r5,r0\n"
    "\tumaal r11,lr,r2,r8\n"
    "\tumaal lr,r6,r3,r8\n"
    "\tumaal r6,r7,r4,r8\n"
    "\tumaal r7,r12,r5,r8\n"
    "\tpush {r6,r7,r12}\n"
    "\tadd r7,sp,#12\n"
    "\tldm r7,{r0-r8}\n"
    "\tmov r12,#0\n"
    "\tadds r3,r0\n"
    "\tadcs r4,r4,r1\n"
    "\tadcs r5,r5,r2\n"
    "\tadcs r6,r6,r0\n"
    "\tadcs r7,r7,r1\n"
    "\tadcs r8,r8,r0\n"
    "\tadcs r9,r9,r1\n"
    "\tadcs r10,r10,#0\n"
    "\tadcs r11,r11,#0\n"
    "\tadcs r12,r12,#0\n"
    "\tadds r6,r3\n"
    "\tadcs r7,r7,r4\n"
    "\tadcs r8,r8,r2\n"
    "\tadcs r9,r9,r3\n"
    "\tadcs r10,r10,r2\n"
    "\tadcs r11,r11,r3\n"
    "\tadcs r12,r12,#0\n"
    "\tsubs r7,r0\n"
    "\tsbcs r8,r8,r1\n"
    "\tsbcs r9,r9,r2\n"
    "\tsbcs r10,r10,r3\n"
    "\tsbcs r11,r11,#0\n"
    "\tsbcs r12,r12,#0\n"
    "\tpop {r1-r3}\n"
    "\tadds r0,lr,r12\n"
    "\tadcs r1,r1,#0\n"
    "\tmov r12,#0\n"
    "\tadcs r12,r12,#0\n"
    "\tadcs r8,r8,r5\n"
    "\tadcs r9,r9,r6\n"
    "\tadcs r10,r10,r4\n"
    "\tadcs r11,r11,r5\n"
    "\tadcs r0,r0,r4\n"
    "\tadcs r1,r1,r5\n"
    "\tadcs r2,r2,r12\n"
    "\tadcs r3,r3,#0\n"
    "\tmov r12,#0\n"
    "\tadcs r12,r12,#0\n"
    "\tadcs r10,r10,r7\n"
    "\tadcs r11,r11,#0\n"
    "\tadcs r0,r0,r6\n"
    "\tadcs r1,r1,r7\n"
    "\tadcs r2,r2,r6\n"
    "\tadcs r3,r3,r7\n"
    "\tadcs r12,r12,#0\n"
    "\tsubs r11,r4\n"
    "\tsbcs r0,r0,r5\n"
    "\tsbcs r1,r1,r6\n"
    "\tsbcs r2,r2,r7\n"
    "\tsbcs r3,r3,#0\n"
    "\tsbcs r12,r12,#0\n"
    "\tsubs r8,r8,#0xffffffff\n"
    "\tsbcs r9,r9,#0xffffffff\n"
    "\tsbcs r10,r10,#0xffffffff\n"
    "\tsbcs r11,r11,#0\n"
    "\tsbcs r4,r0,#0\n"
    "\tsbcs r5,r1,#0\n"
    "\tsbcs r6,r2,#1\n"
    "\tsbcs r7,r3,#0xffffffff\n"
    "\tsbc r12,r12,#0\n"
    "\tadds r0,r8,r12\n"
    "\tadcs r1,r9,r12\n"
    "\tadcs r2,r10,r12\n"
    "\tadcs r3,r11,#0\n"
    "\tadcs r4,r4,#0\n"
    "\tadcs r5,r5,#0\n"
    "\tadcs r6,r6,r12, lsr #31\n"
    "\tadcs r7,r7,r12\n"
    "\tadd sp,sp,#40\n"
    "\tpop {pc}\n"
    "\t.size P256_mulmod, .-P256_mulmod\n"
    "\t.type P256_sqrmod, %function\n"
    "P256_sqrmod:\n"
    "\tpush {lr}\n"
    "\tumull r9,r10,r0,r0\n"
    "\tumull r11,r12,r0,r1\n"
    "\tadds r11,r11,r11\n"
    "\tmov lr,#0\n"
    "\tumaal r10,r11,lr,lr\n"
    "\tpush {r9,r10}\n"
    "\tmov r9,#0\n"
    "\tumaal r9,r12,r0,r2\n"
    "\tadcs r9,r9,r9\n"
    "\tumaal r9,r11,r1,r1\n"
    "\tpush {r9}\n"
    "\tumull r9,r10,r0,r3\n"
    "\tumaal r9,r12,r1,r2\n"
    "\tadcs r9,r9,r9\n"
    "\tumaal r9,r11,lr,lr\n"
    "\tpush {r9}\n"
    "\tmov r9,#0\n"
    "\tumaal r9,r10,r0,r4\n"
    "\tumaal r9,r12,r1,r3\n"
    "\tadcs r9,r9,r9\n"
    "\tumaal r9,r11,r2,r2\n"
    "\tpush {r9}\n"
    "\tumull r9,r8,r0,r5\n"
    "\tumaal r9,r10,r1,r4\n"
    "\tumaal r9,r12,r2,r3\n"
    "\tadcs r9,r9,r9\n"
    "\tumaal r9,r11,lr,lr\n"
    "\tpush {r9}\n"
    "\tmov r9,#0\n"
    "\tumaal r9,r8,r1,r5\n"
    "\tumaal r9,r12,r2,r4\n"
    "\tumaal r9,r10,r0,r6\n"
    "\tadcs r9,r9,r9\n"
    "\tumaal r9,r11,r3,r3\n"
    "\tpush {r9}\n"
    "\tumull r9,r0,r0,r7\n"
    "\tumaal r9,r10,r1,r6\n"
    "\tumaal r9,r12,r2,r5\n"
    "\tumaal r9,r8,r3,r4\n"
    "\tadcs r9,r9,r9\n"
    "\tumaal r9,r11,lr,lr\n"
    "\tumaal r0,r8,r1,r7\n"
    "\tumaal r0,r10,r2,r6\n"
    "\tumaal r0,r12,r3,r5\n"
    "\tadcs r0,r0,r0\n"
    "\tumaal r11,r0,r4,r4\n"
    "\tumaal r12,r8,r2,r7\n"
    "\tumaal r12,r10,r3,r6\n"
    "\tmovs r2,#0\n"
    "\tumaal r12,r2,r4,r5\n"
    "\tadcs r1,r12,r12\n"
    "\tumaal r0,r1,lr,lr\n"
    "\tumaal r2,r8,r3,r7\n"
    "\tumaal r2,r10,r4,r6\n"
    "\tadcs r2,r2,r2\n"
    "\tumaal r1,r2,r5,r5\n"
    "\tmovs r3,#0\n"
    "\tumaal r3,r8,r4,r7\n"
    "\tumaal r3,r10,r5,r6\n"
    "\tadcs r3,r3,r3\n"
    "\tumaal r2,r3,lr,lr\n"
    "\tumaal r8,r10,r5,r7\n"
    "\tadcs r8,r8,r8\n"
    "\tumaal r3,r8,r6,r6\n"
    "\tumull r4,r5,lr,lr\n"
    "\tumaal r4,r10,r6,r7\n"
    "\tadcs r4,r4,r4\n"
    "\tumaal r4,r8,lr,lr\n"
    "\tadcs r10,r10,r10\n"
    "\tumaal r8,r10,r7,r7\n"
    "\tadcs r10,r10,lr\n"
    "\tpush {r4,r8,r10}\n"
    "\tadd r4,sp,#12\n"
    "\tldm r4,{r4-r8,r10,r12}\n"
    "\tX0 .req r10\n"
    "\tX1 .req r12\n"
    "\tX2 .req r8\n"
    "\tX3 .req r7\n"
    "\tX4 .req r6\n"
    "\tX5 .req r5\n"
    "\tX6 .req r4\n"
    "\tX7 .req r9\n"
    "\tX8 .req r11\n"
    "\tX9 .req r0\n"
    "\tX10 .req r1\n"
    "\tX11 .req r2\n"
    "\tX12 .req r3\n"
    "\tX13 .req r7\n"
    "\tX14 .req r8\n"
    "\tX15 .req r12\n"
    "\tadcs X3,X3,X0\n"
    "\tadcs X4,X4,X1\n"
    "\tadcs X5,X5,X2\n"
    "\tadcs X6,X6,X0\n"
    "\tadcs X7,X7,X1\n"
    "\tadcs X8,X8,X0\n"
    "\tadcs X9,X9,X1\n"
    "\tadcs X10,X10,#0\n"
    "\tadcs X11,X11,#0\n"
    "\tadcs lr,lr,#0\n"
    "\tadds X6,X3\n"
    "\tadcs X7,X7,X4\n"
    "\tadcs X8,X8,X2\n"
    "\tadcs X9,X9,X3\n"
    "\tadcs X10,X10,X2\n"
    "\tadcs X11,X11,X3\n"
    "\tadcs lr,lr,#0\n"
    "\tsubs X7,X0\n"
    "\tsbcs X8,X8,X1\n"
    "\tsbcs X9,X9,X2\n"
    "\tsbcs X10,X10,X3\n"
    "\tsbcs X11,X11,#0\n"
    "\tsbcs lr,lr,#0\n"
    "\tpop {X13,X14,X15}\n"
    "\tadds X0,X12,lr\n"
    "\tadcs X13,X13,#0\n"
    "\tmov lr,#0\n"
    "\tadcs lr,lr,#0\n"
    "\tadcs X8,X8,X5\n"
    "\tadcs X9,X9,X6\n"
    "\tadcs X10,X10,X4\n"
    "\tadcs X11,X11,X5\n"
    "\tadcs X0,X0,X4\n"
    "\tadcs X13,X13,X5\n"
    "\tadcs X14,X14,lr\n"
    "\tadcs X15,X15,#0\n"
    "\tmov lr,#0\n"
    "\tadcs lr,lr,#0\n"
    "\tadcs X10,X10,X7\n"
    "\tadcs X11,X11,#0\n"
    "\tadcs X0,X0,X6\n"
    "\tadcs X13,X13,X7\n"
    "\tadcs X14,X14,X6\n"
    "\tadcs X15,X15,X7\n"
    "\tadcs lr,lr,#0\n"
    "\tsubs X11,X4\n"
    "\tsbcs X0,X0,X5\n"
    "\tsbcs X13,X13,X6\n"
    "\tsbcs X14,X14,X7\n"
    "\tsbcs X15,X15,#0\n"
    "\tsbcs lr,lr,#0\n"
    "\tsubs r11,r11,#0xffffffff\n"
    "\tsbcs r9,r0,#0xffffffff\n"
    "\tsbcs r4,r1,#0xffffffff\n"
    "\tsbcs r3,r2,#0\n"
    "\tsbcs r6,r10,#0\n"
    "\tsbcs r5,r7,#0\n"
    "\tsbcs r10,r8,#1\n"
    "\tsbcs r8,r12,#0xffffffff\n"
    "\tsbcs r7,lr,#0\n"
    "\tadds r0,r11,r7\n"
    "\tadcs r1,r9,r7\n"
    "\tadcs r2,r4,r7\n"
    "\tadcs r3,r3,#0\n"
    "\tadcs r4,r6,#0\n"
    "\tadcs r5,r5,#0\n"
    "\tadcs r6,r10,r7, lsr #31\n"
    "\tadcs r7,r8,r7\n"
    "\tadd sp,#28\n"
    "\tpop {pc}\n"
    "\t.size P256_sqrmod, .-P256_sqrmod\n"
    "\t.type P256_submod, %function\n"
    "P256_submod:\n"
    "\tldm r1,{r3-r10}\n"
    "\tldm r2!,{r0,r1,r11,r12}\n"
    "\tsubs r3,r0\n"
    "\tsbcs r4,r4,r1\n"
    "\tsbcs r5,r5,r11\n"
    "\tsbcs r6,r6,r12\n"
    "\tldm r2,{r0,r1,r11,r12}\n"
    "\tsbcs r7,r7,r0\n"
    "\tsbcs r8,r8,r1\n"
    "\tsbcs r9,r9,r11\n"
    "\tsbcs r10,r10,r12\n"
    "\tsbcs r11,r11,r11\n"
    "\tadds r0,r3,r11\n"
    "\tadcs r1,r4,r11\n"
    "\tadcs r2,r5,r11\n"
    "\tadcs r3,r6,#0\n"
    "\tadcs r4,r7,#0\n"
    "\tadcs r5,r8,#0\n"
    "\tadcs r6,r9,r11, lsr #31\n"
    "\tadcs r7,r10,r11\n"
    "\tbx lr\n"
    "\t.size P256_submod, .-P256_submod\n"
    "\t.type P256_addmod, %function\n"
    "P256_addmod:\n"
    "\tldm r2,{r2-r9}\n"
    "\tldm r1!,{r0,r10,r11,r12}\n"
    "\tadds r2,r0\n"
    "\tadcs r3,r3,r10\n"
    "\tadcs r4,r4,r11\n"
    "\tadcs r5,r5,r12\n"
    "\tldm r1,{r0,r1,r11,r12}\n"
    "\tadcs r6,r6,r0\n"
    "\tadcs r7,r7,r1\n"
    "\tadcs r8,r8,r11\n"
    "\tadcs r9,r9,r12\n"
    "\tmovs r10,#0\n"
    "\tadcs r10,r10,r10\n"
    "\tsubs r2,#0xffffffff\n"
    "\tsbcs r3,r3,#0xffffffff\n"
    "\tsbcs r4,r4,#0xffffffff\n"
    "\tsbcs r5,r5,#0\n"
    "\tsbcs r6,r6,#0\n"
    "\tsbcs r7,r7,#0\n"
    "\tsbcs r8,r8,#1\n"
    "\tsbcs r9,r9,#0xffffffff\n"
    "\tsbcs r10,r10,#0\n"
    "\tadds r0,r2,r10\n"
    "\tadcs r1,r3,r10\n"
    "\tadcs r2,r4,r10\n"
    "\tadcs r3,r5,#0\n"
    "\tadcs r4,r6,#0\n"
    "\tadcs r5,r7,#0\n"
    "\tadcs r6,r8,r10, lsr #31\n"
    "\tadcs r7,r9,r10\n"
    "\tbx lr\n"
    "\t.size P256_addmod, .-P256_addmod\n"
    "\t.type P256_sqrmod_many, %function\n"
    "P256_sqrmod_many:\n"
    "\tpush {r8,lr}\n"
    "0:\n"
    "\tbl P256_sqrmod\n"
    "\tldr r8,[sp,#0]\n"
    "\tsubs r8,r8,#1\n"
    "\tstr r8,[sp,#0]\n"
    "\tbne 0b\n"
    "\tpop {r8,pc}\n"
    "\t.size P256_sqrmod_many, .-P256_sqrmod_many\n"
    "\t.type P256_sqrmod_many_and_mulmod, %function\n"
    "P256_sqrmod_many_and_mulmod:\n"
    "\tpush {r9,lr}\n"
    "\tbl P256_sqrmod_many\n"
    "\tpush {r0-r7}\n"
    "\tmov r1,sp\n"
    "\tldr r2,[sp,#32]\n"
    "\tbl P256_mulmod\n"
    "\tadd sp,#36\n"
    "\tpop {pc}\n"
    "\t.size P256_sqrmod_many_and_mulmod, .-P256_sqrmod_many_and_mulmod\n"
    "\t.type P256_modinv_sqrt, %function\n"
    "P256_modinv_sqrt:\n"
    "\tpush {r0-r8,lr}\n"
    "\tmov r8,#1\n"
    "\tmov r9,sp\n"
    "\tbl P256_sqrmod_many_and_mulmod\n"
    "\tpush {r0-r7}\n"
    "\tbl P256_sqrmod\n"
    "\tbl P256_sqrmod\n"
    "\tpush {r0-r7}\n"
    "\tmov r1,sp\n"
    "\tadd r2,sp,#32\n"
    "\tbl P256_mulmod\n"
    "\tadd r8,sp,#32\n"
    "\tstm r8,{r0-r7}\n"
    "\tmov r8,#8-4\n"
    "\tadd r9,sp,#32\n"
    "\tbl P256_sqrmod_many_and_mulmod\n"
    "\tpush {r0-r7}\n"
    "\tmov r8,#16-8\n"
    "\tmov r9,sp\n"
    "\tbl P256_sqrmod_many_and_mulmod\n"
    "\tpush {r0-r7}\n"
    "\tmov r8,#16\n"
    "\tmov r9,sp\n"
    "\tbl P256_sqrmod_many_and_mulmod\n"
    "\tpush {r0-r7}\n"
    "\tmov r8,#32\n"
    "\tadd r9,sp,#5*32\n"
    "\tbl P256_sqrmod_many_and_mulmod\n"
    "\tldr r8,[sp,#6*32]\n"
    "\tcmp r8,#0\n"
    "\tbne 0f\n"
    "\tmov r8,#192-64\n"
    "\tmov r9,sp\n"
    "\tbl P256_sqrmod_many_and_mulmod\n"
    "\tmov r8,#224-192\n"
    "\tmov r9,sp\n"
    "\tbl P256_sqrmod_many_and_mulmod\n"
    "\tmov r8,#240-224\n"
    "\tadd r9,sp,#32\n"
    "\tbl P256_sqrmod_many_and_mulmod\n"
    "\tmov r8,#248-240\n"
    "\tadd r9,sp,#64\n"
    "\tbl P256_sqrmod_many_and_mulmod\n"
    "\tmov r8,#252-248\n"
    "\tadd r9,sp,#128\n"
    "\tbl P256_sqrmod_many_and_mulmod\n"
    "\tmov r8,#256-252\n"
    "\tadd r9,sp,#96\n"
    "\tbl P256_sqrmod_many_and_mulmod\n"
    "\tstm sp,{r0-r7}\n"
    "\tmov r1,sp\n"
    "\tadd r2,sp,#5*32\n"
    "\tbl P256_mulmod\n"
    "\tb 1f\n"
    "0:\n"
    "\tmov r8,#160-64\n"
    "\tadd r9,sp,#5*32\n"
    "\tbl P256_sqrmod_many_and_mulmod\n"
    "\tmov r8,#254-160\n"
    "\tbl P256_sqrmod_many\n"
    "1:\n"
    "\tadd sp,#6*32+4\n"
    "\tpop {pc}\n"
    "\t.size P256_modinv_sqrt, .-P256_modinv_sqrt\n"
    "\t.type P256_times2, %function\n"
    "P256_times2:\n"
    "\tadds r0,r0\n"
    "\tadcs r1,r1,r1\n"
    "\tadcs r2,r2,r2\n"
    "\tadcs r3,r3,r3\n"
    "\tadcs r4,r4,r4\n"
    "\tadcs r5,r5,r5\n"
    "\tadcs r6,r6,r6\n"
    "\tadcs r7,r7,r7\n"
    "\tmov r8,#0\n"
    "\tadcs r8,r8,r8\n"
    "\tsubs r0,#0xffffffff\n"
    "\tsbcs r1,r1,#0xffffffff\n"
    "\tsbcs r2,r2,#0xffffffff\n"
    "\tsbcs r3,r3,#0\n"
    "\tsbcs r4,r4,#0\n"
    "\tsbcs r5,r5,#0\n"
    "\tsbcs r6,r6,#1\n"
    "\tsbcs r7,r7,#0xffffffff\n"
    "\tsbcs r8,r8,#0\n"
    "\tadds r0,r8\n"
    "\tadcs r1,r1,r8\n"
    "\tadcs r2,r2,r8\n"
    "\tadcs r3,r3,#0\n"
    "\tadcs r4,r4,#0\n"
    "\tadcs r5,r5,#0\n"
    "\tadcs r6,r6,r8, lsr #31\n"
    "\tadcs r7,r7,r8\n"
    "\tbx lr\n"
    "\t.size P256_times2, .-P256_times2\n"
    "\t.align 2\n"
    "R2_mod_p:\n"
    "\t.word 3\n"
    "\t.word 0\n"
    "\t.word 0xffffffff\n"
    "\t.word 0xfffffffb\n"
    "\t.word 0xfffffffe\n"
    "\t.word 0xffffffff\n"
    "\t.word 0xfffffffd\n"
    "\t.word 4\n"
    "\t.type P256_to_montgomery, %function\n"
    "P256_to_montgomery:\n"
    "\t.global P256_to_montgomery\n"
    "\tpush {r0,r4-r11,lr}\n"
    "\tadr r2,R2_mod_p\n"
    "\tbl P256_mulmod\n"
    "\tpop {r8}\n"
    "\tstm r8,{r0-r7}\n"
    "\tpop {r4-r11,pc}\n"
    "\t.size P256_to_montgomery, .-P256_to_montgomery\n"
    "\t.type P256_from_montgomery, %function\n"
    "P256_from_montgomery:\n"
    "\t.global P256_from_montgomery\n"
    "\tpush {r0,r4-r11,lr}\n"
    "\tmovs r2,#0\n"
    "\tmovs r3,#0\n"
    "\tpush {r2-r3}\n"
    "\tpush {r2-r3}\n"
    "\tpush {r2-r3}\n"
    "\tmovs r2,#1\n"
    "\tpush {r2-r3}\n"
    "\tmov r2,sp\n"
    "\tbl P256_mulmod\n"
    "\tadd sp,#32\n"
    "\tpop {r8}\n"
    "\tstm r8,{r0-r7}\n"
    "\tpop {r4-r11,pc}\n"
    "\t.size P256_from_montgomery, .-P256_from_montgomery\n"
    "\t.type P256_check_range_p, %function\n"
    "P256_check_range_p:\n"
    "\t.global P256_check_range_p\n"
    "\tpush {r4-r8,lr}\n"
    "\tldm r0,{r1-r8}\n"
    "\tmovs r0,#0xffffffff\n"
    "\tsubs r1,r0\n"
    "\tsbcs r2,r2,r0\n"
    "\tsbcs r3,r3,r0\n"
    "\tsbcs r4,r4,#0\n"
    "\tsbcs r5,r5,#0\n"
    "\tsbcs r6,r6,#0\n"
    "\tsbcs r7,r7,#1\n"
    "\tsbcs r8,r8,r0\n"
    "\tsbcs r0,r0,r0\n"
    "\tlsrs r0,#31\n"
    "\tpop {r4-r8,pc}\n"
    "\t.size P256_check_range_p, .-P256_check_range_p\n"
    "\t.align 2\n"
    "P256_order_mu:\n"
    "\t.word 0xeedf9bfe\n"
    "\t.word 0x012ffd85\n"
    "\t.word 0xdf1a6c21\n"
    "\t.word 0x43190552\n"
    "\t.word 0xffffffff\n"
    "\t.word 0xfffffffe\n"
    "\t.word 0xffffffff\n"
    "\t.word 0x0\n"
    "\t.word 0x1\n"
    "\t.type P256_reduce_mod_n_once, %function\n"
    "P256_reduce_mod_n_once:\n"
    "\tpush {lr}\n"
    "\tadr r10,P256_order\n"
    "\tldm r10,{r10,r11,r12,lr}\n"
    "\tsubs r0,r10\n"
    "\tsbcs r1,r1,r11\n"
    "\tsbcs r2,r2,r12\n"
    "\tsbcs r3,r3,lr\n"
    "\tsbcs r4,r4,#0xffffffff\n"
    "\tsbcs r5,r5,#0xffffffff\n"
    "\tsbcs r6,r6,#0\n"
    "\tsbcs r7,r7,#0xffffffff\n"
    "\tsbcs r8,r8,#0\n"
    "\tsbc r9,r9,r9\n"
    "\tand r10,r9\n"
    "\tand r11,r9\n"
    "\tand r12,r9\n"
    "\tand lr,r9\n"
    "\tadds r0,r10\n"
    "\tadcs r1,r1,r11\n"
    "\tadcs r2,r2,r12\n"
    "\tadcs r3,r3,lr\n"
    "\tadcs r4,r4,r9\n"
    "\tadcs r5,r5,r9\n"
    "\tadcs r6,r6,#0\n"
    "\tadcs r7,r7,r9\n"
    "\tadcs r8,r8,#0\n"
    "\tpop {pc}\n"
    "\t.size P256_reduce_mod_n_once, .-P256_reduce_mod_n_once\n"
    "\t.type P256_reduce_mod_n_64bytes, %function\n"
    "P256_reduce_mod_n_64bytes:\n"
    "\tpush {r0,r4-r11,lr}\n"
    "\tsub sp,sp,#108\n"
    "\tmov r10,r1\n"
    "\tadd r0,sp,#36\n"
    "\tadds r1,r1,#28\n"
    "\tadr r2,P256_order_mu\n"
    "\tbl mul288x288\n"
    "\tmov r0,sp\n"
    "\tadd r1,sp,#72\n"
    "\tadr r2,P256_order\n"
    "\tbl mul288x288\n"
    "\tldm r10,{r0-r8}\n"
    "\tpop {r9-r12}\n"
    "\tsubs r0,r0,r9\n"
    "\tsbcs r1,r1,r10\n"
    "\tsbcs r2,r2,r11\n"
    "\tsbcs r3,r3,r12\n"
    "\tpop {r9-r12,lr}\n"
    "\tsbcs r4,r4,r9\n"
    "\tsbcs r5,r5,r10\n"
    "\tsbcs r6,r6,r11\n"
    "\tsbcs r7,r7,r12\n"
    "\tsbcs r8,r8,lr\n"
    "\tbl P256_reduce_mod_n_once\n"
    "\tbl P256_reduce_mod_n_once\n"
    "\tadd sp,sp,#72\n"
    "\tpop {r9}\n"
    "\tstm r9,{r0-r7}\n"
    "\tpop {r4-r11,pc}\n"
    "\t.size P256_reduce_mod_n_64bytes, .-P256_reduce_mod_n_64bytes\n"
    "\t.type P256_reduce_mod_n_32bytes, %function\n"
    "P256_reduce_mod_n_32bytes:\n"
    "\t.global P256_reduce_mod_n_32bytes\n"
    "\tpush {r0,r4-r11,lr}\n"
    "\tldm r1,{r0-r7}\n"
    "\tmov r8,#0\n"
    "\tbl P256_reduce_mod_n_once\n"
    "\tpop {r8}\n"
    "\tstm r8,{r0-r7}\n"
    "\tpop {r4-r11,pc}\n"
    "\t.size P256_reduce_mod_n_32bytes, .-P256_reduce_mod_n_32bytes\n"
    "\t.type P256_add_mod_n, %function\n"
    "P256_add_mod_n:\n"
    "\t.global P256_add_mod_n\n"
    "\tpush {r0,r4-r11,lr}\n"
    "\tmov r12,r1\n"
    "\tldm r2,{r4-r11}\n"
    "\tldm r12!,{r0-r3}\n"
    "\tadds r0,r4\n"
    "\tadcs r1,r1,r5\n"
    "\tadcs r2,r2,r6\n"
    "\tadcs r3,r3,r7\n"
    "\tldm r12,{r4-r7}\n"
    "\tadcs r4,r4,r8\n"
    "\tadcs r5,r5,r9\n"
    "\tadcs r6,r6,r10\n"
    "\tadcs r7,r7,r11\n"
    "\tmovs r8,#0\n"
    "\tadcs r8,r8,r8\n"
    "\tbl P256_reduce_mod_n_once\n"
    "\tbl P256_reduce_mod_n_once\n"
    "\tpop {r8}\n"
    "\tstm r8,{r0-r7}\n"
    "\tpop {r4-r11,pc}\n"
    "\t.size P256_add_mod_n, .-P256_add_mod_n\n"
    "\t.type P256_mul_mod_n, %function\n"
    "P256_mul_mod_n:\n"
    "\t.global P256_mul_mod_n\n"
    "\tmovs r3,#0\n"
    "\tpush {r3-r10,lr}\n"
    "\tmov r4,r0\n"
    "\tldm r1,{r1,r3,r5-r10}\n"
    "\tpush {r1,r3,r5-r10}\n"
    "\tmovs r1,#0\n"
    "\tpush {r1}\n"
    "\tldm r2,{r1,r3,r5-r10}\n"
    "\tpush {r1,r3,r5-r10}\n"
    "\tsub sp,#72\n"
    "\tmov r0,sp\n"
    "\tadd r1,sp,#72\n"
    "\tadd r2,sp,#108\n"
    "\tbl mul288x288\n"
    "\tmov r0,r4\n"
    "\tmov r1,sp\n"
    "\tbl P256_reduce_mod_n_64bytes\n"
    "\tadd sp,#144\n"
    "\tpop {r4-r10,pc}\n"
    "\t.size P256_mul_mod_n, .-P256_mul_mod_n\n"
    "\t.type P256_divsteps2_31, %function\n"
    "P256_divsteps2_31:\n"
    "\t.global P256_divsteps2_31\n"
    "\tpush {r3,r4-r8,lr}\n"
    "\tmovs r4,#1\n"
    "\tmovs r5,#0\n"
    "\tmovs r6,#0\n"
    "\tmovs r7,#1\n"
    "\tmov lr,#31\n"
    "0:\n"
    "\tsubs r3,r0,#1\n"
    "\tlsl r12,r2,#31\n"
    "\tbic r3,r12,r3\n"
    "\tasrs r3,r3,#31\n"
    "\tlsr r8,r3,#31\n"
    "\teors r0,r0,r3\n"
    "\tsubs r0,r0,r3\n"
    "\tmul r12,r1,r3\n"
    "\tbics r1,r1,r3\n"
    "\tumlal r1,r12,r2,r8\n"
    "\tumaal r2,r12,r2,r3\n"
    "\tmul r12,r4,r3\n"
    "\tbics r4,r4,r3\n"
    "\tumlal r4,r12,r6,r8\n"
    "\tumaal r6,r12,r6,r3\n"
    "\tmul r12,r5,r3\n"
    "\tbics r5,r5,r3\n"
    "\tumlal r5,r12,r7,r8\n"
    "\tumaal r7,r12,r7,r3\n"
    "\tands r12,r2,#1\n"
    "\tadds r0,r0,#1\n"
    "\tmul r3,r12,r1\n"
    "\tadds r2,r2,r3\n"
    "\tlsrs r2,r2,#1\n"
    "\tumlal r6,r8,r12,r4\n"
    "\tumlal r7,r8,r12,r5\n"
    "\tadds r4,r4,r4\n"
    "\tadds r5,r5,r5\n"
    "\tsubs lr,lr,#1\n"
    "\tbne 0b\n"
    "\tpop {r3}\n"
    "\tstm r3!,{r4-r7}\n"
    "\tpop {r4-r8,pc}\n"
    "\t.size P256_divsteps2_31, .-P256_divsteps2_31\n"
    "\t.type P256_matrix_mul_fg_9, %function\n"
    "P256_matrix_mul_fg_9:\n"
    "\t.global P256_matrix_mul_fg_9\n"
    "\tpush {r4-r11,lr}\n"
    "\tand r4,r0,r0,lsl #1\n"
    "\tasrs r4,r4,#31\n"
    "\teors r0,r0,r4\n"
    "\tsubs r0,r0,r4\n"
    "\tand r5,r1,r1,lsl #1\n"
    "\tasrs r5,r5,#31\n"
    "\teors r1,r1,r5\n"
    "\tsubs r1,r1,r5\n"
    "\tldm r2!,{r6}\n"
    "\tldr r7,[r2,#36]\n"
    "\teors r4,r4,r6\n"
    "\teors r5,r5,r7\n"
    "\teors r4,r4,r5\n"
    "\tstm r3!,{r5}\n"
    "\tpush {r1,r2,r3}\n"
    "\tldm r2!,{r1,r3,r5-r11}\n"
    "\teors r1,r1,r4\n"
    "\teors r3,r3,r4\n"
    "\teors r5,r5,r4\n"
    "\teors r6,r6,r4\n"
    "\teors r7,r7,r4\n"
    "\teor r8,r8,r4\n"
    "\teor r9,r9,r4\n"
    "\teor r10,r10,r4\n"
    "\tsubs r1,r1,r4\n"
    "\tsbcs r3,r3,r4\n"
    "\tsbcs r5,r5,r4\n"
    "\tsbcs r6,r6,r4\n"
    "\tsbcs r7,r7,r4\n"
    "\tsbcs r8,r8,r4\n"
    "\tsbcs r9,r9,r4\n"
    "\tsbcs r10,r10,r4\n"
    "\teor r4,r4,r11\n"
    "\tumull r1,lr,r0,r1\n"
    "\tmovs r2,#0\n"
    "\tumull r11,r12,r2,r2\n"
    "\tumaal r2,lr,r0,r3\n"
    "\tumaal r11,lr,r0,r5\n"
    "\tumull r3,r5,r12,r12\n"
    "\tumaal r3,lr,r0,r6\n"
    "\tumaal r5,lr,r0,r7\n"
    "\tumull r6,r7,r12,r12\n"
    "\tumaal r6,lr,r0,r8\n"
    "\tumaal r7,lr,r0,r9\n"
    "\tumaal r12,lr,r0,r10\n"
    "\tmla lr,r0,r4,lr\n"
    "\tpop {r0,r4}\n"
    "\tadds r4,r4,#40\n"
    "\tldm r4!,{r8,r9}\n"
    "\tmov r10,#0\n"
    "\tumaal r1,r10,r0,r8\n"
    "\tumaal r2,r10,r0,r9\n"
    "\tadds r1,r1,r1\n"
    "\tadcs r2,r2,r2\n"
    "\tldm r4!,{r1,r8,r9}\n"
    "\tumaal r10,r11,r0,r1\n"
    "\tumaal r11,r3,r0,r8\n"
    "\tumaal r3,r5,r0,r9\n"
    "\tadcs r10,r10,r10\n"
    "\tadcs r11,r11,r11\n"
    "\tadcs r3,r3,r3\n"
    "\tldm r4,{r1,r4,r8,r9}\n"
    "\tumaal r5,r6,r0,r1\n"
    "\tumaal r6,r7,r0,r4\n"
    "\tumaal r7,r12,r0,r8\n"
    "\tumaal r12,lr,r0,r9\n"
    "\tadcs r5,r5,r5\n"
    "\tadcs r6,r6,r6\n"
    "\tadcs r7,r7,r7\n"
    "\tadcs r12,r12,r12\n"
    "\tsbcs lr,lr,lr\n"
    "\tmvn lr,lr\n"
    "\tpop {r1}\n"
    "\tstm r1!,{r2,r10,r11}\n"
    "\tstm r1!,{r3,r5,r6,r7,r12,lr}\n"
    "\tpop {r4-r11,pc}\n"
    "\t.size P256_matrix_mul_fg_9, .-P256_matrix_mul_fg_9\n"
    "\t.align 2\n"
    "\t.type P256_matrix_mul_mod_n, %function\n"
    "P256_matrix_mul_mod_n:\n"
    "\t.global P256_matrix_mul_mod_n\n"
    "\tpush {r4-r11,lr}\n"
    "\tand r4,r0,r0,lsl #1\n"
    "\tasrs r4,r4,#31\n"
    "\teors r0,r0,r4\n"
    "\tsubs r0,r0,r4\n"
    "\tand r5,r1,r1,lsl #1\n"
    "\tasrs r5,r5,#31\n"
    "\teors r1,r1,r5\n"
    "\tsubs r1,r1,r5\n"
    "\tldm r2!,{r6}\n"
    "\tldr r7,[r2,#32]\n"
    "\teors r4,r4,r6\n"
    "\teors r5,r5,r7\n"
    "\teors r4,r4,r5\n"
    "\tstm r3!,{r5}\n"
    "\tpush {r1,r2,r3}\n"
    "\tldm r2,{r1-r3,r5-r9}\n"
    "\teors r1,r1,r4\n"
    "\teors r2,r2,r4\n"
    "\teors r3,r3,r4\n"
    "\teors r5,r5,r4\n"
    "\teors r6,r6,r4\n"
    "\teors r7,r7,r4\n"
    "\teor r8,r8,r4\n"
    "\teor r9,r9,r4\n"
    "\tsubs r1,r1,r4\n"
    "\tsbcs r2,r2,r4\n"
    "\tsbcs r3,r3,r4\n"
    "\tsbcs r5,r5,r4\n"
    "\tsbcs r6,r6,r4\n"
    "\tsbcs r7,r7,r4\n"
    "\tsbcs r8,r8,r4\n"
    "\tsbcs r9,r9,r4\n"
    "\tsbcs r4,r4,r4\n"
    "\tlsrs lr,r4,#31\n"
    "\tmov r12,#0\n"
    "\tldrd r10,r11,P256_order_local\n"
    "\tumaal r1,r12,lr,r10\n"
    "\tumaal r2,r12,lr,r11\n"
    "\tldrd r10,r11,P256_order_local+8\n"
    "\tumaal r3,r12,lr,r10\n"
    "\tumaal r5,r12,lr,r11\n"
    "\tumaal r6,r12,lr,r4\n"
    "\tumaal r7,r12,lr,r4\n"
    "\tmov r10,#0\n"
    "\tumaal r8,r12,lr,r10\n"
    "\tumaal r9,r12,lr,r4\n"
    "\tumull r11,lr,r10,r10\n"
    "\tumull r10,lr,r0,r1\n"
    "\tumull r1,r12,r11,r11\n"
    "\tumaal r11,lr,r0,r2\n"
    "\tumaal r1,lr,r0,r3\n"
    "\tumull r2,r3,r12,r12\n"
    "\tumaal r2,lr,r0,r5\n"
    "\tumaal r3,lr,r0,r6\n"
    "\tumull r4,r5,r12,r12\n"
    "\tumaal r4,lr,r0,r7\n"
    "\tumaal r5,lr,r0,r8\n"
    "\tumaal r12,lr,r0,r9\n"
    "\tpop {r0,r6}\n"
    "\tadds r6,r6,#36\n"
    "\tldm r6!,{r8,r9}\n"
    "\tmovs r7,#0\n"
    "\tumaal r10,r7,r0,r8\n"
    "\tumaal r11,r7,r0,r9\n"
    "\tldm r6!,{r8,r9}\n"
    "\tumaal r1,r7,r0,r8\n"
    "\tumaal r2,r7,r0,r9\n"
    "\tldm r6!,{r8,r9}\n"
    "\tumaal r3,r7,r0,r8\n"
    "\tumaal r4,r7,r0,r9\n"
    "\tldm r6!,{r8,r9}\n"
    "\tumaal r5,r7,r0,r8\n"
    "\tumaal r12,r7,r0,r9\n"
    "\tadd lr,lr,r7\n"
    "\tldr r0,=0xee00bc4f\n"
    "\tmul r0,r10,r0\n"
    "\tmovs r6,#0\n"
    "\tldrd r8,r9,P256_order_local\n"
    "\tumaal r10,r6,r0,r8\n"
    "\tumaal r11,r6,r0,r9\n"
    "\tsubs r11,r11,r8\n"
    "\tldrd r8,r10,P256_order_local+8\n"
    "\tumaal r1,r6,r0,r8\n"
    "\tsbcs r1,r1,r9\n"
    "\tumaal r2,r6,r0,r10\n"
    "\tmov r9,#-1\n"
    "\tumaal r3,r6,r0,r9\n"
    "\tumaal r4,r6,r0,r9\n"
    "\tmovs r7,#0\n"
    "\tumaal r5,r6,r0,r7\n"
    "\tumaal r12,r6,r0,r9\n"
    "\tumaal lr,r6,r7,r7\n"
    "\tsbcs r2,r2,r8\n"
    "\tsbcs r3,r3,r10\n"
    "\tsbcs r4,r4,r9\n"
    "\tsbcs r5,r5,r9\n"
    "\tsbcs r12,r12,r7\n"
    "\tsbcs lr,lr,r9\n"
    "\tsbcs r6,r6,r7\n"
    "\tldrd r0,r9,P256_order_local\n"
    "\tlsrs r6,r6,#31\n"
    "\tumaal r7,r11,r6,r0\n"
    "\tumaal r1,r11,r6,r9\n"
    "\tumaal r2,r11,r6,r8\n"
    "\tumaal r3,r11,r6,r10\n"
    "\trsbs r0,r6,#0\n"
    "\tumaal r4,r11,r6,r0\n"
    "\tumaal r5,r11,r6,r0\n"
    "\tmov r8,#0\n"
    "\tumaal r11,r12,r6,r8\n"
    "\tumaal r12,lr,r6,r0\n"
    "\tpop {r6}\n"
    "\tstm r6!,{r7}\n"
    "\tstm r6!,{r1,r2,r3,r4,r5,r11,r12}\n"
    "\tpop {r4-r11,pc}\n"
    "\t.ltorg\n"
    "\t.size P256_matrix_mul_mod_n, .-P256_matrix_mul_mod_n\n"
    "\t.align 2\n"
    "P256_order_local:\n"
    "\t.type P256_order, %object\n"
    "P256_order:\n"
    "\t.global P256_order\n"
    "\t.word 0xFC632551\n"
    "\t.word 0xF3B9CAC2\n"
    "\t.word 0xA7179E84\n"
    "\t.word 0xBCE6FAAD\n"
    "\t.word 0xFFFFFFFF\n"
    "\t.word 0xFFFFFFFF\n"
    "\t.word 0\n"
    "\t.word 0xFFFFFFFF\n"
    "\t.word 0\n"
    "\t.size P256_order, .-P256_order\n"
    "\t.type P256_check_range_n, %function\n"
    "P256_check_range_n:\n"
    "\t.global P256_check_range_n\n"
    "\tpush {r4-r11,lr}\n"
    "\tldm r0,{r1-r8}\n"
    "\torrs r0,r1,r2\n"
    "\torrs r0,r3\n"
    "\torrs r0,r4\n"
    "\torrs r0,r5\n"
    "\torrs r0,r6\n"
    "\torrs r0,r7\n"
    "\torrs r0,r8\n"
    "\tbeq 0f\n"
    "\tadr r0,P256_order\n"
    "\tldm r0!,{r9-r12}\n"
    "\tsubs r1,r9\n"
    "\tsbcs r2,r2,r10\n"
    "\tsbcs r3,r3,r11\n"
    "\tsbcs r4,r4,r12\n"
    "\tldm r0,{r0-r3}\n"
    "\tsbcs r5,r5,r0\n"
    "\tsbcs r6,r6,r1\n"
    "\tsbcs r7,r7,r2\n"
    "\tsbcs r8,r8,r3\n"
    "\tsbcs r0,r0,r0\n"
    "\tlsrs r0,#31\n"
    "0:\n"
    "\tpop {r4-r11,pc}\n"
    "\t.size P256_check_range_n, .-P256_check_range_n\n"
    "\t.align 2\n"
    "b_mont:\n"
    "\t.word 0x29c4bddf\n"
    "\t.word 0xd89cdf62\n"
    "\t.word 0x78843090\n"
    "\t.word 0xacf005cd\n"
    "\t.word 0xf7212ed6\n"
    "\t.word 0xe5a220ab\n"
    "\t.word 0x04874834\n"
    "\t.word 0xdc30061d\n"
    "three_mont:\n"
    "\t.word 0x3\n"
    "\t.word 0x0\n"
    "\t.word 0x0\n"
    "\t.word 0xfffffffd\n"
    "\t.word 0xffffffff\n"
    "\t.word 0xffffffff\n"
    "\t.word 0xfffffffc\n"
    "\t.word 0x2\n"
    "\t.type P256_point_is_on_curve, %function\n"
    "P256_point_is_on_curve:\n"
    "\t.global P256_point_is_on_curve\n"
    "\tpush {r0,r4-r11,lr}\n"
    "\tldm r1,{r0-r7}\n"
    "\tbl P256_sqrmod\n"
    "\tpush {r0-r7}\n"
    "\tldr r0,[sp,#32]\n"
    "\tldm r0,{r0-r7}\n"
    "\tbl P256_sqrmod\n"
    "\tpush {r0-r7}\n"
    "\tmov r1,sp\n"
    "\tadr r2,three_mont\n"
    "\tbl P256_submod\n"
    "\tstm sp,{r0-r7}\n"
    "\tldr r1,[sp,#64]\n"
    "\tmov r2,sp\n"
    "\tbl P256_mulmod\n"
    "\tstm sp,{r0-r7}\n"
    "\tadd r1,sp,#32\n"
    "\tmov r2,sp\n"
    "\tbl P256_submod\n"
    "\tadr r8,b_mont\n"
    "\tldm r8!,{r9-r12}\n"
    "\teors r0,r9\n"
    "\tittt eq\n"
    "\teorseq r1,r10\n"
    "\teorseq r2,r11\n"
    "\teorseq r3,r12\n"
    "\tldm r8,{r9-r12}\n"
    "\titttt eq\n"
    "\teorseq r4,r9\n"
    "\teorseq r5,r10\n"
    "\teorseq r6,r11\n"
    "\teorseq r7,r12\n"
    "\tmov r0,#0\n"
    "\tit eq\n"
    "\tmoveq r0,#1\n"
    "\tadd sp,#68\n"
    "\tpop {r4-r11,pc}\n"
    "\t.size P256_point_is_on_curve, .-P256_point_is_on_curve\n"
    "\t.align 2\n"
    "P256_p:\n"
    "\t.word 0xffffffff\n"
    "\t.word 0xffffffff\n"
    "\t.word 0xffffffff\n"
    "\t.word 0\n"
    "\t.word 0\n"
    "\t.word 0\n"
    "\t.word 1\n"
    "\t.word 0xffffffff\n"
    "\t.type P256_jacobian_to_affine, %function\n"
    "P256_jacobian_to_affine:\n"
    "\t.global P256_jacobian_to_affine\n"
    "\tpush {r0,r1,r2,r4-r11,lr}\n"
    "\tadds r2,#64\n"
    "\tldm r2,{r0-r7}\n"
    "\tmov r8,#0\n"
    "\tbl P256_modinv_sqrt\n"
    "\tpush {r0-r7}\n"
    "\tbl P256_sqrmod\n"
    "\tpush {r0-r7}\n"
    "\tadd r1,sp,#32\n"
    "\tmov r2,sp\n"
    "\tbl P256_mulmod\n"
    "\tadd r8,sp,#32\n"
    "\tstm r8,{r0-r7}\n"
    "\tmov r1,sp\n"
    "\tldr r2,[sp,#72]\n"
    "\tbl P256_mulmod\n"
    "\tldr r8,[sp,#64]\n"
    "\tstm r8,{r0-r7}\n"
    "\tldr r2,[sp,#72]\n"
    "\tadd r1,sp,#32\n"
    "\tadds r2,r2,#32\n"
    "\tbl P256_mulmod\n"
    "\tldr r8,[sp,#68]\n"
    "\tstm r8,{r0-r7}\n"
    "\tadd sp,#76\n"
    "\tpop {r4-r11,pc}\n"
    "\t.size P256_jacobian_to_affine, .-P256_jacobian_to_affine\n"
    "\t.type P256_double_j, %function\n"
    "P256_double_j:\n"
    "\t.global P256_double_j\n"
    "\tpush {r0,r1,r4-r11,lr}\n"
    "\tadds r1,#64\n"
    "\tldm r1,{r0-r7}\n"
    "\tbl P256_sqrmod\n"
    "\tpush {r0-r7}\n"
    "\tldr r1,[sp,#36]\n"
    "\tadds r1,#32\n"
    "\tadd r2,r1,#32\n"
    "\tbl P256_mulmod\n"
    "\tldr r8,[sp,#32]\n"
    "\tadd r8,#64\n"
    "\tstm r8,{r0-r7}\n"
    "\tldr r1,[sp,#36]\n"
    "\tmov r2,sp\n"
    "\tbl P256_addmod\n"
    "\tpush {r0-r7}\n"
    "\tldr r1,[sp,#68]\n"
    "\tadd r2,sp,#32\n"
    "\tbl P256_submod\n"
    "\tadd r8,sp,#32\n"
    "\tstm r8,{r0-r7}\n"
    "\tadd r1,sp,#32\n"
    "\tmov r2,sp\n"
    "\tbl P256_mulmod\n"
    "\tadd r8,sp,#32\n"
    "\tstm r8,{r0-r7}\n"
    "\tlsl r8,r0,#31\n"
    "\tadds r0,r0,r8, asr #31\n"
    "\tadcs r1,r1,r8, asr #31\n"
    "\tadcs r2,r2,r8, asr #31\n"
    "\tadcs r3,r3,#0\n"
    "\tadcs r4,r4,#0\n"
    "\tadcs r5,r5,#0\n"
    "\tadcs r6,r6,r8, lsr #31\n"
    "\tadcs r7,r7,r8, asr #31\n"
    "\trrxs r7,r7\n"
    "\trrxs r6,r6\n"
    "\trrxs r5,r5\n"
    "\trrxs r4,r4\n"
    "\trrxs r3,r3\n"
    "\trrxs r2,r2\n"
    "\trrxs r1,r1\n"
    "\trrx r0,r0\n"
    "\tstm sp,{r0-r7}\n"
    "\tadd r1,sp,#32\n"
    "\tmov r2,sp\n"
    "\tbl P256_addmod\n"
    "\tadd r8,sp,#32\n"
    "\tstm r8,{r0-r7}\n"
    "\tbl P256_sqrmod\n"
    "\tstm sp,{r0-r7}\n"
    "\tldr r0,[sp,#68]\n"
    "\tadds r0,#32\n"
    "\tldm r0,{r0-r7}\n"
    "\tbl P256_sqrmod\n"
    "\tldr r8,[sp,#64]\n"
    "\tadd r8,#32\n"
    "\tstm r8,{r0-r7}\n"
    "\tbl P256_sqrmod\n"
    "\tpush {r0-r7}\n"
    "\tldrd r0,r1,[sp,#96]\n"
    "\tadd r2,r0,#32\n"
    "\tbl P256_mulmod\n"
    "\tldr r8,[sp,#96]\n"
    "\tadd r8,#32\n"
    "\tstm r8,{r0-r7}\n"
    "\tbl P256_times2\n"
    "\tldr r8,[sp,#96]\n"
    "\tstm r8,{r0-r7}\n"
    "\tadd r1,sp,#32\n"
    "\tmov r2,r8\n"
    "\tbl P256_submod\n"
    "\tldr r8,[sp,#96]\n"
    "\tstm r8,{r0-r7}\n"
    "\tmov r2,r8\n"
    "\tadd r1,r2,#32\n"
    "\tbl P256_submod\n"
    "\tadd r8,sp,#32\n"
    "\tstm r8,{r0-r7}\n"
    "\tadd r1,sp,#64\n"
    "\tadd r2,sp,#32\n"
    "\tbl P256_mulmod\n"
    "\tadd r8,sp,#64\n"
    "\tstm r8,{r0-r7}\n"
    "\tadd r1,sp,#64\n"
    "\tmov r2,sp\n"
    "\tbl P256_submod\n"
    "\tldr r8,[sp,#96]\n"
    "\tadd r8,#32\n"
    "\tstm r8,{r0-r7}\n"
    "\tadd sp,#104\n"
    "\tpop {r4-r11,pc}\n"
    "\t.size P256_double_j, .-P256_double_j\n"
    "\t.type add_sub_helper, %function\n"
    "add_sub_helper:\n"
    "\tpush {lr}\n"
    "\tldm r1!,{r5-r12}\n"
    "\tstm r0!,{r5-r12}\n"
    "\tldm r1!,{r5-r12}\n"
    "\tcbz r2,0f\n"
    "\tmov lr,#0\n"
    "\trsbs r4,r2,#0\n"
    "\tsubs r5,r4,r5\n"
    "\tsbcs r6,r4,r6\n"
    "\tsbcs r7,r4,r7\n"
    "\tsbcs r8,lr,r8\n"
    "\tsbcs r9,lr,r9\n"
    "\tsbcs r10,lr,r10\n"
    "\tsbcs r11,r2,r11\n"
    "\tsbcs r12,r4,r12\n"
    "0:\n"
    "\tstm r0!,{r5-r12}\n"
    "\tcbnz r3,1f\n"
    "\tldm r1,{r5-r12}\n"
    "\tstm r0,{r5-r12}\n"
    "\tb 2f\n"
    "1:\n"
    "\tmovs r4,#0\n"
    "\tumull r5,r10,r4,r4\n"
    "\tmvns r6,r4\n"
    "\tmvns r7,r4\n"
    "\tmov r8,#0xffffffff\n"
    "\tmov r9,#0xfffffffe\n"
    "\tstm r0,{r3-r10}\n"
    "2:\n"
    "\tpop {pc}\n"
    "\t.size add_sub_helper, .-add_sub_helper\n"
    "\t.type P256_add_sub_j, %function\n"
    "P256_add_sub_j:\n"
    "\t.global P256_add_sub_j\n"
    "\tpush {r0-r11,lr}\n"
    "\tadd r4,r0,#64\n"
    "\tldm r4,{r4-r11}\n"
    "\torrs r4,r5\n"
    "\torrs r4,r6\n"
    "\torrs r4,r7\n"
    "\torrs r4,r8\n"
    "\torrs r4,r9\n"
    "\torrs r4,r10\n"
    "\torrs r4,r11\n"
    "\tbne 2f\n"
    "\tbl add_sub_helper\n"
    "\tadd sp,#16\n"
    "\tpop {r4-r11,pc}\n"
    "2:\n"
    "\tcbnz r3,100f\n"
    "\tadds r1,#64\n"
    "\tldm r1,{r0-r7}\n"
    "\tbl P256_sqrmod\n"
    "\tpush {r0-r7}\n"
    "\tldr r1,[sp,#32]\n"
    "\tmov r2,sp\n"
    "\tbl P256_mulmod\n"
    "\tldr r8,[sp,#32]\n"
    "\tstm r8,{r0-r7}\n"
    "\tldr r1,[sp,#36]\n"
    "\tadds r1,#64\n"
    "\tmov r2,sp\n"
    "\tbl P256_mulmod\n"
    "\tstm sp,{r0-r7}\n"
    "\tldr r1,[sp,#32]\n"
    "\tadds r1,#32\n"
    "\tmov r2,sp\n"
    "\tbl P256_mulmod\n"
    "\tldr r8,[sp,#32]\n"
    "\tadd r8,#32\n"
    "\tstm r8,{r0-r7}\n"
    "\tb 101f\n"
    "100:\n"
    "\tsub sp,#32\n"
    "101:\n"
    "\tldr r1,[sp,#32]\n"
    "\tadds r1,#64\n"
    "\tldm r1,{r0-r7}\n"
    "\tbl P256_sqrmod\n"
    "\tpush {r0-r7}\n"
    "\tldr r1,[sp,#68]\n"
    "\tmov r2,sp\n"
    "\tbl P256_mulmod\n"
    "\tadd r8,sp,#32\n"
    "\tstm r8,{r0-r7}\n"
    "\tldr r1,[sp,#64]\n"
    "\tadds r1,#64\n"
    "\tmov r2,sp\n"
    "\tbl P256_mulmod\n"
    "\tstm sp,{r0-r7}\n"
    "\tldr r1,[sp,#68]\n"
    "\tadds r1,#32\n"
    "\tmov r2,sp\n"
    "\tbl P256_mulmod\n"
    "\tstm sp,{r0-r7}\n"
    "\tldr r1,[sp,#64]\n"
    "\tadd r2,sp,#32\n"
    "\tbl P256_submod\n"
    "\tldr r8,[sp,#64]\n"
    "\tstm r8,{r0-r7}\n"
    "\tbl P256_sqrmod\n"
    "\tpush {r0-r7}\n"
    "\tldr r2,[sp,#96]\n"
    "\tadd r1,r2,#64\n"
    "\tbl P256_mulmod\n"
    "\tldr r8,[sp,#96]\n"
    "\tadd r8,#64\n"
    "\tstm r8,{r0-r7}\n"
    "\tldr r1,[sp,#108]\n"
    "\tcbnz r1,102f\n"
    "\tldr r1,[sp,#100]\n"
    "\tadds r1,#64\n"
    "\tmov r2,r8\n"
    "\tbl P256_mulmod\n"
    "\tldr r8,[sp,#96]\n"
    "\tadd r8,#64\n"
    "\tstm r8,{r0-r7}\n"
    "102:\n"
    "\tldr r1,[sp,#96]\n"
    "\tmov r2,sp\n"
    "\tbl P256_mulmod\n"
    "\tldr r8,[sp,#96]\n"
    "\tstm r8,{r0-r7}\n"
    "\torrs r1,r0\n"
    "\torrs r1,r2\n"
    "\torrs r1,r3\n"
    "\torrs r1,r4\n"
    "\torrs r1,r5\n"
    "\torrs r1,r6\n"
    "\torrs r0,r1,r7\n"
    "3:\n"
    "\tpush {r0}\n"
    "\tldr r1,[sp,#100]\n"
    "\tadds r1,#32\n"
    "\tadd r2,sp,#36\n"
    "\tldr r3,[sp,#108]\n"
    "\tcbz r3,4f\n"
    "\tbl P256_addmod\n"
    "\tb 5f\n"
    "4:\n"
    "\tbl P256_submod\n"
    "5:\n"
    "\tldr r8,[sp,#100]\n"
    "\tadd r8,#32\n"
    "\tstm r8,{r0-r7}\n"
    "\tpop {r8}\n"
    "\torrs r1,r0\n"
    "\torrs r1,r2\n"
    "\torrs r1,r3\n"
    "\torrs r1,r4\n"
    "\torrs r1,r5\n"
    "\torrs r1,r6\n"
    "\torrs r1,r7\n"
    "\torrs r1,r8\n"
    "\tbne 6f\n"
    "\tadd sp,#96\n"
    "\tldm sp,{r0-r3}\n"
    "\tbl add_sub_helper\n"
    "\tldr r0,[sp,#0]\n"
    "\tmov r1,r0\n"
    "\tadd sp,#16\n"
    "\tbl P256_double_j\n"
    "\tpop {r4-r11,pc}\n"
    "6:\n"
    "\tadd r1,sp,#64\n"
    "\tmov r2,sp\n"
    "\tbl P256_mulmod\n"
    "\tadd r8,sp,#64\n"
    "\tstm r8,{r0-r7}\n"
    "\tldr r0,[sp,#96]\n"
    "\tadds r0,#32\n"
    "\tldm r0,{r0-r7}\n"
    "\tbl P256_sqrmod\n"
    "\tstm sp,{r0-r7}\n"
    "\tadd r1,sp,#32\n"
    "\tldr r2,[sp,#96]\n"
    "\tbl P256_mulmod\n"
    "\tadd r8,sp,#32\n"
    "\tstm r8,{r0-r7}\n"
    "\tmov r1,sp\n"
    "\tldr r2,[sp,#96]\n"
    "\tbl P256_submod\n"
    "\tldr r8,[sp,#96]\n"
    "\tstm r8,{r0-r7}\n"
    "\tadd r0,sp,#64\n"
    "\tldm r0,{r0-r7}\n"
    "\tbl P256_times2\n"
    "\tstm sp,{r0-r7}\n"
    "\tldr r1,[sp,#96]\n"
    "\tmov r2,sp\n"
    "\tbl P256_submod\n"
    "\tldr r8,[sp,#96]\n"
    "\tstm r8,{r0-r7}\n"
    "\tadd r1,sp,#64\n"
    "\tldr r2,[sp,#96]\n"
    "\tbl P256_submod\n"
    "\tstm sp,{r0-r7}\n"
    "\tldr r1,[sp,#96]\n"
    "\tadds r1,#32\n"
    "\tmov r2,sp\n"
    "\tbl P256_mulmod\n"
    "\tstm sp,{r0-r7}\n"
    "\tldr r0,[sp,#104]\n"
    "\tmov r1,sp\n"
    "\tadd r2,sp,#32\n"
    "\tcbz r0,7f\n"
    "\tbl P256_addmod\n"
    "\tb 8f\n"
    "7:\n"
    "\tbl P256_submod\n"
    "8:\n"
    "\tldr r8,[sp,#96]\n"
    "\tadd r8,#32\n"
    "\tstm r8,{r0-r7}\n"
    "\tadd sp,#112\n"
    "\tpop {r4-r11,pc}\n"
    "\t.size P256_add_sub_j, .-P256_add_sub_j\n"
    "\t.type P256_verify_last_step, %function\n"
    "P256_verify_last_step:\n"
    "\t.global P256_verify_last_step\n"
    "\tpush {r0,r1,r4-r11,lr}\n"
    "\tsub sp,#32\n"
    "\tadd r1,#64\n"
    "\tldm r1,{r0-r7}\n"
    "\tbl P256_sqrmod\n"
    "\tpush {r0-r7}\n"
    "\torrs r0,r1\n"
    "\torrs r0,r2\n"
    "\torrs r0,r3\n"
    "\torrs r0,r4\n"
    "\torrs r0,r5\n"
    "\torrs r0,r6\n"
    "\torrs r0,r7\n"
    "\tbeq 0f\n"
    "\tldr r1,[sp,#64]\n"
    "2:\n"
    "\tadd r0,sp,#32\n"
    "\tbl P256_to_montgomery\n"
    "\tadd r1,sp,#32\n"
    "\tmov r2,sp\n"
    "\tbl P256_mulmod\n"
    "\tldr r8,[sp,#68]\n"
    "\tldm r8!,{r9-r12}\n"
    "\teors r0,r9\n"
    "\tittt eq\n"
    "\teorseq r1,r10\n"
    "\teorseq r2,r11\n"
    "\teorseq r3,r12\n"
    "\tldm r8!,{r9-r12}\n"
    "\titttt eq\n"
    "\teorseq r4,r9\n"
    "\teorseq r5,r10\n"
    "\teorseq r6,r11\n"
    "\teorseq r7,r12\n"
    "\tmov r0,#1\n"
    "\tbeq 1f\n"
    "\tadr r0,P256_order\n"
    "\tldm r0,{r8-r11}\n"
    "\tldr r0,[sp,#64]\n"
    "\tcbz r0,0f\n"
    "\tldm r0,{r0-r7}\n"
    "\tadds r0,r8\n"
    "\tadcs r1,r1,r9\n"
    "\tadcs r2,r2,r10\n"
    "\tadcs r3,r3,r11\n"
    "\tadcs r4,r4,#0xffffffff\n"
    "\tadcs r5,r5,#0xffffffff\n"
    "\tadcs r6,r6,#0\n"
    "\tadcs r7,r7,#0xffffffff\n"
    "\tbcs 0f\n"
    "\tsubs r8,r0,#0xffffffff\n"
    "\tsbcs r8,r1,#0xffffffff\n"
    "\tsbcs r8,r2,#0xffffffff\n"
    "\tsbcs r8,r3,#0\n"
    "\tsbcs r8,r4,#0\n"
    "\tsbcs r8,r5,#0\n"
    "\tsbcs r8,r6,#1\n"
    "\tsbcs r8,r7,#0xffffffff\n"
    "\tbcs 0f\n"
    "\tadd r8,sp,#32\n"
    "\tstm r8,{r0-r7}\n"
    "\tmovs r2,#0\n"
    "\tstr r2,[sp,#64]\n"
    "\tmov r1,r8\n"
    "\tb 2b\n"
    "0:\n"
    "\tmovs r0,#0\n"
    "1:\n"
    "\tadd sp,#72\n"
    "\tpop {r4-r11,pc}\n"
    "\t.size P256_verify_last_step, .-P256_verify_last_step\n"
    "\t.type P256_negate_mod_m_if, %function\n"
    "P256_negate_mod_m_if:\n"
    "\tpush {r4-r8,lr}\n"
    "\trsb r8,r2,#1\n"
    "\tmovs r6,#8\n"
    "\tsubs r7,r7\n"
    "0:\n"
    "\tldm r1!,{r4,r12}\n"
    "\tldm r3!,{r5,lr}\n"
    "\tsbcs r5,r5,r4\n"
    "\tumull r4,r7,r8,r4\n"
    "\tumaal r4,r7,r2,r5\n"
    "\tsbcs lr,lr,r12\n"
    "\tumull r12,r7,r8,r12\n"
    "\tumaal r12,r7,r2,lr\n"
    "\tstm r0!,{r4,r12}\n"
    "\tsub r6,#2\n"
    "\tcbz r6,1f\n"
    "\tb 0b\n"
    "1:\n"
    "\tpop {r4-r8,pc}\n"
    "\t.size P256_negate_mod_m_if, .-P256_negate_mod_m_if\n"
    "\t.type P256_negate_mod_n_if, %function\n"
    "P256_negate_mod_n_if:\n"
    "\t.global P256_negate_mod_n_if\n"
    "\tldr r3,=P256_order\n"
    "\tb P256_negate_mod_m_if\n"
    "\t.size P256_negate_mod_n_if, .-P256_negate_mod_n_if\n"
    "\t.type P256_negate_mod_p_if, %function\n"
    "P256_negate_mod_p_if:\n"
    "\t.global P256_negate_mod_p_if\n"
    "\tadr r3,P256_p\n"
    "\tb P256_negate_mod_m_if\n"
    "\t.size P256_negate_mod_p_if, .-P256_negate_mod_p_if\n"
    "\t.align 2\n"
    ".popsection\n");  // No .end here: it would drop the C code that follows

// mg_p256_*() wrappers, see tls_p256.h. The API uses big endian byte strings,
// this library uses little endian 32-bit words, so convert both ways

int mg_p256_make_key(uint8_t public_key[64], uint8_t private_key[32]) {
  uint32_t x[8], y[8], d[8];
  int tries, ok = 0;
  for (tries = 0; tries < 64 && !ok; tries++) {
    if (!mg_random(d, sizeof(d))) break;
    ok = p256_keygen(x, y, d);
  }
  if (ok) {
    p256_convert_endianness(public_key, x, 32);
    p256_convert_endianness(public_key + 32, y, 32);
    p256_convert_endianness(private_key, d, 32);
  }
  mg_bzero((unsigned char *) d, sizeof(d));
  return ok;
}

int mg_p256_compute_public_key(const uint8_t private_key[32],
                               uint8_t public_key[64]) {
  uint32_t x[8], y[8], d[8];
  int ok;
  p256_convert_endianness(d, private_key, 32);
  ok = p256_keygen(x, y, d);  // False if the key is out of range
  if (ok) {
    p256_convert_endianness(public_key, x, 32);
    p256_convert_endianness(public_key + 32, y, 32);
  }
  mg_bzero((unsigned char *) d, sizeof(d));
  return ok;
}

int mg_p256_shared_secret(const uint8_t public_key[64],
                          const uint8_t private_key[32], uint8_t secret[32]) {
  uint32_t x[8], y[8], d[8];
  int ok;
  p256_convert_endianness(x, public_key, 32);
  p256_convert_endianness(y, public_key + 32, 32);
  p256_convert_endianness(d, private_key, 32);
  ok = p256_ecdh_calc_shared_secret(secret, d, x, y);  // Validates the point
  mg_bzero((unsigned char *) d, sizeof(d));
  return ok;
}

int mg_p256_sign_with_k(const uint8_t private_key[32], const uint8_t *hash,
                        size_t hash_size, const uint8_t k[32],
                        uint8_t signature[64]) {
  uint32_t r[8], s[8], d[8], kw[8];
  int ok;
  p256_convert_endianness(d, private_key, 32);
  p256_convert_endianness(kw, k, 32);
  ok = p256_sign(r, s, hash, (uint32_t) hash_size, d, kw);  // False if k is
                                                             // out of range
  if (ok) {
    p256_convert_endianness(signature, r, 32);
    p256_convert_endianness(signature + 32, s, 32);
  }
  mg_bzero((unsigned char *) d, sizeof(d));
  mg_bzero((unsigned char *) kw, sizeof(kw));
  return ok;
}

int mg_p256_verify(const uint8_t public_key[64], const uint8_t *hash,
                   size_t hash_size, const uint8_t signature[64]) {
  uint32_t x[8], y[8], r[8], s[8];
  p256_convert_endianness(x, public_key, 32);
  p256_convert_endianness(y, public_key + 32, 32);
  p256_convert_endianness(r, signature, 32);
  p256_convert_endianness(s, signature + 32, 32);
  return p256_verify(x, y, hash, (uint32_t) hash_size, r, s);  // Checks point
}
#undef get_bit
#endif  // MG_P256 == MG_P256_LENNGREN

#ifdef MG_ENABLE_LINES
#line 1 "src/tls_p384.c"
#endif
// P-384 from BearSSL, https://bearssl.org commit 7bea48e
// P-384 signature verification, used by built-in TLS for certificates and
// CertificateVerify. Shared bignum and ECDSA code is in tls_p256.c
//
// Copyright (c) 2016 Thomas Pornin <pornin@bolet.org>
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// Mongoose changes: br_ / BR_ names are prefixed with mg_ / MG_, static
// functions with p384_. Trimmed to P-384. Structs are not typedef-ed




#if MG_TLS == MG_TLS_BUILTIN || defined(MG_OTA_PUBLIC_KEY)

// From BearSSL src/ec/ec_secp384r1.c
static const unsigned char P384_N[] = {
	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 
	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 
	0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 
	0xC7, 0x63, 0x4D, 0x81, 0xF4, 0x37, 0x2D, 0xDF, 
	0x58, 0x1A, 0x0D, 0xB2, 0x48, 0xB0, 0xA7, 0x7A, 
	0xEC, 0xEC, 0x19, 0x6A, 0xCC, 0xC5, 0x29, 0x73
};

static const unsigned char P384_G[] = {
	0x04, 0xAA, 0x87, 0xCA, 0x22, 0xBE, 0x8B, 0x05,
	0x37, 0x8E, 0xB1, 0xC7, 0x1E, 0xF3, 0x20, 0xAD,
	0x74, 0x6E, 0x1D, 0x3B, 0x62, 0x8B, 0xA7, 0x9B,
	0x98, 0x59, 0xF7, 0x41, 0xE0, 0x82, 0x54, 0x2A,
	0x38, 0x55, 0x02, 0xF2, 0x5D, 0xBF, 0x55, 0x29,
	0x6C, 0x3A, 0x54, 0x5E, 0x38, 0x72, 0x76, 0x0A,
	0xB7, 0x36, 0x17, 0xDE, 0x4A, 0x96, 0x26, 0x2C,
	0x6F, 0x5D, 0x9E, 0x98, 0xBF, 0x92, 0x92, 0xDC,
	0x29, 0xF8, 0xF4, 0x1D, 0xBD, 0x28, 0x9A, 0x14,
	0x7C, 0xE9, 0xDA, 0x31, 0x13, 0xB5, 0xF0, 0xB8,
	0xC0, 0x0A, 0x60, 0xB1, 0xCE, 0x1D, 0x7E, 0x81,
	0x9D, 0x7A, 0x43, 0x1D, 0x7C, 0x90, 0xEA, 0x0E,
	0x5F
};


// From BearSSL src/ec/ec_prime_i31.c
/*
 * Parameters for supported curves (field modulus, and 'b' equation
 * parameter; both values use the 'i31' format, and 'b' is in Montgomery
 * representation).
 */

static const uint32_t P384_P[] = {
	0x0000018C,
	0x7FFFFFFF, 0x00000001, 0x00000000, 0x7FFFFFF8,
	0x7FFFFFEF, 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF,
	0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF,
	0x00000FFF
};

static const uint32_t P384_R2[] = {
	0x0000018C,
	0x00000000, 0x00000080, 0x7FFFFE00, 0x000001FF,
	0x00000800, 0x00000000, 0x7FFFE000, 0x00001FFF,
	0x00008000, 0x00008000, 0x00000000, 0x00000000,
	0x00000000
};

static const uint32_t P384_B[] = {
	0x0000018C,
	0x6E666840, 0x070D0392, 0x5D810231, 0x7651D50C,
	0x17E218D6, 0x1B192002, 0x44EFE441, 0x3A524E2B,
	0x2719BA5F, 0x41F02209, 0x36C5643E, 0x5813EFFE,
	0x000008A5
};

struct p384_curve_params {
	const uint32_t *p;
	const uint32_t *b;
	const uint32_t *R2;
	uint32_t p0i;
	size_t point_len;
};

static inline const struct p384_curve_params *
p384_id_to_curve(int curve)
{
	static const struct p384_curve_params pp = {
		P384_P, P384_B, P384_R2, 0x00000001,  97
	};

	(void)curve;
	return &pp;
}

#define I31_LEN   ((MG_BR_MAX_EC_SIZE + 61) / 31)

/*
 * Type for a point in Jacobian coordinates:
 * -- three values, x, y and z, in Montgomery representation
 * -- affine coordinates are X = x / z^2 and Y = y / z^3
 * -- for the point at infinity, z = 0
 */
struct p384_jacobian {
	uint32_t c[3][I31_LEN];
};

/*
 * We use a custom interpreter that uses a dozen registers, and
 * only six operations:
 *    MSET(d, a)       copy a into d
 *    MADD(d, a)       d = d+a (modular)
 *    MSUB(d, a)       d = d-a (modular)
 *    MMUL(d, a, b)    d = a*b (Montgomery multiplication)
 *    MINV(d, a, b)    invert d modulo p; a and b are used as scratch registers
 *    MTZ(d)           clear return value if d = 0
 * Destination of MMUL (d) must be distinct from operands (a and b).
 * There is no such constraint for MSUB and MADD.
 *
 * Registers include the operand coordinates, and temporaries.
 */
#define MSET(d, a)      (0x0000 + ((d) << 8) + ((a) << 4))
#define MADD(d, a)      (0x1000 + ((d) << 8) + ((a) << 4))
#define MSUB(d, a)      (0x2000 + ((d) << 8) + ((a) << 4))
#define MMUL(d, a, b)   (0x3000 + ((d) << 8) + ((a) << 4) + (b))
#define MINV(d, a, b)   (0x4000 + ((d) << 8) + ((a) << 4) + (b))
#define MTZ(d)          (0x5000 + ((d) << 8))
#define ENDCODE         0

/*
 * Registers for the input operands.
 */
#define P1x    0
#define P1y    1
#define P1z    2
#define P2x    3
#define P2y    4
#define P2z    5

/*
 * Alternate names for the first input operand.
 */
#define Px     0
#define Py     1
#define Pz     2

/*
 * Temporaries.
 */
#define t1     6
#define t2     7
#define t3     8
#define t4     9
#define t5    10
#define t6    11
#define t7    12

/*
 * Extra scratch registers available when there is no second operand (e.g.
 * for "double" and "affine").
 */
#define t8     3
#define t9     4
#define t10    5

/*
 * Doubling formulas are:
 *
 *   s = 4*x*y^2
 *   m = 3*(x + z^2)*(x - z^2)
 *   x' = m^2 - 2*s
 *   y' = m*(s - x') - 8*y^4
 *   z' = 2*y*z
 *
 * If y = 0 (P has order 2) then this yields infinity (z' = 0), as it
 * should. This case should not happen anyway, because our curves have
 * prime order, and thus do not contain any point of order 2.
 *
 * If P is infinity (z = 0), then again the formulas yield infinity,
 * which is correct. Thus, this code works for all points.
 *
 * Cost: 8 multiplications
 */
static const uint16_t p384_code_double[] = {
	/*
	 * Compute z^2 (in t1).
	 */
	MMUL(t1, Pz, Pz),

	/*
	 * Compute x-z^2 (in t2) and then x+z^2 (in t1).
	 */
	MSET(t2, Px),
	MSUB(t2, t1),
	MADD(t1, Px),

	/*
	 * Compute m = 3*(x+z^2)*(x-z^2) (in t1).
	 */
	MMUL(t3, t1, t2),
	MSET(t1, t3),
	MADD(t1, t3),
	MADD(t1, t3),

	/*
	 * Compute s = 4*x*y^2 (in t2) and 2*y^2 (in t3).
	 */
	MMUL(t3, Py, Py),
	MADD(t3, t3),
	MMUL(t2, Px, t3),
	MADD(t2, t2),

	/*
	 * Compute x' = m^2 - 2*s.
	 */
	MMUL(Px, t1, t1),
	MSUB(Px, t2),
	MSUB(Px, t2),

	/*
	 * Compute z' = 2*y*z.
	 */
	MMUL(t4, Py, Pz),
	MSET(Pz, t4),
	MADD(Pz, t4),

	/*
	 * Compute y' = m*(s - x') - 8*y^4. Note that we already have
	 * 2*y^2 in t3.
	 */
	MSUB(t2, Px),
	MMUL(Py, t1, t2),
	MMUL(t4, t3, t3),
	MSUB(Py, t4),
	MSUB(Py, t4),

	ENDCODE
};

/*
 * Addtions formulas are:
 *
 *   u1 = x1 * z2^2
 *   u2 = x2 * z1^2
 *   s1 = y1 * z2^3
 *   s2 = y2 * z1^3
 *   h = u2 - u1
 *   r = s2 - s1
 *   x3 = r^2 - h^3 - 2 * u1 * h^2
 *   y3 = r * (u1 * h^2 - x3) - s1 * h^3
 *   z3 = h * z1 * z2
 *
 * If both P1 and P2 are infinity, then z1 == 0 and z2 == 0, implying that
 * z3 == 0, so the result is correct.
 * If either of P1 or P2 is infinity, but not both, then z3 == 0, which is
 * not correct.
 * h == 0 only if u1 == u2; this happens in two cases:
 * -- if s1 == s2 then P1 and/or P2 is infinity, or P1 == P2
 * -- if s1 != s2 then P1 + P2 == infinity (but neither P1 or P2 is infinity)
 *
 * Thus, the following situations are not handled correctly:
 * -- P1 = 0 and P2 != 0
 * -- P1 != 0 and P2 = 0
 * -- P1 = P2
 * All other cases are properly computed. However, even in "incorrect"
 * situations, the three coordinates still are properly formed field
 * elements.
 *
 * The returned flag is cleared if r == 0. This happens in the following
 * cases:
 * -- Both points are on the same horizontal line (same Y coordinate).
 * -- Both points are infinity.
 * -- One point is infinity and the other is on line Y = 0.
 * The third case cannot happen with our curves (there is no valid point
 * on line Y = 0 since that would be a point of order 2). If the two
 * source points are non-infinity, then remains only the case where the
 * two points are on the same horizontal line.
 *
 * This allows us to detect the "P1 == P2" case, assuming that P1 != 0 and
 * P2 != 0:
 * -- If the returned value is not the point at infinity, then it was properly
 * computed.
 * -- Otherwise, if the returned flag is 1, then P1+P2 = 0, and the result
 * is indeed the point at infinity.
 * -- Otherwise (result is infinity, flag is 0), then P1 = P2 and we should
 * use the 'double' code.
 *
 * Cost: 16 multiplications
 */
static const uint16_t p384_code_add[] = {
	/*
	 * Compute u1 = x1*z2^2 (in t1) and s1 = y1*z2^3 (in t3).
	 */
	MMUL(t3, P2z, P2z),
	MMUL(t1, P1x, t3),
	MMUL(t4, P2z, t3),
	MMUL(t3, P1y, t4),

	/*
	 * Compute u2 = x2*z1^2 (in t2) and s2 = y2*z1^3 (in t4).
	 */
	MMUL(t4, P1z, P1z),
	MMUL(t2, P2x, t4),
	MMUL(t5, P1z, t4),
	MMUL(t4, P2y, t5),

	/*
	 * Compute h = u2 - u1 (in t2) and r = s2 - s1 (in t4).
	 */
	MSUB(t2, t1),
	MSUB(t4, t3),

	/*
	 * Report cases where r = 0 through the returned flag.
	 */
	MTZ(t4),

	/*
	 * Compute u1*h^2 (in t6) and h^3 (in t5).
	 */
	MMUL(t7, t2, t2),
	MMUL(t6, t1, t7),
	MMUL(t5, t7, t2),

	/*
	 * Compute x3 = r^2 - h^3 - 2*u1*h^2.
	 * t1 and t7 can be used as scratch registers.
	 */
	MMUL(P1x, t4, t4),
	MSUB(P1x, t5),
	MSUB(P1x, t6),
	MSUB(P1x, t6),

	/*
	 * Compute y3 = r*(u1*h^2 - x3) - s1*h^3.
	 */
	MSUB(t6, P1x),
	MMUL(P1y, t4, t6),
	MMUL(t1, t5, t3),
	MSUB(P1y, t1),

	/*
	 * Compute z3 = h*z1*z2.
	 */
	MMUL(t1, P1z, P2z),
	MMUL(P1z, t1, t2),

	ENDCODE
};

/*
 * Check that the point is on the curve. This code snippet assumes the
 * following conventions:
 * -- Coordinates x and y have been freshly decoded in P1 (but not
 * converted to Montgomery coordinates yet).
 * -- P2x, P2y and P2z are set to, respectively, R^2, b*R and 1.
 */
static const uint16_t p384_code_check[] = {

	/* Convert x and y to Montgomery representation. */
	MMUL(t1, P1x, P2x),
	MMUL(t2, P1y, P2x),
	MSET(P1x, t1),
	MSET(P1y, t2),

	/* Compute x^3 in t1. */
	MMUL(t2, P1x, P1x),
	MMUL(t1, P1x, t2),

	/* Subtract 3*x from t1. */
	MSUB(t1, P1x),
	MSUB(t1, P1x),
	MSUB(t1, P1x),

	/* Add b. */
	MADD(t1, P2y),

	/* Compute y^2 in t2. */
	MMUL(t2, P1y, P1y),

	/* Compare y^2 with x^3 - 3*x + b; they must match. */
	MSUB(t1, t2),
	MTZ(t1),

	/* Set z to 1 (in Montgomery representation). */
	MMUL(P1z, P2x, P2z),

	ENDCODE
};

/*
 * Conversion back to affine coordinates. This code snippet assumes that
 * the z coordinate of P2 is set to 1 (not in Montgomery representation).
 */
static const uint16_t p384_code_affine[] = {

	/* Save z*R in t1. */
	MSET(t1, P1z),

	/* Compute z^3 in t2. */
	MMUL(t2, P1z, P1z),
	MMUL(t3, P1z, t2),
	MMUL(t2, t3, P2z),

	/* Invert to (1/z^3) in t2. */
	MINV(t2, t3, t4),

	/* Compute y. */
	MSET(t3, P1y),
	MMUL(P1y, t2, t3),

	/* Compute (1/z^2) in t3. */
	MMUL(t3, t2, t1),

	/* Compute x. */
	MSET(t2, P1x),
	MMUL(P1x, t2, t3),

	ENDCODE
};

static uint32_t
p384_run_code(struct p384_jacobian *P1, const struct p384_jacobian *P2,
	const struct p384_curve_params *cc, const uint16_t *code)
{
	uint32_t r;
	uint32_t t[13][I31_LEN];
	size_t u;

	r = 1;

	/*
	 * Copy the two operands in the dedicated registers.
	 */
	memcpy(t[P1x], P1->c, 3 * I31_LEN * sizeof(uint32_t));
	memcpy(t[P2x], P2->c, 3 * I31_LEN * sizeof(uint32_t));

	/*
	 * Run formulas.
	 */
	for (u = 0;; u ++) {
		unsigned op, d, a, b;

		op = code[u];
		if (op == 0) {
			break;
		}
		d = (op >> 8) & 0x0F;
		a = (op >> 4) & 0x0F;
		b = op & 0x0F;
		op >>= 12;
		switch (op) {
			uint32_t ctl;
			size_t plen;
			unsigned char tp[(MG_BR_MAX_EC_SIZE + 7) >> 3];

		case 0:
			memcpy(t[d], t[a], I31_LEN * sizeof(uint32_t));
			break;
		case 1:
			ctl = mg_br_i31_add(t[d], t[a], 1);
			ctl |= MG_BR_NOT(mg_br_i31_sub(t[d], cc->p, 0));
			mg_br_i31_sub(t[d], cc->p, ctl);
			break;
		case 2:
			mg_br_i31_add(t[d], cc->p, mg_br_i31_sub(t[d], t[a], 1));
			break;
		case 3:
			mg_br_i31_montymul(t[d], t[a], t[b], cc->p, cc->p0i);
			break;
		case 4:
			plen = (cc->p[0] - (cc->p[0] >> 5) + 7) >> 3;
			mg_br_i31_encode(tp, plen, cc->p);
			tp[plen - 1] -= 2;
			mg_br_i31_modpow(t[d], tp, plen,
				cc->p, cc->p0i, t[a], t[b]);
			break;
		default:
			r &= ~mg_br_i31_iszero(t[d]);
			break;
		}
	}

	/*
	 * Copy back result.
	 */
	memcpy(P1->c, t[P1x], 3 * I31_LEN * sizeof(uint32_t));
	return r;
}

static void
p384_set_one(uint32_t *x, const uint32_t *p)
{
	size_t plen;

	plen = (p[0] + 63) >> 5;
	memset(x, 0, plen * sizeof *x);
	x[0] = p[0];
	x[1] = 0x00000001;
}

static void
p384_point_zero(struct p384_jacobian *P, const struct p384_curve_params *cc)
{
	memset(P, 0, sizeof *P);
	P->c[0][0] = P->c[1][0] = P->c[2][0] = cc->p[0];
}

static inline void
p384_point_double(struct p384_jacobian *P, const struct p384_curve_params *cc)
{
	p384_run_code(P, P, cc, p384_code_double);
}

static inline uint32_t
p384_point_add(struct p384_jacobian *P1, const struct p384_jacobian *P2, const struct p384_curve_params *cc)
{
	return p384_run_code(P1, P2, cc, p384_code_add);
}

static void
p384_point_mul(struct p384_jacobian *P, const unsigned char *x, size_t xlen,
	const struct p384_curve_params *cc)
{
	/*
	 * We do a simple double-and-add ladder with a 2-bit window
	 * to make only one add every two doublings. We thus first
	 * precompute 2P and 3P in some local buffers.
	 *
	 * We always perform two doublings and one addition; the
	 * addition is with P, 2P and 3P and is done in a temporary
	 * array.
	 *
	 * The addition code cannot handle cases where one of the
	 * operands is infinity, which is the case at the start of the
	 * ladder. We therefore need to maintain a flag that controls
	 * this situation.
	 */
	uint32_t qz;
	struct p384_jacobian P2, P3, Q, T, U;

	memcpy(&P2, P, sizeof P2);
	p384_point_double(&P2, cc);
	memcpy(&P3, P, sizeof P3);
	p384_point_add(&P3, &P2, cc);

	p384_point_zero(&Q, cc);
	qz = 1;
	while (xlen -- > 0) {
		int k;

		for (k = 6; k >= 0; k -= 2) {
			uint32_t bits;
			uint32_t bnz;

			p384_point_double(&Q, cc);
			p384_point_double(&Q, cc);
			memcpy(&T, P, sizeof T);
			memcpy(&U, &Q, sizeof U);
			bits = (*x >> k) & (uint32_t)3;
			bnz = MG_BR_NEQ(bits, 0);
			MG_BR_CCOPY(MG_BR_EQ(bits, 2), &T, &P2, sizeof T);
			MG_BR_CCOPY(MG_BR_EQ(bits, 3), &T, &P3, sizeof T);
			p384_point_add(&U, &T, cc);
			MG_BR_CCOPY(bnz & qz, &Q, &T, sizeof Q);
			MG_BR_CCOPY(bnz & ~qz, &Q, &U, sizeof Q);
			qz &= ~bnz;
		}
		x ++;
	}
	memcpy(P, &Q, sizeof Q);
}

/*
 * Decode point into Jacobian coordinates. This function does not support
 * the point at infinity. If the point is invalid then this returns 0, but
 * the coordinates are still set to properly formed field elements.
 */
static uint32_t
p384_point_decode(struct p384_jacobian *P, const void *src, size_t len, const struct p384_curve_params *cc)
{
	/*
	 * Points must use uncompressed format:
	 * -- first byte is 0x04;
	 * -- coordinates X and Y use unsigned big-endian, with the same
	 *    length as the field modulus.
	 *
	 * We don't support hybrid format (uncompressed, but first byte
	 * has value 0x06 or 0x07, depending on the least significant bit
	 * of Y) because it is rather useless, and explicitly forbidden
	 * by PKIX (RFC 5480, section 2.2).
	 *
	 * We don't support compressed format either, because it is not
	 * much used in practice (there are or were patent-related
	 * concerns about point compression, which explains the lack of
	 * generalised support). Also, point compression support would
	 * need a bit more code.
	 */
	const unsigned char *buf;
	size_t plen, zlen;
	uint32_t r;
	struct p384_jacobian Q;

	buf = (const unsigned char *)src;
	p384_point_zero(P, cc);
	plen = (cc->p[0] - (cc->p[0] >> 5) + 7) >> 3;
	if (len != 1 + (plen << 1)) {
		return 0;
	}
	r = mg_br_i31_decode_mod(P->c[0], buf + 1, plen, cc->p);
	r &= mg_br_i31_decode_mod(P->c[1], buf + 1 + plen, plen, cc->p);

	/*
	 * Check first byte.
	 */
	r &= MG_BR_EQ(buf[0], 0x04);
	/* obsolete
	r &= MG_BR_EQ(buf[0], 0x04) | (MG_BR_EQ(buf[0] & 0xFE, 0x06)
		& ~(uint32_t)(buf[0] ^ buf[plen << 1]));
	*/

	/*
	 * Convert coordinates and check that the point is valid.
	 */
	zlen = ((cc->p[0] + 63) >> 5) * sizeof(uint32_t);
	memcpy(Q.c[0], cc->R2, zlen);
	memcpy(Q.c[1], cc->b, zlen);
	p384_set_one(Q.c[2], cc->p);
	r &= ~p384_run_code(P, &Q, cc, p384_code_check);
	return r;
}

/*
 * Encode a point. This method assumes that the point is correct and is
 * not the point at infinity. Encoded size is always 1+2*plen, where
 * plen is the field modulus length, in bytes.
 */
static void
p384_point_encode(void *dst, const struct p384_jacobian *P, const struct p384_curve_params *cc)
{
	unsigned char *buf;
	uint32_t xbl;
	size_t plen;
	struct p384_jacobian Q, T;

	buf = (unsigned char *)dst;
	xbl = cc->p[0];
	xbl -= (xbl >> 5);
	plen = (xbl + 7) >> 3;
	buf[0] = 0x04;
	memcpy(&Q, P, sizeof *P);
	p384_set_one(T.c[2], cc->p);
	p384_run_code(&Q, &T, cc, p384_code_affine);
	mg_br_i31_encode(buf + 1, plen, Q.c[0]);
	mg_br_i31_encode(buf + 1 + plen, plen, Q.c[1]);
}

static const unsigned char *
p384_api_generator(int curve, size_t *len)
{
	(void)curve;
	*len = sizeof P384_G;
	return P384_G;
}

static const unsigned char *
p384_api_order(int curve, size_t *len)
{
	(void)curve;
	*len = sizeof P384_N;
	return P384_N;
}

static size_t
p384_api_xoff(int curve, size_t *len)
{
	p384_api_generator(curve, len);
	*len >>= 1;
	return 1;
}

static uint32_t
p384_api_mul(unsigned char *G, size_t Glen,
	const unsigned char *x, size_t xlen, int curve)
{
	uint32_t r;
	const struct p384_curve_params *cc;
	struct p384_jacobian P;

	cc = p384_id_to_curve(curve);
	if (Glen != cc->point_len) {
		return 0;
	}
	r = p384_point_decode(&P, G, Glen, cc);
	p384_point_mul(&P, x, xlen, cc);
	p384_point_encode(G, &P, cc);
	return r;
}

static size_t
p384_api_mulgen(unsigned char *R,
	const unsigned char *x, size_t xlen, int curve)
{
	const unsigned char *G;
	size_t Glen;

	G = p384_api_generator(curve, &Glen);
	memcpy(R, G, Glen);
	p384_api_mul(R, Glen, x, xlen, curve);
	return Glen;
}

static uint32_t
p384_api_muladd(unsigned char *A, const unsigned char *B, size_t len,
	const unsigned char *x, size_t xlen,
	const unsigned char *y, size_t ylen, int curve)
{
	uint32_t r, t, z;
	const struct p384_curve_params *cc;
	struct p384_jacobian P, Q;

	/*
	 * TODO: see about merging the two ladders. Right now, we do
	 * two independent point multiplications, which is a bit
	 * wasteful of CPU resources (but yields short code).
	 */

	cc = p384_id_to_curve(curve);
	if (len != cc->point_len) {
		return 0;
	}
	r = p384_point_decode(&P, A, len, cc);
	if (B == NULL) {
		size_t Glen;

		B = p384_api_generator(curve, &Glen);
	}
	r &= p384_point_decode(&Q, B, len, cc);
	p384_point_mul(&P, x, xlen, cc);
	p384_point_mul(&Q, y, ylen, cc);

	/*
	 * We want to compute P+Q. Since the base points A and B are distinct
	 * from infinity, and the multipliers are non-zero and lower than the
	 * curve order, then we know that P and Q are non-infinity. This
	 * leaves two special situations to test for:
	 * -- If P = Q then we must use p384_point_double().
	 * -- If P+Q = 0 then we must report an error.
	 */
	t = p384_point_add(&P, &Q, cc);
	p384_point_double(&Q, cc);
	z = mg_br_i31_iszero(P.c[2]);

	/*
	 * If z is 1 then either P+Q = 0 (t = 1) or P = Q (t = 0). So we
	 * have the following:
	 *
	 *   z = 0, t = 0   return P (normal addition)
	 *   z = 0, t = 1   return P (normal addition)
	 *   z = 1, t = 0   return Q (a 'double' case)
	 *   z = 1, t = 1   report an error (P+Q = 0)
	 */
	MG_BR_CCOPY(z & ~t, &P, &Q, sizeof Q);
	p384_point_encode(A, &P, cc);
	r &= ~(z & t);

	return r;
}

static const struct mg_br_ec_impl mg_br_ec_prime_i31 = {
	(uint32_t)1 << MG_BR_EC_secp384r1,
	&p384_api_generator,
	&p384_api_order,
	&p384_api_xoff,
	&p384_api_mul,
	&p384_api_mulgen,
	&p384_api_muladd
};

// See tls_p384.h
int mg_p384_verify(const uint8_t public_key[96], const uint8_t *hash,
                   size_t hash_size, const uint8_t signature[96]) {
  unsigned char q[97];
  q[0] = 0x04;
  memcpy(q + 1, public_key, 96);
  return mg_br_ecdsa_i31_vrfy_raw(&mg_br_ec_prime_i31, MG_BR_EC_secp384r1, hash,
                                  hash_size, q, sizeof(q), signature, 96) == 1;
}

// Undefine local macros, mongoose.c is one translation unit
#undef I31_LEN
#undef MSET
#undef MADD
#undef MSUB
#undef MMUL
#undef MINV
#undef MTZ
#undef ENDCODE
#undef P1x
#undef P1y
#undef P1z
#undef P2x
#undef P2y
#undef P2z
#undef Px
#undef Py
#undef Pz
#undef t1
#undef t2
#undef t3
#undef t4
#undef t5
#undef t6
#undef t7
#undef t8
#undef t9
#undef t10
#endif  // MG_TLS == MG_TLS_BUILTIN || defined(MG_OTA_PUBLIC_KEY)

#ifdef MG_ENABLE_LINES
#line 1 "src/tls_rsa.c"
#endif





#if MG_TLS == MG_TLS_BUILTIN

/*
 * The RSA bigint backend below is derived from BearSSL's i31 RSA code.
 *
 * Copyright (c) 2016 Thomas Pornin <pornin@bolet.org>
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#define BR_MAX_RSA_SIZE 4096
#define BR_MAX_RSA_FACTOR ((BR_MAX_RSA_SIZE + 64) >> 1)
#define BR_I31_WORDS(bits) (2 + (((bits) + 30) / 31))
#define BR_RSA_WORDS BR_I31_WORDS(BR_MAX_RSA_SIZE)
#define BR_RSA_FACTOR_WORDS BR_I31_WORDS(BR_MAX_RSA_FACTOR)
#define BR_64 1
#define MUL31(x, y) ((uint64_t) (x) * (uint64_t) (y))
#define MUL31_lo(x, y) (((uint32_t) (x) * (uint32_t) (y)) & 0x7fffffffU)
#define CCOPY br_ccopy

static uint32_t NOT(uint32_t ctl) {
  return ctl ^ 1;
}
static uint32_t MUX(uint32_t ctl, uint32_t x, uint32_t y) {
  return y ^ (-ctl & (x ^ y));
}
static uint32_t EQ(uint32_t x, uint32_t y) {
  uint32_t q = x ^ y;
  return NOT((q | -q) >> 31);
}
static uint32_t NEQ(uint32_t x, uint32_t y) {
  uint32_t q = x ^ y;
  return (q | -q) >> 31;
}
static uint32_t GT(uint32_t x, uint32_t y) {
  uint32_t z = y - x;
  return (z ^ ((x ^ y) & (x ^ z))) >> 31;
}
#define GE(x, y) NOT(GT(y, x))
#define LT(x, y) GT(y, x)
static uint32_t BIT_LENGTH(uint32_t x) {
  uint32_t k, c;
  k = NEQ(x, 0);
  c = GT(x, 0xffff);
  x = MUX(c, x >> 16, x);
  k += c << 4;
  c = GT(x, 0x00ff);
  x = MUX(c, x >> 8, x);
  k += c << 3;
  c = GT(x, 0x000f);
  x = MUX(c, x >> 4, x);
  k += c << 2;
  c = GT(x, 0x0003);
  x = MUX(c, x >> 2, x);
  k += c << 1;
  k += GT(x, 0x0001);
  return k;
}
static void br_enc32be(void *dst, uint32_t x) {
  unsigned char *buf = (unsigned char *) dst;
  buf[0] = (unsigned char) (x >> 24);
  buf[1] = (unsigned char) (x >> 16);
  buf[2] = (unsigned char) (x >> 8);
  buf[3] = (unsigned char) x;
}
static void br_ccopy(uint32_t ctl, void *dst, const void *src, size_t len) {
  unsigned char *d = (unsigned char *) dst;
  const unsigned char *s = (const unsigned char *) src;
  while (len-- > 0) {
    uint32_t x = *s++, y = *d;
    *d++ = (unsigned char) MUX(ctl, x, y);
  }
}

static uint32_t br_i31_add(uint32_t *, const uint32_t *, uint32_t);
static uint32_t br_i31_sub(uint32_t *, const uint32_t *, uint32_t);
static uint32_t br_i31_bit_length(uint32_t *, size_t);
static void br_i31_decode(uint32_t *, const void *, size_t);
static void br_i31_encode(void *, size_t, const uint32_t *);
static uint32_t br_i31_ninv31(uint32_t);
static uint32_t br_divrem(uint32_t, uint32_t, uint32_t, uint32_t *);
static void br_i31_muladd_small(uint32_t *, uint32_t, const uint32_t *);
static void br_i31_reduce(uint32_t *, const uint32_t *, const uint32_t *);
static void br_i31_rshift(uint32_t *, int);
static void br_i31_decode_reduce(uint32_t *, const void *, size_t,
                                 const uint32_t *);
static void br_i31_mulacc(uint32_t *, const uint32_t *, const uint32_t *);
static void br_i31_montymul(uint32_t *, const uint32_t *, const uint32_t *,
                            const uint32_t *, uint32_t);
static void br_i31_to_monty(uint32_t *, const uint32_t *);
static void br_i31_modpow(uint32_t *, const unsigned char *, size_t,
                          const uint32_t *, uint32_t, uint32_t *, uint32_t *);
static void br_i31_zero(uint32_t *x, uint32_t bit_len) {
  *x++ = bit_len;
  memset(x, 0, ((bit_len + 31) >> 5) * sizeof *x);
}
static uint32_t br_rem(uint32_t hi, uint32_t lo, uint32_t d) {
  uint32_t r;
  br_divrem(hi, lo, d, &r);
  return r;
}
static uint32_t br_div(uint32_t hi, uint32_t lo, uint32_t d) {
  uint32_t r;
  return br_divrem(hi, lo, d, &r);
}
static uint32_t br_i31_add(uint32_t *a, const uint32_t *b, uint32_t ctl) {
  uint32_t cc;
  size_t u, m;

  cc = 0;
  m = (a[0] + 63) >> 5;
  for (u = 1; u < m; u++) {
    uint32_t aw, bw, naw;

    aw = a[u];
    bw = b[u];
    naw = aw + bw + cc;
    cc = naw >> 31;
    a[u] = MUX(ctl, naw & (uint32_t) 0x7FFFFFFF, aw);
  }
  return cc;
}
static uint32_t br_i31_sub(uint32_t *a, const uint32_t *b, uint32_t ctl) {
  uint32_t cc;
  size_t u, m;

  cc = 0;
  m = (a[0] + 63) >> 5;
  for (u = 1; u < m; u++) {
    uint32_t aw, bw, naw;

    aw = a[u];
    bw = b[u];
    naw = aw - bw - cc;
    cc = naw >> 31;
    a[u] = MUX(ctl, naw & 0x7FFFFFFF, aw);
  }
  return cc;
}
static uint32_t br_i31_bit_length(uint32_t *x, size_t xlen) {
  uint32_t tw, twk;

  tw = 0;
  twk = 0;
  while (xlen-- > 0) {
    uint32_t w, c;

    c = EQ(tw, 0);
    w = x[xlen];
    tw = MUX(c, w, tw);
    twk = MUX(c, (uint32_t) xlen, twk);
  }
  return (twk << 5) + BIT_LENGTH(tw);
}
static void br_i31_decode(uint32_t *x, const void *src, size_t len) {
  const unsigned char *buf;
  size_t u, v;
  uint32_t acc;
  int acc_len;

  buf = (const unsigned char *) src;
  u = len;
  v = 1;
  acc = 0;
  acc_len = 0;
  while (u-- > 0) {
    uint32_t b;

    b = buf[u];
    acc |= (b << acc_len);
    acc_len += 8;
    if (acc_len >= 31) {
      x[v++] = acc & (uint32_t) 0x7FFFFFFF;
      acc_len -= 31;
      acc = b >> (8 - acc_len);
    }
  }
  if (acc_len != 0) {
    x[v++] = acc;
  }
  x[0] = br_i31_bit_length(x + 1, v - 1);
}
static void br_i31_encode(void *dst, size_t len, const uint32_t *x) {
  unsigned char *buf;
  size_t k, xlen;
  uint32_t acc;
  int acc_len;

  xlen = (x[0] + 31) >> 5;
  if (xlen == 0) {
    memset(dst, 0, len);
    return;
  }
  buf = (unsigned char *) dst + len;
  k = 1;
  acc = 0;
  acc_len = 0;
  while (len != 0) {
    uint32_t w;

    w = (k <= xlen) ? x[k] : 0;
    k++;
    if (acc_len == 0) {
      acc = w;
      acc_len = 31;
    } else {
      uint32_t z;

      z = acc | (w << acc_len);
      acc_len--;
      acc = w >> (31 - acc_len);
      if (len >= 4) {
        buf -= 4;
        len -= 4;
        br_enc32be(buf, z);
      } else {
        switch (len) {
          case 3:
            buf[-3] = (unsigned char) (z >> 16);
            /* fall through */
          case 2:
            buf[-2] = (unsigned char) (z >> 8);
            /* fall through */
          case 1:
            buf[-1] = (unsigned char) z;
            break;
        }
        return;
      }
    }
  }
}
static uint32_t br_i31_ninv31(uint32_t x) {
  uint32_t y;

  y = 2 - x;
  y *= 2 - y * x;
  y *= 2 - y * x;
  y *= 2 - y * x;
  y *= 2 - y * x;
  return MUX(x & 1, -y, 0) & 0x7FFFFFFF;
}
static uint32_t br_divrem(uint32_t hi, uint32_t lo, uint32_t d, uint32_t *r) {
  /* TODO: optimize this */
  uint32_t q;
  uint32_t ch, cf;
  int k;

  q = 0;
  ch = EQ(hi, d);
  hi = MUX(ch, 0, hi);
  for (k = 31; k > 0; k--) {
    int j;
    uint32_t w, ctl, hi2, lo2;

    j = 32 - k;
    w = (hi << j) | (lo >> k);
    ctl = GE(w, d) | (hi >> k);
    hi2 = (w - d) >> j;
    lo2 = lo - (d << k);
    hi = MUX(ctl, hi2, hi);
    lo = MUX(ctl, lo2, lo);
    q |= ctl << k;
  }
  cf = GE(lo, d) | hi;
  q |= cf;
  *r = MUX(cf, lo - d, lo);
  return q;
}
static void br_i31_muladd_small(uint32_t *x, uint32_t z, const uint32_t *m) {
  uint32_t m_bitlen;
  unsigned mblr;
  size_t u, mlen;
  uint32_t a0, a1, b0, hi, g, q, tb;
  uint32_t under, over;
  uint32_t cc;

  /*
   * We can test on the modulus bit length since we accept to
   * leak that length.
   */
  m_bitlen = m[0];
  if (m_bitlen == 0) {
    return;
  }
  if (m_bitlen <= 31) {
    uint32_t lo;

    hi = x[1] >> 1;
    lo = (x[1] << 31) | z;
    x[1] = br_rem(hi, lo, m[1]);
    return;
  }
  mlen = (m_bitlen + 31) >> 5;
  mblr = (unsigned) m_bitlen & 31;

  /*
   * Principle: we estimate the quotient (x*2^31+z)/m by
   * doing a 64/32 division with the high words.
   *
   * Let:
   *   w = 2^31
   *   a = (w*a0 + a1) * w^N + a2
   *   b = b0 * w^N + b2
   * such that:
   *   0 <= a0 < w
   *   0 <= a1 < w
   *   0 <= a2 < w^N
   *   w/2 <= b0 < w
   *   0 <= b2 < w^N
   *   a < w*b
   * I.e. the two top words of a are a0:a1, the top word of b is
   * b0, we ensured that b0 is "full" (high bit set), and a is
   * such that the quotient q = a/b fits on one word (0 <= q < w).
   *
   * If a = b*q + r (with 0 <= r < q), we can estimate q by
   * doing an Euclidean division on the top words:
   *   a0*w+a1 = b0*u + v  (with 0 <= v < b0)
   * Then the following holds:
   *   0 <= u <= w
   *   u-2 <= q <= u
   */
  hi = x[mlen];
  if (mblr == 0) {
    a0 = x[mlen];
    memmove(x + 2, x + 1, (mlen - 1) * sizeof *x);
    x[1] = z;
    a1 = x[mlen];
    b0 = m[mlen];
  } else {
    a0 = ((x[mlen] << (31 - mblr)) | (x[mlen - 1] >> mblr)) & 0x7FFFFFFF;
    memmove(x + 2, x + 1, (mlen - 1) * sizeof *x);
    x[1] = z;
    a1 = ((x[mlen] << (31 - mblr)) | (x[mlen - 1] >> mblr)) & 0x7FFFFFFF;
    b0 = ((m[mlen] << (31 - mblr)) | (m[mlen - 1] >> mblr)) & 0x7FFFFFFF;
  }

  /*
   * We estimate a divisor q. If the quotient returned by br_div()
   * is g:
   * -- If a0 == b0 then g == 0; we want q = 0x7FFFFFFF.
   * -- Otherwise:
   *    -- if g == 0 then we set q = 0;
   *    -- otherwise, we set q = g - 1.
   * The properties described above then ensure that the true
   * quotient is q-1, q or q+1.
   *
   * Take care that a0, a1 and b0 are 31-bit words, not 32-bit. We
   * must adjust the parameters to br_div() accordingly.
   */
  g = br_div(a0 >> 1, a1 | (a0 << 31), b0);
  q = MUX(EQ(a0, b0), 0x7FFFFFFF, MUX(EQ(g, 0), 0, g - 1));

  /*
   * We subtract q*m from x (with the extra high word of value 'hi').
   * Since q may be off by 1 (in either direction), we may have to
   * add or subtract m afterwards.
   *
   * The 'tb' flag will be true (1) at the end of the loop if the
   * result is greater than or equal to the modulus (not counting
   * 'hi' or the carry).
   */
  cc = 0;
  tb = 1;
  for (u = 1; u <= mlen; u++) {
    uint32_t mw, zw, xw, nxw;
    uint64_t zl;

    mw = m[u];
    zl = MUL31(mw, q) + cc;
    cc = (uint32_t) (zl >> 31);
    zw = (uint32_t) zl & (uint32_t) 0x7FFFFFFF;
    xw = x[u];
    nxw = xw - zw;
    cc += nxw >> 31;
    nxw &= 0x7FFFFFFF;
    x[u] = nxw;
    tb = MUX(EQ(nxw, mw), tb, GT(nxw, mw));
  }

  /*
   * If we underestimated q, then either cc < hi (one extra bit
   * beyond the top array word), or cc == hi and tb is true (no
   * extra bit, but the result is not lower than the modulus). In
   * these cases we must subtract m once.
   *
   * Otherwise, we may have overestimated, which will show as
   * cc > hi (thus a negative result). Correction is adding m once.
   */
  over = GT(cc, hi);
  under = ~over & (tb | LT(cc, hi));
  br_i31_add(x, m, over);
  br_i31_sub(x, m, under);
}
static void br_i31_reduce(uint32_t *x, const uint32_t *a, const uint32_t *m) {
  uint32_t m_bitlen, a_bitlen;
  size_t mlen, alen, u;

  m_bitlen = m[0];
  mlen = (m_bitlen + 31) >> 5;

  x[0] = m_bitlen;
  if (m_bitlen == 0) {
    return;
  }

  /*
   * If the source is shorter, then simply copy all words from a[]
   * and zero out the upper words.
   */
  a_bitlen = a[0];
  alen = (a_bitlen + 31) >> 5;
  if (a_bitlen < m_bitlen) {
    memcpy(x + 1, a + 1, alen * sizeof *a);
    for (u = alen; u < mlen; u++) {
      x[u + 1] = 0;
    }
    return;
  }

  /*
   * The source length is at least equal to that of the modulus.
   * We must thus copy N-1 words, and input the remaining words
   * one by one.
   */
  memcpy(x + 1, a + 2 + (alen - mlen), (mlen - 1) * sizeof *a);
  x[mlen] = 0;
  for (u = 1 + alen - mlen; u > 0; u--) {
    br_i31_muladd_small(x, a[u], m);
  }
}
static void br_i31_rshift(uint32_t *x, int count) {
  size_t u, len;
  uint32_t r;

  len = (x[0] + 31) >> 5;
  if (len == 0) {
    return;
  }
  r = x[1] >> count;
  for (u = 2; u <= len; u++) {
    uint32_t w;

    w = x[u];
    x[u - 1] = ((w << (31 - count)) | r) & 0x7FFFFFFF;
    r = w >> count;
  }
  x[len] = r;
}
static void br_i31_decode_reduce(uint32_t *x, const void *src, size_t len,
                                 const uint32_t *m) {
  uint32_t m_ebitlen, m_rbitlen;
  size_t mblen, k;
  const unsigned char *buf;
  uint32_t acc;
  int acc_len;

  /*
   * Get the encoded bit length.
   */
  m_ebitlen = m[0];

  /*
   * Special case for an invalid (null) modulus.
   */
  if (m_ebitlen == 0) {
    x[0] = 0;
    return;
  }

  /*
   * Clear the destination.
   */
  br_i31_zero(x, m_ebitlen);

  /*
   * First decode directly as many bytes as possible. This requires
   * computing the actual bit length.
   */
  m_rbitlen = m_ebitlen >> 5;
  m_rbitlen = (m_ebitlen & 31) + (m_rbitlen << 5) - m_rbitlen;
  mblen = (m_rbitlen + 7) >> 3;
  k = mblen - 1;
  if (k >= len) {
    br_i31_decode(x, src, len);
    x[0] = m_ebitlen;
    return;
  }
  buf = (const unsigned char *) src;
  br_i31_decode(x, buf, k);
  x[0] = m_ebitlen;

  /*
   * Input remaining bytes, using 31-bit words.
   */
  acc = 0;
  acc_len = 0;
  while (k < len) {
    uint32_t v;

    v = buf[k++];
    if (acc_len >= 23) {
      acc_len -= 23;
      acc <<= (8 - acc_len);
      acc |= v >> acc_len;
      br_i31_muladd_small(x, acc, m);
      acc = v & (0xFF >> (8 - acc_len));
    } else {
      acc = (acc << 8) | v;
      acc_len += 8;
    }
  }

  /*
   * We may have some bits accumulated. We then perform a shift to
   * be able to inject these bits as a full 31-bit word.
   */
  if (acc_len != 0) {
    acc = (acc | (x[1] << acc_len)) & 0x7FFFFFFF;
    br_i31_rshift(x, 31 - acc_len);
    br_i31_muladd_small(x, acc, m);
  }
}
static void br_i31_mulacc(uint32_t *d, const uint32_t *a, const uint32_t *b) {
  size_t alen, blen, u;
  uint32_t dl, dh;

  alen = (a[0] + 31) >> 5;
  blen = (b[0] + 31) >> 5;

  /*
   * We want to add the two bit lengths, but these are encoded,
   * which requires some extra care.
   */
  dl = (a[0] & 31) + (b[0] & 31);
  dh = (a[0] >> 5) + (b[0] >> 5);
  d[0] = (dh << 5) + dl + (~(uint32_t) (dl - 31) >> 31);

  for (u = 0; u < blen; u++) {
    uint32_t f;
    size_t v;

    /*
     * Carry always fits on 31 bits; we want to keep it in a
     * 32-bit register on 32-bit architectures (on a 64-bit
     * architecture, cast down from 64 to 32 bits means
     * clearing the high bits, which is not free; on a 32-bit
     * architecture, the same operation really means ignoring
     * the top register, which has negative or zero cost).
     */
#if BR_64
    uint64_t cc;
#else
    uint32_t cc;
#endif

    f = b[1 + u];
    cc = 0;
    for (v = 0; v < alen; v++) {
      uint64_t z;

      z = (uint64_t) d[1 + u + v] + MUL31(f, a[1 + v]) + cc;
      cc = z >> 31;
      d[1 + u + v] = (uint32_t) z & 0x7FFFFFFF;
    }
    d[1 + u + alen] = (uint32_t) cc;
  }
}
static void br_i31_montymul(uint32_t *d, const uint32_t *x, const uint32_t *y,
                            const uint32_t *m, uint32_t m0i) {
  /*
   * Each outer loop iteration computes:
   *   d <- (d + xu*y + f*m) / 2^31
   * We have xu <= 2^31-1 and f <= 2^31-1.
   * Thus, if d <= 2*m-1 on input, then:
   *   2*m-1 + 2*(2^31-1)*m <= (2^32)*m-1
   * and the new d value is less than 2*m.
   *
   * We represent d over 31-bit words, with an extra word 'dh'
   * which can thus be only 0 or 1.
   */
  size_t len, len4, u, v;
  uint32_t dh;

  len = (m[0] + 31) >> 5;
  len4 = len & ~(size_t) 3;
  br_i31_zero(d, m[0]);
  dh = 0;
  for (u = 0; u < len; u++) {
    /*
     * The carry for each operation fits on 32 bits:
     *   d[v+1] <= 2^31-1
     *   xu*y[v+1] <= (2^31-1)*(2^31-1)
     *   f*m[v+1] <= (2^31-1)*(2^31-1)
     *   r <= 2^32-1
     *   (2^31-1) + 2*(2^31-1)*(2^31-1) + (2^32-1) = 2^63 - 2^31
     * After division by 2^31, the new r is then at most 2^32-1
     *
     * Using a 32-bit carry has performance benefits on 32-bit
     * systems; however, on 64-bit architectures, we prefer to
     * keep the carry (r) in a 64-bit register, thus avoiding some
     * "clear high bits" operations.
     */
    uint32_t f, xu;
#if BR_64
    uint64_t r;
#else
    uint32_t r;
#endif

    xu = x[u + 1];
    f = MUL31_lo((d[1] + MUL31_lo(x[u + 1], y[1])), m0i);

    r = 0;
    for (v = 0; v < len4; v += 4) {
      uint64_t z;

      z = (uint64_t) d[v + 1] + MUL31(xu, y[v + 1]) + MUL31(f, m[v + 1]) + r;
      r = z >> 31;
      d[v + 0] = (uint32_t) z & 0x7FFFFFFF;
      z = (uint64_t) d[v + 2] + MUL31(xu, y[v + 2]) + MUL31(f, m[v + 2]) + r;
      r = z >> 31;
      d[v + 1] = (uint32_t) z & 0x7FFFFFFF;
      z = (uint64_t) d[v + 3] + MUL31(xu, y[v + 3]) + MUL31(f, m[v + 3]) + r;
      r = z >> 31;
      d[v + 2] = (uint32_t) z & 0x7FFFFFFF;
      z = (uint64_t) d[v + 4] + MUL31(xu, y[v + 4]) + MUL31(f, m[v + 4]) + r;
      r = z >> 31;
      d[v + 3] = (uint32_t) z & 0x7FFFFFFF;
    }
    for (; v < len; v++) {
      uint64_t z;

      z = (uint64_t) d[v + 1] + MUL31(xu, y[v + 1]) + MUL31(f, m[v + 1]) + r;
      r = z >> 31;
      d[v] = (uint32_t) z & 0x7FFFFFFF;
    }

    /*
     * Since the new dh can only be 0 or 1, the addition of
     * the old dh with the carry MUST fit on 32 bits, and
     * thus can be done into dh itself.
     */
    dh += (uint32_t) r;
    d[len] = dh & 0x7FFFFFFF;
    dh >>= 31;
  }

  /*
   * We must write back the bit length because it was overwritten in
   * the loop (not overwriting it would require a test in the loop,
   * which would yield bigger and slower code).
   */
  d[0] = m[0];

  /*
   * d[] may still be greater than m[] at that point; notably, the
   * 'dh' word may be non-zero.
   */
  br_i31_sub(d, m, NEQ(dh, 0) | NOT(br_i31_sub(d, m, 0)));
}
static void br_i31_to_monty(uint32_t *x, const uint32_t *m) {
  uint32_t k;

  for (k = (m[0] + 31) >> 5; k > 0; k--) {
    br_i31_muladd_small(x, 0, m);
  }
}
static void br_i31_modpow(uint32_t *x, const unsigned char *e, size_t elen,
                          const uint32_t *m, uint32_t m0i, uint32_t *t1,
                          uint32_t *t2) {
  size_t mlen;
  uint32_t k;

  /*
   * 'mlen' is the length of m[] expressed in bytes (including
   * the "bit length" first field).
   */
  mlen = ((m[0] + 63) >> 5) * sizeof m[0];

  /*
   * Throughout the algorithm:
   * -- t1[] is in Montgomery representation; it contains x, x^2,
   * x^4, x^8...
   * -- The result is accumulated, in normal representation, in
   * the x[] array.
   * -- t2[] is used as destination buffer for each multiplication.
   *
   * Note that there is no need to call br_i32_from_monty().
   */
  memcpy(t1, x, mlen);
  br_i31_to_monty(t1, m);
  br_i31_zero(x, m[0]);
  x[1] = 1;
  for (k = 0; k < ((uint32_t) elen << 3); k++) {
    uint32_t ctl;

    ctl = (e[elen - 1 - (k >> 3)] >> (k & 7)) & 1;
    br_i31_montymul(t2, x, t1, m, m0i);
    CCOPY(ctl, x, t2, mlen);
    br_i31_montymul(t2, t1, t1, m, m0i);
    memcpy(t1, t2, mlen);
  }
}

static size_t rsa_trim(const uint8_t **p, size_t n) {
  while (n > 0 && **p == 0) (*p)++, n--;
  return n;
}

static int rsa_i31_len(size_t nbytes) {
  int z = (int) (nbytes * 8U);
  int n = 1;
  while (z > 0) z -= 31, n++;
  return n + (n & 1);
}

static int rsa_fail(uint8_t *out, size_t outsz) {
  mg_bzero(out, outsz);
  return -1;
}

static int rsa_i31_eq(const uint32_t *a, const uint32_t *b) {
  uint32_t r = a[0] ^ b[0];
  size_t u, n = (a[0] + 31) >> 5;
  for (u = 1; u <= n; u++) r |= a[u] ^ b[u];
  return r == 0;
}

int mg_rsa_mod_pow(const uint8_t *mod, size_t modsz, const uint8_t *exp,
                   size_t expsz, const uint8_t *msg, size_t msgsz, uint8_t *out,
                   size_t outsz) {
  uint32_t tmp[4 * BR_RSA_WORDS], *m = tmp, *x, *t1, *t2, m0i;
  int fwlen;
  if (out == NULL) return -1;
  mg_bzero(out, outsz);
  if (mod == NULL || exp == NULL || msg == NULL) return -1;
  modsz = rsa_trim(&mod, modsz);
  expsz = rsa_trim(&exp, expsz);
  if (modsz < 256 || modsz > (BR_MAX_RSA_SIZE >> 3) || expsz == 0 ||
      expsz > modsz || outsz != modsz || msgsz != modsz) {
    return -1;
  }
  if (memcmp(msg, mod, modsz) >= 0) return -1;
  fwlen = rsa_i31_len(modsz);
  if (4 * (size_t) fwlen > sizeof(tmp) / sizeof(tmp[0])) return -1;
  x = m + fwlen, t1 = x + fwlen, t2 = t1 + fwlen;
  br_i31_decode(m, mod, modsz);
  m0i = br_i31_ninv31(m[1]);
  if ((m0i & 1) == 0) return rsa_fail((uint8_t *) tmp, sizeof(tmp));
  br_i31_decode_reduce(x, msg, msgsz, m);
  br_i31_modpow(x, exp, expsz, m, m0i, t1, t2);
  br_i31_encode(out, outsz, x);
  mg_bzero((uint8_t *) tmp, sizeof(tmp));
  return 0;
}

int mg_rsa_crt_sign(const uint8_t *em, size_t em_len, const uint8_t *dP,
                    size_t dP_len, const uint8_t *dQ, size_t dQ_len,
                    const uint8_t *p, size_t p_len, const uint8_t *q,
                    size_t q_len, const uint8_t *qInv, size_t qInv_len,
                    uint8_t *signature, size_t sig_len) {
  uint32_t tmp[8 * BR_RSA_FACTOR_WORDS], *mq = tmp, *s2, *mp, *s1, *t1, *t2;
  uint32_t *c1, *c2;
  uint32_t p0i, q0i;
  size_t plen, qlen, xlen = sig_len, u;
  int fwlen, ok = 0;
  uint8_t x[BR_MAX_RSA_SIZE >> 3];
  if (signature == NULL) return -1;
  mg_bzero(signature, sig_len);
  if (em == NULL || dP == NULL || dQ == NULL || p == NULL || q == NULL ||
      qInv == NULL) {
    return -1;
  }
  p_len = rsa_trim(&p, p_len);
  q_len = rsa_trim(&q, q_len);
  dP_len = rsa_trim(&dP, dP_len);
  dQ_len = rsa_trim(&dQ, dQ_len);
  qInv_len = rsa_trim(&qInv, qInv_len);
  if (p_len < 128 || q_len < 128 || p_len > (BR_MAX_RSA_FACTOR >> 3) ||
      q_len > (BR_MAX_RSA_FACTOR >> 3) || dP_len == 0 || dQ_len == 0 ||
      qInv_len == 0 || dP_len > p_len || dQ_len > q_len || qInv_len > p_len ||
      xlen > sizeof(x) || em_len == 0 || em_len > xlen) {
    return -1;
  }
  plen = p_len, qlen = q_len;
  fwlen = rsa_i31_len(plen > qlen ? plen : qlen);
  if (8 * fwlen > (int) (sizeof(tmp) / sizeof(tmp[0]))) return -1;
  s2 = mq + fwlen, mp = mq + 2 * fwlen, s1 = mq + 3 * fwlen;
  t1 = mq + 4 * fwlen, t2 = mq + 5 * fwlen;
  c1 = mq + 6 * fwlen, c2 = mq + 7 * fwlen;
  memset(x, 0, xlen - em_len);
  memcpy(x + xlen - em_len, em, em_len);
  br_i31_decode(mq, q, qlen);
  br_i31_decode(mp, p, plen);
  q0i = br_i31_ninv31(mq[1]);
  p0i = br_i31_ninv31(mp[1]);
  if (((p0i & q0i) & 1) == 0) goto done;
  memset(t2, 0, 2 * (size_t) fwlen * sizeof(*t2));
  t2[0] = mq[0];
  br_i31_mulacc(t2, mq, mp);
  // Convert i31-encoded bit length to byte length, and check
  if (xlen != ((t2[0] - (t2[0] >> 5) + 7) >> 3)) goto done;
  br_i31_encode(signature, xlen, t2);
  for (u = xlen, ok = 0; u > 0; u--) {
    uint32_t wn = signature[u - 1], wx = x[u - 1];
    ok = (int) (((wx - (wn + (uint32_t) ok)) >> 8) & 1);
  }
  if (!ok) goto done;
  ok = 0;
  br_i31_decode_reduce(s2, x, xlen, mq);
  br_i31_modpow(s2, dQ, dQ_len, mq, q0i, t1, t2);
  br_i31_decode_reduce(s1, x, xlen, mp);
  br_i31_modpow(s1, dP, dP_len, mp, p0i, t1, t2);
  memcpy(c1, s1, (size_t) fwlen * sizeof(*s1));
  memcpy(c2, s2, (size_t) fwlen * sizeof(*s2));
  br_i31_reduce(t2, s2, mp);
  br_i31_add(s1, mp, br_i31_sub(s1, t2, 1));
  br_i31_to_monty(s1, mp);
  br_i31_decode_reduce(t1, qInv, qInv_len, mp);
  br_i31_montymul(t2, s1, t1, mp, p0i);
  br_i31_mulacc(s2, mq, t2);
  br_i31_reduce(t1, s2, mq);
  ok = rsa_i31_eq(t1, c2);     // fault check s mod q == m2
  br_i31_decode(t2, p, plen);  // recombined sig in s2 overlaps mp, keep p in t2
  br_i31_reduce(t1, s2, t2);
  ok &= rsa_i31_eq(t1, c1);    // fault check s mod p == m1
  if (!ok) goto done;
  ok = 0;
  br_i31_encode(signature, sig_len, s2);
  ok = 1;
done:
  mg_bzero((uint8_t *) tmp, sizeof(tmp));
  mg_bzero(x, sizeof(x));
  if (!ok) return rsa_fail(signature, sig_len);
  return 0;
}

// MGF1-SHA256 RFC 8017 B.2.1
static void mgf1_sha256(const uint8_t *seed, size_t seed_len, uint8_t *mask,
                        size_t mask_len) {
  uint8_t cnt[4];   // I2OSP(counter, 4), big-endian
  uint8_t tmp[32];  // one SHA-256 output block
  uint32_t counter = 0;
  size_t off = 0;
  mg_sha256_ctx ctx;
  while (off < mask_len) {
    size_t n = mask_len - off;
    if (n > 32) n = 32;
    br_enc32be(cnt, counter);
    mg_sha256_init(&ctx);
    mg_sha256_update(&ctx, seed, seed_len);
    mg_sha256_update(&ctx, cnt, 4);
    mg_sha256_final(tmp, &ctx);
    memcpy(mask + off, tmp, n);
    off += n;
    ++counter;
  }
}

// Full RSASSA-PSS-VERIFY for rsa_pss_rsae_sha256 (RFC 8017 9.1.2)
// em      - output of RSA public-key primitive (nlen bytes)
// nlen    - modulus byte length (66 to 512)
// mhash   - SHA-256 of the TLS 1.3 signed content (tls->sighash, 32 bytes)
static bool pss_verify_sha256(const uint8_t *em, size_t nlen,
                              const uint8_t *mhash) {
  // TLS 1.3: H_len = salt_len = 32
  uint8_t db[479];              // unmasked DB after MGF1 XOR
  uint8_t dbmask[479];          // MGF1-SHA256(H, db_len); nlen - hLen - 1 = 479
  uint8_t Hprime[32];           // SHA-256(0x00*8 || mhash || salt)
  uint8_t Mprime[8 + 32 + 32];  // 0*8 || mhash || salt
  const uint8_t *H;
  const uint8_t *salt;
  size_t db_len;
  size_t i;
  uint8_t bad;

  if (nlen < 32 + 32 + 2 || nlen - 32 - 1 > sizeof(db)) return -1;
  if (em[nlen - 1] != 0xbc) return false;
  if (em[0] & 0x80) return false;
  db_len = nlen - 32 - 1;
  H = em + db_len;
  mgf1_sha256(H, 32, dbmask, db_len);
  for (i = 0; i < db_len; i++) db[i] = em[i] ^ dbmask[i];
  db[0] &= 0x7f;  // DB = 0x00^(db_len-salt_len-1) || 0x01 || salt(salt_Len)
  bad = 0;
  for (i = 0; i < db_len - 32 - 1; i++) bad |= db[i];
  bad |= db[db_len - 32 - 1] ^ 0x01;
  if (bad != 0) return false;
  salt = db + db_len - 32;
  memset(Mprime, 0, 8);
  memcpy(Mprime + 8, mhash, 32);
  memcpy(Mprime + 8 + 32, salt, 32);
  mg_sha256(Hprime, Mprime, sizeof(Mprime));
  bad = 0;
  for (i = 0; i < 32; i++) bad |= Hprime[i] ^ H[i];
  return (bad == 0);
}

bool mg_rsa_verify(const uint8_t *em, size_t nlen, const uint8_t *mhash) {
  return pss_verify_sha256(em, nlen, mhash);
}

// Full RSASSA-PKCS1-v1_5-VERIFY for rsa_pkcs1_sha256/sha384 (RFC 8017 9.2)
// em      - output of RSA public-key primitive (nlen bytes)
// nlen    - modulus byte length
// hash    - SHA-256 or SHA-384 of the certificate TBS data
// hashlen - hash length, 32 for SHA-256 or 48 for SHA-384
static bool pkcs1_v15_verify(const uint8_t *em, size_t nlen,
                             const uint8_t *hash, size_t hashlen) {
  static const uint8_t sha256_di[] = {0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60,
                                      0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02,
                                      0x01, 0x05, 0x00, 0x04, 0x20};
  static const uint8_t sha384_di[] = {0x30, 0x41, 0x30, 0x0d, 0x06, 0x09, 0x60,
                                      0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02,
                                      0x02, 0x05, 0x00, 0x04, 0x30};
  const uint8_t *di = hashlen == 32 ? sha256_di : sha384_di;
  size_t disz = hashlen == 32 ? sizeof(sha256_di) : sizeof(sha384_di);
  size_t pslen, i;
  uint8_t bad = 0;

  if (hashlen != 32 && hashlen != 48) return false;
  if (nlen < 3 + 8 + disz + hashlen) return false;
  pslen = nlen - 3 - disz - hashlen;

  bad |= (uint8_t) (em[0] | (em[1] ^ 0x01));
  for (i = 0; i < pslen; i++) bad |= (uint8_t) (em[2 + i] ^ 0xff);
  bad |= em[2 + pslen];
  for (i = 0; i < disz; i++) bad |= (uint8_t) (em[3 + pslen + i] ^ di[i]);
  for (i = 0; i < hashlen; i++)
    bad |= (uint8_t) (em[3 + pslen + disz + i] ^ hash[i]);

  return bad == 0;
}

bool mg_rsa_pkcs_verify(const uint8_t *em, size_t nlen, const uint8_t *hash,
                        size_t hashlen) {
  return pkcs1_v15_verify(em, nlen, hash, hashlen);
}

#endif /* MG_TLS == MG_TLS_BUILTIN */

#ifdef MG_ENABLE_LINES
#line 1 "src/url.c"
#endif


struct url {
  size_t key, user, pass, host, port, uri, end;
};

bool mg_url_is_ssl(const char *url) {
  return strncmp(url, "wss:", 4) == 0 || strncmp(url, "https:", 6) == 0 ||
         strncmp(url, "mqtts:", 6) == 0 || strncmp(url, "ssl:", 4) == 0 ||
         strncmp(url, "tls:", 4) == 0 || strncmp(url, "tcps:", 5) == 0;
}

static struct url urlparse(const char *url) {
  size_t i;
  struct url u;
  memset(&u, 0, sizeof(u));
  for (i = 0; url[i] != '\0'; i++) {
    if (url[i] == '/' && i > 0 && u.host == 0 && url[i - 1] == '/') {
      u.host = i + 1;
      u.port = 0;
    } else if (url[i] == ']') {
      u.port = 0;  // IPv6 URLs, like http://[::1]/bar
    } else if (url[i] == ':' && u.port == 0 && u.uri == 0) {
      u.port = i + 1;
    } else if (url[i] == '@' && u.user == 0 && u.pass == 0 && u.uri == 0) {
      u.user = u.host;
      u.pass = u.port;
      u.host = i + 1;
      u.port = 0;
    } else if (url[i] == '/' && u.host && u.uri == 0) {
      u.uri = i;
    }
  }
  u.end = i;
#if 0
  printf("[%s] %d %d %d %d %d\n", url, u.user, u.pass, u.host, u.port, u.uri);
#endif
  return u;
}

struct mg_str mg_url_host(const char *url) {
  struct url u = urlparse(url);
  size_t n = u.port  ? u.port - u.host - 1
             : u.uri ? u.uri - u.host
                     : u.end - u.host;
  struct mg_str s = mg_str_n(url + u.host, n);
  return s;
}

const char *mg_url_uri(const char *url) {
  struct url u = urlparse(url);
  return u.uri ? url + u.uri : "/";
}

unsigned short mg_url_port(const char *url) {
  struct url u = urlparse(url);
  unsigned short port = 0;
  if (strncmp(url, "http:", 5) == 0 || strncmp(url, "ws:", 3) == 0) port = 80;
  if (strncmp(url, "wss:", 4) == 0 || strncmp(url, "https:", 6) == 0)
    port = 443;
  if (strncmp(url, "mqtt:", 5) == 0) port = 1883;
  if (strncmp(url, "mqtts:", 6) == 0) port = 8883;
  if (u.port) port = (unsigned short) atoi(url + u.port);
  return port;
}

struct mg_str mg_url_user(const char *url) {
  struct url u = urlparse(url);
  struct mg_str s = mg_str("");
  if (u.user && (u.pass || u.host)) {
    size_t n = u.pass ? u.pass - u.user - 1 : u.host - u.user - 1;
    s = mg_str_n(url + u.user, n);
  }
  return s;
}

struct mg_str mg_url_pass(const char *url) {
  struct url u = urlparse(url);
  struct mg_str s = mg_str_n("", 0UL);
  if (u.pass && u.host) {
    size_t n = u.host - u.pass - 1;
    s = mg_str_n(url + u.pass, n);
  }
  return s;
}

#ifdef MG_ENABLE_LINES
#line 1 "src/util.c"
#endif




// Not using memset for zeroing memory, cause it can be dropped by compiler
// See https://github.com/cesanta/mongoose/pull/1265
void mg_bzero(volatile unsigned char *buf, size_t len) {
  if (buf != NULL) {
    while (len--) *buf++ = 0;
  }
}

uint64_t mg_timegm(unsigned int year, unsigned int month, unsigned int day,
                   unsigned int hour, unsigned int min, unsigned int sec) {
  static const uint16_t dm[12] = {0,   31,  59,  90,  120, 151,
                                  181, 212, 243, 273, 304, 334};
  const uint64_t day_secs = 86400;
  const uint64_t year_secs = 31536000;
  unsigned int y, ly;
  if (year < 1970) return 0;
  y = year - 1900;
  ly = month > 2 ? y + 1 : y;
  return (uint64_t) sec + 60 * min + 3600 * hour +
         day_secs * (dm[month - 1] + day - 1) + year_secs * (y - 70) +
         day_secs * ((ly - 69) / 4) - day_secs * ((ly - 1) / 100) +
         day_secs * ((ly + 299) / 400);
}

bool mg_memeq(const void *a, const void *b, size_t n) {
  const uint8_t *p = (const uint8_t *) a, *q = (const uint8_t *) b;
  uint8_t r = 0;
  size_t i;
  for (i = 0; i < n; i++) r = (uint8_t) (r | (uint8_t) (p[i] ^ q[i]));
  return r == 0;
}

#if MG_ENABLE_CUSTOM_RANDOM
#else
bool mg_random(void *buf, size_t len) {
  bool success = false;
  unsigned char *p = (unsigned char *) buf;
#if MG_ARCH == MG_ARCH_ESP32
  while (len--) *p++ = (unsigned char) (esp_random() & 255);
  success = true;
#elif MG_ARCH == MG_ARCH_CUBE && defined(HAL_RNG_MODULE_ENABLED)
  extern RNG_HandleTypeDef hrng;
  success = true;
  for (size_t n = 0; n < len; n += sizeof(uint32_t)) {
    uint32_t r = 0;
    if (HAL_RNG_GenerateRandomNumber(&hrng, &r) != HAL_OK) success = false;
    memcpy((char *) buf + n, &r, n + sizeof(r) > len ? len - n : sizeof(r));
  }
#elif MG_ARCH == MG_ARCH_PICOSDK
  while (len--) *p++ = (unsigned char) (get_rand_32() & 255);
  success = true;
#elif MG_ARCH == MG_ARCH_ZEPHYR
#if MG_TLS == MG_TLS_BUILTIN ||                                    \
    (MG_TLS == MG_TLS_MBED && (!defined(MBEDTLS_VERSION_NUMBER) || \
                               MBEDTLS_VERSION_NUMBER < 0x04000000))
  return (sys_csrand_get(buf, len) == 0);  // do not fallback on reseed error
#else
  sys_rand_get(buf, len);
  success = true;
#endif
#elif MG_ARCH == MG_ARCH_WIN32
#if defined(_MSC_VER) && _MSC_VER < 1700
  static bool initialised = false;
  static HCRYPTPROV hProv;
  // CryptGenRandom() implementation earlier than 2008 is weak, see
  // https://en.wikipedia.org/wiki/CryptGenRandom
  if (!initialised) {
    initialised = CryptAcquireContext(&hProv, NULL, NULL, PROV_RSA_FULL,
                                      CRYPT_VERIFYCONTEXT);
  }
  if (initialised) success = CryptGenRandom(hProv, len, p);
#else
  size_t i;
  for (i = 0; i < len; i++) {
    unsigned int rand_v;
    if (rand_s(&rand_v) == 0) {
      p[i] = (unsigned char) (rand_v & 255);
    } else {
      break;
    }
  }
  success = (i == len);
#endif

#elif MG_ARCH == MG_ARCH_UNIX
  FILE *fp = fopen("/dev/urandom", "rb");
  if (fp != NULL) {
    if (fread(buf, 1, len, fp) == len) success = true;
    fclose(fp);
  }
#endif
  // If everything above did not work, fallback to a pseudo random generator
  if (success == false) {
    MG_ERROR(("Weak RNG: using rand()"));
    while (len--) *p++ = (unsigned char) (rand() & 255);
  }
  return success;
}
#endif

char *mg_random_str(char *buf, size_t len) {
  size_t i;
  mg_random(buf, len);
  for (i = 0; i < len; i++) {
    uint8_t c = ((uint8_t *) buf)[i] % 62U;
    buf[i] = i == len - 1 ? (char) '\0'            // 0-terminate last byte
             : c < 26     ? (char) ('a' + c)       // lowercase
             : c < 52     ? (char) ('A' + c - 26)  // uppercase
                          : (char) ('0' + c - 52);     // numeric
  }
  return buf;
}

uint32_t mg_crc32(uint32_t crc, const char *buf, size_t len) {
  static const uint32_t crclut[16] = {
      // table for polynomial 0xEDB88320 (reflected)
      0x00000000, 0x1DB71064, 0x3B6E20C8, 0x26D930AC, 0x76DC4190, 0x6B6B51F4,
      0x4DB26158, 0x5005713C, 0xEDB88320, 0xF00F9344, 0xD6D6A3E8, 0xCB61B38C,
      0x9B64C2B0, 0x86D3D2D4, 0xA00AE278, 0xBDBDF21C};
  crc = ~crc;
  while (len--) {
    uint8_t b = *(uint8_t *) buf++;
    crc = crclut[(crc ^ b) & 0x0F] ^ (crc >> 4);
    crc = crclut[(crc ^ (b >> 4)) & 0x0F] ^ (crc >> 4);
  }
  return ~crc;
}

uint16_t mg_crc16(uint16_t crc, const char *buf, size_t len) {
  static const uint16_t crclut[16] = {
      // table for polynomial 0x8408 (reflected)
      0x0000, 0x1081, 0x2102, 0x3183, 0x4204, 0x5285, 0x6306, 0x7387,
      0x8408, 0x9489, 0xA50A, 0xB58B, 0xC60C, 0xD68D, 0xE70E, 0xF78F};
  unsigned int c = (unsigned int) ~crc & 0xffff;
  while (len--) {
    uint8_t b = *(uint8_t *) buf++;
    c = crclut[(c ^ b) & 0x0F] ^ (c >> 4);
    c = crclut[(c ^ (b >> 4)) & 0x0F] ^ (c >> 4);
  }
  return (uint16_t) ~c;
}

// Parse "IP" or "IP/BITS" into net and mask. The whole spec must be valid
static bool parse_net(struct mg_str spec, uint32_t *net, uint32_t *mask) {
  struct mg_str caps[3], ip = spec, bits = mg_str("32");
  struct mg_addr addr;
  uint8_t n = 0;
  if (mg_match(spec, mg_str("*/*"), caps)) ip = caps[0], bits = caps[1];
  // ip.len == 0 check: mg_aton() parses an empty string as 0.0.0.0
  if (ip.len == 0 || !mg_aton(ip, &addr) || addr.is_ip6 ||
      !mg_str_to_num(bits, 10, &n, sizeof(n)) || n > 32) {
    return false;
  }
  memcpy(net, addr.addr.ip, sizeof(*net));
  *net = mg_ntohl(*net);
  *mask = n ? (uint32_t) (0xffffffffU << (32 - n)) : (uint32_t) 0;
  return true;
}

int mg_check_ip_acl(struct mg_str acl, struct mg_addr *remote_ip) {
  struct mg_str entry;
  int allowed = acl.len == 0 ? '+' : '-';  // If any ACL is set, deny by default
  uint32_t remote_ip4;
  if (remote_ip->is_ip6) {
    return -1;  // TODO(): handle IPv6 ACL and addresses
  } else {      // IPv4
    memcpy((void *) &remote_ip4, remote_ip->addr.ip, sizeof(remote_ip4));
    while (mg_span(acl, &entry, &acl, ',')) {
      uint32_t net, mask;
      if (entry.buf[0] != '+' && entry.buf[0] != '-') return -1;
      if (!parse_net(mg_str_n(entry.buf + 1, entry.len - 1), &net, &mask)) {
        return -2;
      }
      if ((mg_ntohl(remote_ip4) & mask) == net) allowed = entry.buf[0];
    }
  }
  return allowed == '+';
}

bool mg_path_is_sane(const struct mg_str path) {
  const char *s = path.buf;
  size_t n = path.len;
  if (n == 0 || path.buf[0] == '\0') return true;
  if (s[0] == '~') return false;  // Starts with ~
  if (s[0] == '.' && n > 1 && s[1] == '.')
    return false;  // Starts with ..
  for (; n > 0 && s[0] != '\0'; s++, n--) {
    if ((s[0] == '/' || s[0] == '\\') && n >= 2 && s[1] == '.' && n > 2 &&
        s[2] == '.')
      return false;  // Subdir starts with ..
  }
  if (n > 0) return false;  // embedded nul (terminator not counted in len)
  return true;
}

#if MG_ENABLE_CUSTOM_MILLIS
#else
uint64_t mg_millis(void) {
#if MG_ARCH == MG_ARCH_ESP32 || MG_ENABLE_FREERTOS
  return xTaskGetTickCount() * portTICK_PERIOD_MS;
#elif MG_ARCH == MG_ARCH_THREADX
  return tx_time_get() * (1000 /* MS per SEC */ / TX_TIMER_TICKS_PER_SECOND);
#elif MG_ARCH == MG_ARCH_TIRTOS
  return (uint64_t) Clock_getTicks();
#elif MG_ARCH == MG_ARCH_ZEPHYR
  return (uint64_t) k_uptime_get();
#elif MG_ARCH == MG_ARCH_CMSIS_RTOS1
  return (uint64_t) rt_time_get();
#elif MG_ARCH == MG_ARCH_CMSIS_RTOS2
  return (uint64_t) ((osKernelGetTickCount() * 1000) / osKernelGetTickFreq());
#elif MG_ARCH == MG_ARCH_RTTHREAD
  return (uint64_t) ((rt_tick_get() * 1000) / RT_TICK_PER_SECOND);
#elif MG_ARCH == MG_ARCH_WIN32
  return GetTickCount();
#elif MG_ARCH == MG_ARCH_PICOSDK
  return time_us_64() / 1000;
#elif MG_ARCH == MG_ARCH_CUBE
  return (uint64_t) HAL_GetTick();
#elif MG_ARCH == MG_ARCH_UNIX && defined(__APPLE__)
  // Apple CLOCK_MONOTONIC_RAW is equivalent to CLOCK_BOOTTIME on linux
  // Apple CLOCK_UPTIME_RAW is equivalent to CLOCK_MONOTONIC_RAW on linux
  return clock_gettime_nsec_np(CLOCK_UPTIME_RAW) / 1000000;
#elif MG_ARCH == MG_ARCH_UNIX
  struct timespec ts = {0, 0};
  // See #1615 - prefer monotonic clock
#if defined(CLOCK_MONOTONIC_RAW)
  // Raw hardware-based time that is not subject to NTP adjustment
  clock_gettime(CLOCK_MONOTONIC_RAW, &ts);
#elif defined(CLOCK_MONOTONIC)
  // Affected by the incremental adjustments performed by adjtime and NTP
  clock_gettime(CLOCK_MONOTONIC, &ts);
#else
  // Affected by discontinuous jumps in the system time and by the incremental
  // adjustments performed by adjtime and NTP
  clock_gettime(CLOCK_REALTIME, &ts);
#endif
  return ((uint64_t) ts.tv_sec * 1000 + (uint64_t) ts.tv_nsec / 1000000);
#elif defined(ARDUINO)
  return (uint64_t) millis();
#else
  return (uint64_t) (time(NULL) * 1000);
#endif
}
#endif

// network format equates big endian order
uint16_t mg_ntohs(uint16_t net) {
  return MG_LOAD_BE16(&net);
}

uint32_t mg_ntohl(uint32_t net) {
  return MG_LOAD_BE32(&net);
}

uint64_t mg_ntohll(uint64_t net) {
  return MG_LOAD_BE64(&net);
}

void mg_delayms(unsigned int ms) {
  uint64_t to = mg_millis() + ms + 1;
  while (mg_millis() < to) (void) 0;
}

#if MG_ENABLE_CUSTOM_CALLOC
#else
void *mg_calloc(size_t count, size_t size) {
  return calloc(count, size);
}

void mg_free(void *ptr) {
  free(ptr);
}
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/wifi_dummy.c"
#endif


#if (!defined(MG_ENABLE_DRIVER_PICO_W) || !MG_ENABLE_DRIVER_PICO_W) && \
    (!defined(MG_ENABLE_DRIVER_CYW) || !MG_ENABLE_DRIVER_CYW) && \
    (!defined(MG_ENABLE_DRIVER_CYW_SDIO) || !MG_ENABLE_DRIVER_CYW_SDIO) && \
    (!defined(MG_ENABLE_DRIVER_NXP_WIFI) || !MG_ENABLE_DRIVER_NXP_WIFI) && \
    (!defined(MG_ENABLE_DRIVER_ST67W6) || !MG_ENABLE_DRIVER_ST67W6)


bool mg_wifi_scan(void) {
  MG_ERROR(("No Wi-Fi driver enabled"));
  return false;
}

bool mg_wifi_connect(struct mg_wifi_data *wifi) {
  (void) wifi;
  return mg_wifi_scan();
}

bool mg_wifi_disconnect(void) {
  return mg_wifi_scan();
}

bool mg_wifi_ap_start(struct mg_wifi_data *wifi) {
  (void) wifi;
  return mg_wifi_scan();
}

bool mg_wifi_ap_stop(void) {
  return mg_wifi_scan();
}

#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/ws.c"
#endif











struct ws_msg {
  uint8_t flags;
  size_t header_len;
  size_t data_len;
};

size_t mg_ws_vprintf(struct mg_connection *c, int op, const char *fmt,
                     va_list *ap) {
  size_t len = c->send.len;
  size_t n = mg_vxprintf(mg_pfn_iobuf, &c->send, fmt, ap);
  mg_ws_wrap(c, c->send.len - len, op);
  return n;
}

size_t mg_ws_printf(struct mg_connection *c, int op, const char *fmt, ...) {
  size_t len = 0;
  va_list ap;
  va_start(ap, fmt);
  len = mg_ws_vprintf(c, op, fmt, &ap);
  va_end(ap);
  return len;
}

static void ws_handshake(struct mg_connection *c, const struct mg_str *wskey,
                         const struct mg_str *wsproto, const char *fmt,
                         va_list *ap) {
  const char *magic = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11";
  unsigned char sha[20], b64_sha[30];

  mg_sha1_ctx sha_ctx;
  mg_sha1_init(&sha_ctx);
  mg_sha1_update(&sha_ctx, (unsigned char *) wskey->buf, wskey->len);
  mg_sha1_update(&sha_ctx, (unsigned char *) magic, 36);
  mg_sha1_final(sha, &sha_ctx);
  mg_base64_encode(sha, sizeof(sha), (char *) b64_sha, sizeof(b64_sha));
  mg_xprintf(mg_pfn_iobuf, &c->send,
             "HTTP/1.1 101 Switching Protocols\r\n"
             "Upgrade: websocket\r\n"
             "Connection: Upgrade\r\n"
             "Sec-WebSocket-Accept: %s\r\n",
             b64_sha);
  if (fmt != NULL) mg_vxprintf(mg_pfn_iobuf, &c->send, fmt, ap);
  if (wsproto != NULL) {
    mg_printf(c, "Sec-WebSocket-Protocol: %.*s\r\n", (int) wsproto->len,
              wsproto->buf);
  }
  if (!mg_send(c, "\r\n", 2)) mg_error(c, "OOM");
}

static uint32_t be32(const uint8_t *p) {
  return (((uint32_t) p[3]) << 0) | (((uint32_t) p[2]) << 8) |
         (((uint32_t) p[1]) << 16) | (((uint32_t) p[0]) << 24);
}

static size_t ws_process(uint8_t *buf, size_t len, struct ws_msg *msg) {
  size_t i, n = 0, mask_len = 0;
  memset(msg, 0, sizeof(*msg));
  if (len >= 2) {
    n = buf[1] & 0x7f;                // Frame length
    mask_len = buf[1] & 128 ? 4 : 0;  // last bit is a mask bit
    msg->flags = buf[0];
    if (n < 126 && len >= mask_len) {
      msg->data_len = n;
      msg->header_len = 2 + mask_len;
    } else if (n == 126 && len >= 4 + mask_len) {
      msg->header_len = 4 + mask_len;
      msg->data_len = (((size_t) buf[2]) << 8) | buf[3];
    } else if (len >= 10 + mask_len) {
      msg->header_len = 10 + mask_len;
      msg->data_len =
          (size_t) (((uint64_t) be32(buf + 2) << 32) + be32(buf + 6));
    }
  }
  // Sanity check, and integer overflow protection for the boundary check below
  // data_len should not be larger than 1 Gb
  if (msg->data_len > 1024 * 1024 * 1024) return 0;
  if (msg->header_len + msg->data_len > len) return 0;
  if (mask_len > 0) {
    uint8_t *p = buf + msg->header_len, *m = p - mask_len;
    for (i = 0; i < msg->data_len; i++) p[i] ^= m[i & 3];
  }
  return msg->header_len + msg->data_len;
}

static size_t mkhdr(size_t len, int op, bool is_client, uint8_t *buf) {
  size_t n = 0;
  buf[0] = (uint8_t) (op | 128);
  if (len < 126) {
    buf[1] = (unsigned char) len;
    n = 2;
  } else if (len < 65536) {
    uint16_t tmp = mg_htons((uint16_t) len);
    buf[1] = 126;
    memcpy(&buf[2], &tmp, sizeof(tmp));
    n = 4;
  } else {
    uint32_t tmp;
    buf[1] = 127;
    tmp = mg_htonl((uint32_t) (((uint64_t) len) >> 32));
    memcpy(&buf[2], &tmp, sizeof(tmp));
    tmp = mg_htonl((uint32_t) (len & 0xffffffffU));
    memcpy(&buf[6], &tmp, sizeof(tmp));
    n = 10;
  }
  if (is_client) {
    buf[1] |= 1 << 7;  // Set masking flag
    mg_random(&buf[n], 4);
    n += 4;
  }
  return n;
}

static void mg_ws_mask(struct mg_connection *c, size_t len) {
  if (c->is_client && c->send.buf != NULL) {
    size_t i;
    uint8_t *p = c->send.buf + c->send.len - len, *mask = p - 4;
    for (i = 0; i < len; i++) p[i] ^= mask[i & 3];
  }
}

size_t mg_ws_send(struct mg_connection *c, const void *buf, size_t len,
                  int op) {
  uint8_t header[14];
  size_t header_len = mkhdr(len, op, c->is_client, header);
  if (!mg_send(c, header, header_len)) return 0;
  if (!mg_send(c, buf, len)) return header_len;
  MG_VERBOSE(("WS out: %d [%.*s]", (int) len, (int) len, buf));
  mg_ws_mask(c, len);
  return header_len + len;
}

static bool mg_ws_client_handshake(struct mg_connection *c) {
  int n = mg_http_get_request_len(c->recv.buf, c->recv.len);
  if (n < 0) {
    mg_error(c, "not http");  // Some just, not an HTTP request
  } else if (n > 0) {
    if (n < 15 || memcmp(c->recv.buf + 9, "101", 3) != 0) {
      mg_error(c, "ws handshake error");
    } else {
      struct mg_http_message hm;
      if (mg_http_parse((char *) c->recv.buf, c->recv.len, &hm)) {
        c->is_websocket = 1;
        mg_call(c, MG_EV_WS_OPEN, &hm);
      } else {
        mg_error(c, "ws handshake error");
      }
    }
    mg_iobuf_del(&c->recv, 0, (size_t) n);
  } else {
    return true;  // Request is not yet received, quit event handler
  }
  return false;  // Continue event handler
}

static void mg_ws_cb(struct mg_connection *c, int ev, void *ev_data) {
  struct ws_msg msg;
  size_t ofs = (size_t) c->pfn_data;

  // assert(ofs < c->recv.len);
  if (ev == MG_EV_READ) {
    if (c->is_client && !c->is_websocket && mg_ws_client_handshake(c)) return;

    while (ws_process(c->recv.buf + ofs, c->recv.len - ofs, &msg) > 0) {
      char *s = (char *) c->recv.buf + ofs + msg.header_len;
      struct mg_ws_message m;
      size_t len;
      uint8_t final, op;
      m.data.buf = s, m.data.len = msg.data_len, m.flags = msg.flags;
      len = msg.header_len + msg.data_len;
      final = msg.flags & 128;
      op = msg.flags & 15;
      // MG_VERBOSE ("fin %d op %d len %d [%.*s]", final, op,
      //                       (int) m.data.len, (int) m.data.len, m.data.buf));
      if ((op == WEBSOCKET_OP_CONTINUE && ofs == 0) ||
          ((op == WEBSOCKET_OP_TEXT || op == WEBSOCKET_OP_BINARY) &&
           ofs != 0)) {
        mg_error(c, "invalid WS fragment sequence");
        return;
      }
      switch (op) {
        case WEBSOCKET_OP_CONTINUE:
          mg_call(c, MG_EV_WS_CTL, &m);
          break;
        case WEBSOCKET_OP_PING:
          MG_DEBUG(("%s", "WS PONG"));
          mg_ws_send(c, s, msg.data_len, WEBSOCKET_OP_PONG);
          mg_call(c, MG_EV_WS_CTL, &m);
          break;
        case WEBSOCKET_OP_PONG:
          mg_call(c, MG_EV_WS_CTL, &m);
          break;
        case WEBSOCKET_OP_TEXT:
        case WEBSOCKET_OP_BINARY:
          if (final) mg_call(c, MG_EV_WS_MSG, &m);
          break;
        case WEBSOCKET_OP_CLOSE:
          MG_DEBUG(("%lu WS CLOSE", c->id));
          mg_call(c, MG_EV_WS_CTL, &m);
          // Echo the payload of the received CLOSE message back to the sender
          mg_ws_send(c, m.data.buf, m.data.len, WEBSOCKET_OP_CLOSE);
          c->is_draining = 1;
          break;
        default:
          // Per RFC6455, close conn when an unknown op is recvd
          mg_error(c, "unknown WS op %d", op);
          break;
      }

      // Handle fragmented frames: strip header, keep in c->recv
      if (final == 0 || op == 0) {
        if (op) ofs++, len--, msg.header_len--;       // First frame
        mg_iobuf_del(&c->recv, ofs, msg.header_len);  // Strip header
        len -= msg.header_len;
        ofs += len;
        c->pfn_data = (void *) ofs;
        // MG_INFO(("FRAG %d [%.*s]", (int) ofs, (int) ofs, c->recv.buf));
      }
      // Remove non-fragmented frame
      if (final && op) mg_iobuf_del(&c->recv, ofs, len);
      // Last chunk of the fragmented frame
      if (final && !op && (ofs > 0)) {
        m.flags = c->recv.buf[0];
        m.data = mg_str_n((char *) &c->recv.buf[1], (size_t) (ofs - 1));
        mg_call(c, MG_EV_WS_MSG, &m);
        mg_iobuf_del(&c->recv, 0, ofs);
        ofs = 0;
        c->pfn_data = NULL;
      }
    }
  }
  (void) ev_data;
}

struct mg_connection *mg_ws_connect(struct mg_mgr *mgr, const char *url,
                                    mg_event_handler_t fn, void *fn_data,
                                    const char *fmt, ...) {
  struct mg_connection *c = mg_connect(mgr, url, fn, fn_data);
  if (c != NULL) {
    char nonce[16], key[30];
    struct mg_str host = mg_url_host(url);
    mg_random(nonce, sizeof(nonce));
    mg_base64_encode((unsigned char *) nonce, sizeof(nonce), key, sizeof(key));
    mg_xprintf(mg_pfn_iobuf, &c->send,
               "GET %s HTTP/1.1\r\n"
               "Upgrade: websocket\r\n"
               "Host: %.*s\r\n"
               "Connection: Upgrade\r\n"
               "Sec-WebSocket-Version: 13\r\n"
               "Sec-WebSocket-Key: %s\r\n",
               mg_url_uri(url), (int) host.len, host.buf, key);
    if (fmt != NULL) {
      va_list ap;
      va_start(ap, fmt);
      mg_vxprintf(mg_pfn_iobuf, &c->send, fmt, &ap);
      va_end(ap);
    }
    mg_xprintf(mg_pfn_iobuf, &c->send, "\r\n");
    c->pfn = mg_ws_cb;
    c->pfn_data = NULL;
  }
  return c;
}

void mg_ws_upgrade(struct mg_connection *c, struct mg_http_message *hm,
                   const char *fmt, ...) {
  struct mg_str *wskey = mg_http_get_header(hm, "Sec-WebSocket-Key");
  c->pfn = mg_ws_cb;
  c->pfn_data = NULL;
  if (wskey == NULL) {
    mg_http_reply(c, 426, "", "WS upgrade expected\n");
    c->is_draining = 1;
  } else {
    struct mg_str *wsproto = mg_http_get_header(hm, "Sec-WebSocket-Protocol");
    va_list ap;
    va_start(ap, fmt);
    ws_handshake(c, wskey, wsproto, fmt, &ap);
    va_end(ap);
    c->is_websocket = 1;
    c->pfn_data = NULL;  // HTTP response done; mg_ws_cb uses it as an offset
    mg_call(c, MG_EV_WS_OPEN, hm);
  }
}

size_t mg_ws_wrap(struct mg_connection *c, size_t len, int op) {
  uint8_t header[14], *p;
  size_t header_len = mkhdr(len, op, c->is_client, header);

  // NOTE: order of operations is important!
  if (mg_iobuf_add(&c->send, c->send.len, NULL, header_len) != 0) {
    p = &c->send.buf[c->send.len - len];         // p points to data
    memmove(p, p - header_len, len);             // Shift data
    memcpy(p - header_len, header, header_len);  // Prepend header
    mg_ws_mask(c, len);                          // Mask data
  }  // returning 0 means an OOM condition (iobuf couldn't resize), yet this is
  return c->send.len;  // so far recoverable, let the caller decide
}

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/at_cmd.c"
#endif


#if MG_ENABLE_TCPIP && defined(MG_ENABLE_DRIVER_ATCMD) && MG_ENABLE_DRIVER_ATCMD

#define MG_HDLC_FLAG 0x7e  // HDLC frame delimiter
#define MG_HDLC_ESC 0x7d   // HDLC escape byte for byte stuffing

#define MG_ATCMD_AT_TIMEOUT 2000

static size_t print_atcmd(void (*out)(char, void *), void *arg, va_list *ap) {
  struct mg_str s = va_arg(*ap, struct mg_str);
  size_t i;
  for (i = 0; i < s.len; i++)
    out((char) (s.buf[i] < 0x20 ? '.' : s.buf[i]), arg);
  return s.len;
}

static void mg_atcmd_reset(struct mg_tcpip_driver_atcmd_data *dd) {
  dd->script_index = 0;
  dd->deadline = 0;
  if (dd->reset) dd->reset(dd->usart);
}

static bool mg_atcmd_handle(struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_atcmd_data *dd = (struct mg_tcpip_driver_atcmd_data *) ifp->driver_data;
  if (dd->script == NULL || dd->script_index < 0) return true;
  if (dd->deadline == 0) dd->deadline = mg_millis() + MG_ATCMD_AT_TIMEOUT;
  for (;;) {
    if (dd->script_index % 2 == 0) {  // send AT command
      const char *cmd = dd->script[dd->script_index];
      MG_DEBUG(("send AT[%d]: %M", dd->script_index, print_atcmd, mg_str(cmd)));
      while (*cmd != '\0') dd->tx(dd->usart, (uint8_t) *cmd++);
      dd->script_index++;
      ifp->recv_queue.head = 0;
    } else {  // check AT command response
      const char *expect = dd->script[dd->script_index];
      struct mg_queue *q = &ifp->recv_queue;
      for (;;) {
        int c;
        bool is_timeout = dd->deadline > 0 && mg_millis() > dd->deadline;
        bool is_overflow = q->head >= q->size - 1;
        if (is_timeout || is_overflow) {
          MG_ERROR(("AT error: %s, retrying...", is_timeout ? "timeout" : "overflow"));
          mg_atcmd_reset(dd);
          return false;  // FAIL
        }
        if ((c = dd->rx(dd->usart)) < 0) return false;  // no data
        q->buf[q->head++] = (char) c;
        if (mg_match(mg_str_n(q->buf, q->head), mg_str(expect), NULL)) {
          MG_DEBUG(("recv AT[%d]: %M", dd->script_index, print_atcmd,
                    mg_str_n(q->buf, q->head)));
          dd->script_index++;
          q->head = 0;
          break;
        }
      }
    }
    if (dd->script[dd->script_index] == NULL) {
      MG_DEBUG(("finished AT script"));
      dd->script_index = -1;
      return true;
    }
  }
}

static void byte_stuff(struct mg_tcpip_driver_atcmd_data *dd, uint8_t b);
static bool byte_unstuff(struct mg_tcpip_driver_atcmd_data *dd, uint8_t *p);

static size_t mg_atcmd_rx(void *buf, size_t len, struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_atcmd_data *dd = (struct mg_tcpip_driver_atcmd_data *) ifp->driver_data;
  struct mg_queue *q = &ifp->recv_queue;
  
  if (!dd->link) return 0;

  while (q->head < q->size) { // read as many bytes as possible
    uint8_t b;
    int c = dd->rx(dd->usart);
    if (c < 0) return 0;      // no more bytes, exit
    b = (uint8_t) c;
    if (b == MG_HDLC_FLAG) {
      if (q->head == 0) { // first flag: skip
        dd->unstuffing = false;
        continue;
      } else { // last flag: end of frame, exit loop
        break;
      }
    }
    if (!dd->no_byte_stuff && !byte_unstuff(dd, &b)) continue;
    q->buf[q->head++] = (char) b;
  }
  len = (q->head <= len) ? q->head : 0;
  memmove(buf, q->buf, len);
  q->head = 0;
  return len;
}

static size_t mg_atcmd_tx(const void *buf, size_t len, struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_atcmd_data *dd = (struct mg_tcpip_driver_atcmd_data *) ifp->driver_data;
  uint8_t *p = (uint8_t *) buf;
  size_t n = len;

  if (!dd->link) return 0;

  dd->tx(dd->usart, MG_HDLC_FLAG);
  while (n--) {
    if (dd->no_byte_stuff) {
      dd->tx(dd->usart, *p++);
    } else {
      byte_stuff(dd, *p++);
    }
  }
  dd->tx(dd->usart, MG_HDLC_FLAG);
  return len;
}

static bool mg_atcmd_init(struct mg_tcpip_if *ifp) {
  ifp->recv_queue.size = 1506;  // PPP MTU=1500 + header + HDLC w/ no flags
  return true;
}

static bool mg_atcmd_poll(struct mg_tcpip_if *ifp, bool s1) {
  struct mg_tcpip_driver_atcmd_data *dd = (struct mg_tcpip_driver_atcmd_data *) ifp->driver_data;
  dd->link = mg_atcmd_handle(ifp);
  return s1 ? dd->link :false;
}


struct mg_tcpip_driver mg_tcpip_driver_atcmd = {mg_atcmd_init, mg_atcmd_tx, mg_atcmd_rx, mg_atcmd_poll};



static void byte_stuff(struct mg_tcpip_driver_atcmd_data *dd, uint8_t b) {
  if ((b < 0x20) || (b == MG_HDLC_ESC) || (b == MG_HDLC_FLAG)) {
    dd->tx(dd->usart, MG_HDLC_ESC);
    dd->tx(dd->usart, (uint8_t) (b ^ 0x20));
  } else {
    dd->tx(dd->usart, b);
  }
}

static bool byte_unstuff(struct mg_tcpip_driver_atcmd_data *dd, uint8_t *p) {
  if (!dd->unstuffing) {
    if (*p == MG_HDLC_ESC) {
      dd->unstuffing = true;
      return false;
    } else {
      return true;
    }
  }
  dd->unstuffing = false;
  *p ^= 0x20;
  return true;
}

#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/cmsis.c"
#endif
// https://arm-software.github.io/CMSIS_5/Driver/html/index.html

#if MG_ENABLE_TCPIP && defined(MG_ENABLE_DRIVER_CMSIS) && MG_ENABLE_DRIVER_CMSIS





extern ARM_DRIVER_ETH_MAC Driver_ETH_MAC0;
extern ARM_DRIVER_ETH_PHY Driver_ETH_PHY0;

static struct mg_tcpip_if *s_ifp;

static void mac_cb(uint32_t);
static bool cmsis_init(struct mg_tcpip_if *);
static bool cmsis_poll(struct mg_tcpip_if *, bool);
static size_t cmsis_tx(const void *, size_t, struct mg_tcpip_if *);
static size_t cmsis_rx(void *, size_t, struct mg_tcpip_if *);

struct mg_tcpip_driver mg_tcpip_driver_cmsis = {cmsis_init, cmsis_tx, NULL,
                                                cmsis_poll};

static bool cmsis_init(struct mg_tcpip_if *ifp) {
  ARM_ETH_MAC_ADDR addr;
  s_ifp = ifp;

  ARM_DRIVER_ETH_MAC *mac = &Driver_ETH_MAC0;
  ARM_DRIVER_ETH_PHY *phy = &Driver_ETH_PHY0;
  ARM_ETH_MAC_CAPABILITIES cap = mac->GetCapabilities();
  if (mac->Initialize(mac_cb) != ARM_DRIVER_OK) return false;
  if (phy->Initialize(mac->PHY_Read, mac->PHY_Write) != ARM_DRIVER_OK)
    return false;
  if (cap.event_rx_frame == 0)  // polled mode driver
    mg_tcpip_driver_cmsis.rx = cmsis_rx;
  mac->PowerControl(ARM_POWER_FULL);
  if (cap.mac_address) {  // driver provides MAC address
    mac->GetMacAddress(&addr);
    memcpy(ifp->mac, &addr, sizeof(addr));
  } else {  // we provide MAC address
    memcpy(&addr, ifp->mac, sizeof(addr));
    mac->SetMacAddress(&addr);
  }
  phy->PowerControl(ARM_POWER_FULL);
  phy->SetInterface(cap.media_interface);
  phy->SetMode(ARM_ETH_PHY_AUTO_NEGOTIATE);
  return true;
}

static size_t cmsis_tx(const void *buf, size_t len, struct mg_tcpip_if *ifp) {
  ARM_DRIVER_ETH_MAC *mac = &Driver_ETH_MAC0;
  if (mac->SendFrame(buf, (uint32_t) len, 0) != ARM_DRIVER_OK) {
    ifp->nerr++;
    return 0;
  }
  ifp->nsent++;
  return len;
}

static void cmsis_update_hash_table(struct mg_tcpip_if *ifp) {
  // TODO(): read database, rebuild hash table
  ARM_DRIVER_ETH_MAC *mac = &Driver_ETH_MAC0;
  ARM_ETH_MAC_ADDR addr;
  memcpy(&addr, mcast_addr, sizeof(addr));
  mac->SetAddressFilter(&addr, 1);
  (void) ifp;
}

static bool cmsis_poll(struct mg_tcpip_if *ifp, bool s1) {
  if (ifp->update_mac_hash_table) {
    cmsis_update_hash_table(ifp);
    ifp->update_mac_hash_table = false;
  }
  if (!s1) return false;
  ARM_DRIVER_ETH_PHY *phy = &Driver_ETH_PHY0;
  ARM_DRIVER_ETH_MAC *mac = &Driver_ETH_MAC0;
  bool up = (phy->GetLinkState() == ARM_ETH_LINK_UP) ? 1 : 0;  // link state
  if ((ifp->state == MG_TCPIP_STATE_DOWN) && up) {             // just went up
    ARM_ETH_LINK_INFO st = phy->GetLinkInfo();
    mac->Control(ARM_ETH_MAC_CONFIGURE,
                 (st.speed << ARM_ETH_MAC_SPEED_Pos) |
                     (st.duplex << ARM_ETH_MAC_DUPLEX_Pos) |
                     ARM_ETH_MAC_ADDRESS_BROADCAST);
    MG_DEBUG(("Link is %uM %s-duplex",
              (st.speed == 2) ? 1000
              : st.speed      ? 100
                              : 10,
              st.duplex ? "full" : "half"));
    mac->Control(ARM_ETH_MAC_CONTROL_TX, 1);
    mac->Control(ARM_ETH_MAC_CONTROL_RX, 1);
  } else if ((ifp->state != MG_TCPIP_STATE_DOWN) && !up) {  // just went down
    mac->Control(ARM_ETH_MAC_FLUSH,
                 ARM_ETH_MAC_FLUSH_TX | ARM_ETH_MAC_FLUSH_RX);
    mac->Control(ARM_ETH_MAC_CONTROL_TX, 0);
    mac->Control(ARM_ETH_MAC_CONTROL_RX, 0);
  }
  return up;
}

static void mac_cb(uint32_t ev) {
  if ((ev & ARM_ETH_MAC_EVENT_RX_FRAME) == 0) return;
  ARM_DRIVER_ETH_MAC *mac = &Driver_ETH_MAC0;
  uint32_t len = mac->GetRxFrameSize();  // CRC already stripped
  if (len >= 60 && len <= 1518) {        // proper frame
    char *p;
    if (mg_queue_book(&s_ifp->recv_queue, &p, len) >= len) {  // have room
      if ((len = mac->ReadFrame((uint8_t *) p, len)) > 0) {   // copy succeeds
        mg_queue_add(&s_ifp->recv_queue, len);
        s_ifp->nrecv++;
      }
      return;
    }
    s_ifp->ndrop++;
  }
  mac->ReadFrame(NULL, 0);  // otherwise, discard
}

static size_t cmsis_rx(void *buf, size_t buflen, struct mg_tcpip_if *ifp) {
  ARM_DRIVER_ETH_MAC *mac = &Driver_ETH_MAC0;
  uint32_t len = mac->GetRxFrameSize();  // CRC already stripped
  if (len >= 60 && len <= 1518 &&
      ((len = mac->ReadFrame(buf, (uint32_t) buflen)) > 0))
    return len;
  if (len > 0) mac->ReadFrame(NULL, 0);  // discard bad frames
  (void) ifp;
  return 0;
}

#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/cyw.c"
#endif



#if MG_ENABLE_TCPIP &&                                          \
    ((defined(MG_ENABLE_DRIVER_CYW) && MG_ENABLE_DRIVER_CYW) || \
     (defined(MG_ENABLE_DRIVER_CYW_SDIO) && MG_ENABLE_DRIVER_CYW_SDIO))

#ifndef MG_ENABLE_DRIVER_CYW
#define MG_ENABLE_DRIVER_CYW 0
#endif
#ifndef MG_ENABLE_DRIVER_CYW_SDIO
#define MG_ENABLE_DRIVER_CYW_SDIO 0
#endif

static struct mg_tcpip_if *s_ifp;
static uint32_t s_ip, s_mask;
static bool s_link, s_auth, s_join;

static void wifi_cb(struct mg_tcpip_if *ifp, int ev, void *ev_data) {
  struct mg_wifi_data *wifi = &((struct mg_tcpip_driver_cyw_data *) ifp->driver_data)->wifi;
  if (wifi->apmode && ev == MG_TCPIP_EV_STATE_CHANGE && *(uint8_t *) ev_data == MG_TCPIP_STATE_UP) {
    MG_DEBUG(("Access Point started"));
    s_ip = ifp->ip, ifp->ip = wifi->apip;
    s_mask = ifp->mask, ifp->mask = wifi->apmask;
    ifp->enable_dhcp_client = false;
    ifp->enable_dhcp_server = true;
  }
}

static bool cyw_init(uint8_t *mac);
static void cyw_poll(void);

static bool mg_tcpip_driver_cyw_init(struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_cyw_data *d =
      (struct mg_tcpip_driver_cyw_data *) ifp->driver_data;
  struct mg_wifi_data *wifi = &d->wifi;
  if (MG_BIG_ENDIAN) {
    MG_ERROR(("Big-endian host"));
    return false;
  }
  s_ifp = ifp;
  s_ip = ifp->ip;
  s_mask = ifp->mask;
  s_link = s_auth = s_join = false;
  ifp->pfn = wifi_cb;
  if (!cyw_init(ifp->mac)) return false;

  if (wifi->apmode) {
    return mg_wifi_ap_start(wifi);
  } else if (wifi->ssid != NULL && wifi->pass != NULL) {
    return mg_wifi_connect(wifi);
  }
  return true;
}

static size_t mg_cyw_tx(unsigned int ifc, void *data, size_t len);
size_t mg_tcpip_driver_cyw_output(const void *buf, size_t len,
                                  struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_cyw_data *d =
      (struct mg_tcpip_driver_cyw_data *) ifp->driver_data;
  return mg_cyw_tx(d->wifi.apmode ? 1 : 0, (void *) buf, len) >= len ? len : 0;
}

static bool mg_tcpip_driver_cyw_poll(struct mg_tcpip_if *ifp, bool s1) {
  cyw_poll();
  if (!s1) return false;
  struct mg_tcpip_driver_cyw_data *d =
      (struct mg_tcpip_driver_cyw_data *) ifp->driver_data;
  return d->wifi.apmode ? s_link : s_link && s_auth && s_join;
}

struct mg_tcpip_driver mg_tcpip_driver_cyw = {mg_tcpip_driver_cyw_init,
                                              mg_tcpip_driver_cyw_output, NULL,
                                              mg_tcpip_driver_cyw_poll};

// - DS:
// https://www.mouser.com/datasheet/2/196/Infineon_CYW43439_DataSheet_v03_00_EN-3074791.pdf
// - WHD: https://github.com/Infineon/wifi-host-driver
//
//              |  e   <-- event data
//              |-----
//          net | vnd   <-- network (TCP/IP) | vendor header (Broadcom (bcm))
//         -----|-----
//  IOCTL | ETH | ETH  <-- IOCTL/IOVAR: chip control | ETH: Ethernet header
// -------|-----|-----
//   CDC  | BDC | BDC
// ------- ----- -----
//        SDPCM        <-- includes SDIO bus arbitration, not used in SPI
// -------------------
//    SPI   |  SDIO    <-- padded to 32-bit | 64-bytes
//
// - SDPCM has 3 channels (control, data, and asynchronous data)
// - SPI has 4 "functions", F0 to F3, to access different blocks in the chip,
// like the SPI/SDIO controller, chip backplane, and 2 DMA I/Os; these are
// usually handled by SDPCM but we need to explicitly access the I/O controller
// and chip backplane during initialization
// - SDIO has 3 functions (proper SDIO terminology), F0 to F2, coincident with
// those for SPI, accessed through standard SDIO practices. There is no F3.

// Processor core firmware is loaded to TCM RAM, along with module-dependent
// (hardware design) NVRAM data, via the chip backplane access through the bus
// Once the chip has been initialized, information regarding regulatory
// constraints (CLM blob, “Country Locale Matrix”), is loaded as an IOVAR. This
// is tied to the module being certified, hence it is also module-dependent.
// - Result: chip firmware + module NVRAM data + module CLM blob

#pragma pack(push, 1)
// all little endian

struct cdc_hdr {
  uint32_t cmd;   // ioctl command value
  uint16_t olen;  // output buflen
  uint16_t ilen;  // input buflen (excludes header)
  uint32_t flags;
  uint32_t status;
};

struct bdc_hdr {
  uint8_t flags;     // Flags
  uint8_t priority;  // 802.1d Priority (low 3 bits)
  uint8_t flags2;
  uint8_t data_offset;  // Offset from end of BDC header to packet data, in
                        // 4-uint8_t words. Leaves room for optional headers.
};

struct sdpcm_sw_hdr {
  uint8_t sequence;           // Sequence number of pkt
  uint8_t channel_and_flags;  // IOCTL/IOVAR or User Data or Event
  uint8_t next_length;
  uint8_t header_length;  // Offset to BDC or CDC header
  uint8_t wireless_flow_control;
  uint8_t bus_data_credit;  // Credit from WLAN Chip
  uint8_t _reserved[2];
};

struct sdpcm_hdr {
  uint16_t len;
  uint16_t _len;  // ~len
  struct sdpcm_sw_hdr sw_hdr;
};

struct data_hdr {
  struct sdpcm_hdr sdpcm;
  uint8_t pad[2];
  struct bdc_hdr bdc;
};

// gSPI, CYW43439 DS 4.2.1 Fig.12, 2-bit field
#define CYW_SPID_FUNC_BUS 0   // F0
#define CYW_SPID_FUNC_CHIP 1  // F1
#define CYW_SPID_FUNC_WLAN 2  // F2

// SDIO functions, 3-bit field; CYW4343W and CYW43439 DS 4.1
#define CYW_SDIO_FUNC_BUS 0   // F0
#define CYW_SDIO_FUNC_CHIP 1  // F1
#define CYW_SDIO_FUNC_WLAN 2  // F2

#define CYW_SDPCM_CTRL_HDR 0
#define CYW_SDPCM_ASYNC_HDR 1
#define CYW_SDPCM_DATA_HDR 2

#pragma pack(pop)

static uint8_t s_tx_seqno;
static uint32_t txdata[2048 / 4], resp[2048 / 4];

static void cyw_handle_cdc(struct cdc_hdr *cdc, size_t len);
static void cyw_handle_bdc(struct bdc_hdr *bdc, size_t len);
static void cyw_handle_bdc_evnt(struct bdc_hdr *bdc, size_t len);

static size_t cyw_bus_specific_poll(uint32_t *dest);
static void cyw_update_hash_table(void);

// High-level comm stuff

static void cyw_poll(void) {
  struct sdpcm_hdr *sdpcm = (struct sdpcm_hdr *) resp;
  unsigned int channel;
  if (s_ifp->update_mac_hash_table) {
    // first call to _poll() is after _init(), so this is safe
    cyw_update_hash_table();
    s_ifp->update_mac_hash_table = false;
  }
  if (cyw_bus_specific_poll(resp) == 0) return;
  if ((sdpcm->len ^ sdpcm->_len) != 0xffff || sdpcm->len < sizeof(*sdpcm) ||
      sdpcm->len > 2048 - sizeof(*sdpcm))
    return;
  channel = sdpcm->sw_hdr.channel_and_flags & 0x0F;
  if (channel == CYW_SDPCM_CTRL_HDR) {
    if (sdpcm->len >= sizeof(*sdpcm) + sizeof(struct cdc_hdr)) {
      struct cdc_hdr *cdc =
          (struct cdc_hdr *) ((size_t) sdpcm + sdpcm->sw_hdr.header_length);
      size_t len = sdpcm->len - sdpcm->sw_hdr.header_length;
      cyw_handle_cdc(cdc, len);
    }
  } else if (channel == CYW_SDPCM_DATA_HDR) {
    if (sdpcm->len >= sizeof(*sdpcm) + sizeof(struct bdc_hdr)) {
      struct bdc_hdr *bdc =
          (struct bdc_hdr *) ((size_t) sdpcm + sdpcm->sw_hdr.header_length);
      size_t len = sdpcm->len - sdpcm->sw_hdr.header_length;
      cyw_handle_bdc(bdc, len);
    }
  } else if (channel == CYW_SDPCM_ASYNC_HDR) {
    struct bdc_hdr *bdc =
        (struct bdc_hdr *) ((size_t) sdpcm + sdpcm->sw_hdr.header_length);
    size_t len_ = sdpcm->len - sdpcm->sw_hdr.header_length;
    cyw_handle_bdc_evnt(bdc, len_);
  }  // else silently discard
}

// WLAN frame reception
static void cyw_handle_bdc(struct bdc_hdr *bdc, size_t len) {
  uint8_t *payload = (uint8_t *) &bdc[bdc->data_offset + 1];
  mg_tcpip_qwrite(payload, len - (payload - (uint8_t *) bdc), s_ifp);
}

static size_t cyw_bus_specific_tx(uint32_t *data, uint16_t len);

// WLAN frame transmission
static size_t mg_cyw_tx(unsigned int ifc, void *data, size_t len) {
  struct data_hdr *hdr = (struct data_hdr *) txdata;
  uint16_t txlen = (uint16_t) (len + sizeof(*hdr));
  memset(txdata, 0, sizeof(*hdr));
  memcpy((uint8_t *) txdata + sizeof(*hdr), data, len);
  // TODO(): hdr->bdc.priority = map IP to TOS if supporting QoS/ToS
  hdr->bdc.flags = 2 << 4;          // BDC version 2
  hdr->bdc.flags2 = (uint8_t) ifc;  // 0 -> STA, 1 -> AP
  // hdr->bdc.data_offset = 0; // actually zeroed above
  hdr->sdpcm.len = txlen;
  hdr->sdpcm._len = (uint16_t) ~txlen;
  hdr->sdpcm.sw_hdr.sequence = ++s_tx_seqno;
  hdr->sdpcm.sw_hdr.channel_and_flags = CYW_SDPCM_DATA_HDR,
  hdr->sdpcm.sw_hdr.header_length = offsetof(struct data_hdr, bdc);
  return cyw_bus_specific_tx(txdata, txlen);
}

// WLAN event handling
#pragma pack(push, 1)
// all in network order

struct eth_hdr {  // TODO(scaprile) reuse 'eth' in net_builtin.c
  uint8_t dest[6];
  uint8_t src[6];
  uint16_t type;
};

struct bcm_vendor_hdr {
  uint16_t subtype;  // vendor specific: 0x8001
  uint16_t length;   // bytes following this field
  uint8_t version;   // 0
  uint8_t oui[3];    // vendor specific: 0x00 0x10 0x18
  uint16_t usr_subtype;
};

struct bcm_evnt_hdr {
  uint16_t version;  // 1: fields up to ifname; 2: as shown
  uint16_t flags;
  uint32_t event_type;
  uint32_t status;
  uint32_t reason;
  uint32_t auth_type;
  uint32_t datalen;
  uint8_t addr[6];  // Station address (if applicable)
  char ifname[16];
  uint8_t ifidx;
  uint8_t bss_cfg_idx;
};

struct evnt_msg {
  struct eth_hdr eth;
  //  struct vendor_hdr; but we only handle Broadcom (Wi-Fi processor) events
  struct bcm_vendor_hdr bcm;
  struct bcm_evnt_hdr event;
};

#pragma pack(pop)

struct scan_result;
static void cyw_handle_scan_result(uint32_t status, struct scan_result *data,
                                   size_t len);

// Do not call any IOCTL functions here, otherwise revise cyw_ioctl_wait()
static void cyw_handle_bdc_evnt(struct bdc_hdr *bdc, size_t len) {
  struct evnt_msg *msg = (struct evnt_msg *) &bdc[bdc->data_offset + 1];
  MG_VERBOSE(("%u bytes event", len));
  if (mg_log_level >= MG_LL_VERBOSE) mg_hexdump((void *) bdc, len);
  if (mg_ntohs(msg->eth.type) != 0x886C || msg->bcm.oui[0] != 0x00 ||
      msg->bcm.oui[1] != 0x10 || msg->bcm.oui[2] != 0x18)
    return;  // discard if not Broadcom
  if (mg_ntohl(msg->event.datalen) <=
      len - ((uint8_t *) msg - (uint8_t *) bdc)) {
    uint32_t event_type = mg_ntohl(msg->event.event_type);
    uint32_t status = mg_ntohl(msg->event.status);
    uint32_t reason = mg_ntohl(msg->event.reason);
    uint16_t flags = mg_ntohs(msg->event.flags);
    MG_VERBOSE(("BCM evt %lu %lu %lu %p", event_type, status, reason, flags));
    if (event_type == 16 && status == 0) {  // Link
      s_link = flags & 1;
    } else if (event_type == 46 && s_link) {  // PSK sup with link up
      if (status == 6) {                      // Keyed
      } else if ((status == 4 || status == 8 || status == 10) &&
                 reason == 15) {  // Wait M1/M3/G1
        MG_ERROR(("AUTH TIMEOUT"));
        s_auth = false;
      } else {
        MG_ERROR(("AUTH FAILED"));
        s_auth = false;
      }
    } else if (event_type == 3 && status != 6) {  // Auth (not unsolicited)
      if (status == 0) {                          // Success
        s_auth = true;
      } else {
        MG_ERROR(("AUTH TIMEOUT"));
        s_auth = false;
      }
    } else if (event_type == 1) {  // Join
      if (status == 0) {           // Success
        s_join = true;
      } else {
        MG_ERROR(("%s", status == 3 /* No networks */ ? "SSID NOT FOUND"
                                                      : "JOIN FAILED"));
        s_join = false;
        mg_tcpip_call(s_ifp, MG_TCPIP_EV_WIFI_CONNECT_ERR, &status);
      }
    } else if (event_type == 12 || event_type == 5) {  // Disassoc, Deauth
      s_auth = false;
    } else if (event_type == 69) {  // Scan result
      struct scan_result *data = (struct scan_result *) (&msg->event + 1);
      size_t dlen = mg_ntohl(msg->event.datalen);
      if (dlen > len - ((uint8_t *) data - (uint8_t *) bdc)) return;
      cyw_handle_scan_result(status, data, dlen);
    }
  }  // else silently discard
}

static bool cyw_ioctl_get_(unsigned int ifc, unsigned int cmd, void *data,
                           size_t len);
static bool cyw_ioctl_set_(unsigned int ifc, unsigned int cmd, void *data,
                           size_t len);
static bool cyw_ioctl_iovar_get_(unsigned int ifc, char *var, void *data,
                                 size_t len);
static bool cyw_ioctl_iovar_set_(unsigned int ifc, char *var, void *data,
                                 size_t len);
// clang-format off
// convenience: ioctl funcs on default ifc (0), as only AP needs ifc 1
__attribute__((unused)) static bool cyw_ioctl_get(unsigned int cmd, void *data, size_t len) { return cyw_ioctl_get_(0, cmd, data, len); }
static bool cyw_ioctl_set(unsigned int cmd, void *data, size_t len) { return cyw_ioctl_set_(0, cmd, data, len); }
static bool cyw_ioctl_iovar_get(char *var, void *data, size_t len) { return cyw_ioctl_iovar_get_(0, var, data, len); }
static bool cyw_ioctl_iovar_set(char *var, void *data, size_t len) { return cyw_ioctl_iovar_set_(0, var, data, len); }
// clang-format on

// Wi-Fi network stuff

// clang-format off
static bool cyw_wifi_connect(char *ssid, char *pass) {
  uint32_t sup_wpa[2] = {0, 1}; // bss index 0 = STA, not open
  static const uint32_t eapver[2] = {0, (uint32_t) -1}, // accept AP version
                              tmo[2] = {0, 2500};
  uint32_t data[64/4 + 1]; // max pass length: 64 for WPA, 128 for WPA3 SAE
  uint16_t *da = (uint16_t *) data;
  unsigned int len;
  uint32_t val;
  val = 4; // security type: 0 for none, 2 for WPA, 4 for WPA2/WPA3, 6 for mixed WPA/WPA2
  // sup_wpa[1] = 0 if not using security
  if (!(cyw_ioctl_set(134 /* SET_WSEC */, (uint8_t *)&val, sizeof(val))
        && cyw_ioctl_iovar_set("bsscfg:sup_wpa", (void *)sup_wpa, sizeof(sup_wpa))
        && cyw_ioctl_iovar_set("bsscfg:sup_wpa2_eapver", (void *)eapver, sizeof(eapver))
        && cyw_ioctl_iovar_set("bsscfg:sup_wpa_tmo", (void *)tmo, sizeof(tmo)))
     ) return false;
  mg_delayms(2); // allow radio firmware to be ready
  // skip if not using auth
  memset(data, 0, sizeof(data));
  len = strlen(pass);
  da[0] = (uint16_t) len;
  da[1] = 1; // indicates wireless security key, skip for WPA3 SAE
  memcpy((uint8_t *)data + 2 * sizeof(uint16_t), pass, len); // skip for WPA3 SAE
  if (!cyw_ioctl_set(268 /* SET_WSEC_PMK */, data, sizeof(data))) return false; // skip for WPA3 SAE, sizeof/2 if supporting SAE but using WPA
  // for WPA3 SAE: memcpy((uint8_t *)data + sizeof(uint16_t), pass, len); cyw_ioctl_iovar_set("sae_password", data, sizeof(data));
  // resume if not using auth
  val = 1; if (!cyw_ioctl_set(20 /* SET_INFRA */, (uint8_t *)&val, sizeof(val))) return false;
  val = 0; // auth type: 0 for open, 3 for SAE
  if (!cyw_ioctl_set(22 /* SET_AUTH */, (uint8_t *)&val, sizeof(val))) return false;
  val = 1; // MFP capable: 1 for yes, 0 for no; recommended to be set for WPA2+ (2 for 'required', WPA3)
  cyw_ioctl_iovar_set("mfp", (uint8_t *)&val, sizeof(val)); // Old chipsets do not support MFP
  val = 0x80; // auth type: 0 for none, 4 for WPA PSK, 0x80 for WPA2 PSK, 0x40000 for WPA3 SAE PSK
  if (!cyw_ioctl_set(165 /* SET_WPA_AUTH */, (uint8_t *)&val, sizeof(val))) return false;
  len = strlen(ssid);
  data[0] = (uint32_t) len;
  memcpy((uint8_t *)&data[1], ssid, len);
  if (!cyw_ioctl_set(26 /* SET_SSID */, data, len + sizeof(uint32_t))) return false;
  return true;
}

static bool cyw_wifi_disconnect(void) {
  return cyw_ioctl_set(52 /* DISASSOC */, NULL, 0);
}

// For AP functions, we use explicit ifc selection; both for clarity and maintenance, as some actions are performed on ifc 0, with or without a bss_index, and others are performed on ifc 1

static bool cyw_wifi_ap_start(char *ssid, char *pass, unsigned int channel) {
  uint32_t data[64/4 + 2]; // max pass length: 64 for WPA, 128 for WPA3 SAE
  uint16_t *da = (uint16_t *) data;
  unsigned int len;
  uint32_t val;
  // CHIP DEPENDENCY
  // RPi set the AMPDU parameter for AP (window size = 2) *****************
  // val = 2 ; cyw_ioctl_iovar_set_(0, "ampdu_ba_wsize", (uint8_t *)&val, sizeof(val));
  // some chips might require to turn APSTA off and issue a SET_AP IOCTL
  len = strlen(ssid);
  data[0] = 1; // bss index 1 = AP
  data[1] = (uint32_t) len;
  memcpy((uint8_t *)&data[2], ssid, len);
  // TODO(scaprile): this takes some time to process, or requires a delay before doing it
  if (!cyw_ioctl_iovar_set_(0, "bsscfg:ssid", (uint8_t *)&data, len + 2 * sizeof(uint32_t))) return false;
  // TODO(scaprile): but sometimes this one takes some time to process
  val = (uint32_t) channel; if (!cyw_ioctl_set_(0, 30 /* SET_CHANNEL */, (uint8_t *)&val, sizeof(val))) return false;
  data[0] = 1; // bss index 1 = AP
  data[1] = 0x00400004; // security type: 0 for none, 0x00200002 for WPA, 0x00400004 for WPA2, 0x01000004 for WPA3, 0x01400004 for mixed WPA2/WPA3, 0x00400006 for mixed WPA/WPA2
  // NOTE(): WHD writes & 0xFF if WPS is not enabled (?)
  if (!cyw_ioctl_iovar_set_(0, "bsscfg:wsec", (uint8_t *)&data, 2 * sizeof(uint32_t))) return false;
  val = 1; // MFP capable: 1 for yes, 0 for no; recommended to be set for WPA2+ (2 for 'required', WPA3)
  cyw_ioctl_iovar_set_(1, "mfp", (uint8_t *)&val, sizeof(val)); // Old chipsets do not support MFP
  mg_delayms(2); // allow radio firmware to be ready
  // skip if not using auth
  // WPA, WPA2, mixed WPA/WPA2, mixed WPA2/WPA3
  // NOTE(): WHD does not set SAE password for shared WPA2/WPA3, same do we
  memset(data, 0, sizeof(data));
  len = strlen(pass);
  da[0] = (uint16_t) len; // skip for WPA3 SAE (43430 does NOT support WPA3 in AP)
  da[1] = 1; // indicates wireless security key, skip for WPA3 SAE
  memcpy((uint8_t *)data + 2 * sizeof(uint16_t), pass, len); // skip for WPA3 SAE
  if (!cyw_ioctl_set_(1, 268 /* SET_WSEC_PMK */, data, sizeof(data))) return false; // skip for WPA3 SAE, sizeof/2 if supporting SAE but using WPA
  /* for WPA3 SAE:
    memcpy((uint8_t *)data + sizeof(uint16_t), pass, len);
    cyw_ioctl_iovar_set_(1, "sae_password", data, sizeof(data)); */
  /* for WPA3 or mixed WPA2/WPA3:
    val = 5 ; cyw_ioctl_iovar_set_(1, "sae_max_pwe_loop", (uint8_t *)&val, sizeof(val));  // Some chipsets do not support this */
  // resume if not using auth

  data[0] = 1; // bss index 1 = AP
  data[1] = 0x80; // auth type: 0 for none, 4 for WPA PSK, 0x80 for WPA2 PSK, 0x40000 for WPA3 SAE PSK; ored if mixed
  if (!cyw_ioctl_iovar_set_(0, "bsscfg:wpa_auth", (uint8_t *)&data, 2 * sizeof(uint32_t))) return false;

  val = 1 /* auto */; if (!cyw_ioctl_set_(1, 110 /* SET_GMODE */, (uint8_t *)&val, sizeof(val))) return false;
  // Set multicast tx rate to 11Mbps, may fail in some chipsets, we are enforcing it
  val = 11000000 / 500000; if (!cyw_ioctl_iovar_set_(1, "2g_mrate", (uint8_t *)&val, sizeof(val))) return false;
  val = 1; if (!cyw_ioctl_set_(1, 78 /* SET_DTIMPRD */, (uint8_t *)&val, sizeof(val))) return false;
  data[0] = 1; // bss index 1 = AP
  data[1] = 1; // UP
  // TODO(scaprile): this takes a long time to process
  if (!cyw_ioctl_iovar_set_(0, "bss", (uint8_t *)&data, 2 * sizeof(uint32_t))) return false;
  return true;
}

static bool cyw_wifi_ap_stop(void) {
  uint32_t data[2];
  data[0] = 1; // bss index 1 = AP
  data[1] = 0; // DOWN
  if (!cyw_ioctl_iovar_set_(0, "bss", (uint8_t *)&data, 2 * sizeof(uint32_t))) return false;
  // DO WE NEED TO CLEAR CHANNEL ???
  // CHIP DEPENDENCY
  //val = 8 ; cyw_ioctl_iovar_set_(0, "ampdu_ba_wsize", (uint8_t *)&val, sizeof(val));
  return true;
}

// WLAN scan handling

#pragma pack(push, 1)
// in little endian

struct wifi_scan_opt {
    uint32_t version;
    uint16_t action;
    uint16_t _;
    uint32_t ssid_len;
    uint8_t ssid[32];
    uint8_t bssid[6];
    int8_t bss_type;
    int8_t scan_type;
    int32_t nprobes;
    int32_t active_time;
    int32_t passive_time;
    int32_t home_time;
    int32_t channel_num;
    uint16_t channel_list[1];
};
#pragma pack(pop)

static bool cyw_wifi_scan(void) {
  struct wifi_scan_opt opts;
  memset(&opts, 0, sizeof(opts));
  opts.version = 1;
  opts.action = 1; // start
  opts._ = 0;
  memset(opts.bssid, 0xff, sizeof(opts.bssid));
  opts.bss_type = 2; // any
  opts.nprobes = -1;
  opts.active_time = -1;
  opts.passive_time = -1;
  opts.home_time = -1;
  opts.channel_num = 0;
  opts.channel_list[0] = 0;
  return cyw_ioctl_iovar_set("escan", (uint8_t *)&opts, sizeof(opts));
}


#pragma pack(push, 1)
// in little endian

struct scan_bss {
    uint32_t version;              // version field
    uint32_t length;               // byte length of data in this record, starting at version and including IEs
    uint8_t BSSID[6];              // Unique 6-byte MAC address
    uint16_t beacon_period;        // Interval between two consecutive beacon frames. Units are Kusec
    uint16_t capability;           // Capability information
    uint8_t SSID_len;              // SSID length
    uint8_t SSID[32];              // Array to store SSID
    uint8_t reserved1[1];          // Reserved(padding)
    uint32_t rateset_count;        // Count of rates in this set
    uint8_t rateset_rates[16];     // rates in 500kbps units, higher bit set if basic
    uint16_t chanspec;             // Channel specification for basic service set
    uint16_t atim_window;          // Announcement traffic indication message window size. Units are Kusec
    uint8_t dtim_period;           // Delivery traffic indication message period
    uint8_t reserved2[1];          // Reserved(padding)
    int16_t RSSI;                  // receive signal strength (in dBm)
    int8_t phy_noise;              // noise (in dBm)
    uint8_t n_cap;                 // BSS is 802.11n Capable
    uint8_t reserved3[2];          // Reserved(padding)
    uint32_t nbss_cap;             // 802.11n BSS Capabilities (based on HT_CAP_*)
    uint8_t ctl_ch;                // 802.11n BSS control channel number
    uint8_t reserved4[3];          // Reserved(padding)
    uint32_t reserved32[1];        // Reserved for expansion of BSS properties
    uint8_t flags;                 // flags
    uint8_t vht_cap;               // BSS is vht capable
    uint8_t reserved5[2];          // Reserved(padding)
    uint8_t basic_mcs[16];         // 802.11N BSS required MCS set
    uint16_t ie_offset;            // offset at which IEs start, from beginning
    uint16_t reserved16[1];        // Reserved(padding)
    uint32_t ie_length;            // byte length of Information Elements
    int16_t SNR;                   // Average SNR during frame reception
};

struct scan_result {
    uint32_t buflen;
    uint32_t version;
    uint16_t sync_id;
    uint16_t bss_count;
    struct scan_bss bss[1];
};

#pragma pack(pop)

// CHIP DEPENDENCY
#define CYW_BSS_BANDMASK 0xc000
#define CYW_BSS_BAND2G 0
//

static void cyw_handle_scan_result(uint32_t status, struct scan_result *data, size_t len) {
  MG_VERBOSE(("scan event, status: %ld", status));
  if (status == 0) { // SUCCESS
    MG_VERBOSE(("scan complete"));
    mg_tcpip_call(s_ifp, MG_TCPIP_EV_WIFI_SCAN_END, NULL);
  } else if (status == 8) { // PARTIAL
    struct mg_wifi_scan_bss_data bss;
    struct scan_bss *sbss = data->bss;
    unsigned int band = sbss->chanspec & CYW_BSS_BANDMASK;
    if (data->version != 109 || data->bss_count != 1) {
      MG_ERROR(("Unsupported: %lu %u", data->version, data->bss_count));
      return;
    }
    if (sbss->length > len - offsetof(struct scan_result, bss) || sbss->SSID_len > sizeof(sbss->SSID) || sbss->ie_offset < sizeof(*sbss) || sbss->ie_offset > (sizeof(*sbss) + sbss->ie_length) || sbss->ie_offset + sbss->ie_length > sbss->length)
      return; // silently discard malformed data
    if (!(sbss->flags & MG_BIT(2))) return; // RSSI_ONCHANNEL, ignore off-channel results
    bss.SSID = mg_str_n((char *)sbss->SSID, sbss->SSID_len);
    bss.BSSID = (char *)sbss->BSSID;
    bss.RSSI = (int8_t)sbss->RSSI;
    bss.has_n = sbss->n_cap != 0;
    bss.channel = bss.has_n ? sbss->ctl_ch : (uint8_t)(sbss->chanspec & 0xff); // n 40MHz vs a/b/g and 20MHz
    bss.band = band & CYW_BSS_BAND2G ? MG_WIFI_BAND_2G : MG_WIFI_BAND_5G;
    bss.security = (sbss->capability & MG_BIT(4) /* CAP_PRIVACY */) ? MG_WIFI_SECURITY_WEP : MG_WIFI_SECURITY_OPEN;
    { // travel IEs (Information Elements) in search of security definitions
      const uint8_t wot1[4] = {0x00, 0x50, 0xf2, 0x01}; // WPA_OUI_TYPE1
      uint8_t *ie = (uint8_t *)sbss + sbss->ie_offset;
      int bytes = (int) sbss->ie_length;
      while (bytes > 0 && ie[1] + 2 < bytes) { // ie[0] -> type, ie[1] -> bytes from ie[2]
        if (ie[0] == 48 /* IE_ID_RSN */) bss.security |= MG_WIFI_SECURITY_WPA2;
        if (ie[0] == 221 /* IE_ID_VENDOR_SPECIFIC */ && memcmp(&ie[2], wot1, 4) == 0)
          bss.security |= MG_WIFI_SECURITY_WPA;
        ie += ie[1] + 2;
        bytes -= ie[1] + 2;
      }
    }
    MG_VERBOSE(("BSS: %.*s (%u) (%M) %d dBm %u", bss.SSID.len, bss.SSID.buf, bss.channel, mg_print_mac, bss.BSSID, (int) bss.RSSI, bss.security));
    mg_tcpip_call(s_ifp, MG_TCPIP_EV_WIFI_SCAN_RESULT, &bss);
  } else {
    MG_ERROR(("scan error"));
  }
}
// clang-format on

// IOCTL stuff. All values read and written are in little endian format

static uint16_t s_ioctl_reqid;

// CDC handler for waiting loop
static uint8_t *s_ioctl_resp;
static bool s_ioctl_err;

static void cyw_handle_cdc(struct cdc_hdr *cdc, size_t len) {
  uint8_t *r = (uint8_t *) cdc + sizeof(*cdc);
  MG_VERBOSE(("%u bytes CDC frame", len));
  if ((cdc->flags >> 16) != s_ioctl_reqid) return;
  if (cdc->flags & 1) {
    MG_ERROR(("IOCTL error: %ld", -cdc->status));
    s_ioctl_err = true;
    return;
  }
  if (mg_log_level >= MG_LL_VERBOSE) mg_hexdump((void *) cdc, len);
  MG_VERBOSE(("IOCTL result: %02x %02x %02x %02x ..", r[0], r[1], r[2], r[3]));
  s_ioctl_resp = r;
}
// NOTE(): alt no loop handler dispatching IOCTL response to current handler:
// static void *s_ioctl_hnd; *s_ioctl_hnd(ioctl, len);
// app is a state machine calling get/sets and advancing via these callbacks

#pragma pack(push, 1)
// all little endian

struct ctrl_hdr {
  struct sdpcm_hdr sdpcm;
  struct cdc_hdr cdc;
};

#pragma pack(pop)

// IOCTL command send
static void cyw_ioctl_send_cmd(unsigned int ifc, unsigned int cmd, bool set,
                               size_t len) {
  struct ctrl_hdr *hdr = (struct ctrl_hdr *) txdata;
  uint16_t txlen = (uint16_t) (len + sizeof(*hdr));
  memset(txdata, 0, sizeof(*hdr));
  hdr->cdc.cmd = cmd;
  hdr->cdc.olen = (uint16_t) len;
  // hdr->cdc.ilen = 0; // actually zeroed above
  hdr->cdc.flags = ((uint32_t) ++s_ioctl_reqid << 16) | ((ifc & 0xf) << 12) |
                   (set ? MG_BIT(1) : 0);
  hdr->sdpcm.len = txlen;
  hdr->sdpcm._len = (uint16_t) ~txlen;
  hdr->sdpcm.sw_hdr.sequence = ++s_tx_seqno;
  hdr->sdpcm.sw_hdr.channel_and_flags = CYW_SDPCM_CTRL_HDR;
  hdr->sdpcm.sw_hdr.header_length = offsetof(struct ctrl_hdr, cdc);
  cyw_bus_specific_tx(txdata, txlen);
}

// just send respective commands, response handled via CDC handler
static void cyw_ioctl_send_get(unsigned int ifc, unsigned int cmd) {
  cyw_ioctl_send_cmd(ifc, cmd, false, 0);
}

static void cyw_ioctl_send_set(unsigned int ifc, unsigned int cmd, void *data,
                               size_t len) {
  if (data != NULL && len > 0)
    memcpy((uint8_t *) txdata + sizeof(struct ctrl_hdr), data, len);
  cyw_ioctl_send_cmd(ifc, cmd, true, (uint16_t) len);
}

static void cyw_ioctl_send_iovar_get(unsigned int ifc, char *var, size_t len) {
  unsigned int namelen = strlen(var) + 1;  // include '\0'
  // cmd = GET IOVAR, "set" the name...
  cyw_ioctl_send_set(ifc, 262, var, len > namelen ? len : namelen);
}

static void cyw_ioctl_send_iovar_set2(unsigned int ifc, char *var, void *data1,
                                      size_t len1, void *data2, size_t len2) {
  struct ctrl_hdr *hdr = (struct ctrl_hdr *) txdata;
  unsigned int namelen = strlen(var) + 1;  // include '\0'
  uint16_t txlen, payload_len = (uint16_t) (namelen + len1 + len2);
  memcpy((uint8_t *) txdata + sizeof(*hdr), var, namelen);
  memcpy((uint8_t *) txdata + namelen + sizeof(*hdr), data1, len1);
  if (data2 != NULL)
    memcpy((uint8_t *) txdata + namelen + sizeof(*hdr) + len1, data2, len2);
  txlen = (uint16_t) (payload_len + sizeof(*hdr));
  cyw_ioctl_send_cmd(ifc, 263, true, txlen);  // cmd = SET IOVAR
}

__attribute__((unused)) static void cyw_ioctl_send_iovar_set(unsigned int ifc,
                                                             char *var,
                                                             void *data,
                                                             size_t len) {
  cyw_ioctl_send_iovar_set2(ifc, var, data, len, NULL, 0);
}

static inline bool delayms(unsigned int ms) {
  mg_delayms(ms);
  return true;
}

// wait for a response, meanwhile delivering received frames and events
static bool cyw_ioctl_wait(void) {
  unsigned int times = 100;
  s_ioctl_resp = NULL;
  s_ioctl_err = false;
  do {  // IOCTL response processing does not call any other IOCTL function
    cyw_poll();  // otherwise we can't allow them to pile up here
    // network frames will be pushed to the queue so that is safe
  } while (s_ioctl_resp == NULL && !s_ioctl_err && times-- > 0 && delayms(1));
  MG_VERBOSE(("resp: %lp, err: %c, times: %d", s_ioctl_resp,
              s_ioctl_err ? '1' : '0', (int) times));
  return s_ioctl_resp != NULL;
}

static bool cyw_ioctl_waitdata(void *data, size_t len) {
  if (!cyw_ioctl_wait()) return false;
  memcpy(data, s_ioctl_resp, len);
  return true;
}

// send respective commands, wait for a response or timeout
static bool cyw_ioctl_get_(unsigned int ifc, unsigned int cmd, void *data,
                           size_t len) {
  cyw_ioctl_send_get(ifc, cmd);
  return cyw_ioctl_waitdata(data, len);
}
static bool cyw_ioctl_set_(unsigned int ifc, unsigned int cmd, void *data,
                           size_t len) {
  cyw_ioctl_send_set(ifc, cmd, data, len);
  return cyw_ioctl_wait();
}

static bool cyw_ioctl_iovar_get_(unsigned int ifc, char *var, void *data,
                                 size_t len) {
  cyw_ioctl_send_iovar_get(ifc, var, len);
  return cyw_ioctl_waitdata(data, len);
}
static bool cyw_ioctl_iovar_set2_(unsigned int ifc, char *var, void *data1,
                                  size_t len1, void *data2, size_t len2) {
  cyw_ioctl_send_iovar_set2(ifc, var, data1, len1, data2, len2);
  return cyw_ioctl_wait();
}
static bool cyw_ioctl_iovar_set_(unsigned int ifc, char *var, void *data,
                                 size_t len) {
  return cyw_ioctl_iovar_set2_(ifc, var, data, len, NULL, 0);
}

// CYW43 chipset specifics. All values read and written are in little endian
// format

#pragma pack(push, 1)
// all little endian

struct cyw_country {
  uint32_t a;
  int32_t rev;
  uint32_t c;
};

struct clm_hdr {
  uint16_t flag;
  uint16_t type;
  uint32_t len;
  uint32_t crc;
};

#pragma pack(pop)

// worlwide rev0, TODO(): try rev 17 for 4343W
static const uint32_t country_code = 'X' + ('X' << 8) + (0 << 16);

static bool cyw_bus_specific_init();
static bool cyw_load_clmll(void *data, size_t len);

static bool cyw_load_clm(struct mg_tcpip_driver_cyw_firmware *fw) {
  return cyw_load_clmll((void *) fw->clm_addr, fw->clm_len);
}

// clang-format off
static bool cyw_init(uint8_t *mac) {
  struct mg_tcpip_driver_cyw_data *d = (struct mg_tcpip_driver_cyw_data *) s_ifp->driver_data;
  uint32_t val = 0;
  if (!cyw_bus_specific_init()) return false;
  if (!cyw_load_clm(d->fw)) return false;  // Load CLM blob
  // BT-ENABLED DEPENDENCY
  // set Wi-Fi up
  val = 0 /* disable */; cyw_ioctl_iovar_set("bus:txglom", (uint8_t *)&val, sizeof(val));
  val = 1 /* on */; cyw_ioctl_iovar_set("apsta", (uint8_t *)&val, sizeof(val));
  // CHIP DEPENDENCY
  val = 8 ; cyw_ioctl_iovar_set("ampdu_ba_wsize", (uint8_t *)&val, sizeof(val));
  val = 4 ; cyw_ioctl_iovar_set("ampdu_mpdu", (uint8_t *)&val, sizeof(val));
  val = 0 /* 8K */; cyw_ioctl_iovar_set("ampdu_rx_factor", (uint8_t *)&val, sizeof(val));
  //
  {
    struct cyw_country c;
    unsigned int rev = (unsigned int) (country_code >> 16) & 0xffff;
    c.c = c.a = country_code & 0xffff;
    c.rev = rev == 0 ? -1 : (int32_t) rev; // if rev is 0, set it to -1, the chip will use any NVRAM/OTP configured aggregate or default to rev 0
    cyw_ioctl_iovar_set("country", (void *)&c, sizeof(c));
  } // this takes some time to process
  { // so do some retries while enabling events of interest
    // we care for SET_SSID(0), JOIN(1), AUTH(3), DEAUTH(5), DISASSOC_IND(12), LINK(16), PSK_SUP(46), SCAN_RESULT(69); all < 128
    uint32_t data[128/8/4 + 1];
    data[0] = 0; // bss index: 0 = STA
    memset(&data[1], 0, 128/8); // mark all as not desired
    data[1] = MG_BIT(0) | MG_BIT(1) | MG_BIT(3) | MG_BIT(5) | MG_BIT(12) | MG_BIT(16); // events 0 to 31
    data[2] = MG_BIT(46 - 32); // events 32 to 63
    data[3] = MG_BIT(69 - 64); // events 64 to 95
    unsigned int times = 100;
    while (times --)
      if (cyw_ioctl_iovar_set("bsscfg:event_msgs", (uint8_t *)data, sizeof(data))) break;
    if (times == (unsigned int) ~0) return false;
  }
  val = 0; if (!cyw_ioctl_set(64 /* SET_ANTDIV */, (uint8_t *)&val, sizeof(val))) return false;
  if (!cyw_ioctl_set(2 /* UP, interface up */, NULL, 0)) return false;
  // use PM2 power saving for max throughput
  val = 200 /* ms */; if (!cyw_ioctl_iovar_set("pm2_sleep_ret", (uint8_t *)&val, sizeof(val))) return false;
  // set beacon intervals to reduce power consumption while associated to an AP but idle
  val = 1; if (!cyw_ioctl_iovar_set("bcn_li_bcn", (uint8_t *)&val, sizeof(val))) return false;
  val = 1; if (!cyw_ioctl_iovar_set("bcn_li_dtim", (uint8_t *)&val, sizeof(val))) return false;
  val = 10; if (!cyw_ioctl_iovar_set("assoc_listen", (uint8_t *)&val, sizeof(val))) return false;
  val = 1 /* auto */; if (!cyw_ioctl_set(110 /* SET_GMODE */, (uint8_t *)&val, sizeof(val))) return false;
  val = 0 /* any */; if (!cyw_ioctl_set(142 /* SET_BAND */, (uint8_t *)&val, sizeof(val))) return false;
  if (mg_log_level >= MG_LL_DEBUG) {
    char text[256]; // this is huge, but we're just starting up
    if (cyw_ioctl_iovar_get("ver", (uint8_t *)text, sizeof(text))) {
      unsigned int len = strnlen(text, sizeof(text));
      MG_DEBUG(("Firmware:\n%.*s", len, text));
    }
    text[0] = '\0';
    if (cyw_ioctl_iovar_get("clmver", (uint8_t *)text, sizeof(text)) && text[0] != '\0') {
      unsigned int len = strnlen(text, sizeof(text));
      MG_DEBUG(("CLM:\n%.*s", len, text));
    }
  }
  {
    if(cyw_ioctl_iovar_get("cur_etheraddr", mac, 6)) {
      MG_DEBUG(("MAC: %M", mg_print_mac, mac));
    } else {
      MG_ERROR(("read MAC failed"));
    }
  }
  return true;
}
// clang-format on

static bool cyw_load_fwll(void *fwdata, size_t fwlen, void *nvramdata,
                          size_t nvramlen);

static bool cyw_load_firmware(struct mg_tcpip_driver_cyw_firmware *fw) {
  return cyw_load_fwll((void *) fw->code_addr, fw->code_len,
                       (void *) fw->nvram_addr, fw->nvram_len);
}

// clang-format off
static bool cyw_load_clmll(void *data, size_t len) {
  unsigned int sent = 0, offset = 0;
  struct clm_hdr hdr = {
      .flag = 1 << 12 /* DLOAD_HANDLER_VER */ | MG_BIT(1) /* DL_BEGIN */,
      .type = 2,
      .crc = 0};
  while (sent < len) {
    unsigned int bytes = len - sent;
    if (bytes > 1024) bytes = 1024;
    if (sent + bytes >= len) hdr.flag |= MG_BIT(2);  // DL_END;
    hdr.len = bytes;
    if (!cyw_ioctl_iovar_set2_(0, "clmload", (void *) &hdr, sizeof(hdr), (uint8_t *) data + offset, bytes))
      break;
    sent += bytes;
    offset += bytes;
    hdr.flag &= (uint16_t)~MG_BIT(1);  // DL_BEGIN
  }
  return sent >= len;
}
// clang-format on

static void cyw_update_hash_table(void) {
  // TODO(): read database, rebuild hash table
  uint32_t val = 0;
  val = 1;
  cyw_ioctl_iovar_set2_(0, "mcast_list", (uint8_t *) &val, sizeof(val),
                        (uint8_t *) mcast_addr, sizeof(mcast_addr));
  mg_delayms(50);
}

// CYW43 chip backplane specifics. All values read and written are in little
// endian format

// Access to chip backplane is done windowed in 32KB banks
// - addr = area base address + register offset
// - set the window address to addr & ~ADDRMSK
// - access addr & ADDRMSK for non-32-bit quantities
// - if accesing 32-bit quantities, do it on (addr & ADDRMSK) | ACCSS4B
#define CYW_CHIP_CHIPCOMMON 0x18000000
#define CYW_CHIP_BCKPLN_WINSZ 0x8000
#define CYW_CHIP_BCKPLN_ADDRMSK 0x7fff
#define CYW_CHIP_BCKPLN_ACCSS4B MG_BIT(15)
#define CYW_CHIP_BCKPLN_WRAPPOFF 0x100000
// BUS DEPENDENCY: max bus to backplane transfer size, bus function id
#define CYW_CHIP_BCKPLN_SPIMAX 64
#define CYW_CHIP_BCKPLN_SDIOMAX 1536
#if MG_ENABLE_DRIVER_CYW_SDIO
#define CYW_CHIP_BCKPLN_BUSMAX CYW_CHIP_BCKPLN_SDIOMAX
#define CYW_BUS_FUNC_CHIP CYW_SDIO_FUNC_CHIP
#else
#define CYW_CHIP_BCKPLN_BUSMAX CYW_CHIP_BCKPLN_SPIMAX
#define CYW_BUS_FUNC_CHIP CYW_SPID_FUNC_CHIP
#endif

// CHIP DEPENDENCY
#define CYW_CHIP_ARMCORE_BASE (CYW_CHIP_CHIPCOMMON + 0x3000)
#define CYW_CHIP_SOCSRAM_BASE (CYW_CHIP_CHIPCOMMON + 0x4000)
#define CYW_CHIP_ARMCORE (CYW_CHIP_ARMCORE_BASE + CYW_CHIP_BCKPLN_WRAPPOFF)
#define CYW_CHIP_SOCSRAM (CYW_CHIP_SOCSRAM_BASE + CYW_CHIP_BCKPLN_WRAPPOFF)
#define CYW_CHIP_ATCMRAM_BASE 0
#define CYW_CHIP_RAM_SIZE 0x80000
//

#define CYW_CHIP_ADDRLOW 0x1000a
#define CYW_CHIP_ADDRMID 0x1000b
#define CYW_CHIP_ADDRHIGH 0x1000c
#define CYW_CHIP_SPIFRCTRL 0x1000d
#define CYW_CHIP_CLOCKCSR 0x1000e
#define CYW_CHIP_PULLUP 0x1000f
#define CYW_CHIP_WAKEUPCTL 0x1001e
#define CYW_CHIP_SLEEPCSR 0x1001f

#define CYW_CHIP_SOCSRAM_BANKXIDX 0x010
#define CYW_CHIP_SOCSRAM_BANKXPDA 0x044
#define CYW_CHIP_AI_IOCTRL 0x408
#define CYW_CHIP_AI_RESETCTRL 0x800

static bool cyw_bus_write(unsigned int f, uint32_t addr, void *data,
                          uint16_t len);
static bool cyw_bus_read(unsigned int f, uint32_t addr, void *data,
                         uint16_t len);

// clang-format off
// set backplane window to requested area.
static void cyw_set_backplane_window(uint32_t addr) {
  uint32_t val;
  addr &= ~CYW_CHIP_BCKPLN_ADDRMSK;
  val = (addr >> 24) & 0xff; cyw_bus_write(CYW_BUS_FUNC_CHIP, CYW_CHIP_ADDRHIGH, &val, 1);
  val = (addr >> 16) & 0xff; cyw_bus_write(CYW_BUS_FUNC_CHIP, CYW_CHIP_ADDRMID, &val, 1);
  val = (addr >> 8) & 0xff; cyw_bus_write(CYW_BUS_FUNC_CHIP, CYW_CHIP_ADDRLOW, &val, 1);
}

static bool cyw_core_reset(uint32_t core_base, bool check) {
  uint32_t val = 0;
  // core disabled after chip reset
  cyw_set_backplane_window(core_base); // set backplane window for requested area; we do know offsets fall within that window
  // possible CHIP DEPENDENCY: AI_RESETSTATUS check and wait (instead of these cool reads) to ensure backplane operations end
  cyw_bus_read(CYW_BUS_FUNC_CHIP, (core_base + CYW_CHIP_AI_IOCTRL) & CYW_CHIP_BCKPLN_ADDRMSK, &val, 1); // ensure backplane operations end
  val = MG_BIT(1) | MG_BIT(0) /* SICF_FGC | SICF_CLOCK_EN */; cyw_bus_write(CYW_BUS_FUNC_CHIP, (core_base + CYW_CHIP_AI_IOCTRL) & CYW_CHIP_BCKPLN_ADDRMSK, &val, 1); // reset
  cyw_bus_read(CYW_BUS_FUNC_CHIP, (core_base + CYW_CHIP_AI_IOCTRL) & CYW_CHIP_BCKPLN_ADDRMSK, &val, 1); // ensure backplane operations end
  val = 0x00; cyw_bus_write(CYW_BUS_FUNC_CHIP, (core_base + CYW_CHIP_AI_RESETCTRL) & CYW_CHIP_BCKPLN_ADDRMSK, &val, 1); // release reset
  mg_delayms(1);
  val = MG_BIT(0) /* SICF_CLOCK_EN */; cyw_bus_write(CYW_BUS_FUNC_CHIP, (core_base + CYW_CHIP_AI_IOCTRL) & CYW_CHIP_BCKPLN_ADDRMSK, &val, 1);
  cyw_bus_read(CYW_BUS_FUNC_CHIP, (core_base + CYW_CHIP_AI_IOCTRL) & CYW_CHIP_BCKPLN_ADDRMSK, &val, 1); // ensure backplane operations end
  mg_delayms(1);

  if (check) {
    // Verify only clock is enabled
    cyw_bus_read(CYW_BUS_FUNC_CHIP, (core_base + CYW_CHIP_AI_IOCTRL) & CYW_CHIP_BCKPLN_ADDRMSK, &val, 1);
    if ((val & (MG_BIT(1) | MG_BIT(0)) /* SICF_FGC | SICF_CLOCK_EN) */) != MG_BIT(0)) return false;
    // Verify it is not in reset state
    cyw_bus_read(CYW_BUS_FUNC_CHIP, (core_base + CYW_CHIP_AI_RESETCTRL) & CYW_CHIP_BCKPLN_ADDRMSK, &val, 1);
    if (val & MG_BIT(0)) return false; // AIRC_RESET
  }
  return true;
}

static void cyw_socram_init(void) {
  uint32_t val;
  // CHIP DEPENDENCY: disable remap for SRAM_3: 43430 and 43439 only
  cyw_set_backplane_window(CYW_CHIP_SOCSRAM_BASE); // set backplane window for requested area; we do know offsets fall within that window
  val = 0x03; cyw_bus_write(CYW_BUS_FUNC_CHIP, ((CYW_CHIP_SOCSRAM_BASE + CYW_CHIP_SOCSRAM_BANKXIDX) & CYW_CHIP_BCKPLN_ADDRMSK), &val, sizeof(val));
  val = 0x00; cyw_bus_write(CYW_BUS_FUNC_CHIP, ((CYW_CHIP_SOCSRAM_BASE + CYW_CHIP_SOCSRAM_BANKXPDA) & CYW_CHIP_BCKPLN_ADDRMSK), &val, sizeof(val));
}

// transfer is fractioned in bus-to-backplane-size units within backplane windows
static void cyw_load_data(uint32_t dest, void *data, size_t len) {
  size_t sent = 0, offset = 0;
  uint32_t last_addr = (uint32_t) ~0;
  while (sent < len)  {
    size_t bytes = len - sent, avail;
    uint32_t addr = dest + offset;
    if (addr - last_addr >= CYW_CHIP_BCKPLN_WINSZ || last_addr == (uint32_t) ~0) {
      cyw_set_backplane_window(addr); // set backplane window for requested area
      last_addr = addr & ~CYW_CHIP_BCKPLN_ADDRMSK;
    }
    addr &= CYW_CHIP_BCKPLN_ADDRMSK;
    avail = CYW_CHIP_BCKPLN_WINSZ - (unsigned int) addr; // internal backplane limit
    if (bytes > avail) bytes = avail;
    if (bytes > CYW_CHIP_BCKPLN_BUSMAX) bytes = CYW_CHIP_BCKPLN_BUSMAX; // bus to backplane transfer limit
    cyw_bus_write(CYW_BUS_FUNC_CHIP, addr, (uint8_t *)data + offset, (uint16_t) bytes);
    sent += bytes;
    offset += bytes;
  }
}

// CHIP DEPENDENCY: no SOCSRAM base address; start address in fwdata image (Cortex-R4 chips)
static bool cyw_load_fwll(void *fwdata, size_t fwlen, void *nvramdata, size_t nvramlen) {
  uint32_t val = ((~(nvramlen / 4) & 0xffff) << 16) | (nvramlen / 4); // ~len len in 32-bit words
  cyw_core_reset(CYW_CHIP_SOCSRAM, false);  // cores were disabled at chip reset
  cyw_socram_init();
  cyw_load_data(CYW_CHIP_ATCMRAM_BASE, fwdata, fwlen);
  mg_delayms(5); // TODO(scaprile): CHECK IF THIS IS ACTUALLY NEEDED
  // Load NVRAM and place 'length ~length' at the end; end of chip RAM
  {
    const uint32_t start = CYW_CHIP_RAM_SIZE - 4 - nvramlen;
    cyw_load_data(start, nvramdata, nvramlen); // nvramlen must be a multiple of 4
    // RAM_SIZE is a multiple of WINSZ, so the place for len ~len will be at the end of the window
    cyw_bus_write(CYW_BUS_FUNC_CHIP, (CYW_CHIP_BCKPLN_WINSZ - 4), &val, sizeof(val));
  }
  // Reset ARM core and check it starts
  if (!cyw_core_reset(CYW_CHIP_ARMCORE, true)) return false;
  return true;
}
// clang-format on

#if !MG_ENABLE_DRIVER_CYW_SDIO

// CYW43 SPI bus specifics

#define CYW_BUS_SPI_BUSCTRL 0x00     // 4 regs, 0 to 3
#define CYW_BUS_SPI_INT 0x04         // 2 regs, 4 to 5
#define CYW_BUS_SPI_INTEN 0x06       // 16-bit register
#define CYW_BUS_SPI_STATUS 0x08      // 32-bit register
#define CYW_BUS_SPI_TEST 0x14        // 32-bit register
#define CYW_BUS_SPI_RESPDLY_F1 0x1d  // 8-bit register, F1: chip

#define CYW_BUS_STS_LEN(x) ((x >> 9) & 0x7ff)

static bool cyw_spi_write(unsigned int f, uint32_t addr, void *data,
                          uint16_t len);
static void cyw_spi_read(unsigned int f, uint32_t addr, void *data,
                         uint16_t len);

// clang-format off
static size_t cyw_spi_poll(uint8_t *response) {
  size_t len;
  uint32_t res;
  // SPI poll
  cyw_spi_read(CYW_SPID_FUNC_BUS, CYW_BUS_SPI_STATUS, &res, sizeof(res));
  if (res == (uint32_t) ~0 || !(res & MG_BIT(8) /* packet available */ )) return 0;
  len = CYW_BUS_STS_LEN(res);
  if (len == 0) { // just ack IRQ
    uint16_t val = 1;
    cyw_spi_write(CYW_SPID_FUNC_CHIP, CYW_CHIP_SPIFRCTRL, &val, 1);
    cyw_spi_read(CYW_SPID_FUNC_BUS, CYW_BUS_SPI_INT, &val, sizeof(val));
    cyw_spi_write(CYW_SPID_FUNC_BUS, CYW_BUS_SPI_INT, &val, sizeof(val));
    return 0;
  }
  cyw_spi_read(CYW_SPID_FUNC_WLAN, 0,  response, (uint16_t)len);
  return len;
}

static size_t cyw_spi_tx(uint32_t *data, uint16_t len) {
  while (len & 3) ((uint8_t *)data)[len++] = 0; // SPI 32-bit padding
  return cyw_spi_write(CYW_SPID_FUNC_WLAN, 0, data, len) ? len: 0;
}

// this can be integrated in lowest level SPI read/write _driver_ functions
// (those calling hal SPI transaction functions), though is only used at start
uint32_t sw16_2(uint32_t data) {
  return ((uint32_t)mg_htons((uint16_t)(data >> 16)) << 16) + mg_htons((uint16_t)data);
}

// DS 4.2.2 Table 6: signal we're working in 16-bit mode
#define CYW_SPI_16bMODE MG_BIT(2) // arbitrary bit out of the FUNC space

static bool cyw_spi_init() {
  struct mg_tcpip_driver_cyw_data *d = (struct mg_tcpip_driver_cyw_data *) s_ifp->driver_data;
  uint32_t val = 0;
  // DS 4.2.3 Boot-Up Sequence; WHD: other chips might require more effort
  unsigned int times = 51;
  while (times--) {
    cyw_spi_read(CYW_SPID_FUNC_BUS | CYW_SPI_16bMODE, CYW_BUS_SPI_TEST, &val, sizeof(val));
    if (sw16_2(val) == 0xFEEDBEAD) break;
    mg_delayms(1);
  }
  if (times == (unsigned int) ~0) return false;
  // DS 4.2.3 Table 6. Chip starts in 16-bit little-endian mode.
  // Configure SPI and switch to 32-bit big-endian mode:
  // - High-speed mode: d->hs true
  // - IRQ POLARITY high
  // - SPI RESPONSE DELAY 4 bytes time [not in DS] TODO(scaprile): logic ana
  // - Status not sent after command, IRQ with status
  val = sw16_2(0x000204a3 | (d->hs ? MG_BIT(4) : 0)); // 4 reg content
  cyw_spi_write(CYW_SPID_FUNC_BUS | CYW_SPI_16bMODE, CYW_BUS_SPI_BUSCTRL, &val, sizeof(val));
  mg_tcpip_call(s_ifp, MG_TCPIP_EV_DRIVER, NULL);
  cyw_spi_read(CYW_SPID_FUNC_BUS, CYW_BUS_SPI_TEST, &val, sizeof(val));
  if (val != 0xFEEDBEAD) return false;
  val = 4; cyw_spi_write(CYW_SPID_FUNC_BUS, CYW_BUS_SPI_RESPDLY_F1, &val, 1);
  val = 0x99; // clear error bits DATA_UNAVAILABLE, COMMAND_ERROR, DATA_ERROR, F1_OVERFLOW
  cyw_spi_write(CYW_SPID_FUNC_BUS, CYW_BUS_SPI_INT, &val, 1);
  val = 0x00be; // Enable IRQs F2_F3_FIFO_RD_UNDERFLOW, F2_F3_FIFO_WR_OVERFLOW, COMMAND_ERROR, DATA_ERROR, F2_PACKET_AVAILABLE, F1_OVERFLOW
  // BT-ENABLED DEPENDENCY: add F1_INTR (bit 13)
  cyw_spi_write(CYW_SPID_FUNC_BUS, CYW_BUS_SPI_INTEN, &val, sizeof(uint16_t));

  // chip backplane is ready, initialize it
  // request ALP (Active Low Power) clock
  val = MG_BIT(3) /* ALP_REQ */; cyw_spi_write(CYW_SPID_FUNC_CHIP, CYW_CHIP_CLOCKCSR, &val, 1);
  // BT-ENABLED DEPENDENCY
  times = 10;
  while (times--) {
    cyw_spi_read(CYW_SPID_FUNC_CHIP, CYW_CHIP_CLOCKCSR, &val, 1);
    if (val & MG_BIT(6)) break; // ALP_AVAIL
    mg_delayms(1);
  }
  if (times == (unsigned int) ~0) return false;
  // clear request
  val = 0; cyw_spi_write(CYW_SPID_FUNC_CHIP, CYW_CHIP_CLOCKCSR, &val, 1);
  cyw_set_backplane_window(CYW_CHIP_CHIPCOMMON); // set backplane window to start of CHIPCOMMON area
  cyw_spi_read(CYW_SPID_FUNC_CHIP, (CYW_CHIP_CHIPCOMMON + 0x00) & CYW_CHIP_BCKPLN_ADDRMSK, &val, 2);
  if (val == 43430) val = 4343;
  MG_INFO(("WLAN chip is CYW%u%c", val, val == 4343 ? 'W' : ' '));

  // Load firmware (code and NVRAM)
  if (!cyw_load_firmware(d->fw)) return false;

  // Wait for High Throughput (HT) clock ready
  times = 50;
  while (times--) {
    cyw_spi_read(CYW_SPID_FUNC_CHIP, CYW_CHIP_CLOCKCSR, &val, 1);
    if (val & MG_BIT(7)) break; // HT_AVAIL
    mg_delayms(1);
  }
  if (times == (unsigned int) ~0) return false;
  // Wait for backplane ready
  times = 1000;
  while (times--) {
    cyw_spi_read(CYW_SPID_FUNC_BUS, CYW_BUS_SPI_STATUS, &val, sizeof(val));
    if (val & MG_BIT(5)) break; // F2_RX_READY
    mg_delayms(1);
  }
  if (times == (unsigned int) ~0) return false;

  // CHIP DEPENDENCY
  // Enable save / restore
  // Configure WakeupCtrl, set HT_AVAIL in CLOCK_CSR
  cyw_spi_read(CYW_SPID_FUNC_CHIP, CYW_CHIP_WAKEUPCTL, &val, 1);
  val |= MG_BIT(1) /* WAKE_TILL_HT_AVAIL */; cyw_spi_write(CYW_SPID_FUNC_CHIP, CYW_CHIP_WAKEUPCTL, &val, 1);
#if 0
  // Set BRCM_CARDCAP to CMD_NODEC. NOTE(): This is probably only necessary for SDIO, not SPI
  val = MG_BIT(3); cyw_spi_write(CYW_SPID_FUNC_BUS, 0xf0 /* SDIOD_CCCR_BRCM_CARDCAP */, &val, 1);
#endif
  // Force HT request to chip backplane
  val = MG_BIT(1) /* FORCE_HT */; cyw_spi_write(CYW_SPID_FUNC_CHIP, CYW_CHIP_CLOCKCSR, &val, 1);
  // Enable Keep SDIO On (KSO)
  cyw_spi_read(CYW_SPID_FUNC_CHIP, CYW_CHIP_SLEEPCSR, &val, 1);
  if (!(val & MG_BIT(0))) {
      val |= MG_BIT(0); cyw_spi_write(CYW_SPID_FUNC_CHIP, CYW_CHIP_SLEEPCSR, &val, 1);
  }
  // The SPI bus can be configured for sleep (KSO controls wlan block sleep)
  cyw_spi_read(CYW_SPID_FUNC_BUS, CYW_BUS_SPI_BUSCTRL, &val, sizeof(val));
  val &= ~MG_BIT(7) /* WAKE_UP */; cyw_spi_write(CYW_SPID_FUNC_BUS, CYW_BUS_SPI_BUSCTRL, &val, sizeof(val));
  // Set SPI bus sleep
  val = 0x0f; cyw_spi_write(CYW_SPID_FUNC_CHIP, CYW_CHIP_PULLUP, &val, 1);

  // Clear pullups. NOTE(): ?
  val = 0x00; cyw_spi_write(CYW_SPID_FUNC_CHIP, CYW_CHIP_PULLUP, &val, 1);
  cyw_spi_read(CYW_SPID_FUNC_CHIP, CYW_CHIP_PULLUP, &val, 1);
  // Clear possible data unavailable error
  cyw_spi_read(CYW_SPID_FUNC_BUS, CYW_BUS_SPI_INTEN, &val, sizeof(uint16_t));
  if (val & MG_BIT(0)) cyw_spi_write(CYW_SPID_FUNC_BUS, CYW_BUS_SPI_INTEN, &val, sizeof(uint16_t));

  return true;
}
// clang-format on

// gSPI, CYW43439 DS 4.2.1 Fig.12
#define CYW_SPID_LEN(x) ((x) &0x7FF)             // bits 0-10
#define CYW_SPID_ADDR(x) (((x) &0x1FFFF) << 11)  // bits 11-27,
#define CYW_SPID_FUNC(x) (((x) &3) << 28)        // bits 28-29
#define CYW_SPID_INC MG_BIT(30)
#define CYW_SPID_WR MG_BIT(31)

static bool cyw_spi_write(unsigned int f, uint32_t addr, void *data,
                          uint16_t len) {
  struct mg_tcpip_driver_cyw_data *d =
      (struct mg_tcpip_driver_cyw_data *) s_ifp->driver_data;
  struct mg_tcpip_spi *s = (struct mg_tcpip_spi *) d->bus;
  uint32_t hdr = CYW_SPID_WR | CYW_SPID_INC | CYW_SPID_FUNC(f) |
                 CYW_SPID_ADDR(addr) | CYW_SPID_LEN(len);  // gSPI header
  // TODO(scaprile): check spin in between and timeout values, return false
  if (f == CYW_SPID_FUNC_WLAN) {
    uint32_t val = 0;
    while ((val & MG_BIT(5)) != MG_BIT(5))  // F2 rx ready (FIFO ready)
      cyw_spi_read(CYW_SPID_FUNC_BUS, CYW_BUS_SPI_STATUS, &val, sizeof(val));
  }
  if (f & CYW_SPI_16bMODE)
    hdr = sw16_2(hdr);  // swap half-words in 16-bit little-endian mode

  s->begin(NULL);
  s->txn(NULL, (uint8_t *) &hdr, NULL, sizeof(hdr));
  if (len <= 4) {
    uint32_t pad = 0;
    memcpy(&pad, data, len);
    s->txn(NULL, (uint8_t *) &pad, NULL, sizeof(pad));
  } else {
    s->txn(NULL, (uint8_t *) data, NULL, len);
  }
  s->end(NULL);
  return true;
}

// will write 32-bit aligned quantities to data if f == CYW_SPID_FUNC_WLAN
static void cyw_spi_read(unsigned int f, uint32_t addr, void *data,
                         uint16_t len) {
  struct mg_tcpip_driver_cyw_data *d =
      (struct mg_tcpip_driver_cyw_data *) s_ifp->driver_data;
  struct mg_tcpip_spi *s = (struct mg_tcpip_spi *) d->bus;
  uint32_t padding =
      f == CYW_SPID_FUNC_CHIP
          ? 4
          : 0;  // add padding to chip backplane reads as a response delay
  uint32_t hdr = CYW_SPID_INC | CYW_SPID_FUNC(f) | CYW_SPID_ADDR(addr) |
                 CYW_SPID_LEN(len + padding);  // gSPI header
  if (f == CYW_SPID_FUNC_WLAN && (len & 3))
    len = (len + 4) & ~3;  // align WLAN transfers to 32-bit
  if (f & CYW_SPI_16bMODE)
    hdr = sw16_2(hdr);  // swap half-words in 16-bit little-endian mode

  s->begin(NULL);
  s->txn(NULL, (uint8_t *) &hdr, NULL, sizeof(hdr));
  if (f == CYW_SPID_FUNC_CHIP) {
    uint32_t pad;
    s->txn(NULL, NULL, (uint8_t *) &pad, 4);  // read padding back and discard
  }
  s->txn(NULL, NULL, (uint8_t *) data, len);
  s->end(NULL);
}

static bool cyw_bus_specific_init(void) {
  return cyw_spi_init();
}
static size_t cyw_bus_specific_poll(uint32_t *response) {
  return cyw_spi_poll((uint8_t *) response);
}
static size_t cyw_bus_specific_tx(uint32_t *data, uint16_t len) {
  return cyw_spi_tx(data, len);
}
static bool cyw_bus_write(unsigned int f, uint32_t addr, void *data,
                          uint16_t len) {
  if (f == CYW_SPID_FUNC_CHIP && len >= 4) addr |= CYW_CHIP_BCKPLN_ACCSS4B;
  return cyw_spi_write(f, addr, data, len);
}
static bool cyw_bus_read(unsigned int f, uint32_t addr, void *data,
                         uint16_t len) {
  cyw_spi_read(f, addr, data, len);
  return true;
}

#else  // MG_ENABLE_DRIVER_CYW_SDIO



// CYW43 SDIO bus specifics

// CYW4343W and CYW43439 DS 4.1 SDIO v2.0:
//- F0: max block size is  32 bytes
//- F1: max block size is  64 bytes
//- F2: max block size is 512 bytes

// clang-format off
static bool cyw_sdio_transfer(bool write, unsigned int f, uint32_t addr, void *data, uint32_t len) {
  struct mg_tcpip_driver_cyw_data *d = (struct mg_tcpip_driver_cyw_data *) s_ifp->driver_data;
  struct mg_tcpip_sdio *s = (struct mg_tcpip_sdio *) d->bus;
  uint32_t *ptr = (uint32_t *) data; // assume 32-bit aligned data (all except firmware)
  if (write && (size_t) data & 3) {  // missed, source data is not 32-bit aligned
    memcpy(txdata, data, len);       // copy to an aligned buffer, we know it fits
    ptr = txdata;
  } // all possible read destinations are 32-bit aligned
  // mg_sdio_transfer requires 32-bit alignment for > 1 byte transfers
  return mg_sdio_transfer(s, write, f, addr, ptr, len);
}

static size_t cyw_sdio_poll(uint32_t *response) {
  uint32_t res;
  uint16_t *len = (uint16_t *)&res;
  // WHD: internal docs, "tag" hinting a possible packet.
  // This is actually the len / ~len field of a possible struct sdpcm_hdr, if there is a packet available, or 0 if there is none.
  cyw_sdio_transfer(false, CYW_SDIO_FUNC_WLAN, 0, &res, sizeof(res)); // read "the tag"
  if ((len[0] | len[1]) == 0 || (len[0] ^ len[1]) != 0xffff || *len <= 4) return 0;
  response[0] = res; // copy what we already read, then read the rest
  cyw_sdio_transfer(false, CYW_SDIO_FUNC_WLAN, 0, response + 1, *len - sizeof(res));
  return (size_t)*len;
}

static size_t cyw_sdio_tx(uint32_t *data, uint16_t len) {
  return cyw_sdio_transfer(true, CYW_SDIO_FUNC_WLAN, 0, data, len) ? len: 0;
}

static bool cyw_sdio_init() {
  struct mg_tcpip_driver_cyw_data *d = (struct mg_tcpip_driver_cyw_data *) s_ifp->driver_data;
  struct mg_tcpip_sdio *s = (struct mg_tcpip_sdio *) d->bus;
  uint32_t val = 0;
  if (!mg_sdio_init(s)) return false;
  // no block transfers on F0. if (!mg_sdio_set_blksz(s, CYW_SDIO_FUNC_BUS, 32)) return false;
  if (!mg_sdio_set_blksz(s, CYW_SDIO_FUNC_CHIP, 64)) return false;
  if (!mg_sdio_set_blksz(s, CYW_SDIO_FUNC_WLAN, 64)) return false;
  // TODO(scaprile): we don't handle SDIO interrupts, study CCCR INTEN and SDIO support (SDIO 6.3, 8)
  // Enable chip backplane (F1)
  if (!mg_sdio_enable_f(s, CYW_SDIO_FUNC_CHIP)) return false;
  // Wait for F1 to be ready
  if (!mg_sdio_waitready_f(s, CYW_SDIO_FUNC_CHIP)) return false;
  // chip backplane is ready, initialize it
  // request ALP (Active Low Power) clock
  val = MG_BIT(5) | MG_BIT(3) | MG_BIT(0); // HW_CLKREQ_OFF ALP_REQ FORCE_ALP
  cyw_sdio_transfer(true, CYW_SDIO_FUNC_CHIP, CYW_CHIP_CLOCKCSR, &val, 1);
  // BT-ENABLED DEPENDENCY
  unsigned int times = 10;
  while (times--) {
    if(!cyw_sdio_transfer(false, CYW_SDIO_FUNC_CHIP, CYW_CHIP_CLOCKCSR, &val, 1)) return false;
    if (val & MG_BIT(6)) break; // ALP_AVAIL
    mg_delayms(1);
  }
  if (times == (unsigned int) ~0) return false;
  // clear request
  val = 0; cyw_sdio_transfer(true, CYW_SDIO_FUNC_CHIP, CYW_CHIP_CLOCKCSR, &val, 1);
  // Enable WLAN (F2)
  if (!mg_sdio_enable_f(s, CYW_SDIO_FUNC_WLAN)) return false;
  // Clear pullups. NOTE(): ?
  val = 0x00; cyw_sdio_transfer(true, CYW_SDIO_FUNC_CHIP, CYW_CHIP_PULLUP, &val, 1);
  // we don't handle wake nor OOB interrupts; SEP_INT_CTL is a vendor specific SDIO register
  // SDIO interrupts: enable F2 interrupt only

  cyw_set_backplane_window(CYW_CHIP_CHIPCOMMON); // set backplane window to start of CHIPCOMMON area
  cyw_sdio_transfer(false, CYW_SDIO_FUNC_CHIP, (CYW_CHIP_CHIPCOMMON + 0x00) & CYW_CHIP_BCKPLN_ADDRMSK, &val, 2);
  if (val == 43430) val = 4343;
  MG_INFO(("WLAN chip is CYW%u%c", val, val == 4343 ? 'W' : ' '));
  // Load firmware (code and NVRAM)
  if (!cyw_load_firmware(d->fw)) return false;

  // Wait for High Throughput (HT) clock ready
  times = 50;
  while (times--) {
    if(!cyw_sdio_transfer(false, CYW_SDIO_FUNC_CHIP, CYW_CHIP_CLOCKCSR, &val, 1)) return false;
    if (val & MG_BIT(7)) break; // HT_AVAIL
    mg_delayms(1);
  }
  if (times == (unsigned int) ~0) return false;
  // Wait for WLAN ready
  if (!mg_sdio_waitready_f(s, CYW_SDIO_FUNC_WLAN)) return false;

  // CHIP DEPENDENCY
  // Enable save / restore
  // Configure WakeupCtrl, set HT_AVAIL in CLOCK_CSR
  if(!cyw_sdio_transfer(false, CYW_SDIO_FUNC_CHIP, CYW_CHIP_WAKEUPCTL, &val, 1)) return false;
  val |= MG_BIT(1) /* WAKE_TILL_HT_AVAIL */; cyw_sdio_transfer(true, CYW_SDIO_FUNC_CHIP, CYW_CHIP_WAKEUPCTL, &val, 1);
#if 0 // TODO(scaprile): Check if this is actually necessary
  // Set BRCM_CARDCAP to CMD_NODEC. This is a vendor specific SDIO register
  val = MG_BIT(3); cyw_sdio_transfer(true, CYW_SDIO_FUNC_BUS, 0xf0 /* SDIOD_CCCR_BRCM_CARDCAP */, &val, 1);
#endif
  // Force HT request to chip backplane
  val = MG_BIT(1) /* FORCE_HT */; if(!cyw_sdio_transfer(true, CYW_SDIO_FUNC_CHIP, CYW_CHIP_CLOCKCSR, &val, 1)) return false;
  // Enable Keep SDIO On (KSO)
  cyw_sdio_transfer(false, CYW_SDIO_FUNC_CHIP, CYW_CHIP_SLEEPCSR, &val, 1);
  if (!(val & MG_BIT(0))) {
      val |= MG_BIT(0); cyw_sdio_transfer(true, CYW_SDIO_FUNC_CHIP, CYW_CHIP_SLEEPCSR, &val, 1);
  }
  return true;
}

// clang-format on

static bool cyw_bus_specific_init(void) {
  return cyw_sdio_init();
}
static size_t cyw_bus_specific_poll(uint32_t *response) {
  return cyw_sdio_poll(response);
}
static size_t cyw_bus_specific_tx(uint32_t *data, uint16_t len) {
  return cyw_sdio_tx(data, len);
}
static bool cyw_bus_write(unsigned int f, uint32_t addr, void *data,
                          uint16_t len) {
  if (f == CYW_SDIO_FUNC_CHIP && len == 4) addr |= CYW_CHIP_BCKPLN_ACCSS4B;
  return cyw_sdio_transfer(true, f, addr, data, (uint32_t) len);
}
static bool cyw_bus_read(unsigned int f, uint32_t addr, void *data,
                         uint16_t len) {
  return cyw_sdio_transfer(false, f, addr, data, (uint32_t) len);
}

#endif

// Mongoose Wi-Fi API functions

bool mg_wifi_scan(void) {
  return cyw_wifi_scan();
}

bool mg_wifi_connect(struct mg_wifi_data *wifi) {
  s_ifp->ip = s_ip;
  s_ifp->mask = s_mask;
  if (s_ifp->ip == 0) s_ifp->enable_dhcp_client = true;
  s_ifp->enable_dhcp_server = false;
  MG_DEBUG(("Connecting to '%s'", wifi->ssid));
  return cyw_wifi_connect(wifi->ssid, wifi->pass);
}

bool mg_wifi_disconnect(void) {
  return cyw_wifi_disconnect();
}

bool mg_wifi_ap_start(struct mg_wifi_data *wifi) {
  MG_DEBUG(("Starting AP '%s' (%u)", wifi->apssid, wifi->apchannel));
  return cyw_wifi_ap_start(wifi->apssid, wifi->appass, wifi->apchannel);
}

bool mg_wifi_ap_stop(void) {
  return cyw_wifi_ap_stop();
}

#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/imxrt.c"
#endif


#if MG_ENABLE_TCPIP && \
  (defined(MG_ENABLE_DRIVER_IMXRT10) && MG_ENABLE_DRIVER_IMXRT10) || \
  (defined(MG_ENABLE_DRIVER_IMXRT11) && MG_ENABLE_DRIVER_IMXRT11) || \
  (defined(MG_ENABLE_DRIVER_MCXE) && MG_ENABLE_DRIVER_MCXE)
struct imxrt_enet {
  volatile uint32_t RESERVED0, EIR, EIMR, RESERVED1, RDAR, TDAR, RESERVED2[3],
      ECR, RESERVED3[6], MMFR, MSCR, RESERVED4[7], MIBC, RESERVED5[7], RCR,
      RESERVED6[15], TCR, RESERVED7[7], PALR, PAUR, OPD, TXIC0, TXIC1, TXIC2,
      RESERVED8, RXIC0, RXIC1, RXIC2, RESERVED9[3], IAUR, IALR, GAUR, GALR,
      RESERVED10[7], TFWR, RESERVED11[14], RDSR, TDSR, MRBR[2], RSFL, RSEM,
      RAEM, RAFL, TSEM, TAEM, TAFL, TIPG, FTRL, RESERVED12[3], TACC, RACC,
      RESERVED13[15], RMON_T_PACKETS, RMON_T_BC_PKT, RMON_T_MC_PKT,
      RMON_T_CRC_ALIGN, RMON_T_UNDERSIZE, RMON_T_OVERSIZE, RMON_T_FRAG,
      RMON_T_JAB, RMON_T_COL, RMON_T_P64, RMON_T_P65TO127, RMON_T_P128TO255,
      RMON_T_P256TO511, RMON_T_P512TO1023, RMON_T_P1024TO2048, RMON_T_GTE2048,
      RMON_T_OCTETS, IEEE_T_DROP, IEEE_T_FRAME_OK, IEEE_T_1COL, IEEE_T_MCOL,
      IEEE_T_DEF, IEEE_T_LCOL, IEEE_T_EXCOL, IEEE_T_MACERR, IEEE_T_CSERR,
      IEEE_T_SQE, IEEE_T_FDXFC, IEEE_T_OCTETS_OK, RESERVED14[3], RMON_R_PACKETS,
      RMON_R_BC_PKT, RMON_R_MC_PKT, RMON_R_CRC_ALIGN, RMON_R_UNDERSIZE,
      RMON_R_OVERSIZE, RMON_R_FRAG, RMON_R_JAB, RESERVED15, RMON_R_P64,
      RMON_R_P65TO127, RMON_R_P128TO255, RMON_R_P256TO511, RMON_R_P512TO1023,
      RMON_R_P1024TO2047, RMON_R_GTE2048, RMON_R_OCTETS, IEEE_R_DROP,
      IEEE_R_FRAME_OK, IEEE_R_CRC, IEEE_R_ALIGN, IEEE_R_MACERR, IEEE_R_FDXFC,
      IEEE_R_OCTETS_OK, RESERVED16[71], ATCR, ATVR, ATOFF, ATPER, ATCOR, ATINC,
      ATSTMP, RESERVED17[122], TGSR, TCSR0, TCCR0, TCSR1, TCCR1, TCSR2, TCCR2,
      TCSR3;
};

#undef ENET
#if defined(MG_ENABLE_DRIVER_IMXRT11) && MG_ENABLE_DRIVER_IMXRT11
#define ENET ((struct imxrt_enet *) (uintptr_t) 0x40424000U)
#define ETH_DESC_CNT 5     // Descriptors count
#elif defined(MG_ENABLE_DRIVER_IMXRT10) && MG_ENABLE_DRIVER_IMXRT10
#define ENET ((struct imxrt_enet *) (uintptr_t) 0x402D8000U)
#define ETH_DESC_CNT 4     // Descriptors count
#else // MG_ENABLE_DRIVER_MCXE
#define ENET ((struct imxrt_enet *) (uintptr_t) 0x40079000U)
#define ETH_DESC_CNT 4     // Descriptor count
#endif

#define ETH_PKT_SIZE 1536  // Max frame size, 64-bit aligned

struct enet_desc {
  uint16_t length;   // Data length
  uint16_t control;  // Control and status
  uint32_t *buffer;  // Data ptr
};

// Descriptors: in non-cached area (TODO(scaprile)), (37.5.1.22.2 37.5.1.23.2)
// Buffers: 64-byte aligned (37.3.14)
static volatile struct enet_desc s_rxdesc[ETH_DESC_CNT] MG_ETH_RAM MG_64BYTE_ALIGNED;
static volatile struct enet_desc s_txdesc[ETH_DESC_CNT] MG_ETH_RAM MG_64BYTE_ALIGNED;
static uint8_t s_rxbuf[ETH_DESC_CNT][ETH_PKT_SIZE] MG_ETH_RAM MG_64BYTE_ALIGNED;
static uint8_t s_txbuf[ETH_DESC_CNT][ETH_PKT_SIZE] MG_ETH_RAM MG_64BYTE_ALIGNED;
static struct mg_tcpip_if *s_ifp;  // MIP interface

static uint16_t enet_read_phy(uint8_t addr, uint8_t reg) {
  ENET->EIR |= MG_BIT(23);  // MII interrupt clear
  ENET->MMFR = (1 << 30) | (2 << 28) | (addr << 23) | (reg << 18) | (2 << 16);
  while ((ENET->EIR & MG_BIT(23)) == 0) (void) 0;
  return ENET->MMFR & 0xffff;
}

static void enet_write_phy(uint8_t addr, uint8_t reg, uint16_t val) {
  ENET->EIR |= MG_BIT(23);  // MII interrupt clear
  ENET->MMFR =
      (1 << 30) | (1 << 28) | (addr << 23) | (reg << 18) | (2 << 16) | val;
  while ((ENET->EIR & MG_BIT(23)) == 0) (void) 0;
}

//  MDC clock is generated from IPS Bus clock (ipg_clk); as per 802.3,
//  it must not exceed 2.5MHz
// The PHY receives the PLL6-generated 50MHz clock
static bool mg_tcpip_driver_imxrt_init(struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_imxrt_data *d =
      (struct mg_tcpip_driver_imxrt_data *) ifp->driver_data;
  s_ifp = ifp;

  // Init RX descriptors
  for (int i = 0; i < ETH_DESC_CNT; i++) {
    s_rxdesc[i].control = MG_BIT(15);              // Own (E)
    s_rxdesc[i].buffer = (uint32_t *) s_rxbuf[i];  // Point to data buffer
  }
  s_rxdesc[ETH_DESC_CNT - 1].control |= MG_BIT(13);  // Wrap last descriptor

  // Init TX descriptors
  for (int i = 0; i < ETH_DESC_CNT; i++) {
    // s_txdesc[i].control = MG_BIT(10);  // Own (TC)
    s_txdesc[i].buffer = (uint32_t *) s_txbuf[i];
  }
  s_txdesc[ETH_DESC_CNT - 1].control |= MG_BIT(13);  // Wrap last descriptor

  ENET->ECR = MG_BIT(0);                     // Software reset, disable
  while ((ENET->ECR & MG_BIT(0))) (void) 0;  // Wait until done

  // Set MDC clock divider. If user told us the value, use it.
  // TODO(): Otherwise, guess (currently assuming max freq)
  int cr = (d == NULL || d->mdc_cr < 0) ? 24 : d->mdc_cr;
  ENET->MSCR = (1 << 8) | ((cr & 0x3f) << 1);  // HOLDTIME 2 clks
  struct mg_phy phy = {enet_read_phy, enet_write_phy};
  mg_phy_init(&phy, d->phy_addr, MG_PHY_LEDS_ACTIVE_HIGH); // MAC clocks PHY  
  // Select RMII mode, 100M, keep CRC, set max rx length, disable loop
  ENET->RCR = (1518 << 16) | MG_BIT(8) | MG_BIT(2);
  // ENET->RCR |= MG_BIT(3);     // Receive all
  ENET->TCR = MG_BIT(2);  // Full-duplex
  ENET->RDSR = (uint32_t) (uintptr_t) s_rxdesc;
  ENET->TDSR = (uint32_t) (uintptr_t) s_txdesc;
  ENET->MRBR[0] = ETH_PKT_SIZE;  // Same size for RX/TX buffers
  // MAC address filtering (bytes in reversed order)
  ENET->PAUR = ((uint32_t) ifp->mac[4] << 24U) | (uint32_t) ifp->mac[5] << 16U;
  ENET->PALR = (uint32_t) (ifp->mac[0] << 24U) |
               ((uint32_t) ifp->mac[1] << 16U) |
               ((uint32_t) ifp->mac[2] << 8U) | ifp->mac[3];
  ENET->ECR = MG_BIT(8) | MG_BIT(1);  // Little-endian CPU, Enable
  ENET->EIMR = MG_BIT(25);            // Set interrupt mask
  ENET->RDAR = MG_BIT(24);            // Receive Descriptors have changed
  ENET->TDAR = MG_BIT(24);            // Transmit Descriptors have changed
  // ENET->OPD = 0x10014;
  ENET->IAUR = 0;
  ENET->IALR = 0;
  ENET->GAUR = 0;
  ENET->GALR = 0;
  return true;
}


// Transmit frame
static size_t mg_tcpip_driver_imxrt_tx(const void *buf, size_t len,
                                       struct mg_tcpip_if *ifp) {
  static int s_txno;  // Current descriptor index
  if (len > sizeof(s_txbuf[ETH_DESC_CNT])) {
    MG_ERROR(("Frame too big, %ld", (long) len));
    len = (size_t) -1;  // fail
  } else if ((s_txdesc[s_txno].control & MG_BIT(15))) {
    ifp->nerr++;
    MG_ERROR(("No descriptors available"));
    len = 0;  // retry later
  } else {
    memcpy(s_txbuf[s_txno], buf, len);         // Copy data
    s_txdesc[s_txno].length = (uint16_t) len;  // Set data len
    // Table 37-34, R, L, TC (Ready, last, transmit CRC after frame
    s_txdesc[s_txno].control |=
        (uint16_t) (MG_BIT(15) | MG_BIT(11) | MG_BIT(10));
    ENET->TDAR = MG_BIT(24);  // Descriptor ring updated
    if (++s_txno >= ETH_DESC_CNT) s_txno = 0;
  }
  (void) ifp;
  return len;
}

static void mg_tcpip_driver_imxrt_update_hash_table(struct mg_tcpip_if *ifp) {
  // TODO(): read database, rebuild hash table
  // RM 37.3.4.3.2
  uint32_t hash_table[2] = {0, 0};
  // uint8_t hash64 = ((~mg_crc32(0, mcast_addr, 6)) >> 26) & 0x3f;
  // hash_table[((uint8_t)hash64) >> 5] |= (1 << (hash64 & 0x1f));
  hash_table[1] = MG_BIT(1); // above reduces to this for mDNS addr
  ENET->GAUR = hash_table[1];
  ENET->GALR = hash_table[0];
  (void) ifp;
}

static bool mg_tcpip_driver_imxrt_poll(struct mg_tcpip_if *ifp, bool s1) {
  if (ifp->update_mac_hash_table) {
    mg_tcpip_driver_imxrt_update_hash_table(ifp);
    ifp->update_mac_hash_table = false;
  }
  if (!s1) return false;
  struct mg_tcpip_driver_imxrt_data *d =
      (struct mg_tcpip_driver_imxrt_data *) ifp->driver_data;
  uint8_t speed = MG_PHY_SPEED_10M;
  bool up = false, full_duplex = false;
  struct mg_phy phy = {enet_read_phy, enet_write_phy};
  up = mg_phy_up(&phy, d->phy_addr, &full_duplex, &speed);
  if ((ifp->state == MG_TCPIP_STATE_DOWN) && up) {  // link state just went up
    // tmp = reg with flags set to the most likely situation: 100M full-duplex
    // if(link is slow or half) set flags otherwise
    // reg = tmp
    uint32_t tcr = ENET->TCR | MG_BIT(2);             // Full-duplex
    uint32_t rcr = ENET->RCR & ~MG_BIT(9);            // 100M
    if (speed == MG_PHY_SPEED_10M) rcr |= MG_BIT(9);  // 10M
    if (full_duplex == false) tcr &= ~MG_BIT(2);      // Half-duplex
    ENET->TCR = tcr;  // IRQ handler does not fiddle with these registers
    ENET->RCR = rcr;
    MG_DEBUG(("Link is %uM %s-duplex", rcr & MG_BIT(9) ? 10 : 100,
              tcr & MG_BIT(2) ? "full" : "half"));
  }
  return up;
}

static uint32_t s_rxno;
#if !defined(MG_ENABLE_DRIVER_MCXE)
void ENET_IRQHandler(void);
void ENET_IRQHandler(void) {
#else
void ENET_Receive_IRQHandler(void);
void ENET_Receive_IRQHandler(void) {
#endif
  ENET->EIR = MG_BIT(25);  // Ack IRQ
  // Frame received, loop
  for (uint32_t i = 0; i < 10; i++) {  // read as they arrive but not forever
    uint32_t r = s_rxdesc[s_rxno].control;
    if (r & MG_BIT(15)) break;  // exit when done
    // skip partial/errored frames (Table 37-32)
    if ((r & MG_BIT(11)) &&
        !(r & (MG_BIT(5) | MG_BIT(4) | MG_BIT(2) | MG_BIT(1) | MG_BIT(0)))) {
      size_t len = s_rxdesc[s_rxno].length;
      mg_tcpip_qwrite(s_rxbuf[s_rxno], len > 4 ? len - 4 : len, s_ifp);
    }
    s_rxdesc[s_rxno].control |= MG_BIT(15);
    if (++s_rxno >= ETH_DESC_CNT) s_rxno = 0;
  }
  ENET->RDAR = MG_BIT(24);  // Receive Descriptors have changed
  // If b24 == 0, descriptors were exhausted and probably frames were dropped
}

struct mg_tcpip_driver mg_tcpip_driver_imxrt = {mg_tcpip_driver_imxrt_init,
                                                mg_tcpip_driver_imxrt_tx, NULL,
                                                mg_tcpip_driver_imxrt_poll};

#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/netc.c"
#endif


#if defined(MG_ENABLE_TCPIP) && MG_ENABLE_TCPIP && \
    defined(MG_ENABLE_DRIVER_NETC) && MG_ENABLE_DRIVER_NETC

// IMXRT1180 RM - Chapter 53 - Ethernet Controller (NETC)

#define ETH_DESC_CNT 8 // Descriptor count must be multiple of 8 (53.4.6.11.97)
#define ETH_PKT_SIZE 1536

struct netc_pci_function { // 53.4.6.1.1
  volatile uint32_t DID_VID;
  volatile uint16_t CMD, STAT;
  volatile uint8_t reserved0[0x40];
  volatile uint16_t DEV_CTL, DEV_STAT;
  volatile uint8_t reserved1[0x34];
  volatile uint16_t MSIX_CAP_LIST, MSIX_MSG_CTL;
};

struct netc_emdio { // 53.4.6.12
  volatile uint32_t EMDIO_CFG, EMDIO_CTL, EMDIO_DATA, EMDIO_ADDR, EMDIO_STAT;
};

struct netc_enetc { // 53.4.6.6 
  volatile uint32_t ECAPR0, ECAPR1, ECAPR2, reserved0, PMR;
  volatile uint8_t reserved1[0x1ec];
  volatile uint32_t PSIPMMR, PSIPVMR;
  volatile uint8_t reserved2[0x5f8];
  volatile uint32_t SMCAPR;
  volatile uint8_t reserved3[0x17fc];
  struct {
    volatile uint32_t PSIPMAR0, PSIPMAR1, PSIVLANR, reserved0, PSICFGR0,
      PSICFGR1, PSICFGR2;
    volatile uint8_t reserved1[0x3c];
    volatile uint32_t PSIMMHFR0, PSIMMHFR1;
    volatile uint8_t reserved2[0x20];
  } PSI[1];
};

struct netc_port { // 53.4.6.8
  volatile uint32_t PCAPR, PMCAPR, PIOCAPR, reserved0, PCR;
  volatile uint8_t reserved1[0x0c];
  volatile uint32_t PMAR0, PMAR1;
  volatile uint8_t reserved2[0xd8];
  volatile uint32_t POR, PSR;
  volatile uint8_t reserved3[0x30];
  volatile uint32_t PDGSR;
  volatile uint8_t reserved4[0xc4];
  struct {
    volatile uint32_t PTGSTCSR, reserved0, PTCTMSDUR, reserved1, PTCCBSR0,
      PTCCBSR1;
    volatile uint8_t reserved2[8];
  } TCT[8];
  volatile uint8_t reserved5[0x220];
  volatile uint32_t BPSTGSR;
};

struct netc_port_mac { // 53.4.6.9
  volatile uint8_t reserved0[8];
  volatile uint32_t COMMAND_CONFIG, MAC_ADDR_0, MAC_ADDR_1, MAXFRM, MINFRM;
  volatile uint8_t reserved1[0x24];
  volatile uint32_t IEVENT;
  volatile uint8_t reserved2[0x1dc];
  volatile uint64_t TFRMN;
  volatile uint8_t reserved3[0xd8];
  volatile uint32_t IF_MODE;
  volatile uint8_t reserved4[0xfc];
};

struct netc_eth_link {
  struct netc_port_mac PM[2];
};

struct netc_pseudo_link {
  volatile uint32_t PPMSR;
  volatile uint8_t reserved0[0x0c];
  volatile uint32_t PPMCR;
};

struct netc_switch { // 53.4.6.5
  volatile uint8_t reserved0[0x400];
  volatile uint32_t MPCR;
  volatile uint8_t reserved1[0x1c0c];
  volatile uint32_t VFHTDECR0, VFHTDECR1, VFHTDECR2;
};

struct netc_bdr {
  volatile uint32_t TBMR, TBSR;
  volatile uint8_t reserved0[8];
  volatile uint32_t TBBAR0, TBBAR1, TBPIR, TBCIR, TBLENR;
  volatile uint8_t reserved1[0x7c];
  volatile uint32_t TBIER, TBIDR, TBICR0, TBICR1;
  volatile uint8_t reserved2[0x50];
  volatile uint32_t RBMR, RBSR, RBBSR, RBCIR, RBBAR0, RBBAR1, RBPIR,
    reserved3, RBLENR;
  volatile uint8_t reserved4[0x5c];
  volatile uint32_t RBDCR;
  volatile uint8_t reserved5[0x1c];
  volatile uint32_t RBIER, RBIDR, RBICR0, RBICR1;
  volatile uint8_t reserved6[0x50];
};

struct netc_si { // 53.4.6.11
  volatile uint32_t SIMR, SISR;
  volatile uint8_t reserved0[0x30];
  volatile uint32_t SIRBGCR;
  volatile uint8_t reserved1[0x2ec];
  volatile uint32_t SITFRM0;
  volatile uint8_t reserved2[0x6fc];
  volatile uint32_t SIRXIDR0;
  volatile uint8_t reserved3[0x154];
  volatile uint32_t SIMSIRRVR[1];
  volatile uint8_t reserved4[0x747c];
  struct netc_bdr BDR[1];
};

struct netc_msix_entry { // 53.4.2.11
  volatile uint32_t MSG_ADDR, MSG_UPPER_ADDR, MSG_DATA, VECTOR_CTRL;
};

struct netc_msgintr { // 54.4
  struct {
    volatile uint32_t MSIIR, MSIR;
  } MSI[3];
};

#define NETC_EMDIO_PCI \
  ((struct netc_pci_function *) (uintptr_t) 0x60001000U)
#define NETC_SWITCH_PCI \
  ((struct netc_pci_function *) (uintptr_t) 0x60002000U)
#define NETC_ENETC1_PCI \
  ((struct netc_pci_function *) (uintptr_t) 0x60004000U)
#define NETC_EMDIO ((struct netc_emdio *) (uintptr_t) 0x60ba1c00U)

#define NETC_ENETC1 ((struct netc_enetc *) (uintptr_t) 0x60b50000U)
#define NETC_ENETC1_PORT ((struct netc_port *) (uintptr_t) 0x60b54000U)
#define NETC_ENETC1_PSEUDO_LINK \
  ((struct netc_pseudo_link *) (uintptr_t) 0x60b55000U)
#define NETC_ENETC1_SI0 ((struct netc_si *) (uintptr_t) 0x60b40000U)
#define NETC_ENETC1_MSIX \
  ((struct netc_msix_entry *) (uintptr_t) 0x60c00000U)

#define NETC_SWITCH ((struct netc_switch *) (uintptr_t) 0x60a00000U)
#define NETC_SW0_PORT(link)                                   \
  ((struct netc_port *) (uintptr_t) (0x60a04000U +            \
                                     (uint32_t) (link) * 0x4000U))
#define NETC_SW0_ETH(link)                                    \
  ((struct netc_eth_link *) (uintptr_t) (0x60a05000U +        \
                                         (uint32_t) (link) * 0x4000U))
#define NETC_SW0_PORT4 ((struct netc_port *) (uintptr_t) 0x60a14000U)
#define NETC_MSGINTR1 \
  ((struct netc_msgintr *) (uintptr_t) 0x428a0000U)

enum {
  NETC_RX_MSIX_ENTRY = 0,
  NETC_RX_MSG_INDEX = 2,
};

struct netc_tx_bd {
  uint64_t buffer_address;
  uint16_t buffer_length, frame_length;
  uint32_t control;
};

struct netc_rx_descriptor {
  uint64_t buffer_address, reserved;
};

struct netc_rx_writeback {
  uint16_t internet_checksum, parser_summary;
  uint32_t rss_hash;
  uint16_t buffer_length, vlan;
  uint32_t status;
};

union netc_rx_bd {
  struct netc_rx_descriptor descriptor;
  struct netc_rx_writeback writeback;
};

static volatile struct netc_tx_bd s_txdesc[ETH_DESC_CNT] MG_ETH_RAM
    __attribute__((aligned(128)));
static volatile union netc_rx_bd s_rxdesc[ETH_DESC_CNT] MG_ETH_RAM
    __attribute__((aligned(128)));
static uint8_t s_txbuf[ETH_DESC_CNT][ETH_PKT_SIZE] MG_ETH_RAM
    __attribute__((aligned(64)));
static uint8_t s_rxbuf[ETH_DESC_CNT][ETH_PKT_SIZE] MG_ETH_RAM
    __attribute__((aligned(64)));

static uint16_t s_tx_prod;
static uint16_t s_tx_clean;
static uint16_t s_rx_cons;
static struct mg_tcpip_if *s_ifp;

static uint16_t enet_read_phy(uint8_t addr, uint8_t reg) {
  NETC_EMDIO->EMDIO_CFG &= ~MG_BIT(6);  // ENC45: use Clause 22
  NETC_EMDIO->EMDIO_CTL = MG_BIT(15) | ((uint32_t) (addr & 0x1fU) << 5) |
                           (reg & 0x1fU);  // READ, PORT_ADDR, DEV_ADDR
  while (NETC_EMDIO->EMDIO_CFG & MG_BIT(31)) (void) 0;  // BSY1
  return (uint16_t) (NETC_EMDIO->EMDIO_DATA & 0xffffU);
}

static void enet_write_phy(uint8_t addr, uint8_t reg, uint16_t value) {
  NETC_EMDIO->EMDIO_CFG &= ~MG_BIT(6);  // ENC45: use Clause 22
  NETC_EMDIO->EMDIO_CTL = ((uint32_t) (addr & 0x1fU) << 5) |
                           (reg & 0x1fU);  // PORT_ADDR, DEV_ADDR
  NETC_EMDIO->EMDIO_DATA = value;
  while (NETC_EMDIO->EMDIO_CFG & MG_BIT(31)) (void) 0;  // BSY1
}

static bool mg_tcpip_driver_netc_init(struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_netc_data *d =
      (struct mg_tcpip_driver_netc_data *) ifp->driver_data;
  struct mg_phy phy = {enet_read_phy, enet_write_phy};
  struct netc_port *switch_port;
  struct netc_eth_link *eth_link;
  uint64_t tx_base = (uint64_t) (uintptr_t) s_txdesc;
  uint64_t rx_base = (uint64_t) (uintptr_t) s_rxdesc;
  uint32_t mac0 = (uint32_t) ifp->mac[0] |
                  ((uint32_t) ifp->mac[1] << 8) |
                  ((uint32_t) ifp->mac[2] << 16) |
                  ((uint32_t) ifp->mac[3] << 24);
  uint32_t mac1 = (uint32_t) ifp->mac[4] |
                  ((uint32_t) ifp->mac[5] << 8);
  uint32_t mdc_cr =
      (uint32_t) (d->mdc_cr < 0 ? MG_DRIVER_MDC_CR : d->mdc_cr);

  switch_port = NETC_SW0_PORT(d->link);
  eth_link = NETC_SW0_ETH(d->link);
  s_ifp = ifp;
  for (int i = 0; i < ETH_DESC_CNT; i++) {
    s_txdesc[i].buffer_address = (uint64_t) (uintptr_t) s_txbuf[i];
    s_rxdesc[i].descriptor.buffer_address =
        (uint64_t) (uintptr_t) s_rxbuf[i];
  }

  NETC_EMDIO_PCI->DEV_CTL |= MG_BIT(15);  // INIT_FLR
  while (NETC_EMDIO_PCI->DEV_CTL & MG_BIT(15)) (void) 0;
  NETC_EMDIO_PCI->CMD |= MG_BIT(2) | MG_BIT(1);  // BUS_MASTER_EN, MEM_ACCESS
  NETC_EMDIO->EMDIO_CFG =
      ((mdc_cr & 0x1ffU) << 7) | MG_BIT(2);  // MDIO_CLK_DIV, MDIO_HOLD
  mg_phy_init(&phy, d->phy_addr, 0);

  // Initialize ENETC1 PSI0 and port
  NETC_ENETC1_PCI->CMD |= MG_BIT(2) | MG_BIT(1);  // BUS_MASTER_EN, MEM_ACCESS
  NETC_ENETC1_PCI->MSIX_MSG_CTL = MG_BIT(14);  // FUNC_MASK
  NETC_ENETC1_PORT->POR |= MG_BIT(1) | MG_BIT(0);  // RXDIS, TXDIS
  NETC_ENETC1_PCI->DEV_CTL |= MG_BIT(15);  // INIT_FLR
  while (NETC_ENETC1_PCI->DEV_CTL & MG_BIT(15)) (void) 0;
  NETC_ENETC1_PCI->CMD |= MG_BIT(2) | MG_BIT(1);  // BUS_MASTER_EN, MEM_ACCESS
  NETC_ENETC1_PCI->MSIX_MSG_CTL = MG_BIT(14);  // FUNC_MASK
  NETC_ENETC1_PORT->PCR = 99U << 16;  // PSPEED: 1 Gbps
  NETC_ENETC1_PORT->PMAR0 = mac0;
  NETC_ENETC1_PORT->PMAR1 = mac1;
  NETC_ENETC1_PSEUDO_LINK->PPMCR = MG_BIT(16);  // TXPAD
  NETC_ENETC1->PSIPMMR &= ~(MG_BIT(16) | MG_BIT(0));
  NETC_ENETC1_SI0->SIMR &=
      ~(MG_BIT(31) | MG_BIT(3) | MG_BIT(2) | MG_BIT(1));
  NETC_ENETC1_SI0->SIRBGCR = 0;
  NETC_ENETC1->PSI[0].PSICFGR0 =
      MG_BIT(16) | MG_BIT(0);  // one Rx BDR, one Tx BDR

  NETC_ENETC1->PSI[0].PSICFGR2 = 0;  // NUM_MSIX: one entry
  NETC_ENETC1_MSIX[NETC_RX_MSIX_ENTRY].VECTOR_CTRL = MG_BIT(0);  // MASK
  NETC_ENETC1_MSIX[NETC_RX_MSIX_ENTRY].MSG_ADDR =
      (uint32_t) (uintptr_t) &NETC_MSGINTR1->MSI[0].MSIIR;
  NETC_ENETC1_MSIX[NETC_RX_MSIX_ENTRY].MSG_UPPER_ADDR = 0;
  NETC_ENETC1_MSIX[NETC_RX_MSIX_ENTRY].MSG_DATA = NETC_RX_MSG_INDEX;
  NETC_ENETC1_SI0->SIMSIRRVR[0] = NETC_RX_MSIX_ENTRY;  // Rx ring 0 -> entry 0

  // Initialize switch function 2 to drive the selected physical port
  NETC_SWITCH_PCI->CMD |= MG_BIT(2) | MG_BIT(1);
  NETC_SWITCH_PCI->MSIX_MSG_CTL = MG_BIT(14);  // FUNC_MASK
  switch_port->POR |= MG_BIT(1) | MG_BIT(0);
  eth_link->PM[0].COMMAND_CONFIG |= MG_BIT(26);  // SWR
  while (eth_link->PM[0].COMMAND_CONFIG & MG_BIT(26)) (void) 0;
  eth_link->PM[1].COMMAND_CONFIG |= MG_BIT(26);  // SWR
  while (eth_link->PM[1].COMMAND_CONFIG & MG_BIT(26)) (void) 0;
  NETC_SWITCH_PCI->DEV_CTL |= MG_BIT(15);  // INIT_FLR
  while (NETC_SWITCH_PCI->DEV_CTL & MG_BIT(15)) (void) 0;
  NETC_SWITCH_PCI->CMD |= MG_BIT(2) | MG_BIT(1);
  NETC_SWITCH_PCI->MSIX_MSG_CTL = MG_BIT(14);  // FUNC_MASK

  // Configure switch physical port and ethernet link (ETH_X)
  switch_port->PCR = 99U << 16;  // PSPEED: 1 Gbps
  switch_port->PMAR0 = mac0;
  switch_port->PMAR1 = mac1;
  switch_port->TCT[0].PTCTMSDUR = MG_BIT(24) | MG_BIT(16) | ETH_PKT_SIZE;
  eth_link->PM[0].MAXFRM = ETH_PKT_SIZE;
  eth_link->PM[0].MINFRM = 64;
  eth_link->PM[0].IF_MODE = MG_BIT(14) | 4U;  // 1G RGMII
  eth_link->PM[0].COMMAND_CONFIG =
      MG_BIT(15) | MG_BIT(1) | MG_BIT(0);  // TXP, RX_EN, TX_EN
  eth_link->PM[1].MAXFRM = ETH_PKT_SIZE;
  eth_link->PM[1].MINFRM = 64;
  eth_link->PM[1].IF_MODE = MG_BIT(14) | 4U;
  eth_link->PM[1].COMMAND_CONFIG =
      MG_BIT(15) | MG_BIT(1) | MG_BIT(0);

  // Bridge the selected physical port with management port 4 (54.4.1.1)
  NETC_SWITCH->VFHTDECR0 = MG_BIT(4) | MG_BIT(d->link);
  NETC_SWITCH->VFHTDECR2 = (1UL << 27) | (1UL << 24);  // MFO=1, MLO=1
  switch_port->BPSTGSR = 2U;  // STG0: forwarding
  NETC_SW0_PORT4->BPSTGSR = 2U;  // STG0: forwarding
  switch_port->POR &= ~(MG_BIT(1) | MG_BIT(0));
  NETC_SW0_PORT4->POR &= ~(MG_BIT(1) | MG_BIT(0));

  // Initialize Tx BD ring 0 (direct switch enqueue)
  NETC_ENETC1_SI0->BDR[0].TBBAR0 = (uint32_t) tx_base;
  NETC_ENETC1_SI0->BDR[0].TBBAR1 = (uint32_t) (tx_base >> 32);
  NETC_ENETC1_SI0->BDR[0].TBLENR = ETH_DESC_CNT;

  // Initialize Rx BD ring 0
  NETC_ENETC1_SI0->BDR[0].RBICR0 =
      MG_BIT(31) | MG_BIT(0);  // ICEN, interrupt after one frame
  NETC_ENETC1_SI0->BDR[0].RBBAR0 = (uint32_t) rx_base;
  NETC_ENETC1_SI0->BDR[0].RBBAR1 = (uint32_t) (rx_base >> 32);
  NETC_ENETC1_SI0->BDR[0].RBBSR = ETH_PKT_SIZE;
  NETC_ENETC1_SI0->BDR[0].RBLENR = ETH_DESC_CNT;

  // Clear stale device and MSGINTR state before the ring becomes active
  NETC_ENETC1_SI0->SIRXIDR0 = MG_BIT(0);  // RX0, W1C
  (void) NETC_MSGINTR1->MSI[0].MSIR;  // clear pending messages
  NETC_ENETC1_SI0->BDR[0].RBIER = MG_BIT(0);  // RXTIE
  NETC_ENETC1_SI0->BDR[0].TBMR = MG_BIT(31);
  NETC_ENETC1_SI0->BDR[0].RBMR = MG_BIT(31);  // EN, CRC=0: strip FCS
  NETC_ENETC1_PORT->POR &= ~(MG_BIT(1) | MG_BIT(0));
  NETC_ENETC1->PMR |= MG_BIT(16); // SI0EN
  NETC_ENETC1_SI0->SIMR |= MG_BIT(31);  // EN
  MG_DSB();
  if ((NETC_ENETC1->SMCAPR & MG_BIT(0)) == 0U ||
      (NETC_SWITCH->MPCR & 0x1fU) != 4U) {
    MG_ERROR(("NETC fixed management path unavailable: SMCAPR=%08lx MPCR=%08lx",
              (unsigned long) NETC_ENETC1->SMCAPR,
              (unsigned long) NETC_SWITCH->MPCR));
    return false;
  }

  NETC_ENETC1_PCI->MSIX_MSG_CTL =
    MG_BIT(15);  // Enable RX interrupt, MSIX_EN=1, FUNC_MASK=0
  MG_DSB();
  NETC_ENETC1_MSIX[NETC_RX_MSIX_ENTRY].VECTOR_CTRL = 0;  // MASK=0, last
  MG_DSB();
  return true;
}

static size_t mg_tcpip_driver_netc_tx(const void *buf, size_t len,
                                      struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_netc_data *d =
      (struct mg_tcpip_driver_netc_data *) ifp->driver_data;
  uint16_t next;
  s_tx_clean =
      (uint16_t) (NETC_ENETC1_SI0->BDR[0].TBCIR & 0xffffU);  // BDR_INDEX
  next = (uint16_t) (s_tx_prod + 1U);
  if (next >= ETH_DESC_CNT) next = 0;
  if (len > sizeof(s_txbuf[0])) {
    MG_ERROR(("Frame too big, %ld", (long) len));
    len = (size_t) -1;  // fail
  } else if (next == s_tx_clean) {
    ifp->nerr++;
    MG_ERROR(("No descriptors available"));
    len = 0;  // retry later
  } else {
    memcpy(s_txbuf[s_tx_prod], buf, len);  // Copy data
    s_txdesc[s_tx_prod].buffer_address =
        (uint64_t) (uintptr_t) s_txbuf[s_tx_prod];
    s_txdesc[s_tx_prod].buffer_length = (uint16_t) len;  // BUFF_LEN
    s_txdesc[s_tx_prod].frame_length = (uint16_t) len;   // FRAME_LEN
    s_txdesc[s_tx_prod].control =
        MG_BIT(31) | (2UL << 24) | MG_BIT(23) | ((uint32_t) d->link << 16);
    // F=1, FLQ=2, SMSO=1, EGR_PORT=link, IPV=0, DR=0
    MG_DSB();
    if (++s_tx_prod >= ETH_DESC_CNT) s_tx_prod = 0;
    NETC_ENETC1_SI0->BDR[0].TBPIR = s_tx_prod;  // BDR_INDEX
  }
  return len;
}

static void mg_tcpip_driver_netc_update_hash_table(struct mg_tcpip_if *ifp) {
  // mcast address has NETC XOR hash index 56 (RM 53.4.2.3.1.3.8.4.1)
  NETC_ENETC1->PSI[0].PSIMMHFR0 = 0;
  NETC_ENETC1->PSI[0].PSIMMHFR1 = MG_BIT(24);
  (void) ifp;
}

static bool mg_tcpip_driver_netc_poll(struct mg_tcpip_if *ifp, bool s1) {
  if (ifp->update_mac_hash_table) {
    mg_tcpip_driver_netc_update_hash_table(ifp);
    ifp->update_mac_hash_table = false;
  }
  if (!s1) return false;
  struct mg_tcpip_driver_netc_data *d =
      (struct mg_tcpip_driver_netc_data *) ifp->driver_data;
  struct netc_port *switch_port = NETC_SW0_PORT(d->link);
  struct netc_eth_link *eth_link = NETC_SW0_ETH(d->link);
  uint8_t speed = MG_PHY_SPEED_10M;
  bool up, full_duplex = false;
  struct mg_phy phy = {enet_read_phy, enet_write_phy};

  up = mg_phy_up(&phy, d->phy_addr, &full_duplex, &speed);
  if ((ifp->state == MG_TCPIP_STATE_DOWN) && up) {
    uint16_t port_speed = speed == MG_PHY_SPEED_1000M ? 99U
                          : speed == MG_PHY_SPEED_100M ? 9U
                                                       : 0U;
    uint32_t config = switch_port->PCR;
    uint32_t mode = eth_link->PM[0].IF_MODE;
    config &= ~(0x3fffU << 16);  // PCR[29:16] PSPEED
    config |= (uint32_t) port_speed << 16;
    mode &= ~(MG_BIT(14) | MG_BIT(13) | MG_BIT(6));  // SSP, HD
    if (speed == MG_PHY_SPEED_1000M) {
      mode |= MG_BIT(14);  // SSP: 1 Gbps, full duplex
    } else {
      if (speed == MG_PHY_SPEED_10M) mode |= MG_BIT(13);  // SSP
      if (!full_duplex) mode |= MG_BIT(6);                // HD
    }
    switch_port->PCR = config;
    eth_link->PM[0].IF_MODE = mode;
    eth_link->PM[1].IF_MODE = mode;

    // TX_CSD, RX_CSD, and SPD_DUP are latched while the PHY is reset/down.
    eth_link->PM[0].IEVENT = MG_BIT(23) | MG_BIT(22) | MG_BIT(21);
    MG_DSB();
    MG_DEBUG(("Link is %uM %s-duplex",
              speed == MG_PHY_SPEED_1000M ? 1000U
              : speed == MG_PHY_SPEED_100M ? 100U
                                           : 10U,
              full_duplex ? "full" : "half"));
  }
  return up;
}

void MSGINTR1_IRQHandler(void);
void MSGINTR1_IRQHandler(void) {
  if (NETC_MSGINTR1->MSI[0].MSIR & MG_BIT(NETC_RX_MSG_INDEX)) {
    NETC_ENETC1_SI0->SIRXIDR0 = MG_BIT(0);  // RX0, W1C
    MG_DSB();
    for (int i = 0; i < 10; i++) {  // Read as they arrive but not forever
      uint32_t status;
      size_t len;
      if (s_rx_cons ==
          (uint16_t) (NETC_ENETC1_SI0->BDR[0].RBPIR & 0xffffU))
        break;
      status = s_rxdesc[s_rx_cons].writeback.status;
      if (!(status & MG_BIT(30))) break;  // R: writeback is ready
      len = s_rxdesc[s_rx_cons].writeback.buffer_length;
      if ((status & (MG_BIT(31) | MG_BIT(30))) ==
              (MG_BIT(31) | MG_BIT(30)) &&  // F, R
          !(status & (0xffU << 16)) &&       // ERROR
          !(status & (0xfU << 2)) &&         // HR=0: regular frame
          len > 0 && len <= sizeof(s_rxbuf[0])) {
        mg_tcpip_qwrite(s_rxbuf[s_rx_cons], len, s_ifp);  // FCS stripped
      }
      s_rxdesc[s_rx_cons].descriptor.buffer_address =
          (uint64_t) (uintptr_t) s_rxbuf[s_rx_cons];
      s_rxdesc[s_rx_cons].descriptor.reserved = 0;
      MG_DSB();
      if (++s_rx_cons >= ETH_DESC_CNT) s_rx_cons = 0;
      NETC_ENETC1_SI0->BDR[0].RBCIR = s_rx_cons;
    }
  }
  MG_DSB();
}

struct mg_tcpip_driver mg_tcpip_driver_netc = {mg_tcpip_driver_netc_init,
                                               mg_tcpip_driver_netc_tx, NULL,
                                               mg_tcpip_driver_netc_poll};

#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/nxp_wifi.c"
#endif
#if MG_ENABLE_TCPIP && defined(MG_ENABLE_DRIVER_NXP_WIFI) && \
    MG_ENABLE_DRIVER_NXP_WIFI



bool __attribute__((weak)) netif_init(struct mg_tcpip_if *ifp) {
  (void) ifp;
  MG_ERROR(("Please link wifi/port/net contents"));
  return false;
}
size_t __attribute__((weak))
netif_tx(const void *bfr, size_t len, struct mg_tcpip_if *ifp) {
  (void) bfr;
  (void) len;
  netif_init(ifp);
  return 0;
}
bool __attribute__((weak)) netif_connect(struct mg_wifi_data *wifi) {
  (void) wifi;
  return netif_init(NULL);
}
bool __attribute__((weak))
netif_poll(struct mg_tcpip_if *ifp, bool s1, mg_tcpip_event_handler_t evcb) {
  (void) ifp;
  (void) s1;
  (void) evcb;
  return false;
}

static struct mg_tcpip_if *s_ifp;
static uint32_t s_ip, s_mask;

static void wifi_cb(struct mg_tcpip_if *ifp, int ev, void *ev_data) {
  struct mg_wifi_data *wifi =
      &((struct mg_tcpip_driver_nxp_wifi_data *) ifp->driver_data)->wifi;
  if (wifi->apmode && ev == MG_TCPIP_EV_STATE_CHANGE &&
      *(uint8_t *) ev_data == MG_TCPIP_STATE_UP) {
    MG_DEBUG(("Access Point started"));
    s_ip = ifp->ip, ifp->ip = wifi->apip;
    s_mask = ifp->mask, ifp->mask = wifi->apmask;
    ifp->enable_dhcp_client = false;
    ifp->enable_dhcp_server = true;
  }
}

static bool nxp_wifi_init(struct mg_tcpip_if *ifp) {
  struct mg_wifi_data *wifi =
      &((struct mg_tcpip_driver_nxp_wifi_data *) ifp->driver_data)->wifi;
  s_ifp = ifp;
  s_ip = ifp->ip;
  s_mask = ifp->mask;
  ifp->pfn = wifi_cb;
  if (!netif_init(ifp)) return false;
  if (wifi->apmode) {
    return mg_wifi_ap_start(wifi);
  } else if (wifi->ssid != NULL && wifi->pass != NULL) {
    return mg_wifi_connect(wifi);
  }
  return true;
}

bool nxp_wifi_poll(struct mg_tcpip_if *ifp, bool s1) {
  return netif_poll(ifp, s1, mg_tcpip_call);
}

struct mg_tcpip_driver mg_tcpip_driver_nxp_wifi = {nxp_wifi_init, netif_tx,
                                                   NULL, nxp_wifi_poll};

bool mg_wifi_connect(struct mg_wifi_data *wifi) {
  s_ifp->ip = s_ip;
  s_ifp->mask = s_mask;
  if (s_ifp->ip == 0) s_ifp->enable_dhcp_client = true;
  s_ifp->enable_dhcp_server = false;
  return netif_connect(wifi);
}

bool __attribute__((weak)) mg_wifi_scan(void) {
  return netif_init(NULL);
}
bool __attribute__((weak)) mg_wifi_disconnect(void) {
  return netif_init(NULL);
}
bool __attribute__((weak)) mg_wifi_ap_start(struct mg_wifi_data *wifi) {
  (void) wifi;
  return netif_init(NULL);
}
bool __attribute__((weak)) mg_wifi_ap_stop(void) {
  return netif_init(NULL);
}

#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/phy.c"
#endif


enum {                  // ID1  ID2
  MG_PHY_KSZ8x = 0x22,  // 0022 156x - KSZ8081RNB, KSZ8091RNB
  MG_PHY_DP83x = 0x2000,
  MG_PHY_DP83867 = 0xa231,  // 2000 a231 - TI DP83867I
  MG_PHY_DP83825 = 0xa140,  // 2000 a140 - TI DP83825I
  MG_PHY_DP83848 = 0x5ca2,  // 2000 5ca2 - TI DP83848I
  MG_PHY_LAN87x = 0x7,      // 0007 c0fx - LAN8720
  MG_PHY_RTL82x = 0x1c,
  MG_PHY_RTL8201 = 0xc816,  // 001c c816 - RTL8201F
  MG_PHY_RTL8211 = 0xc916,  // 001c c916 - RTL8211F
  MG_PHY_ICS1894x = 0x15,
  MG_PHY_ICS189432 = 0xf450,  // 0015 f450 - ICS1894
  MG_PHY_YT85x = 0x4f51,
  MG_PHY_YT8531 = 0xe91a  // 4f51 e91a - Motorcomm YT8531
};

enum {
  MG_PHY_REG_BCR = 0,
  MG_PHY_REG_BSR = 1,
  MG_PHY_REG_ID1 = 2,
  MG_PHY_REG_ID2 = 3,
  MG_PHY_DP83x_REG_PHYSTS = 16,
  MG_PHY_DP83867_REG_PHYSTS = 17,
  MG_PHY_DP83x_REG_RCSR = 23,
  MG_PHY_DP83x_REG_LEDCR = 24,
  MG_PHY_KSZ8x_REG_PC1R = 30,
  MG_PHY_KSZ8x_REG_PC2R = 31,
  MG_PHY_LAN87x_REG_SCSR = 31,
  MG_PHY_RTL82x_REG_PAGESEL = 31,
  MG_PHY_RTL8201_REG_RMSR = 16,   // in page 7
  MG_PHY_RTL8211_REG_PHYSR = 26,  // in page a43
  MG_PHY_ICS189432_REG_POLL = 17,
  MG_PHY_YT8531_REG_PHYSTS = 17,
  MG_PHY_YT8531_REG_EXT_ADDR = 30,
  MG_PHY_YT8531_REG_EXT_DATA = 31
};

static const char *mg_phy_id_to_str(uint16_t id1, uint16_t id2) {
  switch (id1) {
    case MG_PHY_DP83x:
      switch (id2) {
        case MG_PHY_DP83867:
          return "DP83867";
        case MG_PHY_DP83848:
          return "DP83848";
        case MG_PHY_DP83825:
          return "DP83825";
        default:
          return "DP83x";
      }
    case MG_PHY_KSZ8x:
      return "KSZ8x";
    case MG_PHY_LAN87x:
      return "LAN87x";
    case MG_PHY_RTL82x:
      switch (id2) {
        case MG_PHY_RTL8201:
          return "RTL8201";
        case MG_PHY_RTL8211:
          return "RTL8211";
        default:
          return "RTL82x";
      }
    case MG_PHY_ICS1894x:
      return "ICS1894x";
    case MG_PHY_YT85x:
      return id2 == MG_PHY_YT8531 ? "YT8531" : "YT85x";
    default:
      return "unknown";
  }
}

void mg_phy_init(struct mg_phy *phy, uint8_t phy_addr, uint8_t config) {
  uint16_t id1, id2;
  phy->write_reg(phy_addr, MG_PHY_REG_BCR, MG_BIT(15));  // Reset PHY
  while (phy->read_reg(phy_addr, MG_PHY_REG_BCR) & MG_BIT(15)) (void) 0;
  // MG_PHY_REG_BCR[12]: Autonegotiation is default unless hw says otherwise

  id1 = phy->read_reg(phy_addr, MG_PHY_REG_ID1);
  id2 = phy->read_reg(phy_addr, MG_PHY_REG_ID2);
  MG_INFO(("PHY ID: %#04x %#04x (%s)", id1, id2, mg_phy_id_to_str(id1, id2)));

  if (id1 == MG_PHY_YT85x && id2 == MG_PHY_YT8531) {
    uint16_t value;
    // TODO (robertc2000): Autonegotiation remains disabled after PHY is reset
    // We temporarily force enabling autoneg, until a workaround is found.
    phy->write_reg(phy_addr, MG_PHY_REG_BCR, MG_BIT(12));
    phy->write_reg(phy_addr, MG_PHY_YT8531_REG_EXT_ADDR, 0xa003);
    value = phy->read_reg(phy_addr, MG_PHY_YT8531_REG_EXT_DATA);
    value &= (uint16_t) ~(MG_BIT(3) | MG_BIT(2) | MG_BIT(1) | MG_BIT(0));
    phy->write_reg(phy_addr, MG_PHY_YT8531_REG_EXT_DATA,
                   (uint16_t) (value | MG_BIT(1)));  // TXDLY: 2 * 150 ps
  }

  if (id1 == MG_PHY_DP83x && id2 == MG_PHY_DP83867) {
    phy->write_reg(phy_addr, 0x0d, 0x1f);  // write 0x10d to IO_MUX_CFG (0x0170)
    phy->write_reg(phy_addr, 0x0e, 0x170);
    phy->write_reg(phy_addr, 0x0d, 0x401f);
    phy->write_reg(phy_addr, 0x0e, 0x10d);
  }

  if (config & MG_PHY_CLOCKS_MAC) {
    // Use PHY crystal oscillator (preserve defaults)
    // nothing to do
  } else {  // MAC clocks PHY, PHY has no xtal
    // Enable 50 MHz external ref clock at XI (preserve defaults)
    if (id1 == MG_PHY_DP83x && id2 != MG_PHY_DP83867 && id2 != MG_PHY_DP83848) {
      phy->write_reg(phy_addr, MG_PHY_DP83x_REG_RCSR, MG_BIT(7) | MG_BIT(0));
    } else if (id1 == MG_PHY_KSZ8x) {
      // Disable isolation (override hw, it doesn't make sense at this point)
      // - #2848, some NXP boards set ISO, even though docs say they don't
      phy->write_reg(phy_addr, MG_PHY_REG_BCR,
                     (uint16_t) (phy->read_reg(phy_addr, MG_PHY_REG_BCR) &
                                 (uint16_t) ~MG_BIT(10)));
      // now do clock stuff
      phy->write_reg(phy_addr, MG_PHY_KSZ8x_REG_PC2R,
                     (uint16_t) (MG_BIT(15) | MG_BIT(8) | MG_BIT(7)));
    } else if (id1 == MG_PHY_LAN87x) {
      // nothing to do
    } else if (id1 == MG_PHY_RTL82x && id2 == MG_PHY_RTL8201) {
      // assume PHY has been hardware strapped properly
#if 0
      phy->write_reg(phy_addr, MG_PHY_RTL82x_REG_PAGESEL, 7);  // Select page 7
      phy->write_reg(phy_addr, MG_PHY_RTL8201_REG_RMSR, 0x1ffa);
      phy->write_reg(phy_addr, MG_PHY_RTL82x_REG_PAGESEL, 0);  // Select page 0
#endif
    } else if (id1 == MG_PHY_RTL82x && id2 == MG_PHY_RTL8211) {
      // assume PHY has been hardware strapped properly
    }
  }

  if (config & MG_PHY_LEDS_ACTIVE_HIGH && id1 == MG_PHY_DP83x) {
    phy->write_reg(phy_addr, MG_PHY_DP83x_REG_LEDCR,
                   MG_BIT(9) | MG_BIT(7));  // LED status, active high
  }  // Other PHYs do not support this feature
}

bool mg_phy_up(struct mg_phy *phy, uint8_t phy_addr, bool *full_duplex,
               uint8_t *speed) {
  bool up = false;
  uint16_t bsr = phy->read_reg(phy_addr, MG_PHY_REG_BSR);
  if ((bsr & MG_BIT(5)) && !(bsr & MG_BIT(2)))  // some PHYs latch down events
    bsr = phy->read_reg(phy_addr, MG_PHY_REG_BSR);  // read again
  up = bsr & MG_BIT(2);
  if (up && full_duplex != NULL && speed != NULL) {
    uint16_t id1 = phy->read_reg(phy_addr, MG_PHY_REG_ID1);
    if (id1 == MG_PHY_DP83x) {
      uint16_t id2 = phy->read_reg(phy_addr, MG_PHY_REG_ID2);
      if (id2 == MG_PHY_DP83867) {
        uint16_t physts = phy->read_reg(phy_addr, MG_PHY_DP83867_REG_PHYSTS);
        *full_duplex = physts & MG_BIT(13);
        *speed = (physts & MG_BIT(15))   ? MG_PHY_SPEED_1000M
                 : (physts & MG_BIT(14)) ? MG_PHY_SPEED_100M
                                         : MG_PHY_SPEED_10M;
      } else {
        uint16_t physts = phy->read_reg(phy_addr, MG_PHY_DP83x_REG_PHYSTS);
        *full_duplex = physts & MG_BIT(2);
        *speed = (physts & MG_BIT(1)) ? MG_PHY_SPEED_10M : MG_PHY_SPEED_100M;
      }
    } else if (id1 == MG_PHY_KSZ8x) {
      uint16_t pc1r = phy->read_reg(phy_addr, MG_PHY_KSZ8x_REG_PC1R);
      *full_duplex = pc1r & MG_BIT(2);
      *speed = (pc1r & 3) == 1 ? MG_PHY_SPEED_10M : MG_PHY_SPEED_100M;
    } else if (id1 == MG_PHY_LAN87x) {
      uint16_t scsr = phy->read_reg(phy_addr, MG_PHY_LAN87x_REG_SCSR);
      *full_duplex = scsr & MG_BIT(4);
      *speed = (scsr & MG_BIT(3)) ? MG_PHY_SPEED_100M : MG_PHY_SPEED_10M;
    } else if (id1 == MG_PHY_RTL82x) {
      uint16_t id2 = phy->read_reg(phy_addr, MG_PHY_REG_ID2);
      if (id2 == MG_PHY_RTL8211) {
        uint16_t physr;
        phy->write_reg(phy_addr, MG_PHY_RTL82x_REG_PAGESEL, 0xa43);
        physr = phy->read_reg(phy_addr, MG_PHY_RTL8211_REG_PHYSR);
        phy->write_reg(phy_addr, MG_PHY_RTL82x_REG_PAGESEL, 0);
        *full_duplex = physr & MG_BIT(3);
        *speed = (physr & MG_BIT(5))   ? MG_PHY_SPEED_1000M
                 : (physr & MG_BIT(4)) ? MG_PHY_SPEED_100M
                                       : MG_PHY_SPEED_10M;
      } else {
        uint16_t bcr = phy->read_reg(phy_addr, MG_PHY_REG_BCR);
        *full_duplex = bcr & MG_BIT(8);
        *speed = (bcr & MG_BIT(13)) ? MG_PHY_SPEED_100M : MG_PHY_SPEED_10M;
      }
    } else if (id1 == MG_PHY_ICS1894x) {
      uint16_t poll_reg = phy->read_reg(phy_addr, MG_PHY_ICS189432_REG_POLL);
      *full_duplex = poll_reg & MG_BIT(14);
      *speed = (poll_reg & MG_BIT(15)) ? MG_PHY_SPEED_100M : MG_PHY_SPEED_10M;
    } else if (id1 == MG_PHY_YT85x) {
      uint16_t id2 = phy->read_reg(phy_addr, MG_PHY_REG_ID2);
      if (id2 == MG_PHY_YT8531) {
        uint16_t physts = phy->read_reg(phy_addr, MG_PHY_YT8531_REG_PHYSTS);
        *full_duplex = physts & MG_BIT(13);
        *speed = (physts & MG_BIT(15))   ? MG_PHY_SPEED_1000M
                 : (physts & MG_BIT(14)) ? MG_PHY_SPEED_100M
                                         : MG_PHY_SPEED_10M;
      }
    }
  }
  return up;
}

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/pico-w.c"
#endif
#if MG_ENABLE_TCPIP && MG_ARCH == MG_ARCH_PICOSDK && \
    defined(MG_ENABLE_DRIVER_PICO_W) && MG_ENABLE_DRIVER_PICO_W





static struct mg_tcpip_if *s_ifp;
static uint32_t s_ip, s_mask;
static bool s_aplink = false, s_scanning = false;
static bool s_stalink = false, s_connecting = false;

static void wifi_cb(struct mg_tcpip_if *ifp, int ev, void *ev_data) {
  struct mg_wifi_data *wifi =
      &((struct mg_tcpip_driver_pico_w_data *) ifp->driver_data)->wifi;
  if (wifi->apmode && ev == MG_TCPIP_EV_STATE_CHANGE &&
      *(uint8_t *) ev_data == MG_TCPIP_STATE_UP) {
    MG_DEBUG(("Access Point started"));
    s_ip = ifp->ip, ifp->ip = wifi->apip;
    s_mask = ifp->mask, ifp->mask = wifi->apmask;
    ifp->enable_dhcp_client = false;
    ifp->enable_dhcp_server = true;
  }
}

static bool mg_tcpip_driver_pico_w_init(struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_pico_w_data *d =
      (struct mg_tcpip_driver_pico_w_data *) ifp->driver_data;
  struct mg_wifi_data *wifi = &d->wifi;
  s_ifp = ifp;
  s_ip = ifp->ip;
  s_mask = ifp->mask;
  ifp->pfn = wifi_cb;
  if (cyw43_arch_init() != 0)
    return false;  // initialize async_context and WiFi chip
  if (wifi->apmode && wifi->apssid != NULL) {
    if (!mg_wifi_ap_start(wifi)) return false;
    cyw43_wifi_get_mac(&cyw43_state, CYW43_ITF_STA, ifp->mac);  // same MAC
  } else {
    cyw43_arch_enable_sta_mode();
    cyw43_wifi_get_mac(&cyw43_state, CYW43_ITF_STA, ifp->mac);
    if (wifi->ssid != NULL) {
      return mg_wifi_connect(wifi);
    } else {
      cyw43_arch_disable_sta_mode();
    }
  }
  return true;
}

static size_t mg_tcpip_driver_pico_w_tx(const void *buf, size_t len,
                                        struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_pico_w_data *d =
      (struct mg_tcpip_driver_pico_w_data *) ifp->driver_data;
  return cyw43_send_ethernet(&cyw43_state,
                             d->wifi.apmode ? CYW43_ITF_AP : CYW43_ITF_STA, len,
                             buf, false) == 0
             ? len
             : 0;
}

static bool mg_tcpip_driver_pico_w_poll(struct mg_tcpip_if *ifp, bool s1) {
  cyw43_arch_poll();  // not necessary, except when IRQs are disabled (OTA)
  if (s_scanning && !cyw43_wifi_scan_active(&cyw43_state)) {
    MG_VERBOSE(("scan complete"));
    s_scanning = 0;
    mg_tcpip_call(ifp, MG_TCPIP_EV_WIFI_SCAN_END, NULL);
  }
  if (ifp->update_mac_hash_table) {
    // first call to _poll() is after _init(), so this is safe
    cyw43_wifi_update_multicast_filter(&cyw43_state, (uint8_t *) mcast_addr,
                                       true);
    ifp->update_mac_hash_table = false;
  }
  if (!s1) return false;
  struct mg_tcpip_driver_pico_w_data *d =
      (struct mg_tcpip_driver_pico_w_data *) ifp->driver_data;
  if (d->wifi.apmode) return s_aplink;
  int sdkstate = cyw43_wifi_link_status(&cyw43_state, CYW43_ITF_STA);
  MG_VERBOSE(("conn: %c state: %d", s_connecting ? '1' : '0', sdkstate));
  if (sdkstate < 0 && s_connecting) {
    mg_tcpip_call(ifp, MG_TCPIP_EV_WIFI_CONNECT_ERR, &sdkstate);
    s_connecting = false;
  }
  return s_stalink;
}

struct mg_tcpip_driver mg_tcpip_driver_pico_w = {
    mg_tcpip_driver_pico_w_init,
    mg_tcpip_driver_pico_w_tx,
    NULL,
    mg_tcpip_driver_pico_w_poll,
};

// Called once per outstanding frame by async_context
void cyw43_cb_process_ethernet(void *cb_data, int itf, size_t len,
                               const uint8_t *buf) {
  mg_tcpip_qwrite((void *) buf, len, s_ifp);
  (void) cb_data;
}

// Called by async_context
void cyw43_cb_tcpip_set_link_up(cyw43_t *self, int itf) {
  if (itf == CYW43_ITF_AP) {
    s_aplink = true;
  } else {
    s_stalink = true;
    s_connecting = false;
  }
}
void cyw43_cb_tcpip_set_link_down(cyw43_t *self, int itf) {
  if (itf == CYW43_ITF_AP) {
    s_aplink = false;
  } else {
    s_stalink = false;
    // SDK calls this before we check status, don't clear s_connecting here
  }
}

// there's life beyond lwIP
void pbuf_copy_partial(void) {
  (void) 0;
}

static int result_cb(void *arg, const cyw43_ev_scan_result_t *data) {
  struct mg_wifi_scan_bss_data bss;
  bss.SSID = mg_str_n(data->ssid, data->ssid_len);
  bss.BSSID = (char *) data->bssid;
  bss.RSSI = (int8_t) data->rssi;
  bss.has_n = 0;  // SDK ignores this
  bss.channel = (uint8_t) data->channel;
  bss.band = MG_WIFI_BAND_2G;
  // SDK-internal dependency, 2.1.0
  bss.security = data->auth_mode & MG_BIT(0) ? MG_WIFI_SECURITY_WEP
                                             : MG_WIFI_SECURITY_OPEN;
  if (data->auth_mode & MG_BIT(1)) bss.security |= MG_WIFI_SECURITY_WPA;
  if (data->auth_mode & MG_BIT(2)) bss.security |= MG_WIFI_SECURITY_WPA2;
  MG_VERBOSE(("BSS: %.*s (%u) (%M) %d dBm %u", bss.SSID.len, bss.SSID.buf,
              bss.channel, mg_print_mac, bss.BSSID, (int) bss.RSSI,
              bss.security));
  mg_tcpip_call(s_ifp, MG_TCPIP_EV_WIFI_SCAN_RESULT, &bss);
  return 0;
}

bool mg_wifi_scan(void) {
  cyw43_wifi_scan_options_t opts;
  memset(&opts, 0, sizeof(opts));
  bool res = (cyw43_wifi_scan(&cyw43_state, &opts, NULL, result_cb) == 0);
  if (res) s_scanning = true;
  return res;
}

bool mg_wifi_connect(struct mg_wifi_data *wifi) {
  s_ifp->ip = s_ip;
  s_ifp->mask = s_mask;
  if (s_ifp->ip == 0) s_ifp->enable_dhcp_client = true;
  s_ifp->enable_dhcp_server = false;
  cyw43_arch_enable_sta_mode();
  MG_DEBUG(("Connecting to '%s'", wifi->ssid));
  int res = cyw43_arch_wifi_connect_async(wifi->ssid, wifi->pass,
                                          CYW43_AUTH_WPA2_AES_PSK);
  MG_VERBOSE(("res: %d", res));
  if (res == 0) s_connecting = true;
  return (res == 0);
}

bool mg_wifi_disconnect(void) {
  cyw43_arch_disable_sta_mode();
  s_connecting = false;
  return true;
}

bool mg_wifi_ap_start(struct mg_wifi_data *wifi) {
  MG_DEBUG(("Starting AP '%s' (%u)", wifi->apssid, wifi->apchannel));
  cyw43_wifi_ap_set_channel(&cyw43_state, wifi->apchannel);
  cyw43_arch_enable_ap_mode(wifi->apssid, wifi->appass,
                            CYW43_AUTH_WPA2_AES_PSK);
  return true;
}

bool mg_wifi_ap_stop(void) {
  cyw43_arch_disable_ap_mode();
  return true;
}

#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/ra.c"
#endif


#if MG_ENABLE_TCPIP && \
  (defined(MG_ENABLE_DRIVER_RA6) && MG_ENABLE_DRIVER_RA6) || \
  (defined(MG_ENABLE_DRIVER_RA8) && MG_ENABLE_DRIVER_RA8)
struct ra_etherc {
  volatile uint32_t ECMR, RESERVED, RFLR, RESERVED1, ECSR, RESERVED2, ECSIPR,
      RESERVED3, PIR, RESERVED4, PSR, RESERVED5[5], RDMLR, RESERVED6[3], IPGR,
      APR, MPR, RESERVED7, RFCF, TPAUSER, TPAUSECR, BCFRR, RESERVED8[20], MAHR,
      RESERVED9, MALR, RESERVED10, TROCR, CDCR, LCCR, CNDCR, RESERVED11, CEFCR,
      FRECR, TSFRCR, TLFRCR, RFCR, MAFCR;
};

struct ra_edmac {
  volatile uint32_t EDMR, RESERVED, EDTRR, RESERVED1, EDRRR, RESERVED2, TDLAR,
      RESERVED3, RDLAR, RESERVED4, EESR, RESERVED5, EESIPR, RESERVED6, TRSCER,
      RESERVED7, RMFCR, RESERVED8, TFTR, RESERVED9, FDR, RESERVED10, RMCR,
      RESERVED11[2], TFUCR, RFOCR, IOSR, FCFTR, RESERVED12, RPADIR, TRIMD,
      RESERVED13[18], RBWAR, RDFAR, RESERVED14, TBRAR, TDFAR;
};

#undef ETHERC
#undef EDMAC
#undef RASYSC
#undef ICU_IELSR
#if defined(MG_ENABLE_DRIVER_RA8) && MG_ENABLE_DRIVER_RA8
#define ETHERC ((struct ra_etherc *) (uintptr_t) 0x40354100U)
#define EDMAC ((struct ra_edmac *) (uintptr_t) 0x40354000U)
#define RASYSC ((uint32_t *) (uintptr_t) 0x4001E000U)
#define ICU_IELSR ((uint32_t *) (uintptr_t) 0x4000C300U)
#else
#define ETHERC ((struct ra_etherc *) (uintptr_t) 0x40114100U)
#define EDMAC ((struct ra_edmac *) (uintptr_t) 0x40114000U)
#define RASYSC ((uint32_t *) (uintptr_t) 0x4001E000U)
#define ICU_IELSR ((uint32_t *) (uintptr_t) 0x40006300U)
#endif

#define ETH_PKT_SIZE 1536  // Max frame size, multiple of 32
#define ETH_DESC_CNT 4     // Descriptors count

// Descriptors: 16-byte aligned
// Buffers: 32-byte aligned (27.3.1)
static volatile uint32_t s_rxdesc[ETH_DESC_CNT][4] MG_ETH_RAM MG_16BYTE_ALIGNED;
static volatile uint32_t s_txdesc[ETH_DESC_CNT][4] MG_ETH_RAM MG_16BYTE_ALIGNED;
static uint8_t s_rxbuf[ETH_DESC_CNT][ETH_PKT_SIZE] MG_ETH_RAM MG_32BYTE_ALIGNED;
static uint8_t s_txbuf[ETH_DESC_CNT][ETH_PKT_SIZE] MG_ETH_RAM MG_32BYTE_ALIGNED;
static struct mg_tcpip_if *s_ifp;  // MIP interface

// fastest is 3 cycles (SUB + BNE) on a 3-stage pipeline or equivalent
static inline void raspin(volatile uint32_t count) {
  while (count--) (void) 0;
}
// count to get the 200ns SMC semi-cycle period (2.5MHz) calling raspin():
// SYS_FREQUENCY * 200ns / 3 = SYS_FREQUENCY / 15000000
static uint32_t s_smispin;

// Bit-banged SMI
static void smi_preamble(void) {
  unsigned int i = 32;
  uint32_t pir = MG_BIT(1) | MG_BIT(2);  // write, mdio = 1, mdc = 0
  ETHERC->PIR = pir;
  while (i--) {
    pir &= ~MG_BIT(0);  // mdc = 0
    ETHERC->PIR = pir;
    raspin(s_smispin);
    pir |= MG_BIT(0);  // mdc = 1
    ETHERC->PIR = pir;
    raspin(s_smispin);
  }
}
static void smi_wr(uint16_t header, uint16_t data) {
  uint32_t word = (header << 16) | data;
  smi_preamble();
  unsigned int i = 32;
  while (i--) {
    uint32_t pir = MG_BIT(1) |
                   (word & 0x80000000 ? MG_BIT(2) : 0);  // write, mdc = 0, data
    ETHERC->PIR = pir;
    raspin(s_smispin);
    pir |= MG_BIT(0);  // mdc = 1
    ETHERC->PIR = pir;
    raspin(s_smispin);
    word <<= 1;
  }
}
static uint16_t smi_rd(uint16_t header) {
  smi_preamble();
  unsigned int i = 16;  // 2 LSb as turnaround
  uint32_t pir;
  while (i--) {
    pir = (i > 1 ? MG_BIT(1) : 0) |
          (header & 0x8000
               ? MG_BIT(2)
               : 0);  // mdc = 0, header, set read direction at turnaround
    ETHERC->PIR = pir;
    raspin(s_smispin);
    pir |= MG_BIT(0);  // mdc = 1
    ETHERC->PIR = pir;
    raspin(s_smispin);
    header <<= 1;
  }
  i = 16;
  uint16_t data = 0;
  while (i--) {
    data <<= 1;
    pir = 0;  // read, mdc = 0
    ETHERC->PIR = pir;
    raspin(s_smispin / 2);  // 1/4 clock period, 300ns max access time
    data |= (uint16_t) (ETHERC->PIR & MG_BIT(3) ? 1 : 0);  // read mdio
    raspin(s_smispin / 2);                                 // 1/4 clock period
    pir |= MG_BIT(0);                                      // mdc = 1
    ETHERC->PIR = pir;
    raspin(s_smispin);
  }
  return data;
}

static uint16_t raeth_read_phy(uint8_t addr, uint8_t reg) {
  return smi_rd(
      (uint16_t) ((1 << 14) | (2 << 12) | (addr << 7) | (reg << 2) | (2 << 0)));
}

static void raeth_write_phy(uint8_t addr, uint8_t reg, uint16_t val) {
  smi_wr(
      (uint16_t) ((1 << 14) | (1 << 12) | (addr << 7) | (reg << 2) | (2 << 0)),
      val);
}

// MDC clock is generated manually; as per 802.3, it must not exceed 2.5MHz
static bool mg_tcpip_driver_ra_init(struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_ra_data *d =
      (struct mg_tcpip_driver_ra_data *) ifp->driver_data;
  s_ifp = ifp;

  // Init SMI clock timing. If user told us the clock value, use it.
  // TODO(): Otherwise, guess
  s_smispin = d->clock / 15000000;

  // Init RX descriptors
  for (int i = 0; i < ETH_DESC_CNT; i++) {
    s_rxdesc[i][0] = MG_BIT(31);             // RACT
    s_rxdesc[i][1] = ETH_PKT_SIZE << 16;     // RBL
    s_rxdesc[i][2] = (uint32_t) s_rxbuf[i];  // Point to data buffer
  }
  s_rxdesc[ETH_DESC_CNT - 1][0] |= MG_BIT(30);  // Wrap last descriptor

  // Init TX descriptors
  for (int i = 0; i < ETH_DESC_CNT; i++) {
    // TACT = 0
    s_txdesc[i][2] = (uint32_t) s_txbuf[i];
  }
  s_txdesc[ETH_DESC_CNT - 1][0] |= MG_BIT(30);  // Wrap last descriptor

  EDMAC->EDMR = MG_BIT(0);  // Software reset, wait 64 PCLKA clocks (27.2.1)
  uint32_t sckdivcr = RASYSC[8];  // get divisors from SCKDIVCR (8.2.2)
  uint32_t ick = 1 << ((sckdivcr >> 24) & 7);   // sys_clock div
  uint32_t pcka = 1 << ((sckdivcr >> 12) & 7);  // pclka div
  raspin((64U * pcka) / (3U * ick));
  EDMAC->EDMR = MG_BIT(6);  // Initialize, little-endian (27.2.1)

  MG_DEBUG(("PHY addr: %d, smispin: %d", d->phy_addr, s_smispin));
  struct mg_phy phy = {raeth_read_phy, raeth_write_phy};
  mg_phy_init(&phy, d->phy_addr, MG_PHY_CLOCKS_MAC);

  // Select RMII mode,
  ETHERC->ECMR = MG_BIT(2) | MG_BIT(1);  // 100M, Full-duplex, CRC
  // ETHERC->ECMR |= MG_BIT(0);             // Receive all
  ETHERC->RFLR = 1518;  // Set max rx length

  EDMAC->RDLAR = (uint32_t) (uintptr_t) s_rxdesc;
  EDMAC->TDLAR = (uint32_t) (uintptr_t) s_txdesc;
  // MAC address filtering (bytes in reversed order)
  ETHERC->MAHR = (uint32_t) (ifp->mac[0] << 24U) |
                 ((uint32_t) ifp->mac[1] << 16U) |
                 ((uint32_t) ifp->mac[2] << 8U) | ifp->mac[3];
  ETHERC->MALR = ((uint32_t) ifp->mac[4] << 8U) | ifp->mac[5];

  EDMAC->TFTR = 0;                        // Store and forward (27.2.10)
  EDMAC->FDR = 0x070f;                    // (27.2.11)
  EDMAC->RMCR = MG_BIT(0);                // (27.2.12)
  ETHERC->ECMR |= MG_BIT(6) | MG_BIT(5);  // TE RE
  EDMAC->EESIPR = MG_BIT(18);  // FR: Enable Rx (frame) IRQ
  EDMAC->EDRRR = MG_BIT(0);  // Receive Descriptors have changed
  EDMAC->EDTRR = MG_BIT(0);  // Transmit Descriptors have changed
  return true;
}

// Transmit frame
static size_t mg_tcpip_driver_ra_tx(const void *buf, size_t len,
                                    struct mg_tcpip_if *ifp) {
  static int s_txno;  // Current descriptor index
  if (len > sizeof(s_txbuf[ETH_DESC_CNT])) {
    MG_ERROR(("Frame too big, %ld", (long) len));
    len = (size_t) -1;  // fail
  } else if ((s_txdesc[s_txno][0] & MG_BIT(31))) {
    ifp->nerr++;
    MG_ERROR(("No descriptors available"));
    len = 0;  // retry later
  } else {
    memcpy(s_txbuf[s_txno], buf, len);            // Copy data
    s_txdesc[s_txno][1] = len << 16;              // Set data len
    s_txdesc[s_txno][0] |= MG_BIT(31) | 3 << 28;  // (27.3.1.1) mark valid
    EDMAC->EDTRR = MG_BIT(0);                     // Transmit request
    if (++s_txno >= ETH_DESC_CNT) s_txno = 0;
  }
  return len;
}

static bool mg_tcpip_driver_ra_poll(struct mg_tcpip_if *ifp, bool s1) {
  if (ifp->update_mac_hash_table) {
    EDMAC->EESIPR = MG_BIT(18) | MG_BIT(7);  // FR, RMAF: Frame and mcast IRQ
    ifp->update_mac_hash_table = false;
  }
  if (!s1) return false;
  struct mg_tcpip_driver_ra_data *d =
      (struct mg_tcpip_driver_ra_data *) ifp->driver_data;
  uint8_t speed = MG_PHY_SPEED_10M;
  bool up = false, full_duplex = false;
  struct mg_phy phy = {raeth_read_phy, raeth_write_phy};
  up = mg_phy_up(&phy, d->phy_addr, &full_duplex, &speed);
  if ((ifp->state == MG_TCPIP_STATE_DOWN) && up) {  // link state just went up
    // tmp = reg with flags set to the most likely situation: 100M full-duplex
    // if(link is slow or half) set flags otherwise
    // reg = tmp
    uint32_t ecmr = ETHERC->ECMR | MG_BIT(2) | MG_BIT(1);  // 100M Full-duplex
    if (speed == MG_PHY_SPEED_10M) ecmr &= ~MG_BIT(2);     // 10M
    if (full_duplex == false) ecmr &= ~MG_BIT(1);          // Half-duplex
    ETHERC->ECMR = ecmr;  // IRQ handler does not fiddle with these registers
    MG_DEBUG(("Link is %uM %s-duplex", ecmr & MG_BIT(2) ? 100 : 10,
              ecmr & MG_BIT(1) ? "full" : "half"));
  }
  return up;
}

void EDMAC_IRQHandler(void);
static uint32_t s_rxno;
void EDMAC_IRQHandler(void) {
  struct mg_tcpip_driver_ra_data *d =
      (struct mg_tcpip_driver_ra_data *) s_ifp->driver_data;
  EDMAC->EESR = MG_BIT(18) | MG_BIT(7);  // Ack IRQ in EDMAC 1st
  ICU_IELSR[d->irqno] &= ~MG_BIT(16);  // Ack IRQ in ICU last
  // Frame received, loop
  for (uint32_t i = 0; i < 10; i++) {  // read as they arrive but not forever
    uint32_t r = s_rxdesc[s_rxno][0];
    if (r & MG_BIT(31)) break;  // exit when done
    // skip partial/errored frames (27.3.1.2)
    if ((r & (MG_BIT(29) | MG_BIT(28)) && !(r & MG_BIT(27)))) {
      size_t len = s_rxdesc[s_rxno][1] & 0xffff;
      mg_tcpip_qwrite(s_rxbuf[s_rxno], len, s_ifp);  // CRC already stripped
    }
    s_rxdesc[s_rxno][0] |= MG_BIT(31);
    if (++s_rxno >= ETH_DESC_CNT) s_rxno = 0;
  }
  EDMAC->EDRRR = MG_BIT(0);  // Receive Descriptors have changed
  // If b0 == 0, descriptors were exhausted and probably frames were dropped,
  // (27.2.9 RMFCR counts them)
}

struct mg_tcpip_driver mg_tcpip_driver_ra = {mg_tcpip_driver_ra_init,
                                             mg_tcpip_driver_ra_tx, NULL,
                                             mg_tcpip_driver_ra_poll};

#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/rw612.c"
#endif


#if MG_ENABLE_TCPIP && defined(MG_ENABLE_DRIVER_RW612) && MG_ENABLE_DRIVER_RW612

struct ENET_Type {
  volatile uint32_t RESERVED_0[1], EIR, EIMR, RESERVED_1[1], RDAR, TDAR,
      RESERVED_2[3], ECR, RESERVED_3[6], MMFR, MSCR, RESERVED_4[7], MIBC,
      RESERVED_5[7], RCR, RESERVED_6[15], TCR, RESERVED_7[7], PALR, PAUR, OPD,
      TXIC[1], RESERVED_8[3], RXIC[1], RESERVED_9[5], IAUR, IALR, GAUR, GALR,
      RESERVED_10[7], TFWR, RESERVED_11[14], RDSR, TDSR, MRBR, RESERVED_12[1],
      RSFL, RSEM, RAEM, RAFL, TSEM, TAEM, TAFL, TIPG, FTRL, RESERVED_13[3],
      TACC, RACC, RESERVED_14[15], RMON_T_PACKETS, RMON_T_BC_PKT, RMON_T_MC_PKT,
      RMON_T_CRC_ALIGN, RMON_T_UNDERSIZE, RMON_T_OVERSIZE, RMON_T_FRAG,
      RMON_T_JAB, RMON_T_COL, RMON_T_P64, RMON_T_P65TO127, RMON_T_P128TO255,
      RMON_T_P256TO511, RMON_T_P512TO1023, RMON_T_P1024TO2047, RMON_T_P_GTE2048,
      RMON_T_OCTETS, IEEE_T_DROP, IEEE_T_FRAME_OK, IEEE_T_1COL, IEEE_T_MCOL,
      IEEE_T_DEF, IEEE_T_LCOL, IEEE_T_EXCOL, IEEE_T_MACERR, IEEE_T_CSERR,
      IEEE_T_SQE, IEEE_T_FDXFC, IEEE_T_OCTETS_OK, RESERVED_15[3],
      RMON_R_PACKETS, RMON_R_BC_PKT, RMON_R_MC_PKT, RMON_R_CRC_ALIGN,
      RMON_R_UNDERSIZE, RMON_R_OVERSIZE, RMON_R_FRAG, RMON_R_JAB,
      RESERVED_16[1], RMON_R_P64, RMON_R_P65TO127, RMON_R_P128TO255,
      RMON_R_P256TO511, RMON_R_P512TO1023, RMON_R_P1024TO2047, RMON_R_P_GTE2048,
      RMON_R_OCTETS, IEEE_R_DROP, IEEE_R_FRAME_OK, IEEE_R_CRC, IEEE_R_ALIGN,
      IEEE_R_MACERR, IEEE_R_FDXFC, IEEE_R_OCTETS_OK, RESERVED_17[71], ATCR,
      ATVR, ATOFF, ATPER, ATCOR, ATINC, ATSTMP, RESERVED_18[122], TGSR,
      CHANNEL_TCSR[4], CHANNEL_TCCR[4];
};

#undef ENET
#define ENET ((struct ENET_Type *) 0x40138000)

#define ETH_PKT_SIZE 1536  // Max frame size
#define ETH_DESC_CNT 4     // Descriptors count
#define ETH_DS 2           // Descriptor size (words)

static uint8_t s_rxbuf[ETH_DESC_CNT][ETH_PKT_SIZE] MG_ETH_RAM MG_8BYTE_ALIGNED;
static uint8_t s_txbuf[ETH_DESC_CNT][ETH_PKT_SIZE] MG_ETH_RAM MG_8BYTE_ALIGNED;
static uint32_t s_rxdesc[ETH_DESC_CNT][ETH_DS] MG_ETH_RAM MG_8BYTE_ALIGNED;
static uint32_t s_txdesc[ETH_DESC_CNT][ETH_DS] MG_ETH_RAM MG_8BYTE_ALIGNED;
static uint8_t s_txno;  // Current TX descriptor
static uint8_t s_rxno;  // Current RX descriptor

static struct mg_tcpip_if *s_ifp;  // MIP interface

static uint16_t eth_read_phy(uint8_t addr, uint8_t reg) {
  ENET->MMFR = MG_BIT(30) |  // Start of frame delimiter
               MG_BIT(29) |  // Opcode
               ((addr & 0x1f) << 23) | ((reg & 0x1f) << 18) | MG_BIT(17);
  while ((ENET->EIR & MG_BIT(23)) == 0) (void) 0;
  ENET->EIR |= MG_BIT(23);
  return ENET->MMFR & 0xffff;
}

static void eth_write_phy(uint8_t addr, uint8_t reg, uint16_t val) {
  ENET->MMFR = MG_BIT(30) |  // Start of frame delimiter
               MG_BIT(28) |  // Opcode
               ((addr & 0x1f) << 23) | ((reg & 0x1f) << 18) | MG_BIT(17) | val;
  while ((ENET->EIR & MG_BIT(23)) == 0) (void) 0;
  ENET->EIR |= MG_BIT(23);
}

static bool mg_tcpip_driver_rw612_init(struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_rw612_data *d =
      (struct mg_tcpip_driver_rw612_data *) ifp->driver_data;
  s_ifp = ifp;
  ENET->MSCR = ((d->mdc_cr & 0x3f) << 1) | ((d->mdc_holdtime & 7) << 8);
  struct mg_phy phy = {eth_read_phy, eth_write_phy};
  mg_phy_init(&phy, d->phy_addr, 0);
  ENET->ECR |= MG_BIT(0);  // reset ETH

  // initialize descriptors
  for (int i = 0; i < ETH_DESC_CNT; i++) {
    s_rxdesc[i][1] = (uint32_t) s_rxbuf[i];
    s_rxdesc[i][0] = MG_BIT(31);  // OWN
    if (i == ETH_DESC_CNT - 1) {
      s_rxdesc[i][0] |= MG_BIT(29);  // mark last descriptor
    }
    s_txdesc[i][1] = (uint32_t) s_txbuf[i];
    if (i == ETH_DESC_CNT - 1) {
      s_txdesc[i][0] |= MG_BIT(29);  // mark last descriptor
    }
  }

  ENET->RCR = (ENET->RCR & (0xffff << 16)) | MG_BIT(14) | MG_BIT(8) | MG_BIT(2);
  ENET->TCR = MG_BIT(2);  // full duplex
  ENET->TDSR = (uint32_t) &s_txdesc[0][0];
  ENET->RDSR = (uint32_t) &s_rxdesc[0][0];
  ENET->MRBR = ETH_PKT_SIZE;
  ENET->PALR =
      ifp->mac[0] << 24 | ifp->mac[1] << 16 | ifp->mac[2] << 8 | ifp->mac[3];
  ENET->PAUR |= (ifp->mac[4] << 24 | ifp->mac[5] << 16);
  ENET->IALR = 0;
  ENET->IAUR = 0;
  ENET->GALR = 0;
  ENET->GAUR = 0;
  ENET->MSCR = ((d->mdc_cr & 0x3f) << 1) | ((d->mdc_holdtime & 7) << 8);
  ENET->EIMR = MG_BIT(25);             // Enable RX interrupt
  ENET->ECR |= MG_BIT(8) | MG_BIT(1);  // DBSWP, Enable
  ENET->RDAR = 0;                      // activate RX descriptors ring
  return true;
}

static size_t mg_tcpip_driver_rw612_tx(const void *buf, size_t len,
                                       struct mg_tcpip_if *ifp) {
  if (len > sizeof(s_txbuf[s_txno])) {
    MG_ERROR(("Frame too big, %ld", (long) len));
    len = 0;  // Frame is too big
  } else if (((s_txdesc[s_txno][0] & MG_BIT(31)) != 0)) {
    ifp->nerr++;
    MG_ERROR(("No free descriptors"));
    len = 0;  // All descriptors are busy, fail
  } else {
    memcpy(s_txbuf[s_txno], buf, len);
    s_txdesc[s_txno][0] = len | MG_BIT(27) | MG_BIT(26);  // last buffer, crc
    if (s_txno == ETH_DESC_CNT - 1) {
      s_txdesc[s_txno][0] |= MG_BIT(29);  // wrap
    }
    s_txdesc[s_txno][0] |= MG_BIT(31);  // release ownership
    MG_DSB();
    ENET->TDAR = 0;
    // MG_INFO(("s_txdesc[%d][0]: 0x%x", s_txno, s_txdesc[s_txno][0]));
    if (++s_txno >= ETH_DESC_CNT) s_txno = 0;
  }

  return len;
}


static void mg_tcpip_driver_rw612_update_hash_table(struct mg_tcpip_if *ifp) {
  // TODO(): read database, rebuild hash table
  ENET->GAUR = MG_BIT(1); // see imxrt, it reduces to this for mDNS
  (void) ifp;
}

static bool mg_tcpip_driver_rw612_poll(struct mg_tcpip_if *ifp, bool s1) {
  if (ifp->update_mac_hash_table) {
    mg_tcpip_driver_rw612_update_hash_table(ifp);
    ifp->update_mac_hash_table = false;
  }
  if (!s1) return false;
  struct mg_tcpip_driver_rw612_data *d =
      (struct mg_tcpip_driver_rw612_data *) ifp->driver_data;
  uint8_t speed = MG_PHY_SPEED_10M;
  bool up = false, full_duplex = false;
  struct mg_phy phy = {eth_read_phy, eth_write_phy};
  up = mg_phy_up(&phy, d->phy_addr, &full_duplex, &speed);
  if ((ifp->state == MG_TCPIP_STATE_DOWN) && up) {  // link state just went up
    // tmp = reg with flags set to the most likely situation: 100M full-duplex
    // if(link is slow or half) set flags otherwise
    // reg = tmp
    if (speed == MG_PHY_SPEED_100M && (ENET->RCR & MG_BIT(9))) {
      ENET->RCR &= ~MG_BIT(9);
    } else if (speed == MG_PHY_SPEED_10M && (ENET->RCR & MG_BIT(9)) == 0) {
      ENET->RCR |= MG_BIT(9);
    }
    if (full_duplex && (ENET->TCR & MG_BIT(2)) == 0) {
      ENET->ECR &= ~MG_BIT(1);
      ENET->TCR |= MG_BIT(2);
      ENET->ECR |= MG_BIT(1);
    } else if (!full_duplex && (ENET->TCR & MG_BIT(2))) {
      ENET->ECR &= ~MG_BIT(1);
      ENET->TCR &= ~MG_BIT(2);
      ENET->ECR |= MG_BIT(1);
    }
    MG_INFO(("Link is %uM %s-duplex",
             speed == MG_PHY_SPEED_10M
                 ? 10
                 : (speed == MG_PHY_SPEED_100M ? 100 : 1000),
             full_duplex ? "full" : "half"));
  }
  return up;
}

void ENET_IRQHandler(void) {
  if (ENET->EIR & MG_BIT(25)) {
    ENET->EIR = MG_BIT(25);              // Ack RX
    for (uint32_t i = 0; i < 10; i++) {  // read as they arrive but not forever
      if ((s_rxdesc[s_rxno][0] & MG_BIT(31)) != 0) break;  // exit when done
      // skip partial/errored frames
      if ((s_rxdesc[s_rxno][0] & MG_BIT(27)) &&
          !(s_rxdesc[s_rxno][0] &
            (MG_BIT(21) | MG_BIT(20) | MG_BIT(18) | MG_BIT(17) | MG_BIT(16)))) {
        size_t len = s_rxdesc[s_rxno][0] & 0xffff;
        mg_tcpip_qwrite(s_rxbuf[s_rxno], len, s_ifp);
      }
      s_rxdesc[s_rxno][0] |= MG_BIT(31);  // OWN bit: handle control to DMA
      MG_DSB();
      ENET->RDAR = 0;
      if (++s_rxno >= ETH_DESC_CNT) s_rxno = 0;
    }
  }
}

struct mg_tcpip_driver mg_tcpip_driver_rw612 = {mg_tcpip_driver_rw612_init,
                                                mg_tcpip_driver_rw612_tx, NULL,
                                                mg_tcpip_driver_rw612_poll};
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/same54.c"
#endif


#if MG_ENABLE_TCPIP && defined(MG_ENABLE_DRIVER_SAME54) && \
    MG_ENABLE_DRIVER_SAME54

struct GMAC_REGS_ {
  uint32_t GMAC_NCR, GMAC_NCFGR, GMAC_NSR, GMAC_UR, GMAC_DCFGR, GMAC_TSR,
      GMAC_RBQB, GMAC_TBQB, GMAC_RSR, GMAC_ISR, GMAC_IER, GMAC_IDR, GMAC_IMR,
      GMAC_MAN, GMAC_RPQ, GMAC_TPQ, GMAC_TPSF, GMAC_RPSF, GMAC_RJFML,
      Reserved1[13], GMAC_HRB, GMAC_HRT, GMAC_SAB0, GMAC_SAT0, GMAC_SAB1,
      GMAC_SAT1, GMAC_SAB2, GMAC_SAT2, GMAC_SAB3, GMAC_SAT3, GMAC_TIDM[4],
      GMAC_WOL, GMAC_IPGS, GMAC_SVLAN, GMAC_TPFCP, GMAC_SAMB1, GMAC_SAMT1,
      Reserved2[3], GMAC_NSC, GMAC_SCL, GMAC_SCH, GMAC_EFTSH, GMAC_EFRSH,
      GMAC_PEFTSH, GMAC_PEFRSH, Reserved3[2], GMAC_OTLO, GMAC_OTHI, GMAC_FT,
      GMAC_BCFT, GMAC_MFT, GMAC_PFT, GMAC_BFT64, GMAC_TBFT127, GMAC_TBFT255,
      GMAC_TBFT511, GMAC_TBFT1023, GMAC_TBFT1518, GMAC_GTBFT1518, GMAC_TUR,
      GMAC_SCF, GMAC_MCF, GMAC_EC, GMAC_LC, GMAC_DTF, GMAC_CSE, GMAC_ORLO,
      GMAC_ORHI, GMAC_FR, GMAC_BCFR, GMAC_MFR, GMAC_PFR, GMAC_BFR64,
      GMAC_TBFR127, GMAC_TBFR255, GMAC_TBFR511, GMAC_TBFR1023, GMAC_TBFR1518,
      GMAC_TMXBFR, GMAC_UFR, GMAC_OFR, GMAC_JR, GMAC_FCSE, GMAC_LFFE, GMAC_RSE,
      GMAC_AE, GMAC_RRE, GMAC_ROE, GMAC_IHCE, GMAC_TCE, GMAC_UCE, Reserved4[2],
      GMAC_TISUBN, GMAC_TSH, Reserved5, GMAC_TSSSL, GMAC_TSSN, GMAC_TSL,
      GMAC_TN, GMAC_TA, GMAC_TI, GMAC_EFTSL, GMAC_EFTN, GMAC_EFRSL, GMAC_EFRN,
      GMAC_PEFTSL, GMAC_PEFTN, GMAC_PEFRSL, GMAC_PEFRN, Reserved6[28],
      GMAC_RLPITR, GMAC_RLPITI, GMAC_TLPITR, GMAC_TLPITI;
};

struct GCLK_REGS_ {
  uint32_t GCLK_CTRLA_RESERVED, GCLK_SYNCBUSY, Reserved2[6], GCLK_GENCTRL[12],
      Reserved3[12], GCLK_PCHCTRL[48];
};

#define GMAC_REGS ((struct GMAC_REGS_ *) 0x42000800)
#define GCLK_REGS ((struct GCLK_REGS_ *) 0x40001c00)

#define ETH_PKT_SIZE 1536  // Max frame size
#define ETH_DESC_CNT 4     // Descriptors count
#define ETH_DS 2           // Descriptor size (words)

static uint8_t s_rxbuf[ETH_DESC_CNT][ETH_PKT_SIZE];
static uint8_t s_txbuf[ETH_DESC_CNT][ETH_PKT_SIZE];
static uint32_t s_rxdesc[ETH_DESC_CNT][ETH_DS];  // RX descriptors
static uint32_t s_txdesc[ETH_DESC_CNT][ETH_DS];  // TX descriptors
static uint8_t s_txno;                           // Current TX descriptor
static uint8_t s_rxno;                           // Current RX descriptor

static struct mg_tcpip_if *s_ifp;  // MIP interface

static uint16_t eth_read_phy(uint8_t addr, uint8_t reg) {
  GMAC_REGS->GMAC_MAN = MG_BIT(30) |  // Clause 22
                        MG_BIT(29) |  // Setting the read operation
                        MG_BIT(17) | ((addr & 0x1f) << 23) |  // PHY address
                        ((reg & 0x1f) << 18);  // Setting the register
  while (!(GMAC_REGS->GMAC_NSR & MG_BIT(2))) (void) 0;
  return GMAC_REGS->GMAC_MAN & 0xffff;  // Getting the read value
}

static void eth_write_phy(uint8_t addr, uint8_t reg, uint16_t val) {
  GMAC_REGS->GMAC_MAN = MG_BIT(30) |
                        MG_BIT(28) |  // Setting the write operation
                        MG_BIT(17) | ((addr & 0x1f) << 23) |  // PHY address
                        ((reg & 0x1f) << 18) | val;  // Setting the register
  while (!(GMAC_REGS->GMAC_NSR & MG_BIT(2)))
    ;  // Waiting until the write op is complete
}

uint32_t get_clock_rate(struct mg_tcpip_driver_same54_data *d) {
  if (d && d->mdc_cr >= 0 && d->mdc_cr <= 5) {
    return d->mdc_cr;
  } else {
    // get MCLK from GCLK_GENERATOR 0
    uint32_t div = 512;
    uint32_t mclk;
    if (!(GCLK_REGS->GCLK_GENCTRL[0] & MG_BIT(12))) {
      div = ((GCLK_REGS->GCLK_GENCTRL[0] & 0x00FF0000) >> 16);
      if (div == 0) div = 1;
    }
    switch (GCLK_REGS->GCLK_GENCTRL[0] & 0xF) {
      case 0:               // GCLK_GENCTRL_SRC_XOSC0_Val
        mclk = 32000000UL;  // 32MHz
        break;
      case 1:               // GCLK_GENCTRL_SRC_XOSC1_Val
        mclk = 32000000UL;  // 32MHz
        break;
      case 4:            // GCLK_GENCTRL_SRC_OSCULP32K_Val
        mclk = 32000UL;  // 32Khz
        break;
      case 5:            // GCLK_GENCTRL_SRC_XOSC32K_Val
        mclk = 32000UL;  // 32Khz
        break;
      case 6:               // GCLK_GENCTRL_SRC_DFLL_Val
        mclk = 48000000UL;  // 48MHz
        break;
      case 7:                // GCLK_GENCTRL_SRC_DPLL0_Val:
        mclk = 200000000UL;  // 200MHz
        break;
      case 8:                // GCLK_GENCTRL_SRC_DPLL1_Val
        mclk = 200000000UL;  // 200MHz
        break;
      default:
        mclk = 200000000UL;  // 200MHz
    }

    mclk /= div;
    uint8_t crs[] = {0, 1, 2, 3, 4, 5};            // GMAC->NCFGR::CLK values
    uint8_t dividers[] = {8, 16, 32, 48, 64, 96};  // Respective CLK dividers
    for (int i = 0; i < 6; i++) {
      if (mclk / dividers[i] <= 2375000UL) {  // 2.5MHz - 5%
        return crs[i];
      }
    }

    return 5;
  }
}

static bool mg_tcpip_driver_same54_init(struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_same54_data *d =
      (struct mg_tcpip_driver_same54_data *) ifp->driver_data;
  s_ifp = ifp;

  GMAC_REGS->GMAC_NCFGR = get_clock_rate(d) << 18;  // Set MDC divider
  GMAC_REGS->GMAC_NCR = 0;                          // Disable RX & TX
  GMAC_REGS->GMAC_NCR |= MG_BIT(4);                 // Enable MDC & MDIO

  struct mg_phy phy = {eth_read_phy, eth_write_phy};
  mg_phy_init(&phy, d->phy_addr, 0);

  for (int i = 0; i < ETH_DESC_CNT; i++) {   // Init TX descriptors
    s_txdesc[i][0] = (uint32_t) s_txbuf[i];  // Point to data buffer
    s_txdesc[i][1] = MG_BIT(31);             // OWN bit
  }
  s_txdesc[ETH_DESC_CNT - 1][1] |= MG_BIT(30);  // Last tx descriptor - wrap

  GMAC_REGS->GMAC_DCFGR = (0x18 << 16) |     // DMA recv buf 1536
                          (3 << 8) |         // RXBMS
                          MG_BIT(10);        // See #2487
  for (int i = 0; i < ETH_DESC_CNT; i++) {   // Init RX descriptors
    s_rxdesc[i][0] = (uint32_t) s_rxbuf[i];  // Address of the data buffer
    s_rxdesc[i][1] = 0;                      // Clear status
  }
  s_rxdesc[ETH_DESC_CNT - 1][0] |= MG_BIT(1);  // Last rx descriptor - wrap

  GMAC_REGS->GMAC_TBQB = (uint32_t) s_txdesc;  // about the descriptor addresses
  GMAC_REGS->GMAC_RBQB = (uint32_t) s_rxdesc;  // Let the controller know

  GMAC_REGS->GMAC_SAB0 =
      MG_U32(ifp->mac[3], ifp->mac[2], ifp->mac[1], ifp->mac[0]);
  GMAC_REGS->GMAC_SAT0 = MG_U32(0, 0, ifp->mac[5], ifp->mac[4]);

  GMAC_REGS->GMAC_UR &= ~MG_BIT(0);                 // Disable MII, use RMII
  GMAC_REGS->GMAC_NCFGR |= MG_BIT(8) | MG_BIT(6) |  // MAXFX, MTIHEN
                           MG_BIT(25) | MG_BIT(4);  // EFRHD, CAF
  GMAC_REGS->GMAC_TSR = 0x17f;                      // all transmit statuses
  GMAC_REGS->GMAC_RSR = 0xf;                        // all recv statuses
  GMAC_REGS->GMAC_IDR = ~0U;  // Disable interrupts, then enable required
  GMAC_REGS->GMAC_IER = MG_BIT(11) | MG_BIT(10) |  // HRESP, ROVR
                        MG_BIT(7) | MG_BIT(6) |    // TCOMP, TFC
                        MG_BIT(5) | MG_BIT(4) |    // RLEX, TUR
                        MG_BIT(2) | MG_BIT(1);     // RXUBR, RCOMP
  GMAC_REGS->GMAC_NCR |= MG_BIT(3) | MG_BIT(2);    // TXEN, RXEN

  return true;
}

static size_t mg_tcpip_driver_same54_tx(const void *buf, size_t len,
                                        struct mg_tcpip_if *ifp) {
  if (len > sizeof(s_txbuf[s_txno])) {
    MG_ERROR(("Frame too big, %ld", (long) len));
    len = 0;  // Frame is too big
  } else if ((s_txdesc[s_txno][1] & MG_BIT(31)) == 0) {
    ifp->nerr++;
    MG_ERROR(("No free descriptors"));
    len = 0;  // All descriptors are busy, fail
  } else {
    uint32_t status = len | MG_BIT(15);  // Frame length, last chunk
    if (s_txno == ETH_DESC_CNT - 1) status |= MG_BIT(30);  // wrap
    memcpy(s_txbuf[s_txno], buf, len);                     // Copy data
    s_txdesc[s_txno][1] = status;
    if (++s_txno >= ETH_DESC_CNT) s_txno = 0;
  }
  MG_DSB();                          // Ensure descriptors have been written
  GMAC_REGS->GMAC_NCR |= MG_BIT(9);  // Enable transmission
  return len;
}

static void mg_tcpip_driver_same54_update_hash_table(struct mg_tcpip_if *ifp) {
  // TODO(): read database, rebuild hash table
  // Setting Hash Index for 01:00:5e:00:00:fb (multicast)
  // 24.6.9 Hash addressing
  // computed hash is 55, which means bit 23 (55 - 32) in
  // HRT register must be set
  GMAC_REGS->GMAC_HRT = MG_BIT(23);
  GMAC_REGS->GMAC_NCFGR |= MG_BIT(6);  // enable multicast hash filtering
  (void) ifp;
}

static bool mg_tcpip_driver_same54_poll(struct mg_tcpip_if *ifp, bool s1) {
  if (ifp->update_mac_hash_table) {
    mg_tcpip_driver_same54_update_hash_table(ifp);
    ifp->update_mac_hash_table = false;
  }

  bool up = false;
  if (s1) {
    uint8_t speed = MG_PHY_SPEED_10M;
    bool full_duplex = false;
    struct mg_phy phy = {eth_read_phy, eth_write_phy};
    up = mg_phy_up(&phy, 0, &full_duplex, &speed);

    // If PHY is ready, update NCFGR accordingly
    if (ifp->state == MG_TCPIP_STATE_DOWN && up) {
      GMAC_REGS->GMAC_NCFGR =
          (GMAC_REGS->GMAC_NCFGR & ~(MG_BIT(0) | MG_BIT(1))) | (speed & 1) |
          (full_duplex << 1);
    }
  }
  return up;
}

void GMAC_Handler(void);
void GMAC_Handler(void) {
  uint32_t isr = GMAC_REGS->GMAC_ISR;
  uint32_t rsr = GMAC_REGS->GMAC_RSR;
  uint32_t tsr = GMAC_REGS->GMAC_TSR;
  if (isr & MG_BIT(1)) {
    if (rsr & MG_BIT(1)) {
      for (uint8_t i = 0; i < ETH_DESC_CNT; i++) {
        if ((s_rxdesc[s_rxno][0] & MG_BIT(0)) == 0) break;
        size_t len = s_rxdesc[s_rxno][1] & (MG_BIT(13) - 1);
        mg_tcpip_qwrite(s_rxbuf[s_rxno], len, s_ifp);
        s_rxdesc[s_rxno][0] &= ~MG_BIT(0);  // Disown
        if (++s_rxno >= ETH_DESC_CNT) s_rxno = 0;
      }
    }
  }

  if (tsr != 0) {
    // MG_INFO((" --> %#x %#x", s_txdesc[s_txno][1], tsr));
    if (!(s_txdesc[s_txno][1] & MG_BIT(31))) s_txdesc[s_txno][1] |= MG_BIT(31);
  }

  GMAC_REGS->GMAC_RSR = rsr;
  GMAC_REGS->GMAC_TSR = tsr;
}

struct mg_tcpip_driver mg_tcpip_driver_same54 = {
    mg_tcpip_driver_same54_init, mg_tcpip_driver_same54_tx, NULL,
    mg_tcpip_driver_same54_poll};
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/sdio.c"
#endif


#if MG_ENABLE_TCPIP && \
    (defined(MG_ENABLE_DRIVER_CYW_SDIO) && MG_ENABLE_DRIVER_CYW_SDIO)

// SDIO 6.9 Table 6-1 CCCR (Common Card Control Registers)
#define MG_SDIO_CCCR_SDIOREV 0x000
#define MG_SDIO_CCCR_SDREV 0x001
#define MG_SDIO_CCCR_IOEN 0x002
#define MG_SDIO_CCCR_IORDY 0x003
#define MG_SDIO_CCCR_INTEN 0x004
#define MG_SDIO_CCCR_BIC 0x007
#define MG_SDIO_CCCR_CCAP 0x008
#define MG_SDIO_CCCR_CCIS 0x009     // 3 registers
#define MG_SDIO_CCCR_F0BLKSZ 0x010  // 2 registers
#define MG_SDIO_CCCR_HISPD 0x013
// SDIO 6.10 Table 6-3 FBR (Function Basic Registers)
#define MG_SDIO_FBR_FnBLKSZ(n) (((n) &7) * 0x100 + 0x10)  // 2 registers

// SDIO 5.1 IO_RW_DIRECT Command (CMD52)
#define MG_SDIO_DATA(x) ((x) &0xFF)            // bits 0-7
#define MG_SDIO_ADDR(x) (((x) &0x1FFFF) << 9)  // bits 9-25
#define MG_SDIO_FUNC(x) (((x) &3) << 28)       // bits 28-30 (30 unused here)
#define MG_SDIO_WR MG_BIT(31)

// SDIO 5.3 IO_RW_EXTENDED Command (CMD53)
#define MG_SDIO_LEN(x) ((x) &0x1FF)  // bits 0-8
#define MG_SDIO_OPINC MG_BIT(26)
#define MG_SDIO_BLKMODE MG_BIT(27)

// - Drivers set blocksize, drivers request transfers. Requesting a read
// transfer > blocksize means block transfer will be used.
// - To simplify the use of DMA transfers and avoid intermediate buffers,
// drivers must have room to accomodate a whole block transfer, e.g.: blocksize
// = 64, read 65 => 2 blocks = 128 bytes
// - Transfers of more than 1 byte assume (uint32_t *) data. 1-byte transfers
// use (uint8_t *) data
// - 'len' is the number of _bytes_ to transfer
bool mg_sdio_transfer(struct mg_tcpip_sdio *sdio, bool write, unsigned int f,
                      uint32_t addr, void *data, uint32_t len) {
  uint32_t arg, val = 0;
  unsigned int blksz = 64;  // TODO(): mg_sdio_set_blksz() stores in an array,
                            // index on f, skip if 0
  if (len == 1) {
    arg = (write ? MG_SDIO_WR : 0) | MG_SDIO_FUNC(f) | MG_SDIO_ADDR(addr) |
          (write ? MG_SDIO_DATA(*(uint8_t *) data) : 0);
    bool res = sdio->txn(sdio, 52, arg, &val);  // IO_RW_DIRECT
    if (!write) *(uint8_t *) data = (uint8_t) val;
    return res;
  }
  // IO_RW_EXTENDED
  arg = (write ? MG_SDIO_WR : 0) | MG_SDIO_OPINC | MG_SDIO_FUNC(f) |
        MG_SDIO_ADDR(addr);
  if (len > 512 || (blksz != 0 && len > blksz)) {  // SDIO 5.3 512 -> len=0
    unsigned int blkcnt;
    if (blksz == 0) return false;  // > 512 requires block size set
    blkcnt = (len + blksz - 1) / blksz;
    if (blkcnt > 511) return false;  // we don't support "infinite" blocks
    arg |= MG_SDIO_BLKMODE | MG_SDIO_LEN(blkcnt);  // block transfer
    len = blksz * blkcnt;
  } else {
    arg |= MG_SDIO_LEN(len);  // multi-byte transfer
  }
  return sdio->xfr(sdio, write, arg,
                   (arg & MG_SDIO_BLKMODE) ? (uint16_t) blksz : 0,
                   (uint32_t *) data, len, &val);
}

bool mg_sdio_set_blksz(struct mg_tcpip_sdio *sdio, unsigned int f,
                       uint16_t blksz) {
  uint32_t val = blksz & 0xff;
  if (!mg_sdio_transfer(sdio, true, 0, MG_SDIO_FBR_FnBLKSZ(f), &val, 1))
    return false;
  val = (blksz >> 8) & 0x0f;  // SDIO 6.10 Table 6-4, max 2048
  if (!mg_sdio_transfer(sdio, true, 0, MG_SDIO_FBR_FnBLKSZ(f) + 1, &val, 1))
    return false;
  // TODO(): store in an array, index on f. Static 8-element array
  MG_VERBOSE(("F%c block size set", (f & 7) + '0'));
  return true;
}

// Enable Fx
bool mg_sdio_enable_f(struct mg_tcpip_sdio *sdio, unsigned int f) {
  uint8_t bit = 1U << (f & 7), bits;
  uint32_t val = 0;
  if (!mg_sdio_transfer(sdio, false, 0, MG_SDIO_CCCR_IOEN, &val, 1))
    return false;
  bits = (uint8_t) val | bit;
  unsigned int times = 501;
  while (times--) {
    val = bits; /* IOEf */
    ;
    if (!mg_sdio_transfer(sdio, true, 0, MG_SDIO_CCCR_IOEN, &val, 1))
      return false;
    mg_delayms(1);
    val = 0;
    if (!mg_sdio_transfer(sdio, false, 0, MG_SDIO_CCCR_IOEN, &val, 1))
      return false;
    if (val & bit) break;
  }
  if (times == (unsigned int) ~0) return false;
  MG_VERBOSE(("F%c enabled", (f & 7) + '0'));
  return true;
}

// Wait for Fx to be ready
bool mg_sdio_waitready_f(struct mg_tcpip_sdio *sdio, unsigned int f) {
  uint8_t bit = 1U << (f & 7);
  unsigned int times = 501;
  while (times--) {
    uint32_t val;
    if (!mg_sdio_transfer(sdio, false, 0, MG_SDIO_CCCR_IORDY, &val, 1))
      return false;
    if (val & bit) break;  // IORf
    mg_delayms(1);
  }
  if (times == (unsigned int) ~0) return false;
  MG_VERBOSE(("F%c ready", (f & 7) + '0'));
  return true;
}

// SDIO 6.14 Bus State Diagram
bool mg_sdio_init(struct mg_tcpip_sdio *sdio) {
  uint32_t val = 0;
  if (!sdio->txn(sdio, 0, 0, NULL)) return false;  // GO_IDLE_STATE
  sdio->txn(sdio, 5, 0, &val);                     // IO_SEND_OP_COND, no CRC
  MG_VERBOSE(("IO Functions: %u, Memory: %c", 1 + ((val >> 28) & 7),
              (val & MG_BIT(27)) ? 'Y' : 'N'));
  if (!sdio->txn(sdio, 3, 0, &val)) return false;  // SEND_RELATIVE_ADDR
  val = ((uint32_t) val) >> 16;                    // RCA
  if (!sdio->txn(sdio, 7, val << 16, &val))
    return false;  // SELECT/DESELECT_CARD
  mg_sdio_transfer(sdio, false, 0, MG_SDIO_CCCR_SDIOREV, &val, 1);
  MG_DEBUG(("CCCR: %u.%u, SDIO: %u.%u", 1 + ((val >> 2) & 3), (val >> 0) & 3,
            1 + ((val >> 6) & 3), (val >> 4) & 3));
  mg_sdio_transfer(sdio, false, 0, MG_SDIO_CCCR_SDREV, &val, 1);
  MG_VERBOSE(("SD: %u.%u", 1 + ((val >> 2) & 3), (val >> 0) & 3));
  mg_sdio_transfer(sdio, false, 0, MG_SDIO_CCCR_BIC, &val, 1);
  MG_SET_BITS(val, 3,
              MG_BIT(7) | MG_BIT(1));  // SDIO 6.9 Tables 6-1 6-2, 4-bit bus
  mg_sdio_transfer(sdio, true, 0, MG_SDIO_CCCR_BIC, &val, 1);
  // All Full-Speed SDIO cards support a 4-bit bus. Skip for Low-Speed SDIO
  // cards, we don't provide separate low-level functions for width and speed
  sdio->cfg(sdio, 0);  // set DS;
  if (!mg_sdio_transfer(sdio, false, 0, MG_SDIO_CCCR_HISPD, &val, 1))
    return false;
  if (val & MG_BIT(0) /* SHS */) {
    val = MG_BIT(1); /* EHS */
    if (!mg_sdio_transfer(sdio, true, 0, MG_SDIO_CCCR_HISPD, &val, 1))
      return false;
    sdio->cfg(sdio, 1);  // set HS;
    MG_VERBOSE(("Bus set to 4-bit @50MHz"));
  } else {
    MG_VERBOSE(("Bus set to 4-bit @25MHz"));
  }
  return true;
}

// - 6.11 Card Information Structure (CIS): 0x0001000-0x017FF; for card common
// and all functions
// - 16.5 SDIO Card Metaformat
// - 16.7.2 CISTPL_FUNCE (0x22): Function Extension Tuple, provides standard
// information about the card (common) and each individual function. One
// CISTPL_FUNCE in each function’s CIS, immediately following the CISTPL_FUNCID
// tuple
// - 16.7.3 CISTPL_FUNCE Tuple for Function 0 (common)
// - 16.7.4 CISTPL_FUNCE Tuple for Function 1-7

#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/st67w6.c"
#endif



#if MG_ENABLE_TCPIP && defined(MG_ENABLE_DRIVER_ST67W6) && \
    MG_ENABLE_DRIVER_ST67W6

static struct mg_tcpip_if *s_ifp;
static uint32_t s_ip, s_mask;
static bool s_link = false, s_connecting = false;

static void wifi_cb(struct mg_tcpip_if *ifp, int ev, void *ev_data) {
  struct mg_wifi_data *wifi =
      &((struct mg_tcpip_driver_st67w6_data *) ifp->driver_data)->wifi;
  if (wifi->apmode && ev == MG_TCPIP_EV_STATE_CHANGE &&
      *(uint8_t *) ev_data == MG_TCPIP_STATE_UP) {
    MG_DEBUG(("Access Point started"));
    s_ip = ifp->ip, ifp->ip = wifi->apip;
    s_mask = ifp->mask, ifp->mask = wifi->apmask;
    ifp->enable_dhcp_client = false;
    ifp->enable_dhcp_server = true;
  }
}

static bool st67w6_init(uint8_t *mac);
static void st67w6_poll(bool is_at);

static bool mg_tcpip_driver_st67w6_init(struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_st67w6_data *d =
      (struct mg_tcpip_driver_st67w6_data *) ifp->driver_data;
  struct mg_wifi_data *wifi = &d->wifi;
  if (MG_BIG_ENDIAN) {
    MG_ERROR(("Big-endian host"));
    return false;
  }
  if (d->is_ready == NULL) return false;
  s_ifp = ifp;
  s_ip = ifp->ip;
  s_mask = ifp->mask;
  s_link = false;
  ifp->pfn = wifi_cb;
  if (!st67w6_init(ifp->mac)) return false;

  if (d->send_queue.size == 0) d->send_queue.size = 8192;
  d->send_queue.buf = (char *) mg_calloc(1, d->send_queue.size);
  if (d->send_queue.buf == NULL) {
    MG_ERROR(("OOM"));
    return false;
  }

  if (wifi->apmode) {
    return mg_wifi_ap_start(wifi);
  } else if (wifi->ssid != NULL && wifi->pass != NULL) {
    return mg_wifi_connect(wifi);
  }
  return true;
}

// Decouple; module access depends on it being RDY. See st67w6_poll()
size_t mg_tcpip_driver_st67w6_output(const void *buf, size_t len,
                                     struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_st67w6_data *d =
      (struct mg_tcpip_driver_st67w6_data *) ifp->driver_data;
  char *p;
  if (mg_queue_book(&d->send_queue, &p, len) < len) return 0;
  memcpy(p, buf, len);
  mg_queue_add(&d->send_queue, len);
  return len;
}

static bool mg_tcpip_driver_st67w6_poll(struct mg_tcpip_if *ifp, bool s1) {
  struct mg_tcpip_driver_st67w6_data *d =
      (struct mg_tcpip_driver_st67w6_data *) ifp->driver_data;
  if (d->is_ready == NULL) return false;
  st67w6_poll(false);
  if (!s1) return false;
  return s_link;
}

struct mg_tcpip_driver mg_tcpip_driver_st67w6 = {
    mg_tcpip_driver_st67w6_init, mg_tcpip_driver_st67w6_output, NULL,
    mg_tcpip_driver_st67w6_poll};

//  AT | STA | AP | HCI | OT
// --------------------------
//       framing             <-- includes rx stall indication
// --------------------------
//         SPI               <-- padded to 32-bit
//
// Transactions take place when the module signals RDY, 'tx' and 'write'
// functions actually just write to memory
// - AT: handles configuration and events (unsolicited +foo:). Responses may
// come as:
//   - 1 frame: just OK or ERROR
//   - 2 frames: 1 text line each, a response and OK
//   - 3 frames: 2 text lines: 1 frame with text and no CR/LF, 1 frame with
//   binary and CR/LF, 1 text line with OK
// - STA and AP contain plain Ethernet frames to/from the STA and AP networks,
// respectively
// - HCI: BLE stuff
// - OT: ?

#pragma pack(push, 1)
// little endian

struct spi_hdr {
  uint16_t magic;
  uint16_t len;
  uint8_t vflags;  // version :2, rx_stall :1, flags :5
  uint8_t type;
  uint16_t reserved;
};

#define ST67W6_SPI_TYPE_AT 0
#define ST67W6_SPI_TYPE_STA 1
#define ST67W6_SPI_TYPE_AP 2
#define ST67W6_SPI_TYPE_HCI 3
#define ST67W6_SPI_TYPE_OT 4

#pragma pack(pop)

static uint32_t txdata[2048 / 4], rxdata[2048 / 4];
static uint8_t at_resp[300];

static bool s_at_ok, s_at_err;
static size_t s_at_resp_len;
static void st67w6_handle_wifi_evnt(char *, size_t len);
static void st67w6_handle_scan_result(char *, size_t len);

static size_t st67w6_spi_poll(uint8_t *write, uint8_t *read);
static void st67w6_write(unsigned int f, void *data, uint16_t len);
static void st67w6_update_hash_table(void);

// High-level comm stuff

static void st67w6_poll(bool is_at) {
  struct mg_tcpip_driver_st67w6_data *d =
      (struct mg_tcpip_driver_st67w6_data *) s_ifp->driver_data;
  struct spi_hdr *h = (struct spi_hdr *) rxdata;
  unsigned int type;
  uint8_t *txsource = (uint8_t *) txdata;
  if (s_ifp->update_mac_hash_table) {
    // first call to _poll() is after _init(), so this is safe
    st67w6_update_hash_table();
    s_ifp->update_mac_hash_table = false;
  }
  if (!is_at) {  // send outstanding WLAN frames in the queue
    char *buf;
    size_t len;
    // NOTE(): have traffic-dependent queues or queue traffic type with data
    if ((len = mg_queue_next(&d->send_queue, &buf)) > 0) {
      st67w6_write(d->wifi.apmode ? ST67W6_SPI_TYPE_AP : ST67W6_SPI_TYPE_STA,
                   buf, (uint16_t) len);
      mg_queue_del(&d->send_queue, len);
    } else {  // nothing to send
      txsource = NULL;
    }
  }
  if (st67w6_spi_poll(txsource, (uint8_t *) rxdata) == 0) return;
  if (h->len == 0) return;
  type = h->type;
  if (type == ST67W6_SPI_TYPE_AT) {
    char *p = (char *) (h + 1);
    size_t len = h->len;
    if (len > 7 && strncmp(p, "+CWLAP:", 7) == 0) {  // scan result
      st67w6_handle_scan_result(p + 7, len - 7);
    } else if (len > 4 && strncmp(p, "+CW:", 4) == 0) {
      st67w6_handle_wifi_evnt(p + 4, len - 4);
    } else {
      MG_VERBOSE(("AT partial: %.*s", (int) len, p));
      if (s_at_resp_len + len >= sizeof(at_resp))
        s_at_resp_len = 0;  // truncate response, error will be caught later
      memcpy(at_resp + s_at_resp_len, p, len);
      s_at_resp_len += len;
      if (mg_match(mg_str_n((char *) at_resp, s_at_resp_len),
                   mg_str_n("*ERROR*", 7), NULL)) {
        s_at_err = true;
      } else if (mg_match(mg_str_n((char *) at_resp, s_at_resp_len),
                          mg_str_n("*OK*", 4), NULL)) {
        s_at_ok = true;
      }
    }
  } else if (type == ST67W6_SPI_TYPE_STA || type == ST67W6_SPI_TYPE_AP) {
    // WLAN frame reception
    mg_tcpip_qwrite(h + 1, h->len, s_ifp);
  }  // else silently discard
}

// WLAN event handling

// - Do not call any AT functions here, otherwise revise st67w6_at_wait()
// - The module likes to send ERROR along with other events
static void st67w6_handle_wifi_evnt(char *p, size_t len) {
  struct mg_str data[2];
  MG_VERBOSE(("event: %.*s", (int) len, p));
  if (len > 9 && strncmp(p, "CONNECTED", 9) == 0) {
    s_link = true;
    s_connecting = false;
  } else if (len > 12 && strncmp(p, "DISCONNECTED", 12) == 0) {
    s_link = false;
    s_connecting = false;  // should not be needed
  } else if (s_connecting && len > 6 &&
             mg_match(mg_str_n(p, len), mg_str_n("ERROR,*\r\n*", 10), data)) {
    size_t reason = 0;
    bool ok = mg_to_size_t(data[0], &reason);
    s_connecting = false;
    MG_ERROR(("CONNECT FAILED"));
    mg_tcpip_call(s_ifp, MG_TCPIP_EV_WIFI_CONNECT_ERR, ok ? &reason : NULL);
  } else if (len > 9 && strncmp(p, "SCAN_DONE", 9) == 0) {
    MG_VERBOSE(("scan complete"));
    mg_tcpip_call(s_ifp, MG_TCPIP_EV_WIFI_SCAN_END, NULL);
  }  // else silently discard: CONNECTING; STA_CONNECTED,"MAC";
     // STA_DISCONNECTED,"MAC"; DIST_STA_IP,"MAC","IP"
}

static bool st67w6_at_cmd(char *cmd, size_t len);

// Wi-Fi network stuff

static bool st67w6_wifi_connect(char *ssid, char *pass) {
  char cmd[90];  // ssid + pass + AT
  size_t cmd_len;
  if (!st67w6_at_cmd("AT+CWMODE=1,0\r\n", 15)) return false;
  cmd_len = mg_snprintf(cmd, sizeof(cmd), "AT+CWJAP=\"%s\",\"%s\",,0\r\n", ssid,
                        pass);  // takes >700ms
  if (!st67w6_at_cmd(cmd, cmd_len)) return false;
  st67w6_at_cmd("AT+CWRECONNCFG=0,0\r\n", 20);  // disregard error, connecting
  s_connecting = true;
  return true;
}

static bool st67w6_wifi_disconnect(void) {
  s_connecting = false;
  if (!st67w6_at_cmd("AT+CWQAP=0\r\n", 12)) return false;  // takes >800ms
  return st67w6_at_cmd("AT+CWMODE=0,0\r\n", 15);           // takes >550ms
}

static bool st67w6_wifi_ap_start(char *ssid, char *pass, unsigned int channel) {
  char cmd[90];  // ssid + pass + AT
  size_t cmd_len;
  if (!st67w6_at_cmd("AT+CWMODE=2,0\r\n", 15)) return false;  // takes >800ms
  cmd_len = mg_snprintf(
      cmd, sizeof(cmd), "AT+CWSAP=\"%s\",\"%s\",%u,3,2,0\r\n", ssid, pass,
      channel);  // 3: WPA2_PSK; 2: max stations  // takes >350ms
  s_link = true;
  return st67w6_at_cmd(cmd, cmd_len);
}

static bool st67w6_wifi_ap_stop(void) {
  s_link = false;
  return st67w6_at_cmd("AT+CWMODE=0,0\r\n", 15);  // takes >550ms
}

// WLAN scan handling

// +CWLAP:(security,"SSID",RSSI,"BSSID",channel,cipher,proto,wps)\r\n
// security: OPEN, WEP, WPA, WPA2, WPA-WPA2, WPA-EAP, WPA3-SAE, WPA2-WPA3-SAE
// cipher: NONE, WEP, AES/CCMP, TKIP, TKIP and AES/CCMP
// proto: 4-bit bitmap AX,N,G,B; all set from right to left

static bool st67w6_wifi_scan(void) {
  return st67w6_at_cmd("AT+CWLAPOPT=1,1695,-100,255,50\r\n", 32) &&
         st67w6_at_cmd("AT+CWLAP=0,,,0\r\n", 16);
}

static void st67w6_handle_scan_result(char *data, size_t len) {
  struct mg_wifi_scan_bss_data bss;
  struct mg_str fields[2];
  char mac[6];
  uint8_t val;
  unsigned int i;
  MG_VERBOSE(("scan result event: %.*s", (int) len, data));
  ++data, --len;  // skip '('
  if (!mg_span(mg_str_n(data, len), &fields[0], &fields[1], ',') ||
      !mg_str_to_num(fields[0], 10, &val, 1))
    return;
  bss.security =
      (val == 0) ? MG_WIFI_SECURITY_OPEN : (uint8_t) MG_WIFI_SECURITY_WEP;
  if (val == 2 || val == 4) bss.security |= MG_WIFI_SECURITY_WPA;
  if (val == 3 || val == 4 || val == 7) bss.security |= MG_WIFI_SECURITY_WPA2;
  if (val == 6 || val == 7) bss.security |= MG_WIFI_SECURITY_WPA3;
  if (val == 5) bss.security |= MG_WIFI_SECURITY_WPA_ENTERPRISE;
  if (!mg_span(fields[1], &fields[0], &fields[1], ',')) return;
  bss.SSID.buf = fields[0].buf + 1, bss.SSID.len = fields[0].len - 2;
  if (!mg_span(fields[1], &fields[0], &fields[1], ',')) return;
  while (fields[0].buf[0] == ' ') ++fields[0].buf, --fields[0].len;
  if (fields[0].buf[0] != '-') return;  // positive RSSI would be great
  ++fields[0].buf, --fields[0].len;
  if (!mg_str_to_num(fields[0], 10, &val, 1)) return;
  bss.RSSI = (int8_t) - (int8_t) val;
  if (!mg_span(fields[1], &fields[0], &fields[1], ',')) return;
  if (fields[0].len < 19) return;
  ++fields[0].buf, --fields[0].len;  // skip '"'
  for (i = 0; i < 6; i++) {
    struct mg_str str;
    str.buf = fields[0].buf + 3 * i;
    str.len = 2;
    if (!mg_str_to_num(str, 16, &mac[i], 1)) return;
  }
  bss.BSSID = mac;
  if (!mg_span(fields[1], &fields[0], &fields[1], ',') ||
      !mg_str_to_num(fields[0], 10, &bss.channel, 1))
    return;
  if (!mg_span(fields[1], &fields[0], &fields[1], ',') ||
      !mg_str_to_num(fields[0], 10, &val, 1))
    return;
  // ignore cypher
  if (!mg_span(fields[1], &fields[0], &fields[1], ',') ||
      !mg_str_to_num(fields[0], 10, &val, 1))
    return;
  bss.has_n = (val & 4) != 0;
  bss.has_ax = (val & 8) != 0;
  bss.band = MG_WIFI_BAND_2G;  // NOT INFORMED with default options, no docs
  MG_VERBOSE(("BSS: %.*s (%u) (%M) %d dBm %u", bss.SSID.len, bss.SSID.buf,
              bss.channel, mg_print_mac, bss.BSSID, (int) bss.RSSI,
              bss.security));
  mg_tcpip_call(s_ifp, MG_TCPIP_EV_WIFI_SCAN_RESULT, &bss);
}

// AT stuff

static inline bool delayms(unsigned int ms) {
  mg_delayms(ms);
  return true;
}

// send AT command, wait for a response or timeout, meanwhile delivering
// received frames and events

static bool st67w6_at_cmd(char *cmd, size_t len) {
  bool is_at = true;
  unsigned int times = 1000;
  s_at_resp_len = 0;
  st67w6_write(ST67W6_SPI_TYPE_AT, cmd, (uint16_t) len);
  s_at_ok = false, s_at_err = false;
  do {  // AT response processing does not call any other AT function
    st67w6_poll(is_at);  // otherwise we can't allow them to pile up here
    is_at = false;       // avoid repeating, allow queued WLAN frames to be sent
    // network frames will be pushed to the queue so that is safe
  } while (!s_at_ok && !s_at_err && times-- > 0 && delayms(1));
  MG_VERBOSE(("AT response:\n%.*s", s_at_resp_len, at_resp));
  MG_VERBOSE(("ok: %c, err: %c, times: %d", s_at_ok ? '1' : '0',
              s_at_err ? '1' : '0', (int) times));
  return s_at_ok;
}

static bool st67w6_spi_init(void);
bool mg_to_size_t(struct mg_str str, size_t *val);

static bool st67w6_init(uint8_t *mac) {
  //  struct mg_tcpip_driver_st67w6_data *d = (struct
  //  mg_tcpip_driver_st67w6_data *) s_ifp->driver_data;
  struct mg_str data[3];
  size_t val;
  bool is_b = false;
  if (!st67w6_spi_init()) return false;
  if (!st67w6_at_cmd("AT\r\n", 4)) return false;
  if (!st67w6_at_cmd("AT+CWNETMODE?\r\n", 15) ||
      !mg_match(mg_str_n((char *) at_resp, s_at_resp_len),
                mg_str_n("*:*\r\n*", 6), data) ||
      !mg_to_size_t(data[1], &val))
    return false;
  if (val != 0) {
    MG_ERROR(("Wrong firmware, T02 is needed"));
    return false;
  }
  // set clock, who cares ???
  if (!st67w6_at_cmd("AT+GET_CLOCK\r\n", 14)) return false;
  MG_DEBUG(("%.*s", s_at_resp_len, at_resp));  // TODO(scaprile): --> VERBOSE
  // BT-ENABLED DEPENDENCY

  // MODULE DEPENDENCY
  if (!st67w6_at_cmd("AT+EFUSE-R=24,\"0x100\"\r\n", 23) ||
      !mg_match(mg_str_n((char *) at_resp, s_at_resp_len), mg_str_n("*,*", 3),
                data))
    return false;
  if (data[1].buf[0] == 'C' && data[1].buf[1] == '6') is_b = true;
  MG_DEBUG(("WLAN module is %sB type", is_b ? "" : "not"));  // --> VERBOSE
  if (is_b) {
    // Disable the antenna diversity pin
    if (!st67w6_at_cmd("AT+IORST=0\r\n", 12)) return false;
    // Apparently they intend to disable some antenna ...
    if (!st67w6_at_cmd("AT+CWANTENABLE?\r\n", 17)) return false;
    MG_DEBUG(("%.*s", s_at_resp_len, at_resp));  // --> VERBOSE
  }

  // Do not set wake-up pin (AT+SLWKIO)
  // Disable power save mode
  // NOTE(scaprile): (no response if in hibernate mode, though I guess we
  // wouldn't have reached this point in that case either)
  if (!st67w6_at_cmd("AT+PWR=0\r\n", 12)) return false;
  // set Wi-Fi
  // set country code
  if (!st67w6_at_cmd("AT+CWCOUNTRY=0,\"00\"\r\n", 21)) return false;
#if 0
	// set DTIM
  if (!st67w6_at_cmd("AT+SLWKDTIM=1\r\n", 15)) return false;
#endif
  // Read only default MAC, ignore set bit count (data[6] & 0x3F). Custom MACs
  // reside at @0x64 and 0x70
  if (st67w6_at_cmd("AT+EFUSE-R=7,\"0x014\"\r\n", 22) &&
      mg_match(mg_str_n((char *) at_resp, s_at_resp_len), mg_str_n("*,*", 3),
               data)) {
    int i;
    for (i = 0; i < 6; i++) mac[i] = data[1].buf[5 - i];
    MG_DEBUG(("MAC: %M", mg_print_mac, mac));
  } else {
    MG_ERROR(("read MAC failed"));
  }
  return true;
}

static void st67w6_update_hash_table(void) {
  // TODO(): read database, rebuild hash table
  //  uint32_t val = 0;
  //  val = 1;
  //  st67w6_at_iovar_set2_(0, "mcast_list", (uint8_t *) &val, sizeof(val),
  //  (uint8_t *) mcast_addr, sizeof(mcast_addr)); mg_delayms(50);
}

// SPI specifics

#define ST67W6_SPI_MAGIC 0x55AA
// #define ST67W6_SPI_VERSION(x), IS_STALL, FLAGS(x), ...

static const uint8_t idlehdr[sizeof(struct spi_hdr)] = {0xaa, 0x55, 0, 0,
                                                        0,    0,    0, 0};

static void st67w6_write(unsigned int f, void *data, uint16_t len) {
  struct spi_hdr *h = (struct spi_hdr *) txdata;
  h->magic = ST67W6_SPI_MAGIC;
  h->type = (uint8_t) f;
  h->vflags = 0;
  h->len = len;
  h->reserved = 0;
  memmove(h + 1, data, len);
}

static size_t st67w6_spi_poll(uint8_t *write, uint8_t *read) {
  struct mg_tcpip_driver_st67w6_data *d =
      (struct mg_tcpip_driver_st67w6_data *) s_ifp->driver_data;
  struct spi_hdr *th, *rh = (struct spi_hdr *) read;
  struct mg_tcpip_spi *s = (struct mg_tcpip_spi *) d->spi;
  size_t padded;
  unsigned int times;

  th = (write != NULL) ? (struct spi_hdr *) write : (struct spi_hdr *) idlehdr;
  s->begin(s->spi);
  times = 50;
  while (times--) {
    if (d->is_ready()) break;
    if (times == 0) {
      MG_ERROR(("RDY TIMEOUT"));
      s->end(s->spi);
      return 0;
    }
    mg_delayms(1);
  }

  padded = (th->len + 3) & ~3;
  s->txn(s->spi, (uint8_t *) th, (uint8_t *) rh, sizeof(*th) + padded);
  if (rh->magic == ST67W6_SPI_MAGIC && rh->len > padded) {
    size_t remaining_padded = (rh->len - padded + 3) & ~3;
    if (remaining_padded > (2048 - sizeof(*rh) - padded))
      remaining_padded = 2048 - sizeof(*rh) - padded;
    s->txn(s->spi, NULL, read + sizeof(*rh) + padded, remaining_padded);
  }
  times = 50;
  while (times--) {
    if (!d->is_ready()) break;
    if (times == 0) {
      MG_ERROR(("!RDY TIMEOUT"));
      break;
    }
    mg_delayms(1);
  }
  s->end(s->spi);
  return (size_t) rh->len;
}

static bool st67w6_spi_init(void) {
  struct mg_tcpip_driver_st67w6_data *d =
      (struct mg_tcpip_driver_st67w6_data *) s_ifp->driver_data;
  size_t len;
  unsigned int times = 1000;
  while (times--) {
    if (d->is_ready()) break;
    if (times == 0) return false;
    mg_delayms(1);
  }
  if (((len = st67w6_spi_poll(NULL, (uint8_t *) rxdata)) == 0) ||
      !mg_match(mg_str_n(((char *) rxdata) + sizeof(struct spi_hdr), len),
                mg_str_n("*ready*", 7), NULL))
    return false;
  return true;
}

// Mongoose Wi-Fi API functions

bool mg_wifi_scan(void) {
  return st67w6_wifi_scan();
}

bool mg_wifi_connect(struct mg_wifi_data *wifi) {
  s_ifp->ip = s_ip;
  s_ifp->mask = s_mask;
  if (s_ifp->ip == 0) s_ifp->enable_dhcp_client = true;
  s_ifp->enable_dhcp_server = false;
  MG_DEBUG(("Connecting to '%s'", wifi->ssid));
  return st67w6_wifi_connect(wifi->ssid, wifi->pass);
}

bool mg_wifi_disconnect(void) {
  return st67w6_wifi_disconnect();
}

bool mg_wifi_ap_start(struct mg_wifi_data *wifi) {
  MG_DEBUG(("Starting AP '%s' (%u)", wifi->apssid, wifi->apchannel));
  return st67w6_wifi_ap_start(wifi->apssid, wifi->appass, wifi->apchannel);
}

bool mg_wifi_ap_stop(void) {
  return st67w6_wifi_ap_stop();
}

#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/stm32f.c"
#endif


#if MG_ENABLE_TCPIP && defined(MG_ENABLE_DRIVER_STM32F) && \
    MG_ENABLE_DRIVER_STM32F
struct stm32f_eth {
  volatile uint32_t MACCR, MACFFR, MACHTHR, MACHTLR, MACMIIAR, MACMIIDR, MACFCR,
      MACVLANTR, RESERVED0[2], MACRWUFFR, MACPMTCSR, RESERVED1, MACDBGR, MACSR,
      MACIMR, MACA0HR, MACA0LR, MACA1HR, MACA1LR, MACA2HR, MACA2LR, MACA3HR,
      MACA3LR, RESERVED2[40], MMCCR, MMCRIR, MMCTIR, MMCRIMR, MMCTIMR,
      RESERVED3[14], MMCTGFSCCR, MMCTGFMSCCR, RESERVED4[5], MMCTGFCR,
      RESERVED5[10], MMCRFCECR, MMCRFAECR, RESERVED6[10], MMCRGUFCR,
      RESERVED7[334], PTPTSCR, PTPSSIR, PTPTSHR, PTPTSLR, PTPTSHUR, PTPTSLUR,
      PTPTSAR, PTPTTHR, PTPTTLR, RESERVED8, PTPTSSR, PTPPPSCR, RESERVED9[564],
      DMABMR, DMATPDR, DMARPDR, DMARDLAR, DMATDLAR, DMASR, DMAOMR, DMAIER,
      DMAMFBOCR, DMARSWTR, RESERVED10[8], DMACHTDR, DMACHRDR, DMACHTBAR,
      DMACHRBAR;
};
#undef ETH
#define ETH ((struct stm32f_eth *) (uintptr_t) 0x40028000)

#define ETH_PKT_SIZE 1540  // Max frame size
#define ETH_DESC_CNT 4     // Descriptors count
#define ETH_DS 4           // Descriptor size (words)

static uint32_t s_rxdesc[ETH_DESC_CNT][ETH_DS] MG_ETH_RAM;      // RX descriptors
static uint32_t s_txdesc[ETH_DESC_CNT][ETH_DS] MG_ETH_RAM;      // TX descriptors
static uint8_t s_rxbuf[ETH_DESC_CNT][ETH_PKT_SIZE] MG_ETH_RAM;  // RX ethernet buffers
static uint8_t s_txbuf[ETH_DESC_CNT][ETH_PKT_SIZE] MG_ETH_RAM;  // TX ethernet buffers
static uint8_t s_txno;                               // Current TX descriptor
static uint8_t s_rxno;                               // Current RX descriptor

static struct mg_tcpip_if *s_ifp;  // MIP interface

static uint16_t eth_read_phy(uint8_t addr, uint8_t reg) {
  ETH->MACMIIAR &= (7 << 2);
  ETH->MACMIIAR |= ((uint32_t) addr << 11) | ((uint32_t) reg << 6);
  ETH->MACMIIAR |= MG_BIT(0);
  while (ETH->MACMIIAR & MG_BIT(0)) (void) 0;
  return ETH->MACMIIDR & 0xffff;
}

static void eth_write_phy(uint8_t addr, uint8_t reg, uint16_t val) {
  ETH->MACMIIDR = val;
  ETH->MACMIIAR &= (7 << 2);
  ETH->MACMIIAR |= ((uint32_t) addr << 11) | ((uint32_t) reg << 6) | MG_BIT(1);
  ETH->MACMIIAR |= MG_BIT(0);
  while (ETH->MACMIIAR & MG_BIT(0)) (void) 0;
}

static uint32_t get_hclk(void) {
  struct rcc {
    volatile uint32_t CR, PLLCFGR, CFGR;
  } *rcc = (struct rcc *) 0x40023800;
  uint32_t clk = 0, hsi = 16000000 /* 16 MHz */, hse = 8000000 /* 8MHz */;

  if (rcc->CFGR & (1 << 2)) {
    clk = hse;
  } else if (rcc->CFGR & (1 << 3)) {
    uint32_t vco, m, n, p;
    m = (rcc->PLLCFGR & (0x3f << 0)) >> 0;
    n = (rcc->PLLCFGR & (0x1ff << 6)) >> 6;
    p = (((rcc->PLLCFGR & (3 << 16)) >> 16) + 1) * 2;
    clk = (rcc->PLLCFGR & (1 << 22)) ? hse : hsi;
    vco = (uint32_t) ((uint64_t) clk * n / m);
    clk = vco / p;
  } else {
    clk = hsi;
  }
  uint32_t hpre = (rcc->CFGR & (15 << 4)) >> 4;
  if (hpre < 8) return clk;

  uint8_t ahbptab[8] = {1, 2, 3, 4, 6, 7, 8, 9};  // log2(div)
  return ((uint32_t) clk) >> ahbptab[hpre - 8];
}

//  Guess CR from HCLK. MDC clock is generated from HCLK (AHB); as per 802.3,
//  it must not exceed 2.5MHz As the AHB clock can be (and usually is) derived
//  from the HSI (internal RC), and it can go above specs, the datasheets
//  specify a range of frequencies and activate one of a series of dividers to
//  keep the MDC clock safely below 2.5MHz. We guess a divider setting based on
//  HCLK with a +5% drift. If the user uses a different clock from our
//  defaults, needs to set the macros on top Valid for STM32F74xxx/75xxx
//  (38.8.1) and STM32F42xxx/43xxx (33.8.1) (both 4.5% worst case drift)
static int guess_mdc_cr(void) {
  uint8_t crs[] = {2, 3, 0, 1, 4, 5};          // ETH->MACMIIAR::CR values
  uint8_t div[] = {16, 26, 42, 62, 102, 124};  // Respective HCLK dividers
  uint32_t hclk = get_hclk();                  // Guess system HCLK
  int result = -1;                             // Invalid CR value
  if (hclk < 25000000) {
    MG_ERROR(("HCLK too low"));
  } else {
    for (int i = 0; i < 6; i++) {
      if (hclk / div[i] <= 2375000UL /* 2.5MHz - 5% */) {
        result = crs[i];
        break;
      }
    }
    if (result < 0) MG_ERROR(("HCLK too high"));
  }
  MG_DEBUG(("HCLK: %u, CR: %d", hclk, result));
  return result;
}

static bool mg_tcpip_driver_stm32f_init(struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_stm32f_data *d =
      (struct mg_tcpip_driver_stm32f_data *) ifp->driver_data;
  uint8_t phy_addr = d == NULL ? 0 : d->phy_addr;
  s_ifp = ifp;

  // Init RX descriptors
  for (int i = 0; i < ETH_DESC_CNT; i++) {
    s_rxdesc[i][0] = MG_BIT(31);                         // Own
    s_rxdesc[i][1] = sizeof(s_rxbuf[i]) | MG_BIT(14);    // 2nd address chained
    s_rxdesc[i][2] = (uint32_t) (uintptr_t) s_rxbuf[i];  // Point to data buffer
    s_rxdesc[i][3] =
        (uint32_t) (uintptr_t) s_rxdesc[(i + 1) % ETH_DESC_CNT];  // Chain
  }

  // Init TX descriptors
  for (int i = 0; i < ETH_DESC_CNT; i++) {
    s_txdesc[i][2] = (uint32_t) (uintptr_t) s_txbuf[i];  // Buf pointer
    s_txdesc[i][3] =
        (uint32_t) (uintptr_t) s_txdesc[(i + 1) % ETH_DESC_CNT];  // Chain
  }

  ETH->DMABMR |= MG_BIT(0);                         // Software reset
  while ((ETH->DMABMR & MG_BIT(0)) != 0) (void) 0;  // Wait until done

  // Set MDC clock divider. If user told us the value, use it. Otherwise, guess
  int cr = (d == NULL || d->mdc_cr < 0) ? guess_mdc_cr() : d->mdc_cr;
  ETH->MACMIIAR = ((uint32_t) cr & 7) << 2;

  // NOTE(cpq): we do not use extended descriptor bit 7, and do not use
  // hardware checksum. Therefore, descriptor size is 4, not 8
  // ETH->DMABMR = MG_BIT(13) | MG_BIT(16) | MG_BIT(22) | MG_BIT(23) |
  // MG_BIT(25);
  ETH->MACIMR = MG_BIT(3) | MG_BIT(9);  // Mask timestamp & PMT IT
  ETH->MACFCR = MG_BIT(7);              // Disable zero quarta pause
  ETH->MACFFR = MG_BIT(10);             // Perfect filtering
  struct mg_phy phy = {eth_read_phy, eth_write_phy};
  mg_phy_init(&phy, phy_addr, MG_PHY_CLOCKS_MAC);
  ETH->DMARDLAR = (uint32_t) (uintptr_t) s_rxdesc;  // RX descriptors
  ETH->DMATDLAR = (uint32_t) (uintptr_t) s_txdesc;  // RX descriptors
  ETH->DMAIER = MG_BIT(6) | MG_BIT(16);             // RIE, NISE
  ETH->MACCR =
      MG_BIT(2) | MG_BIT(3) | MG_BIT(11) | MG_BIT(14);  // RE, TE, Duplex, Fast
  ETH->DMAOMR =
      MG_BIT(1) | MG_BIT(13) | MG_BIT(21) | MG_BIT(25);  // SR, ST, TSF, RSF

  // MAC address filtering
  ETH->MACA0HR = ((uint32_t) ifp->mac[5] << 8U) | ifp->mac[4];
  ETH->MACA0LR = (uint32_t) (ifp->mac[3] << 24) |
                 ((uint32_t) ifp->mac[2] << 16) |
                 ((uint32_t) ifp->mac[1] << 8) | ifp->mac[0];
  return true;
}

static size_t mg_tcpip_driver_stm32f_tx(const void *buf, size_t len,
                                        struct mg_tcpip_if *ifp) {
  if (len > sizeof(s_txbuf[s_txno])) {
    MG_ERROR(("Frame too big, %ld", (long) len));
    len = 0;  // Frame is too big
  } else if ((s_txdesc[s_txno][0] & MG_BIT(31))) {
    ifp->nerr++;
    MG_ERROR(("No free descriptors"));
    // printf("D0 %lx SR %lx\n", (long) s_txdesc[0][0], (long) ETH->DMASR);
    len = 0;  // All descriptors are busy, fail
  } else {
    memcpy(s_txbuf[s_txno], buf, len);                           // Copy data
    s_txdesc[s_txno][1] = (uint32_t) len;                        // Set data len
    s_txdesc[s_txno][0] = MG_BIT(20) | MG_BIT(28) | MG_BIT(29);  // Chain,FS,LS
    s_txdesc[s_txno][0] |= MG_BIT(31);  // Set OWN bit - let DMA take over
    if (++s_txno >= ETH_DESC_CNT) s_txno = 0;
  }
  MG_DSB();                            // ensure descriptors have been written
  ETH->DMASR = MG_BIT(2) | MG_BIT(5);  // Clear any prior TBUS/TUS
  ETH->DMATPDR = 0;                    // and resume
  return len;
}

static void mg_tcpip_driver_stm32f_update_hash_table(struct mg_tcpip_if *ifp) {
  // TODO(): read database, rebuild hash table
  ETH->MACA1LR = (uint32_t) mcast_addr[3] << 24 |
                 (uint32_t) mcast_addr[2] << 16 |
                 (uint32_t) mcast_addr[1] << 8 | (uint32_t) mcast_addr[0];
  ETH->MACA1HR = (uint32_t) mcast_addr[5] << 8 | (uint32_t) mcast_addr[4];
  ETH->MACA1HR |= MG_BIT(31);  // AE
  (void) ifp;
}

static bool mg_tcpip_driver_stm32f_poll(struct mg_tcpip_if *ifp, bool s1) {
  if (ifp->update_mac_hash_table) {
    mg_tcpip_driver_stm32f_update_hash_table(ifp);
    ifp->update_mac_hash_table = false;
  }
  if (!s1) return false;
  struct mg_tcpip_driver_stm32f_data *d =
      (struct mg_tcpip_driver_stm32f_data *) ifp->driver_data;
  uint8_t phy_addr = d == NULL ? 0 : d->phy_addr;
  uint8_t speed = MG_PHY_SPEED_10M;
  bool up = false, full_duplex = false;
  struct mg_phy phy = {eth_read_phy, eth_write_phy};
  up = mg_phy_up(&phy, phy_addr, &full_duplex, &speed);
  if ((ifp->state == MG_TCPIP_STATE_DOWN) && up) {  // link state just went up
    // tmp = reg with flags set to the most likely situation: 100M full-duplex
    // if(link is slow or half) set flags otherwise
    // reg = tmp
    uint32_t maccr = ETH->MACCR | MG_BIT(14) | MG_BIT(11);  // 100M, Full-duplex
    if (speed == MG_PHY_SPEED_10M) maccr &= ~MG_BIT(14);    // 10M
    if (full_duplex == false) maccr &= ~MG_BIT(11);         // Half-duplex
    ETH->MACCR = maccr;  // IRQ handler does not fiddle with this register
    MG_DEBUG(("Link is %uM %s-duplex", maccr & MG_BIT(14) ? 100 : 10,
              maccr & MG_BIT(11) ? "full" : "half"));
  }
  return up;
}

#ifdef __riscv
__attribute__((interrupt()))  // For RISCV CH32V307, which share the same MAC
#endif
void ETH_IRQHandler(void);
void ETH_IRQHandler(void) {
  if (ETH->DMASR & MG_BIT(6)) {           // Frame received, loop
    ETH->DMASR = MG_BIT(16) | MG_BIT(6);  // Clear flag
    for (uint32_t i = 0; i < 10; i++) {   // read as they arrive but not forever
      if (s_rxdesc[s_rxno][0] & MG_BIT(31)) break;  // exit when done
      if (((s_rxdesc[s_rxno][0] & (MG_BIT(8) | MG_BIT(9))) ==
           (MG_BIT(8) | MG_BIT(9))) &&
          !(s_rxdesc[s_rxno][0] & MG_BIT(15))) {  // skip partial/errored frames
        uint32_t len = ((s_rxdesc[s_rxno][0] >> 16) & (MG_BIT(14) - 1));
        //  printf("%lx %lu %lx %.8lx\n", s_rxno, len, s_rxdesc[s_rxno][0],
        //  ETH->DMASR);
        mg_tcpip_qwrite(s_rxbuf[s_rxno], len > 4 ? len - 4 : len, s_ifp);
      }
      s_rxdesc[s_rxno][0] = MG_BIT(31);
      if (++s_rxno >= ETH_DESC_CNT) s_rxno = 0;
    }
  }
  // Cleanup flags
  ETH->DMASR = MG_BIT(16)    // NIS, normal interrupt summary
               | MG_BIT(7);  // Clear possible RBUS while processing
  ETH->DMARPDR = 0;          // and resume RX
}

struct mg_tcpip_driver mg_tcpip_driver_stm32f = {
    mg_tcpip_driver_stm32f_init, mg_tcpip_driver_stm32f_tx, NULL,
    mg_tcpip_driver_stm32f_poll};
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/stm32h.c"
#endif


#if MG_ENABLE_TCPIP && (MG_ENABLE_DRIVER_STM32H || MG_ENABLE_DRIVER_MCXN || \
                        MG_ENABLE_DRIVER_STM32N)
// STM32H: vendor modded single-queue Synopsys v4.2
// STM32N: dual-queue GbE Synopsys v5.2 with no hash table option, 64-bit AXI
// MCXNx4x: dual-queue Synopsys v5.2 with no hash table option
// RT1170: ENET_QOS: quad-queue Synopsys v5.1
#if MG_ENABLE_DRIVER_STM32H
#define SYNOPSYS_ENET_V5 0
#define SYNOPSYS_ENET_SINGLEQ 1
#define SYNOPSYS_ENET_NOHASHTABLE 0
#define SYNOPSYS_ENET_GbE 0
#elif MG_ENABLE_DRIVER_STM32N
#define SYNOPSYS_ENET_V5 1
#define SYNOPSYS_ENET_SINGLEQ 0
#define SYNOPSYS_ENET_NOHASHTABLE 1
#define SYNOPSYS_ENET_GbE 1
#elif MG_ENABLE_DRIVER_MCXN
#define SYNOPSYS_ENET_V5 1
#define SYNOPSYS_ENET_SINGLEQ 0
#define SYNOPSYS_ENET_NOHASHTABLE 1
#define SYNOPSYS_ENET_GbE 0
#endif

struct synopsys_enet_qos {
  volatile uint32_t MACCR, MACECR, MACPFR, MACWTR, MACHT0R, MACHT1R,
      RESERVED1[14], MACVTR, RESERVED2, MACVHTR, RESERVED3, MACVIR, MACIVIR,
      RESERVED4[2], MACTFCR, RESERVED5[7], MACRFCR, RESERVED6[7], MACISR,
      MACIER, MACRXTXSR, RESERVED7, MACPCSR, MACRWKPFR, RESERVED8[2], MACLCSR,
      MACLTCR, MACLETR, MAC1USTCR, RESERVED9[12], MACVR, MACDR, RESERVED10,
      MACHWF0R, MACHWF1R, MACHWF2R, RESERVED11[54], MACMDIOAR, MACMDIODR,
      RESERVED12[2], MACARPAR, RESERVED13[59], MACA0HR, MACA0LR, MACA1HR,
      MACA1LR, MACA2HR, MACA2LR, MACA3HR, MACA3LR, RESERVED14[248], MMCCR,
      MMCRIR, MMCTIR, MMCRIMR, MMCTIMR, RESERVED15[14], MMCTSCGPR, MMCTMCGPR,
      RESERVED16[5], MMCTPCGR, RESERVED17[10], MMCRCRCEPR, MMCRAEPR,
      RESERVED18[10], MMCRUPGR, RESERVED19[9], MMCTLPIMSTR, MMCTLPITCR,
      MMCRLPIMSTR, MMCRLPITCR, RESERVED20[65], MACL3L4C0R, MACL4A0R,
      RESERVED21[2], MACL3A0R0R, MACL3A1R0R, MACL3A2R0R, MACL3A3R0R,
      RESERVED22[4], MACL3L4C1R, MACL4A1R, RESERVED23[2], MACL3A0R1R,
      MACL3A1R1R, MACL3A2R1R, MACL3A3R1R, RESERVED24[108], MACTSCR, MACSSIR,
      MACSTSR, MACSTNR, MACSTSUR, MACSTNUR, MACTSAR, RESERVED25, MACTSSR,
      RESERVED26[3], MACTTSSNR, MACTTSSSR, RESERVED27[2], MACACR, RESERVED28,
      MACATSNR, MACATSSR, MACTSIACR, MACTSEACR, MACTSICNR, MACTSECNR,
      RESERVED29[4], MACPPSCR, RESERVED30[3], MACPPSTTSR, MACPPSTTNR, MACPPSIR,
      MACPPSWR, RESERVED31[12], MACPOCR, MACSPI0R, MACSPI1R, MACSPI2R, MACLMIR,
      RESERVED32[11], MTLOMR, RESERVED33[7], MTLISR, RESERVED34[55], MTLTQOMR,
      MTLTQUR, MTLTQDR, RESERVED35[8], MTLQICSR, MTLRQOMR, MTLRQMPOCR, MTLRQDR,
      RESERVED36[177], DMAMR, DMASBMR, DMAISR, DMADSR, RESERVED37[60], DMACCR,
      DMACTCR, DMACRCR, RESERVED38[2], DMACTDLAR, RESERVED39, DMACRDLAR,
      DMACTDTPR, RESERVED40, DMACRDTPR, DMACTDRLR, DMACRDRLR, DMACIER,
      DMACRIWTR, DMACSFCSR, RESERVED41, DMACCATDR, RESERVED42, DMACCARDR,
      RESERVED43, DMACCATBR, RESERVED44, DMACCARBR, DMACSR, RESERVED45[2],
      DMACMFCR;
};
#undef ETH
#if MG_ENABLE_DRIVER_STM32H
#define ETH ((struct synopsys_enet_qos *) (uintptr_t) 0x40028000UL)
#elif MG_ENABLE_DRIVER_STM32N
#define ETH ((struct synopsys_enet_qos *) (uintptr_t) 0x48036000UL)
#elif MG_ENABLE_DRIVER_MCXN
#define ETH ((struct synopsys_enet_qos *) (uintptr_t) 0x40100000UL)
#endif

#define ETH_PKT_SIZE 1540  // Max frame size
#define ETH_DESC_CNT 4     // Descriptors count
#define ETH_DS 4           // Descriptor size (words)

#if MG_ENABLE_DRIVER_STM32H || MG_ENABLE_DRIVER_STM32N
#define CACHE_LINESZ 32  // must be a whole number of (d)words (see DESC_SZW)
#ifndef SCB
struct m7_scb {
  volatile uint32_t CPUID, RESERVED1[4], CCR, RESERVED2[145], DCIMVAC, DCISW,
      DCCMVAU, DCCMVAC, DCCSW, DCCIMVAC, DCCISW, RESERVED3[10], CACR,
      RESERVED4[3];
};
#define SCB ((struct m7_scb *) (uintptr_t) 0xE000ED00UL)
#endif
// ending ISB is not needed because we don't cache instructions in data space
static inline void MG_CACHE_INVAL(uint8_t *addr, int32_t len) {
#if MG_ENABLE_DRIVER_STM32H
  if ((SCB->CPUID & 0xfff0) != 0xc270) return;  // not a Cortex-M7 => not an H7
#endif
  if ((SCB->CCR & MG_BIT(16)) == 0) return;     // cache not enabled
  MG_DSB();
  while (len > 0) {
    SCB->DCIMVAC = (uint32_t) addr;
    addr += CACHE_LINESZ;
    len -= CACHE_LINESZ;
  }
  MG_DSB();
}
static inline void MG_CACHE_FLUSH(uint8_t *addr, int32_t len) {
#if MG_ENABLE_DRIVER_STM32H
  if ((SCB->CPUID & 0xfff0) != 0xc270) return;  // not a Cortex-M7 => not an H7
#endif
  if ((SCB->CCR & MG_BIT(16)) == 0) return;     // cache not enabled
  MG_DSB();
  while (len > 0) {
    SCB->DCCMVAC = (uint32_t) addr;
    addr += CACHE_LINESZ;
    len -= CACHE_LINESZ;
  }
  MG_DSB();
}
#define ETH_RAM_ALIGNED MG_32BYTE_ALIGNED  // depends on CACHE_LINESZ and ETH
#define CACHE_ALIGN(x) \
  ((((size_t) (x)) + CACHE_LINESZ - 1) & ~(CACHE_LINESZ - 1))
#define DESC_SZ CACHE_ALIGN(4 * ETH_DS)  // grow descriptors to fit a line
#define DESC_SZW (DESC_SZ / 4)
#define BUFF_SZ CACHE_ALIGN(ETH_PKT_SIZE)  // grow buffers to fit n lines
#if MG_ENABLE_DRIVER_STM32H
#define DESC_SKIPW (DESC_SZW - ETH_DS)  // tell DMA the descriptor size, words
#else
// MG_ENABLE_DRIVER_STM32N, DMA is AXI and specs skip in 64-bit double-words
#define DESC_SKIPW ((DESC_SZW - ETH_DS) / 2)
#endif
#else
#define MG_CACHE_FLUSH(a, b)
#define MG_CACHE_INVAL(a, b)
#define ETH_RAM_ALIGNED MG_8BYTE_ALIGNED  // depends on ETH DMA alone
#define DESC_SZ 0
#define DESC_SZW ETH_DS
#define BUFF_SZ ETH_PKT_SIZE
#define DESC_SKIPW 0  // no need to skip, as we're not aligning to cache lines
#endif

// array[rows][cols] = coldata coldata ... for all rows, keeps alignment
static volatile uint32_t s_rxdesc[ETH_DESC_CNT][DESC_SZW] MG_ETH_RAM
    ETH_RAM_ALIGNED;
static volatile uint32_t s_txdesc[ETH_DESC_CNT][DESC_SZW] MG_ETH_RAM
    ETH_RAM_ALIGNED;
static uint8_t s_rxbuf[ETH_DESC_CNT][BUFF_SZ] MG_ETH_RAM ETH_RAM_ALIGNED;
static uint8_t s_txbuf[ETH_DESC_CNT][BUFF_SZ] MG_ETH_RAM ETH_RAM_ALIGNED;

static struct mg_tcpip_if *s_ifp;  // MIP interface

static uint16_t eth_read_phy(uint8_t addr, uint8_t reg) {
  ETH->MACMDIOAR &= (0xF << 8);
  ETH->MACMDIOAR |= ((uint32_t) addr << 21) | ((uint32_t) reg << 16) | 3 << 2;
  ETH->MACMDIOAR |= MG_BIT(0);
  while (ETH->MACMDIOAR & MG_BIT(0)) (void) 0;
  return (uint16_t) ETH->MACMDIODR;
}

static void eth_write_phy(uint8_t addr, uint8_t reg, uint16_t val) {
  ETH->MACMDIODR = val;
  ETH->MACMDIOAR &= (0xF << 8);
  ETH->MACMDIOAR |= ((uint32_t) addr << 21) | ((uint32_t) reg << 16) | 1 << 2;
  ETH->MACMDIOAR |= MG_BIT(0);
  while (ETH->MACMDIOAR & MG_BIT(0)) (void) 0;
}

static bool mg_tcpip_driver_stm32h_init(struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_stm32h_data *d =
      (struct mg_tcpip_driver_stm32h_data *) ifp->driver_data;
  s_ifp = ifp;
  uint8_t phy_addr = d == NULL ? 0 : d->phy_addr;
  uint8_t phy_conf = d == NULL ? MG_PHY_CLOCKS_MAC : d->phy_conf;

  // Init RX descriptors
  memset((char *) s_rxdesc, 0, sizeof(s_rxdesc));  // manual init
  for (int i = 0; i < ETH_DESC_CNT; i++) {
    s_rxdesc[i][0] = (uint32_t) (uintptr_t) s_rxbuf[i];  // Point to data buffer
    s_rxdesc[i][3] = MG_BIT(31) | MG_BIT(30) | MG_BIT(24);  // OWN, IOC, BUF1V
  }
  MG_CACHE_FLUSH((uint8_t *) s_rxdesc, sizeof(s_rxdesc));

  // Init TX descriptors
  memset((char *) s_txdesc, 0, sizeof(s_txdesc));  // manual init
  for (int i = 0; i < ETH_DESC_CNT; i++) {
    s_txdesc[i][0] = (uint32_t) (uintptr_t) s_txbuf[i];  // Buf pointer
  }
  MG_CACHE_FLUSH((uint8_t *) s_txdesc, sizeof(s_txdesc));

  ETH->DMAMR |= MG_BIT(0);  // Software reset
  for (int i = 0; i < 4; i++)
    (void) 0;  // wait at least 4 clocks before reading
  while ((ETH->DMAMR & MG_BIT(0)) != 0) (void) 0;  // Wait until done

  // Set MDC clock divider. Get user value, else, assume max freq
  int cr = (d == NULL || d->mdc_cr < 0) ? 7 : d->mdc_cr;
  ETH->MACMDIOAR = ((uint32_t) cr & 0xF) << 8;

  // NOTE(scaprile): We do not use timing facilities so the DMA engine does not
  // re-write buffer address
  ETH->DMAMR = 0 << 16;        // use interrupt mode 0 (58.8.1) (reset value)
  ETH->DMASBMR |= MG_BIT(12);  // AAL NOTE(scaprile): is this actually needed
  ETH->MACIER = 0;  // Do not enable additional irq sources (reset value)
  ETH->MACTFCR = MG_BIT(7);  // Disable zero-quanta pause
#if !SYNOPSYS_ENET_V5
  ETH->MACPFR = MG_BIT(10);  // Perfect filtering
#endif
  struct mg_phy phy = {eth_read_phy, eth_write_phy};
  mg_phy_init(&phy, phy_addr, phy_conf);
  ETH->DMACRDLAR =
      (uint32_t) (uintptr_t) s_rxdesc;  // RX descriptors start address
  ETH->DMACRDRLR = ETH_DESC_CNT - 1;    // ring length
  ETH->DMACRDTPR =
      (uint32_t) (uintptr_t) &s_rxdesc[ETH_DESC_CNT -
                                       1];  // last valid descriptor address
  ETH->DMACTDLAR =
      (uint32_t) (uintptr_t) s_txdesc;  // TX descriptors start address
  ETH->DMACTDRLR = ETH_DESC_CNT - 1;    // ring length
  ETH->DMACTDTPR =
      (uint32_t) (uintptr_t) s_txdesc;  // first available descriptor address
  ETH->DMACCR = DESC_SKIPW << 18;  // DSL (contiguous/sparse descriptor table)
#if SYNOPSYS_ENET_V5
  MG_SET_BITS(ETH->DMACTCR, 0x3F << 16, MG_BIT(16));
  MG_SET_BITS(ETH->DMACRCR, 0x3F << 16, MG_BIT(16));
#endif
  ETH->DMACIER = MG_BIT(6) | MG_BIT(15);  // RIE, NIE
  ETH->MACCR = MG_BIT(0) | MG_BIT(1) | MG_BIT(13) | MG_BIT(14) |
               MG_BIT(15);  // RE, TE, Duplex, Fast, (10/100)/Reserved
#if SYNOPSYS_ENET_SINGLEQ
  ETH->MTLTQOMR |= MG_BIT(1);  // TSF
  ETH->MTLRQOMR |= MG_BIT(5);  // RSF
#else
  ETH->MTLTQOMR |= (7 << 16) | MG_BIT(3) | MG_BIT(1);  // 2KB Q0, TSF
  ETH->MTLRQOMR |= (7 << 20) | MG_BIT(5);              // 2KB Q, RSF
  MG_SET_BITS(ETH->RESERVED6[3], 3, 2);  // Enable RxQ0 (MAC_RXQ_CTRL0)
#endif
  ETH->DMACTCR |= MG_BIT(0);  // ST
  ETH->DMACRCR |= MG_BIT(0);  // SR

  // MAC address filtering
  ETH->MACA0HR = ((uint32_t) ifp->mac[5] << 8U) | ifp->mac[4];
  ETH->MACA0LR = (uint32_t) (ifp->mac[3] << 24) |
                 ((uint32_t) ifp->mac[2] << 16) |
                 ((uint32_t) ifp->mac[1] << 8) | ifp->mac[0];
  return true;
}

static uint32_t s_txno;
static size_t mg_tcpip_driver_stm32h_tx(const void *buf, size_t len,
                                        struct mg_tcpip_if *ifp) {
  if (len > sizeof(s_txbuf[s_txno])) {
    MG_ERROR(("Frame too big, %ld", (long) len));
    return 0;  // Frame is too big
  }
  MG_CACHE_INVAL((uint8_t *) &s_txdesc[s_txno], DESC_SZ);
  if ((s_txdesc[s_txno][3] & MG_BIT(31))) {
    ifp->nerr++;
    MG_ERROR(("No free descriptors: %u %08X %08X %08X", s_txno,
              s_txdesc[s_txno][3], ETH->DMACSR, ETH->DMACTCR));
    MG_CACHE_INVAL((uint8_t *) s_txdesc, sizeof(s_txdesc));
    for (int i = 0; i < ETH_DESC_CNT; i++) MG_ERROR(("%08X", s_txdesc[i][3]));
    len = 0;  // All descriptors are busy, fail
  } else {
    memcpy(s_txbuf[s_txno], buf, len);  // Copy data
    MG_CACHE_FLUSH((uint8_t *) &s_txbuf[s_txno], BUFF_SZ);
    s_txdesc[s_txno][2] = (uint32_t) len;           // Set data len
    s_txdesc[s_txno][3] = MG_BIT(28) | MG_BIT(29);  // FD, LD
    s_txdesc[s_txno][3] |= MG_BIT(31);  // Set OWN bit - let DMA take over
    MG_CACHE_FLUSH((uint8_t *) &s_txdesc[s_txno], DESC_SZ);
    if (++s_txno >= ETH_DESC_CNT) s_txno = 0;
  }
  ETH->DMACSR |= MG_BIT(2) | MG_BIT(1);  // Clear any prior TBU, TPS
  ETH->DMACTDTPR = (uint32_t) (uintptr_t) &s_txdesc[s_txno];  // and resume
  return len;
  (void) ifp;
}

static void mg_tcpip_driver_stm32h_update_hash_table(struct mg_tcpip_if *ifp) {
#if SYNOPSYS_ENET_NOHASHTABLE
  ETH->MACPFR = MG_BIT(4);  // Pass Multicast (pass all multicast frames)
#else
  // TODO(): read database, rebuild hash table
  // add mDNS / DNS-SD multicast address
  ETH->MACA1LR = (uint32_t) mcast_addr[3] << 24 |
                 (uint32_t) mcast_addr[2] << 16 |
                 (uint32_t) mcast_addr[1] << 8 | (uint32_t) mcast_addr[0];
  ETH->MACA1HR = (uint32_t) mcast_addr[5] << 8 | (uint32_t) mcast_addr[4];
  ETH->MACA1HR |= MG_BIT(31);  // AE
#endif
  (void) ifp;
}

static bool mg_tcpip_driver_stm32h_poll(struct mg_tcpip_if *ifp, bool s1) {
  if (ifp->update_mac_hash_table) {
    mg_tcpip_driver_stm32h_update_hash_table(ifp);
    ifp->update_mac_hash_table = false;
  }
  if (!s1) return false;
  struct mg_tcpip_driver_stm32h_data *d =
      (struct mg_tcpip_driver_stm32h_data *) ifp->driver_data;
  uint8_t phy_addr = d == NULL ? 0 : d->phy_addr;
  uint8_t speed = MG_PHY_SPEED_10M;
  bool up = false, full_duplex = false;
  struct mg_phy phy = {eth_read_phy, eth_write_phy};
  up = mg_phy_up(&phy, phy_addr, &full_duplex, &speed);
  if ((ifp->state == MG_TCPIP_STATE_DOWN) && up) {  // link state just went up
    // tmp = reg with flags set to the most likely situation: 100M full-duplex
    // if(link is slow or half) set flags otherwise
    // reg = tmp
    uint32_t maccr = ETH->MACCR | MG_BIT(14) | MG_BIT(13);  // 100M, Full-duplex
#if SYNOPSYS_ENET_GbE
    if (speed == MG_PHY_SPEED_1000M) maccr &= ~MG_BIT(15);  // 1000M
#endif
    if (speed == MG_PHY_SPEED_10M) maccr &= ~MG_BIT(14);  // 10M
    if (full_duplex == false) maccr &= ~MG_BIT(13);       // Half-duplex
    ETH->MACCR = maccr;  // IRQ handler does not fiddle with this register
    MG_DEBUG(("Link is %uM %s-duplex", maccr & MG_BIT(14) ? 100 : 10,
              maccr & MG_BIT(13) ? "full" : "half"));
  }
  return up;
}

static uint32_t s_rxno;
#if MG_ENABLE_DRIVER_MCXN
void ETHERNET_IRQHandler(void);
void ETHERNET_IRQHandler(void) {
#elif MG_ENABLE_DRIVER_STM32H
void ETH_IRQHandler(void);
void ETH_IRQHandler(void) {
#else
void ETH1_IRQHandler(void);
void ETH1_IRQHandler(void) {
#endif
  if (ETH->DMACSR & MG_BIT(6)) {           // Frame received, loop
    ETH->DMACSR = MG_BIT(15) | MG_BIT(6);  // Clear flag
    for (uint32_t i = 0; i < 10; i++) {  // read as they arrive but not forever
      MG_CACHE_INVAL((uint8_t *) &s_rxdesc[s_rxno], DESC_SZ);
      if (s_rxdesc[s_rxno][3] & MG_BIT(31)) break;  // exit when done
      if (((s_rxdesc[s_rxno][3] & (MG_BIT(28) | MG_BIT(29))) ==
           (MG_BIT(28) | MG_BIT(29))) &&
          !(s_rxdesc[s_rxno][3] & MG_BIT(15))) {  // skip partial/errored frames
        uint32_t len = s_rxdesc[s_rxno][3] & (MG_BIT(15) - 1);
        // MG_DEBUG(("%lx %lu %lx %08lx", s_rxno, len, s_rxdesc[s_rxno][3],
        // ETH->DMACSR));
        MG_CACHE_INVAL((uint8_t *) &s_rxbuf[s_rxno], BUFF_SZ);
        mg_tcpip_qwrite(s_rxbuf[s_rxno], len > 4 ? len - 4 : len, s_ifp);
      }
      s_rxdesc[s_rxno][3] =
          MG_BIT(31) | MG_BIT(30) | MG_BIT(24);  // OWN, IOC, BUF1V
      MG_CACHE_FLUSH((uint8_t *) &s_rxdesc[s_rxno], DESC_SZ);
      if (++s_rxno >= ETH_DESC_CNT) s_rxno = 0;
    }
  }
  ETH->DMACSR =
      MG_BIT(7) | MG_BIT(8);  // Clear possible RBU RPS while processing
  ETH->DMACRDTPR =
      (uint32_t) (uintptr_t) &s_rxdesc[ETH_DESC_CNT - 1];  // and resume RX
}

struct mg_tcpip_driver mg_tcpip_driver_stm32h = {
    mg_tcpip_driver_stm32h_init, mg_tcpip_driver_stm32h_tx, NULL,
    mg_tcpip_driver_stm32h_poll};
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/tm4c.c"
#endif


#if MG_ENABLE_TCPIP && defined(MG_ENABLE_DRIVER_TM4C) && MG_ENABLE_DRIVER_TM4C
struct tm4c_emac {
  volatile uint32_t EMACCFG, EMACFRAMEFLTR, EMACHASHTBLH, EMACHASHTBLL,
      EMACMIIADDR, EMACMIIDATA, EMACFLOWCTL, EMACVLANTG, RESERVED0, EMACSTATUS,
      EMACRWUFF, EMACPMTCTLSTAT, RESERVED1[2], EMACRIS, EMACIM, EMACADDR0H,
      EMACADDR0L, EMACADDR1H, EMACADDR1L, EMACADDR2H, EMACADDR2L, EMACADDR3H,
      EMACADDR3L, RESERVED2[31], EMACWDOGTO, RESERVED3[8], EMACMMCCTRL,
      EMACMMCRXRIS, EMACMMCTXRIS, EMACMMCRXIM, EMACMMCTXIM, RESERVED4,
      EMACTXCNTGB, RESERVED5[12], EMACTXCNTSCOL, EMACTXCNTMCOL, RESERVED6[4],
      EMACTXOCTCNTG, RESERVED7[6], EMACRXCNTGB, RESERVED8[4], EMACRXCNTCRCERR,
      EMACRXCNTALGNERR, RESERVED9[10], EMACRXCNTGUNI, RESERVED10[239],
      EMACVLNINCREP, EMACVLANHASH, RESERVED11[93], EMACTIMSTCTRL, EMACSUBSECINC,
      EMACTIMSEC, EMACTIMNANO, EMACTIMSECU, EMACTIMNANOU, EMACTIMADD,
      EMACTARGSEC, EMACTARGNANO, EMACHWORDSEC, EMACTIMSTAT, EMACPPSCTRL,
      RESERVED12[12], EMACPPS0INTVL, EMACPPS0WIDTH, RESERVED13[294],
      EMACDMABUSMOD, EMACTXPOLLD, EMACRXPOLLD, EMACRXDLADDR, EMACTXDLADDR,
      EMACDMARIS, EMACDMAOPMODE, EMACDMAIM, EMACMFBOC, EMACRXINTWDT,
      RESERVED14[8], EMACHOSTXDESC, EMACHOSRXDESC, EMACHOSTXBA, EMACHOSRXBA,
      RESERVED15[218], EMACPP, EMACPC, EMACCC, RESERVED16, EMACEPHYRIS,
      EMACEPHYIM, EMACEPHYIMSC;
};
#undef EMAC
#define EMAC ((struct tm4c_emac *) (uintptr_t) 0x400EC000)

#define ETH_PKT_SIZE 1540  // Max frame size
#define ETH_DESC_CNT 4     // Descriptors count
#define ETH_DS 4           // Descriptor size (words)

static uint32_t s_rxdesc[ETH_DESC_CNT][ETH_DS];      // RX descriptors
static uint32_t s_txdesc[ETH_DESC_CNT][ETH_DS];      // TX descriptors
static uint8_t s_rxbuf[ETH_DESC_CNT][ETH_PKT_SIZE];  // RX ethernet buffers
static uint8_t s_txbuf[ETH_DESC_CNT][ETH_PKT_SIZE];  // TX ethernet buffers
static struct mg_tcpip_if *s_ifp;                    // MIP interface
enum {
  EPHY_ADDR = 0,
  EPHYBMCR = 0,
  EPHYBMSR = 1,
  EPHYSTS = 16
};  // PHY constants

static inline void tm4cspin(volatile uint32_t count) {
  while (count--) (void) 0;
}

static uint32_t emac_read_phy(uint8_t addr, uint8_t reg) {
  EMAC->EMACMIIADDR &= (0xf << 2);
  EMAC->EMACMIIADDR |= ((uint32_t) addr << 11) | ((uint32_t) reg << 6);
  EMAC->EMACMIIADDR |= MG_BIT(0);
  while (EMAC->EMACMIIADDR & MG_BIT(0)) tm4cspin(1);
  return EMAC->EMACMIIDATA;
}

static void emac_write_phy(uint8_t addr, uint8_t reg, uint32_t val) {
  EMAC->EMACMIIDATA = val;
  EMAC->EMACMIIADDR &= (0xf << 2);
  EMAC->EMACMIIADDR |=
      ((uint32_t) addr << 11) | ((uint32_t) reg << 6) | MG_BIT(1);
  EMAC->EMACMIIADDR |= MG_BIT(0);
  while (EMAC->EMACMIIADDR & MG_BIT(0)) tm4cspin(1);
}

static uint32_t get_sysclk(void) {
  struct sysctl {
    volatile uint32_t DONTCARE0[44], RSCLKCFG, DONTCARE1[43], PLLFREQ0,
        PLLFREQ1;
  } *sysctl = (struct sysctl *) 0x400FE000;
  uint32_t clk = 0, piosc = 16000000 /* 16 MHz */, mosc = 25000000 /* 25MHz */;
  if (sysctl->RSCLKCFG & (1 << 28)) {  // USEPLL
    uint32_t fin, vco, mdiv, n, q, psysdiv;
    uint32_t pllsrc = (sysctl->RSCLKCFG & (0xf << 24)) >> 24;
    if (pllsrc == 0) {
      clk = piosc;
    } else if (pllsrc == 3) {
      clk = mosc;
    } else {
      MG_ERROR(("Unsupported clock source"));
    }
    q = (sysctl->PLLFREQ1 & (0x1f << 8)) >> 8;
    n = (sysctl->PLLFREQ1 & (0x1f << 0)) >> 0;
    fin = clk / ((q + 1) * (n + 1));
    mdiv = (sysctl->PLLFREQ0 & (0x3ff << 0)) >>
           0;  // mint + (mfrac / 1024); MFRAC not supported
    psysdiv = (sysctl->RSCLKCFG & (0x3f << 0)) >> 0;
    vco = (uint32_t) ((uint64_t) fin * mdiv);
    return vco / (psysdiv + 1);
  }
  uint32_t oscsrc = (sysctl->RSCLKCFG & (0xf << 20)) >> 20;
  if (oscsrc == 0) {
    clk = piosc;
  } else if (oscsrc == 3) {
    clk = mosc;
  } else {
    MG_ERROR(("Unsupported clock source"));
  }
  uint32_t osysdiv = (sysctl->RSCLKCFG & (0xf << 16)) >> 16;
  return clk / (osysdiv + 1);
}

//  Guess CR from SYSCLK. MDC clock is generated from SYSCLK (AHB); as per
//  802.3, it must not exceed 2.5MHz (also 20.4.2.6) As the AHB clock can be
//  derived from the PIOSC (internal RC), and it can go above  specs, the
//  datasheets specify a range of frequencies and activate one of a series of
//  dividers to keep the MDC clock safely below 2.5MHz. We guess a divider
//  setting based on SYSCLK with a +5% drift. If the user uses a different clock
//  from our defaults, needs to set the macros on top Valid for TM4C129x (20.7)
//  (4.5% worst case drift)
// The PHY receives the main oscillator (MOSC) (20.3.1)
static int guess_mdc_cr(void) {
  uint8_t crs[] = {2, 3, 0, 1};      // EMAC->MACMIIAR::CR values
  uint8_t div[] = {16, 26, 42, 62};  // Respective HCLK dividers
  uint32_t sysclk = get_sysclk();    // Guess system SYSCLK
  int i, result = -1;                // Invalid CR value
  if (sysclk < 25000000) {
    MG_ERROR(("SYSCLK too low"));
  } else {
    for (i = 0; i < 4; i++) {
      if (sysclk / div[i] <= 2375000UL /* 2.5MHz - 5% */) {
        result = crs[i];
        break;
      }
    }
    if (result < 0) MG_ERROR(("SYSCLK too high"));
  }
  MG_DEBUG(("SYSCLK: %u, CR: %d", sysclk, result));
  return result;
}

static bool mg_tcpip_driver_tm4c_init(struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_tm4c_data *d =
      (struct mg_tcpip_driver_tm4c_data *) ifp->driver_data;
  int i;
  s_ifp = ifp;

  // Init RX descriptors
  for (i = 0; i < ETH_DESC_CNT; i++) {
    s_rxdesc[i][0] = MG_BIT(31);                         // Own
    s_rxdesc[i][1] = sizeof(s_rxbuf[i]) | MG_BIT(14);    // 2nd address chained
    s_rxdesc[i][2] = (uint32_t) (uintptr_t) s_rxbuf[i];  // Point to data buffer
    s_rxdesc[i][3] =
        (uint32_t) (uintptr_t) s_rxdesc[(i + 1) % ETH_DESC_CNT];  // Chain
    // MG_DEBUG(("%d %p", i, s_rxdesc[i]));
  }

  // Init TX descriptors
  for (i = 0; i < ETH_DESC_CNT; i++) {
    s_txdesc[i][2] = (uint32_t) (uintptr_t) s_txbuf[i];  // Buf pointer
    s_txdesc[i][3] =
        (uint32_t) (uintptr_t) s_txdesc[(i + 1) % ETH_DESC_CNT];  // Chain
  }

  EMAC->EMACDMABUSMOD |= MG_BIT(0);  // Software reset
  while ((EMAC->EMACDMABUSMOD & MG_BIT(0)) != 0)
    tm4cspin(1);  // Wait until done

  // Set MDC clock divider. If user told us the value, use it. Otherwise, guess
  int cr = (d == NULL || d->mdc_cr < 0) ? guess_mdc_cr() : d->mdc_cr;
  EMAC->EMACMIIADDR = ((uint32_t) cr & 0xf) << 2;

  // NOTE(cpq): we do not use extended descriptor bit 7, and do not use
  // hardware checksum. Therefore, descriptor size is 4, not 8
  // EMAC->EMACDMABUSMOD = MG_BIT(13) | MG_BIT(16) | MG_BIT(22) | MG_BIT(23) |
  // MG_BIT(25);
  EMAC->EMACIM = MG_BIT(3) | MG_BIT(9);  // Mask timestamp & PMT IT
  EMAC->EMACFLOWCTL = MG_BIT(7);         // Disable zero-quanta pause
  EMAC->EMACFRAMEFLTR = MG_BIT(10);      // Perfect filtering
  // EMAC->EMACPC defaults to internal PHY (EPHY) in MMI mode
  emac_write_phy(EPHY_ADDR, EPHYBMCR, MG_BIT(15));  // Reset internal PHY (EPHY)
  emac_write_phy(EPHY_ADDR, EPHYBMCR, MG_BIT(12));  // Set autonegotiation
  EMAC->EMACRXDLADDR = (uint32_t) (uintptr_t) s_rxdesc;  // RX descriptors
  EMAC->EMACTXDLADDR = (uint32_t) (uintptr_t) s_txdesc;  // TX descriptors
  EMAC->EMACDMAIM = MG_BIT(6) | MG_BIT(16);              // RIE, NIE
  EMAC->EMACCFG =
      MG_BIT(2) | MG_BIT(3) | MG_BIT(11) | MG_BIT(14);  // RE, TE, Duplex, Fast
  EMAC->EMACDMAOPMODE =
      MG_BIT(1) | MG_BIT(13) | MG_BIT(21) | MG_BIT(25);  // SR, ST, TSF, RSF
  EMAC->EMACADDR0H = ((uint32_t) ifp->mac[5] << 8U) | ifp->mac[4];
  EMAC->EMACADDR0L = (uint32_t) (ifp->mac[3] << 24) |
                     ((uint32_t) ifp->mac[2] << 16) |
                     ((uint32_t) ifp->mac[1] << 8) | ifp->mac[0];
  return true;
}

static uint32_t s_txno;
static size_t mg_tcpip_driver_tm4c_tx(const void *buf, size_t len,
                                      struct mg_tcpip_if *ifp) {
  if (len > sizeof(s_txbuf[s_txno])) {
    MG_ERROR(("Frame too big, %ld", (long) len));
    len = 0;  // fail
  } else if ((s_txdesc[s_txno][0] & MG_BIT(31))) {
    ifp->nerr++;
    MG_ERROR(("No descriptors available"));
    // printf("D0 %lx SR %lx\n", (long) s_txdesc[0][0], (long)
    // EMAC->EMACDMARIS);
    len = 0;  // fail
  } else {
    memcpy(s_txbuf[s_txno], buf, len);     // Copy data
    s_txdesc[s_txno][1] = (uint32_t) len;  // Set data len
    s_txdesc[s_txno][0] =
        MG_BIT(20) | MG_BIT(28) | MG_BIT(29) | MG_BIT(30);  // Chain,FS,LS,IC
    s_txdesc[s_txno][0] |= MG_BIT(31);  // Set OWN bit - let DMA take over
    if (++s_txno >= ETH_DESC_CNT) s_txno = 0;
  }
  EMAC->EMACDMARIS = MG_BIT(2) | MG_BIT(5);  // Clear any prior TU/UNF
  EMAC->EMACTXPOLLD = 0;                     // and resume
  return len;
}

static void mg_tcpip_driver_tm4c_update_hash_table(struct mg_tcpip_if *ifp) {
  // TODO(): read database, rebuild hash table
  // add mDNS / DNS-SD multicast address
  EMAC->EMACADDR1L = (uint32_t) mcast_addr[3] << 24 |
                     (uint32_t) mcast_addr[2] << 16 |
                     (uint32_t) mcast_addr[1] << 8 | (uint32_t) mcast_addr[0];
  EMAC->EMACADDR1H = (uint32_t) mcast_addr[5] << 8 | (uint32_t) mcast_addr[4];
  EMAC->EMACADDR1H |= MG_BIT(31);  // AE
  (void) ifp;
}

static bool mg_tcpip_driver_tm4c_poll(struct mg_tcpip_if *ifp, bool s1) {
  if (ifp->update_mac_hash_table) {
    mg_tcpip_driver_tm4c_update_hash_table(ifp);
    ifp->update_mac_hash_table = false;
  }
  if (!s1) return false;
  uint32_t bmsr = emac_read_phy(EPHY_ADDR, EPHYBMSR);
  bool up = (bmsr & MG_BIT(2)) ? 1 : 0;
  if ((ifp->state == MG_TCPIP_STATE_DOWN) && up) {  // link state just went up
    uint32_t sts = emac_read_phy(EPHY_ADDR, EPHYSTS);
    // tmp = reg with flags set to the most likely situation: 100M full-duplex
    // if(link is slow or half) set flags otherwise
    // reg = tmp
    uint32_t emaccfg =
        EMAC->EMACCFG | MG_BIT(14) | MG_BIT(11);         // 100M, Full-duplex
    if (sts & MG_BIT(1)) emaccfg &= ~MG_BIT(14);         // 10M
    if ((sts & MG_BIT(2)) == 0) emaccfg &= ~MG_BIT(11);  // Half-duplex
    EMAC->EMACCFG = emaccfg;  // IRQ handler does not fiddle with this register
    MG_DEBUG(("Link is %uM %s-duplex", emaccfg & MG_BIT(14) ? 100 : 10,
              emaccfg & MG_BIT(11) ? "full" : "half"));
  }
  return up;
}

void EMAC0_IRQHandler(void);
static uint32_t s_rxno;
void EMAC0_IRQHandler(void) {
  int i;
  if (EMAC->EMACDMARIS & MG_BIT(6)) {           // Frame received, loop
    EMAC->EMACDMARIS = MG_BIT(16) | MG_BIT(6);  // Clear flag
    for (i = 0; i < 10; i++) {  // read as they arrive but not forever
      if (s_rxdesc[s_rxno][0] & MG_BIT(31)) break;  // exit when done
      if (((s_rxdesc[s_rxno][0] & (MG_BIT(8) | MG_BIT(9))) ==
           (MG_BIT(8) | MG_BIT(9))) &&
          !(s_rxdesc[s_rxno][0] & MG_BIT(15))) {  // skip partial/errored frames
        uint32_t len = ((s_rxdesc[s_rxno][0] >> 16) & (MG_BIT(14) - 1));
        //  printf("%lx %lu %lx %.8lx\n", s_rxno, len, s_rxdesc[s_rxno][0],
        //  EMAC->EMACDMARIS);
        mg_tcpip_qwrite(s_rxbuf[s_rxno], len > 4 ? len - 4 : len, s_ifp);
      }
      s_rxdesc[s_rxno][0] = MG_BIT(31);
      if (++s_rxno >= ETH_DESC_CNT) s_rxno = 0;
    }
  }
  EMAC->EMACDMARIS = MG_BIT(7);  // Clear possible RU while processing
  EMAC->EMACRXPOLLD = 0;         // and resume RX
}

struct mg_tcpip_driver mg_tcpip_driver_tm4c = {mg_tcpip_driver_tm4c_init,
                                               mg_tcpip_driver_tm4c_tx, NULL,
                                               mg_tcpip_driver_tm4c_poll};
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/tms570.c"
#endif


#if MG_ENABLE_TCPIP && defined(MG_ENABLE_DRIVER_TMS570) && MG_ENABLE_DRIVER_TMS570
struct tms570_emac_ctrl {
  volatile uint32_t REVID, SOFTRESET, RESERVED1[1], INTCONTROL, C0RXTHRESHEN,
  C0RXEN, C0TXEN, C0MISCEN, RESERVED2[8],
  C0RXTHRESHSTAT, C0RXSTAT, C0TXSTAT, C0MISCSTAT,
  RESERVED3[8],
  C0RXIMAX, C0TXIMAX;
};
struct tms570_emac {
  volatile uint32_t TXREVID, TXCONTROL, TXTEARDOWN, RESERVED1[1], RXREVID,
  RXCONTROL, RXTEARDOWN, RESERVED2[25], TXINTSTATRAW,TXINTSTATMASKED,
  TXINTMASKSET, TXINTMASKCLEAR, MACINVECTOR, MACEOIVECTOR, RESERVED8[2], RXINTSTATRAW,
  RXINTSTATMASKED, RXINTMASKSET, RXINTMASKCLEAR, MACINTSTATRAW, MACINTSTATMASKED,
  MACINTMASKSET, MACINTMASKCLEAR, RESERVED3[16], RXMBPENABLE, RXUNICASTSET,
  RXUNICASTCLEAR, RXMAXLEN, RXBUFFEROFFSET, RXFILTERLOWTHRESH, RESERVED9[2], RXFLOWTHRESH[8],
  RXFREEBUFFER[8], MACCONTROL, MACSTATUS, EMCONTROL, FIFOCONTROL, MACCONFIG,
  SOFTRESET, RESERVED4[22], MACSRCADDRLO, MACSRCADDRHI, MACHASH1, MACHASH2,
  BOFFTEST, TPACETEST, RXPAUSE, TXPAUSE, RESERVED5[4], RXGOODFRAMES, RXBCASTFRAMES,
  RXMCASTFRAMES, RXPAUSEFRAMES, RXCRCERRORS, RXALIGNCODEERRORS, RXOVERSIZED,
  RXJABBER, RXUNDERSIZED, RXFRAGMENTS, RXFILTERED, RXQOSFILTERED, RXOCTETS,
  TXGOODFRAMES, TXBCASTFRAMES, TXMCASTFRAMES, TXPAUSEFRAMES, TXDEFERRED,
  TXCOLLISION, TXSINGLECOLL, TXMULTICOLL, TXEXCESSIVECOLL, TXLATECOLL,
  TXUNDERRUN, TXCARRIERSENSE, TXOCTETS, FRAME64, FRAME65T127, FRAME128T255,
  FRAME256T511, FRAME512T1023, FRAME1024TUP, NETOCTETS, RXSOFOVERRUNS,
  RXMOFOVERRUNS, RXDMAOVERRUNS, RESERVED6[156], MACADDRLO, MACADDRHI,
  MACINDEX, RESERVED7[61], TXHDP[8], RXHDP[8], TXCP[8], RXCP[8];
};
struct tms570_mdio {
  volatile uint32_t REVID, CONTROL, ALIVE, LINK, LINKINTRAW, LINKINTMASKED,
  RESERVED1[2], USERINTRAW, USERINTMASKED, USERINTMASKSET, USERINTMASKCLEAR,
  RESERVED2[20], USERACCESS0, USERPHYSEL0, USERACCESS1, USERPHYSEL1;
};
#define SWAP32(x) ( (((x) & 0x000000FF) << 24) | \
                              (((x) & 0x0000FF00) << 8)  | \
                              (((x) & 0x00FF0000) >> 8)  | \
                              (((x) & 0xFF000000) >> 24) )
#undef EMAC
#undef EMAC_CTRL
#undef MDIO
#define EMAC ((struct tms570_emac *) (uintptr_t) 0xFCF78000)
#define EMAC_CTRL ((struct tms570_emac_ctrl *) (uintptr_t) 0xFCF78800)
#define MDIO ((struct tms570_mdio *) (uintptr_t) 0xFCF78900)
#define ETH_PKT_SIZE 1540  // Max frame size
#define ETH_DESC_CNT 4     // Descriptors count
#define ETH_DS 4           // Descriptor size (words)
static uint32_t s_txdesc[ETH_DESC_CNT][ETH_DS] 
  __attribute__((section(".ETH_CPPI"), aligned(4)));      // TX descriptors
static uint32_t s_rxdesc[ETH_DESC_CNT][ETH_DS] 
  __attribute__((section(".ETH_CPPI"), aligned(4)));      // RX descriptors
static uint8_t s_rxbuf[ETH_DESC_CNT][ETH_PKT_SIZE] 
  __attribute__((aligned(4)));  // RX ethernet buffers
static uint8_t s_txbuf[ETH_DESC_CNT][ETH_PKT_SIZE] 
  __attribute__((aligned(4)));  // TX ethernet buffers
static struct mg_tcpip_if *s_ifp;                    // MIP interface
static uint16_t emac_read_phy(uint8_t addr, uint8_t reg) {
  while(MDIO->USERACCESS0 & MG_BIT(31)) (void) 0;
  MDIO->USERACCESS0 = MG_BIT(31) | ((reg & 0x1f) << 21) |
                      ((addr & 0x1f) << 16);
  while(MDIO->USERACCESS0 & MG_BIT(31)) (void) 0;
  return MDIO->USERACCESS0 & 0xffff;
}
static void emac_write_phy(uint8_t addr, uint8_t reg, uint16_t val) {
  while(MDIO->USERACCESS0 & MG_BIT(31)) (void) 0;
  MDIO->USERACCESS0 = MG_BIT(31) | MG_BIT(30) | ((reg & 0x1f) << 21) |
                      ((addr & 0x1f) << 16) | (val & 0xffff);
  while(MDIO->USERACCESS0 & MG_BIT(31)) (void) 0;
}
static bool mg_tcpip_driver_tms570_init(struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_tms570_data *d =
      (struct mg_tcpip_driver_tms570_data *) ifp->driver_data;
  s_ifp = ifp;
  EMAC_CTRL->SOFTRESET = MG_BIT(0); // Reset the EMAC Control Module
  while(EMAC_CTRL->SOFTRESET & MG_BIT(0)) (void) 0; // wait
  EMAC->SOFTRESET = MG_BIT(0); // Reset the EMAC Module
  while(EMAC->SOFTRESET & MG_BIT(0)) (void) 0;
  EMAC->MACCONTROL = 0;
  EMAC->RXCONTROL = 0;
  EMAC->TXCONTROL = 0;
  // Initialize all the header descriptor pointer registers
  uint32_t i;
  for(i =  0; i < ETH_DESC_CNT; i++) {
    EMAC->RXHDP[i] = 0;
    EMAC->TXHDP[i] = 0;
    EMAC->RXCP[i] = 0;
    EMAC->TXCP[i] = 0;
    ///EMAC->RXFREEBUFFER[i] = 0xff;
  }
  // Clear the interrupt enable for all the channels
  EMAC->TXINTMASKCLEAR = 0xff;
  EMAC->RXINTMASKCLEAR = 0xff;
  EMAC->MACHASH1 = 0;
  EMAC->MACHASH2 = 0;
  EMAC->RXBUFFEROFFSET = 0;
  EMAC->RXUNICASTCLEAR = 0xff;
  EMAC->RXUNICASTSET = 0;
  EMAC->RXMBPENABLE = 0;
  // init MDIO
  // MDIO_CLK frequency = VCLK3/(CLKDIV + 1). (MDIO must be between 1.0 - 2.5Mhz)
  uint32_t clkdiv = 75; // VCLK is configured to 75Mhz
  // CLKDIV, ENABLE, PREAMBLE, FAULTENB
  MDIO->CONTROL = (clkdiv - 1) | MG_BIT(30) | MG_BIT(20) | MG_BIT(18);
  volatile int delay = 0xfff;
  while (delay-- != 0) (void) 0;
  struct mg_phy phy = {emac_read_phy, emac_write_phy};
  mg_phy_init(&phy, d->phy_addr, MG_PHY_CLOCKS_MAC);
  uint32_t channel;
  for (channel = 0; channel < 8; channel++) {
    EMAC->MACINDEX = channel;
    EMAC->MACADDRHI = ifp->mac[0] | (ifp->mac[1] << 8) | (ifp->mac[2] << 16) |
                       (ifp->mac[3] << 24);
    EMAC->MACADDRLO = ifp->mac[4] | (ifp->mac[5] << 8) | MG_BIT(20) |
                      MG_BIT(19) | (channel << 16);
  }
  EMAC->RXUNICASTSET = 1; // accept unicast frames;

  EMAC->RXMBPENABLE |= MG_BIT(30) | MG_BIT(13); // CRC, broadcast

  // Initialize the descriptors
  for (i = 0; i < ETH_DESC_CNT; i++) {
    if (i < ETH_DESC_CNT - 1) {
      s_txdesc[i][0] = 0;
      s_rxdesc[i][0] = SWAP32(((uint32_t) &s_rxdesc[i + 1][0]));
    }
    s_txdesc[i][1] = SWAP32(((uint32_t) s_txbuf[i]));
    s_rxdesc[i][1] = SWAP32(((uint32_t) s_rxbuf[i]));
    s_txdesc[i][2] = 0;
    s_rxdesc[i][2] = SWAP32(ETH_PKT_SIZE);
    s_txdesc[i][3] = 0;
    s_rxdesc[i][3] = SWAP32(MG_BIT(29)); // OWN
  }
  s_txdesc[ETH_DESC_CNT - 1][0] = 0;
  s_rxdesc[ETH_DESC_CNT - 1][0] = 0;
  
  EMAC->MACCONTROL = MG_BIT(5) | MG_BIT(0); // Enable MII, Full-duplex
  //EMAC->TXINTMASKSET = 1; // Enable TX interrupt
  EMAC->RXINTMASKSET = 1; // Enable RX interrupt
  //EMAC_CTRL->C0TXEN = 1; // TX completion interrupt
  EMAC_CTRL->C0RXEN = 1; // RX completion interrupt
  EMAC->TXCONTROL = 1; // TXEN
  EMAC->RXCONTROL = 1; // RXEN
  EMAC->RXHDP[0] = (uint32_t) &s_rxdesc[0][0];
  return true;
}
static uint32_t s_txno;
static size_t mg_tcpip_driver_tms570_tx(const void *buf, size_t len,
                                      struct mg_tcpip_if *ifp) {
  if (len > sizeof(s_txbuf[s_txno])) {
    MG_ERROR(("Frame too big, %ld", (long) len));
    len = 0;  // fail
  } else if ((s_txdesc[s_txno][3] & SWAP32(MG_BIT(29)))) {
    ifp->nerr++;
    MG_ERROR(("No descriptors available"));
    len = 0;  // fail
  } else {
    memcpy(s_txbuf[s_txno], buf, len);     // Copy data
    if (len < 128) {
      memset(s_txbuf[s_txno] + len, 0, 128 - len);
      len = 128;
    }
    s_txdesc[s_txno][2] = SWAP32((uint32_t) len);  // Set data len
    s_txdesc[s_txno][3] =
        SWAP32(MG_BIT(31) | MG_BIT(30) | MG_BIT(29) | len);  // SOP, EOP, OWN, length
    
    while(EMAC->TXHDP[0] != 0) (void) 0;
    EMAC->TXHDP[0] = (uint32_t) &s_txdesc[s_txno][0];
    if(++s_txno == ETH_DESC_CNT) {
      s_txno = 0;
    }
  }
  return len;
  (void) ifp;
}

static void mg_tcpip_driver_tms570_update_hash_table(struct mg_tcpip_if *ifp) {
  // TODO(): read database, rebuild hash table
  // Setting Hash Index for 01:00:5e:00:00:fb (multicast)
  // using TMS570 XOR method (32.5.37).
  // computed hash is 55, which means bit 23 (55 - 32) in
  // HASH2 register must be set
  EMAC->MACHASH2 = MG_BIT(23);
  EMAC->RXMBPENABLE = MG_BIT(5); // enable hash filtering
  (void) ifp;
}

static bool mg_tcpip_driver_tms570_poll(struct mg_tcpip_if *ifp, bool s1) {
  if (ifp->update_mac_hash_table) {
    mg_tcpip_driver_tms570_update_hash_table(ifp);
    ifp->update_mac_hash_table = false;
  }
  if (!s1) return false;
  struct mg_tcpip_driver_tms570_data *d =
      (struct mg_tcpip_driver_tms570_data *) ifp->driver_data;
  uint8_t speed = MG_PHY_SPEED_10M;
  bool up = false, full_duplex = false;
  struct mg_phy phy = {emac_read_phy, emac_write_phy};
  if (!s1) return false;
  up = mg_phy_up(&phy, d->phy_addr, &full_duplex, &speed);
  if ((ifp->state == MG_TCPIP_STATE_DOWN) && up) {
    // link state just went up
    MG_DEBUG(("Link is %uM %s-duplex", speed == MG_PHY_SPEED_10M ? 10 : 100,
              full_duplex ? "full" : "half"));
  }
  return up;
}

#pragma CODE_STATE(EMAC_TX_IRQHandler, 32)
#pragma INTERRUPT(EMAC_TX_IRQHandler, IRQ)
void EMAC_TX_IRQHandler(void) {
  uint32_t status = EMAC_CTRL->C0TXSTAT;
  if (status & 1) { // interrupt caused on channel 0
    while(s_txdesc[s_txno][3] & SWAP32(MG_BIT(29))) (void) 0;
    EMAC->TXCP[0] = (uint32_t) &s_txdesc[s_txno][0];
  }
  //Write the DMA end of interrupt vector
  EMAC->MACEOIVECTOR = 2;
}
static uint32_t s_rxno;
#pragma CODE_STATE(EMAC_RX_IRQHandler, 32)
#pragma INTERRUPT(EMAC_RX_IRQHandler, IRQ)
void EMAC_RX_IRQHandler(void) {
  uint32_t status = EMAC_CTRL->C0RXSTAT;
  if (status & 1) { // Frame received, loop
    uint32_t i;
    //MG_INFO(("RX interrupt"));
    for (i = 0; i < 10; i++) {   // read as they arrive but not forever
      if (s_rxdesc[s_rxno][3] & SWAP32(MG_BIT(29))) break;
      uint32_t len = SWAP32(s_rxdesc[s_rxno][3]) & 0xffff;
      //MG_INFO(("recv len: %d", len));
      //mg_hexdump(s_rxbuf[s_rxno], len);
      mg_tcpip_qwrite(s_rxbuf[s_rxno], len > 4 ? len - 4 : len, s_ifp);
      uint32_t flags = s_rxdesc[s_rxno][3];
      s_rxdesc[s_rxno][3] = SWAP32(MG_BIT(29));
      s_rxdesc[s_rxno][2] = SWAP32(ETH_PKT_SIZE);
      EMAC->RXCP[0] = (uint32_t) &s_rxdesc[s_rxno][0];
      if (flags & SWAP32(MG_BIT(28))) {
        //MG_INFO(("EOQ detected"));
        EMAC->RXHDP[0] = (uint32_t) &s_rxdesc[0][0];
      }
      if (++s_rxno >= ETH_DESC_CNT) s_rxno = 0;
    }
  }
  //Write the DMA end of interrupt vector
  EMAC->MACEOIVECTOR = 1;
}
struct mg_tcpip_driver mg_tcpip_driver_tms570 = {mg_tcpip_driver_tms570_init,
                                               mg_tcpip_driver_tms570_tx, NULL,
                                               mg_tcpip_driver_tms570_poll};
#endif


#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/w5100.c"
#endif


#if MG_ENABLE_TCPIP && defined(MG_ENABLE_DRIVER_W5100) && MG_ENABLE_DRIVER_W5100

static void w5100_txn(struct mg_tcpip_spi *s, uint16_t addr, bool wr, void *buf,
                      size_t len) {
  uint8_t control = wr ? 0xF0 : 0x0F;
  uint8_t cmd[] = {control, (uint8_t) (addr >> 8), (uint8_t) (addr & 255)};
  s->begin(s->spi);
  s->txn(s->spi, cmd, NULL, sizeof(cmd));
  if (wr) s->txn(s->spi, (uint8_t *) buf, NULL, len);
  if (!wr) s->txn(s->spi, NULL, (uint8_t *) buf, len);
  s->end(s->spi);
}

// clang-format off
static  void w5100_wn(struct mg_tcpip_spi *s, uint16_t addr, void *buf, size_t len) { w5100_txn(s, addr, true, buf, len); }
static  void w5100_w1(struct mg_tcpip_spi *s, uint16_t addr, uint8_t val) { w5100_wn(s, addr, &val, 1); }
static  void w5100_w2(struct mg_tcpip_spi *s, uint16_t addr, uint16_t val) { uint8_t buf[2] = {(uint8_t) (val >> 8), (uint8_t) (val & 255)}; w5100_wn(s, addr, buf, sizeof(buf)); }
static  void w5100_rn(struct mg_tcpip_spi *s, uint16_t addr, void *buf, size_t len) { w5100_txn(s, addr, false, buf, len); }
static  uint8_t w5100_r1(struct mg_tcpip_spi *s, uint16_t addr) { uint8_t r = 0; w5100_rn(s, addr, &r, 1); return r; }
static  uint16_t w5100_r2(struct mg_tcpip_spi *s, uint16_t addr) { uint8_t buf[2] = {0, 0}; w5100_rn(s, addr, buf, sizeof(buf)); return (uint16_t) ((buf[0] << 8) | buf[1]); }
// clang-format on

static size_t w5100_rx(void *buf, size_t buflen, struct mg_tcpip_if *ifp) {
  struct mg_tcpip_spi *s = (struct mg_tcpip_spi *) ifp->driver_data;
  uint16_t r = 0, n = 0, len = (uint16_t) buflen, n2;  // Read recv len
  while ((n2 = w5100_r2(s, 0x426)) > n) n = n2;        // Until it is stable
  if (n > 0) {
    uint16_t ptr = w5100_r2(s, 0x428);  // Get read pointer
    if (n <= len + 2 && n > 1) {
      r = (uint16_t) (n - 2);
    }
    uint16_t rxbuf_size = (1 << (w5100_r1(s, 0x1a) & 3)) * 1024;
    uint16_t rxbuf_addr = 0x6000;
    uint16_t ptr_ofs = (ptr + 2) & (rxbuf_size - 1);
    if (ptr_ofs + r < rxbuf_size) {
      w5100_rn(s, rxbuf_addr + ptr_ofs, buf, r);
    } else {
      uint16_t remaining_len = rxbuf_size - ptr_ofs;
      w5100_rn(s, rxbuf_addr + ptr_ofs, buf, remaining_len);
      w5100_rn(s, rxbuf_addr, buf + remaining_len, r - remaining_len);
    }
    w5100_w2(s, 0x428, (uint16_t) (ptr + n));
    w5100_w1(s, 0x401, 0x40);  // Sock0 CR -> RECV
  }
  return r;
}

static size_t w5100_tx(const void *buf, size_t buflen,
                       struct mg_tcpip_if *ifp) {
  struct mg_tcpip_spi *s = (struct mg_tcpip_spi *) ifp->driver_data;
  uint16_t i, n = 0, ptr = 0, len = (uint16_t) buflen;
  while (n < len) n = w5100_r2(s, 0x420);  // Wait for space
  ptr = w5100_r2(s, 0x424);                // Get write pointer
  uint16_t txbuf_size = (1 << (w5100_r1(s, 0x1b) & 3)) * 1024;
  uint16_t ptr_ofs = ptr & (txbuf_size - 1);
  uint16_t txbuf_addr = 0x4000;
  if (ptr_ofs + len > txbuf_size) {
    uint16_t size = txbuf_size - ptr_ofs;
    w5100_wn(s, txbuf_addr + ptr_ofs, (char *) buf, size);
    w5100_wn(s, txbuf_addr, (char *) buf + size, len - size);
  } else {
    w5100_wn(s, txbuf_addr + ptr_ofs, (char *) buf, len);
  }
  w5100_w2(s, 0x424, (uint16_t) (ptr + len));  // Advance write pointer
  w5100_w1(s, 0x401, 0x20);                    // Sock0 CR -> SEND
  for (i = 0; i < 40; i++) {
    uint8_t ir = w5100_r1(s, 0x402);  // Read S0 IR
    if (ir == 0) continue;
    // printf("IR %d, len=%d, free=%d, ptr %d\n", ir, (int) len, (int) n, ptr);
    w5100_w1(s, 0x402, ir);    // Write S0 IR: clear it!
    if (ir & 8) len = 0;       // Timeout. Report error
    if (ir & (16 | 8)) break;  // Stop on SEND_OK or timeout
  }
  return len;
}

static bool w5100_init(struct mg_tcpip_if *ifp) {
  struct mg_tcpip_spi *s = (struct mg_tcpip_spi *) ifp->driver_data;
  s->end(s->spi);
  w5100_w1(s, 0, 0x80);               // Reset chip: CR -> 0x80
  w5100_w1(s, 0x72, 0x53);            // CR PHYLCKR -> unlock PHY
  w5100_w1(s, 0x46, 0);               // CR PHYCR0 -> autonegotiation
  w5100_w1(s, 0x47, 0);               // CR PHYCR1 -> reset
  w5100_w1(s, 0x72, 0x00);            // CR PHYLCKR -> lock PHY
  w5100_wn(s, 0x09, ifp->mac, 6);     // SHAR
  w5100_w1(s, 0x1a, 6);               // Sock0 RX buf size - 4KB
  w5100_w1(s, 0x1b, 6);               // Sock0 TX buf size - 4KB
  w5100_w1(s, 0x400, 0x44);           // Sock0 MR -> MACRAW, MAC filter
  w5100_w1(s, 0x401, 1);              // Sock0 CR -> OPEN
  return w5100_r1(s, 0x403) == 0x42;  // Sock0 SR == MACRAW
}

static bool w5100_poll(struct mg_tcpip_if *ifp, bool s1) {
  struct mg_tcpip_spi *spi = (struct mg_tcpip_spi *) ifp->driver_data;
  return s1 ? w5100_r1(spi, 0x3c /* PHYSR */) & 1
            : false;  // Bit 0 of PHYSR is LNK (0 - down, 1 - up)
}

struct mg_tcpip_driver mg_tcpip_driver_w5100 = {w5100_init, w5100_tx, w5100_rx,
                                                w5100_poll};
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/w5500.c"
#endif


#if MG_ENABLE_TCPIP && defined(MG_ENABLE_DRIVER_W5500) && MG_ENABLE_DRIVER_W5500

enum { W5500_CR = 0, W5500_S0 = 1, W5500_TX0 = 2, W5500_RX0 = 3 };

static void w5500_txn(struct mg_tcpip_spi *s, uint8_t block, uint16_t addr,
                      bool wr, void *buf, size_t len) {
  uint8_t cmd[] = {(uint8_t) (addr >> 8), (uint8_t) (addr & 255),
                   (uint8_t) ((block << 3) | (wr ? 4 : 0))};
  s->begin(s->spi);
  s->txn(s->spi, cmd, NULL, sizeof(cmd));
  if (wr) s->txn(s->spi, (uint8_t *) buf, NULL, len);
  if (!wr) s->txn(s->spi, NULL, (uint8_t *) buf, len);
  s->end(s->spi);
}

// clang-format off
static  void w5500_wn(struct mg_tcpip_spi *s, uint8_t block, uint16_t addr, void *buf, size_t len) { w5500_txn(s, block, addr, true, buf, len); }
static  void w5500_w1(struct mg_tcpip_spi *s, uint8_t block, uint16_t addr, uint8_t val) { w5500_wn(s, block, addr, &val, 1); }
static  void w5500_w2(struct mg_tcpip_spi *s, uint8_t block, uint16_t addr, uint16_t val) { uint8_t buf[2] = {(uint8_t) (val >> 8), (uint8_t) (val & 255)}; w5500_wn(s, block, addr, buf, sizeof(buf)); }
static  void w5500_rn(struct mg_tcpip_spi *s, uint8_t block, uint16_t addr, void *buf, size_t len) { w5500_txn(s, block, addr, false, buf, len); }
static  uint8_t w5500_r1(struct mg_tcpip_spi *s, uint8_t block, uint16_t addr) { uint8_t r = 0; w5500_rn(s, block, addr, &r, 1); return r; }
static  uint16_t w5500_r2(struct mg_tcpip_spi *s, uint8_t block, uint16_t addr) { uint8_t buf[2] = {0, 0}; w5500_rn(s, block, addr, buf, sizeof(buf)); return (uint16_t) ((buf[0] << 8) | buf[1]); }
// clang-format on

static size_t w5500_rx(void *buf, size_t buflen, struct mg_tcpip_if *ifp) {
  struct mg_tcpip_spi *s = (struct mg_tcpip_spi *) ifp->driver_data;
  uint16_t r = 0, n = 0, len = (uint16_t) buflen, n2;     // Read recv len
  while ((n2 = w5500_r2(s, W5500_S0, 0x26)) > n) n = n2;  // Until it is stable
  // printf("RSR: %d\n", (int) n);
  if (n > 0) {
    uint16_t ptr = w5500_r2(s, W5500_S0, 0x28);  // Get read pointer
    n = w5500_r2(s, W5500_RX0, ptr);             // Read frame length
    if (n <= len + 2 && n > 1) {
      r = (uint16_t) (n - 2);
      w5500_rn(s, W5500_RX0, (uint16_t) (ptr + 2), buf, r);
    }
    w5500_w2(s, W5500_S0, 0x28, (uint16_t) (ptr + n));  // Advance read pointer
    w5500_w1(s, W5500_S0, 1, 0x40);                     // Sock0 CR -> RECV
    // printf("  RX_RD: tot=%u n=%u r=%u\n", n2, n, r);
  }
  return r;
}

static size_t w5500_tx(const void *buf, size_t buflen,
                       struct mg_tcpip_if *ifp) {
  struct mg_tcpip_spi *s = (struct mg_tcpip_spi *) ifp->driver_data;
  uint16_t i, ptr, n = 0, len = (uint16_t) buflen;
  while (n < len) n = w5500_r2(s, W5500_S0, 0x20);      // Wait for space
  ptr = w5500_r2(s, W5500_S0, 0x24);                    // Get write pointer
  w5500_wn(s, W5500_TX0, ptr, (void *) buf, len);       // Write data
  w5500_w2(s, W5500_S0, 0x24, (uint16_t) (ptr + len));  // Advance write pointer
  w5500_w1(s, W5500_S0, 1, 0x20);                       // Sock0 CR -> SEND
  for (i = 0; i < 40; i++) {
    uint8_t ir = w5500_r1(s, W5500_S0, 2);  // Read S0 IR
    if (ir == 0) continue;
    // printf("IR %d, len=%d, free=%d, ptr %d\n", ir, (int) len, (int) n, ptr);
    w5500_w1(s, W5500_S0, 2, ir);  // Write S0 IR: clear it!
    if (ir & 8) len = 0;           // Timeout. Report error
    if (ir & (16 | 8)) break;      // Stop on SEND_OK or timeout
  }
  return len;
}

static bool w5500_init(struct mg_tcpip_if *ifp) {
  struct mg_tcpip_spi *s = (struct mg_tcpip_spi *) ifp->driver_data;
  s->end(s->spi);
  w5500_w1(s, W5500_CR, 0, 0x80);     // Reset chip: CR -> 0x80
  w5500_w1(s, W5500_CR, 0x2e, 0);     // CR PHYCFGR -> reset
  w5500_w1(s, W5500_CR, 0x2e, 0xf8);  // CR PHYCFGR -> set
  // w5500_wn(s, W5500_CR, 9, s->mac, 6);      // Set source MAC
  w5500_w1(s, W5500_S0, 0x1e, 16);          // Sock0 RX buf size
  w5500_w1(s, W5500_S0, 0x1f, 16);          // Sock0 TX buf size
  w5500_w1(s, W5500_S0, 0, 4);              // Sock0 MR -> MACRAW
  w5500_w1(s, W5500_S0, 1, 1);              // Sock0 CR -> OPEN
  return w5500_r1(s, W5500_S0, 3) == 0x42;  // Sock0 SR == MACRAW
}

static bool w5500_poll(struct mg_tcpip_if *ifp, bool s1) {
  struct mg_tcpip_spi *spi = (struct mg_tcpip_spi *) ifp->driver_data;
  return s1 ? w5500_r1(spi, W5500_CR, 0x2e /* PHYCFGR */) & 1
            : false;  // Bit 0 of PHYCFGR is LNK (0 - down, 1 - up)
}

struct mg_tcpip_driver mg_tcpip_driver_w5500 = {w5500_init, w5500_tx, w5500_rx,
                                                w5500_poll};
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/xmc.c"
#endif


#if MG_ENABLE_TCPIP && defined(MG_ENABLE_DRIVER_XMC) && MG_ENABLE_DRIVER_XMC

struct ETH_GLOBAL_TypeDef {
  volatile uint32_t MAC_CONFIGURATION, MAC_FRAME_FILTER, HASH_TABLE_HIGH,
      HASH_TABLE_LOW, GMII_ADDRESS, GMII_DATA, FLOW_CONTROL, VLAN_TAG, VERSION,
      DEBUG, REMOTE_WAKE_UP_FRAME_FILTER, PMT_CONTROL_STATUS, RESERVED[2],
      INTERRUPT_STATUS, INTERRUPT_MASK, MAC_ADDRESS0_HIGH, MAC_ADDRESS0_LOW,
      MAC_ADDRESS1_HIGH, MAC_ADDRESS1_LOW, MAC_ADDRESS2_HIGH, MAC_ADDRESS2_LOW,
      MAC_ADDRESS3_HIGH, MAC_ADDRESS3_LOW, RESERVED1[40], MMC_CONTROL,
      MMC_RECEIVE_INTERRUPT, MMC_TRANSMIT_INTERRUPT, MMC_RECEIVE_INTERRUPT_MASK,
      MMC_TRANSMIT_INTERRUPT_MASK, TX_STATISTICS[26], RESERVED2,
      RX_STATISTICS_1[26], RESERVED3[6], MMC_IPC_RECEIVE_INTERRUPT_MASK,
      RESERVED4, MMC_IPC_RECEIVE_INTERRUPT, RESERVED5, RX_STATISTICS_2[30],
      RESERVED7[286], TIMESTAMP_CONTROL, SUB_SECOND_INCREMENT,
      SYSTEM_TIME_SECONDS, SYSTEM_TIME_NANOSECONDS, SYSTEM_TIME_SECONDS_UPDATE,
      SYSTEM_TIME_NANOSECONDS_UPDATE, TIMESTAMP_ADDEND, TARGET_TIME_SECONDS,
      TARGET_TIME_NANOSECONDS, SYSTEM_TIME_HIGHER_WORD_SECONDS,
      TIMESTAMP_STATUS, PPS_CONTROL, RESERVED8[564], BUS_MODE,
      TRANSMIT_POLL_DEMAND, RECEIVE_POLL_DEMAND,
      RECEIVE_DESCRIPTOR_LIST_ADDRESS, TRANSMIT_DESCRIPTOR_LIST_ADDRESS, STATUS,
      OPERATION_MODE, INTERRUPT_ENABLE,
      MISSED_FRAME_AND_BUFFER_OVERFLOW_COUNTER,
      RECEIVE_INTERRUPT_WATCHDOG_TIMER, RESERVED9, AHB_STATUS, RESERVED10[6],
      CURRENT_HOST_TRANSMIT_DESCRIPTOR, CURRENT_HOST_RECEIVE_DESCRIPTOR,
      CURRENT_HOST_TRANSMIT_BUFFER_ADDRESS, CURRENT_HOST_RECEIVE_BUFFER_ADDRESS,
      HW_FEATURE;
};

#undef ETH0
#define ETH0 ((struct ETH_GLOBAL_TypeDef *) 0x5000C000UL)

#define ETH_PKT_SIZE 1536  // Max frame size
#define ETH_DESC_CNT 4     // Descriptors count
#define ETH_DS 4           // Descriptor size (words)

static uint8_t s_rxbuf[ETH_DESC_CNT][ETH_PKT_SIZE] MG_ETH_RAM;
static uint8_t s_txbuf[ETH_DESC_CNT][ETH_PKT_SIZE] MG_ETH_RAM;
static uint32_t s_rxdesc[ETH_DESC_CNT]
                        [ETH_DS] MG_ETH_RAM;  // RX descriptors
static uint32_t s_txdesc[ETH_DESC_CNT]
                        [ETH_DS] MG_ETH_RAM;  // TX descriptors
static uint8_t s_txno;                             // Current TX descriptor
static uint8_t s_rxno;                             // Current RX descriptor

static struct mg_tcpip_if *s_ifp;  // MIP interface
enum { MG_PHY_ADDR = 0, MG_PHYREG_BCR = 0, MG_PHYREG_BSR = 1 };

static uint16_t eth_read_phy(uint8_t addr, uint8_t reg) {
  ETH0->GMII_ADDRESS = (ETH0->GMII_ADDRESS & 0x3c) | ((uint32_t) addr << 11) |
                       ((uint32_t) reg << 6) | 1;
  while ((ETH0->GMII_ADDRESS & 1) != 0) (void) 0;
  return (uint16_t) (ETH0->GMII_DATA & 0xffff);
}

static void eth_write_phy(uint8_t addr, uint8_t reg, uint16_t val) {
  ETH0->GMII_DATA = val;
  ETH0->GMII_ADDRESS = (ETH0->GMII_ADDRESS & 0x3c) | ((uint32_t) addr << 11) |
                       ((uint32_t) reg << 6) | 3;
  while ((ETH0->GMII_ADDRESS & 1) != 0) (void) 0;
}

static uint32_t get_clock_rate(struct mg_tcpip_driver_xmc_data *d) {
  if (d->mdc_cr == -1) {
    // assume ETH clock is 60MHz by default
    // then according to 13.2.8.1, we need to set value 3
    return 3;
  }

  return d->mdc_cr;
}

static bool mg_tcpip_driver_xmc_init(struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_xmc_data *d =
      (struct mg_tcpip_driver_xmc_data *) ifp->driver_data;
  s_ifp = ifp;

  // reset MAC
  ETH0->BUS_MODE |= 1;
  while (ETH0->BUS_MODE & 1) (void) 0;

  // set clock rate
  ETH0->GMII_ADDRESS = get_clock_rate(d) << 2;

  // init phy
  struct mg_phy phy = {eth_read_phy, eth_write_phy};
  mg_phy_init(&phy, d->phy_addr, MG_PHY_CLOCKS_MAC);

  // configure MAC: DO, DM, FES, TC
  ETH0->MAC_CONFIGURATION = MG_BIT(13) | MG_BIT(11) | MG_BIT(14) | MG_BIT(24);

  // set the MAC address
  ETH0->MAC_ADDRESS0_HIGH = MG_U32(0, 0, ifp->mac[5], ifp->mac[4]);
  ETH0->MAC_ADDRESS0_LOW =
      MG_U32(ifp->mac[3], ifp->mac[2], ifp->mac[1], ifp->mac[0]);

  // Configure the receive filter
  ETH0->MAC_FRAME_FILTER = MG_BIT(10);  // Perfect filter
  // Disable flow control
  ETH0->FLOW_CONTROL = 0;
  // Enable store and forward mode
  ETH0->OPERATION_MODE = MG_BIT(25) | MG_BIT(21);  // RSF, TSF

  // Configure DMA bus mode (AAL, USP, RPBL, PBL)
  ETH0->BUS_MODE = MG_BIT(25) | MG_BIT(23) | (32 << 17) | (32 << 8);

  // init RX descriptors
  for (int i = 0; i < ETH_DESC_CNT; i++) {
    s_rxdesc[i][0] = MG_BIT(31);  // OWN descriptor
    s_rxdesc[i][1] = MG_BIT(14) | ETH_PKT_SIZE;
    s_rxdesc[i][2] = (uint32_t) s_rxbuf[i];
    if (i == ETH_DESC_CNT - 1) {
      s_rxdesc[i][3] = (uint32_t) &s_rxdesc[0][0];
    } else {
      s_rxdesc[i][3] = (uint32_t) &s_rxdesc[i + 1][0];
    }
  }
  ETH0->RECEIVE_DESCRIPTOR_LIST_ADDRESS = (uint32_t) &s_rxdesc[0][0];

  // init TX descriptors
  for (int i = 0; i < ETH_DESC_CNT; i++) {
    s_txdesc[i][0] = MG_BIT(30) | MG_BIT(20);
    s_txdesc[i][2] = (uint32_t) s_txbuf[i];
    if (i == ETH_DESC_CNT - 1) {
      s_txdesc[i][3] = (uint32_t) &s_txdesc[0][0];
    } else {
      s_txdesc[i][3] = (uint32_t) &s_txdesc[i + 1][0];
    }
  }
  ETH0->TRANSMIT_DESCRIPTOR_LIST_ADDRESS = (uint32_t) &s_txdesc[0][0];

  // Clear interrupts
  ETH0->STATUS = 0xFFFFFFFF;

  // Disable MAC interrupts
  ETH0->MMC_TRANSMIT_INTERRUPT_MASK = 0xFFFFFFFF;
  ETH0->MMC_RECEIVE_INTERRUPT_MASK = 0xFFFFFFFF;
  ETH0->MMC_IPC_RECEIVE_INTERRUPT_MASK = 0xFFFFFFFF;
  ETH0->INTERRUPT_MASK = MG_BIT(9) | MG_BIT(3);  // TSIM, PMTIM

  // Enable interrupts (NIE, RIE, TIE)
  ETH0->INTERRUPT_ENABLE = MG_BIT(16) | MG_BIT(6) | MG_BIT(0);

  // Enable MAC transmission and reception (TE, RE)
  ETH0->MAC_CONFIGURATION |= MG_BIT(3) | MG_BIT(2);
  // Enable DMA transmission and reception (ST, SR)
  ETH0->OPERATION_MODE |= MG_BIT(13) | MG_BIT(1);
  return true;
}

static size_t mg_tcpip_driver_xmc_tx(const void *buf, size_t len,
                                     struct mg_tcpip_if *ifp) {
  if (len > sizeof(s_txbuf[s_txno])) {
    MG_ERROR(("Frame too big, %ld", (long) len));
    len = 0;  // Frame is too big
  } else if ((s_txdesc[s_txno][0] & MG_BIT(31))) {
    ifp->nerr++;
    MG_ERROR(("No free descriptors"));
    len = 0;  // All descriptors are busy, fail
  } else {
    memcpy(s_txbuf[s_txno], buf, len);
    s_txdesc[s_txno][1] = len;
    // Table 13-19 Transmit Descriptor Word 0 (IC, LS, FS, TCH)
    s_txdesc[s_txno][0] = MG_BIT(30) | MG_BIT(29) | MG_BIT(28) | MG_BIT(20);
    s_txdesc[s_txno][0] |= MG_BIT(31);  // OWN bit: handle control to DMA
    if (++s_txno >= ETH_DESC_CNT) s_txno = 0;
  }

  // Resume processing
  ETH0->STATUS = MG_BIT(2);  // clear Transmit unavailable
  ETH0->TRANSMIT_POLL_DEMAND = 0;
  return len;
}

static void mg_tcpip_driver_xmc_update_hash_table(struct mg_tcpip_if *ifp) {
  // TODO(): read database, rebuild hash table
  // set the multicast address filter
  ETH0->MAC_ADDRESS1_HIGH =
      MG_U32(0, 0, mcast_addr[5], mcast_addr[4]) | MG_BIT(31);
  ETH0->MAC_ADDRESS1_LOW =
      MG_U32(mcast_addr[3], mcast_addr[2], mcast_addr[1], mcast_addr[0]);
  (void) ifp;
}

static bool mg_tcpip_driver_xmc_poll(struct mg_tcpip_if *ifp, bool s1) {
  if (ifp->update_mac_hash_table) {
    mg_tcpip_driver_xmc_update_hash_table(ifp);
    ifp->update_mac_hash_table = false;
  }
  if (!s1) return false;
  struct mg_tcpip_driver_xmc_data *d =
      (struct mg_tcpip_driver_xmc_data *) ifp->driver_data;
  uint8_t speed = MG_PHY_SPEED_10M;
  bool up = false, full_duplex = false;
  struct mg_phy phy = {eth_read_phy, eth_write_phy};
  up = mg_phy_up(&phy, d->phy_addr, &full_duplex, &speed);
  if ((ifp->state == MG_TCPIP_STATE_DOWN) && up) {  // link state just went up
    MG_DEBUG(("Link is %uM %s-duplex", speed == MG_PHY_SPEED_10M ? 10 : 100,
              full_duplex ? "full" : "half"));
  }
  return up;
}

void ETH0_0_IRQHandler(void);
void ETH0_0_IRQHandler(void) {
  uint32_t irq_status = ETH0->STATUS;

  // check if a frame was received
  if (irq_status & MG_BIT(6)) {
    for (uint8_t i = 0; i < 10; i++) {  // read as they arrive, but not forever
      if (s_rxdesc[s_rxno][0] & MG_BIT(31)) break;
      size_t len = (s_rxdesc[s_rxno][0] & 0x3fff0000) >> 16;
      mg_tcpip_qwrite(s_rxbuf[s_rxno], len, s_ifp);
      s_rxdesc[s_rxno][0] = MG_BIT(31);  // OWN bit: handle control to DMA
      // Resume processing
      ETH0->STATUS = MG_BIT(7) | MG_BIT(6);  // clear RU and RI
      ETH0->RECEIVE_POLL_DEMAND = 0;
      if (++s_rxno >= ETH_DESC_CNT) s_rxno = 0;
    }
    ETH0->STATUS = MG_BIT(6);
  }

  // clear Successful transmission interrupt
  if (irq_status & 1) {
    ETH0->STATUS = 1;
  }

  // clear normal interrupt
  if (irq_status & MG_BIT(16)) {
    ETH0->STATUS = MG_BIT(16);
  }
}

struct mg_tcpip_driver mg_tcpip_driver_xmc = {mg_tcpip_driver_xmc_init,
                                              mg_tcpip_driver_xmc_tx, NULL,
                                              mg_tcpip_driver_xmc_poll};
#endif

#ifdef MG_ENABLE_LINES
#line 1 "src/drivers/xmc7.c"
#endif


#if MG_ENABLE_TCPIP && defined(MG_ENABLE_DRIVER_XMC7) && MG_ENABLE_DRIVER_XMC7

struct ETH_Type {
  volatile uint32_t CTL, STATUS, RESERVED[1022], NETWORK_CONTROL,
      NETWORK_CONFIG, NETWORK_STATUS, USER_IO_REGISTER, DMA_CONFIG,
      TRANSMIT_STATUS, RECEIVE_Q_PTR, TRANSMIT_Q_PTR, RECEIVE_STATUS,
      INT_STATUS, INT_ENABLE, INT_DISABLE, INT_MASK, PHY_MANAGEMENT, PAUSE_TIME,
      TX_PAUSE_QUANTUM, PBUF_TXCUTTHRU, PBUF_RXCUTTHRU, JUMBO_MAX_LENGTH,
      EXTERNAL_FIFO_INTERFACE, RESERVED1, AXI_MAX_PIPELINE, RSC_CONTROL,
      INT_MODERATION, SYS_WAKE_TIME, RESERVED2[7], HASH_BOTTOM, HASH_TOP,
      SPEC_ADD1_BOTTOM, SPEC_ADD1_TOP, SPEC_ADD2_BOTTOM, SPEC_ADD2_TOP,
      SPEC_ADD3_BOTTOM, SPEC_ADD3_TOP, SPEC_ADD4_BOTTOM, SPEC_ADD4_TOP,
      SPEC_TYPE1, SPEC_TYPE2, SPEC_TYPE3, SPEC_TYPE4, WOL_REGISTER,
      STRETCH_RATIO, STACKED_VLAN, TX_PFC_PAUSE, MASK_ADD1_BOTTOM,
      MASK_ADD1_TOP, DMA_ADDR_OR_MASK, RX_PTP_UNICAST, TX_PTP_UNICAST,
      TSU_NSEC_CMP, TSU_SEC_CMP, TSU_MSB_SEC_CMP, TSU_PTP_TX_MSB_SEC,
      TSU_PTP_RX_MSB_SEC, TSU_PEER_TX_MSB_SEC, TSU_PEER_RX_MSB_SEC,
      DPRAM_FILL_DBG, REVISION_REG, OCTETS_TXED_BOTTOM, OCTETS_TXED_TOP,
      FRAMES_TXED_OK, BROADCAST_TXED, MULTICAST_TXED, PAUSE_FRAMES_TXED,
      FRAMES_TXED_64, FRAMES_TXED_65, FRAMES_TXED_128, FRAMES_TXED_256,
      FRAMES_TXED_512, FRAMES_TXED_1024, FRAMES_TXED_1519, TX_UNDERRUNS,
      SINGLE_COLLISIONS, MULTIPLE_COLLISIONS, EXCESSIVE_COLLISIONS,
      LATE_COLLISIONS, DEFERRED_FRAMES, CRS_ERRORS, OCTETS_RXED_BOTTOM,
      OCTETS_RXED_TOP, FRAMES_RXED_OK, BROADCAST_RXED, MULTICAST_RXED,
      PAUSE_FRAMES_RXED, FRAMES_RXED_64, FRAMES_RXED_65, FRAMES_RXED_128,
      FRAMES_RXED_256, FRAMES_RXED_512, FRAMES_RXED_1024, FRAMES_RXED_1519,
      UNDERSIZE_FRAMES, EXCESSIVE_RX_LENGTH, RX_JABBERS, FCS_ERRORS,
      RX_LENGTH_ERRORS, RX_SYMBOL_ERRORS, ALIGNMENT_ERRORS, RX_RESOURCE_ERRORS,
      RX_OVERRUNS, RX_IP_CK_ERRORS, RX_TCP_CK_ERRORS, RX_UDP_CK_ERRORS,
      AUTO_FLUSHED_PKTS, RESERVED3, TSU_TIMER_INCR_SUB_NSEC, TSU_TIMER_MSB_SEC,
      TSU_STROBE_MSB_SEC, TSU_STROBE_SEC, TSU_STROBE_NSEC, TSU_TIMER_SEC,
      TSU_TIMER_NSEC, TSU_TIMER_ADJUST, TSU_TIMER_INCR, TSU_PTP_TX_SEC,
      TSU_PTP_TX_NSEC, TSU_PTP_RX_SEC, TSU_PTP_RX_NSEC, TSU_PEER_TX_SEC,
      TSU_PEER_TX_NSEC, TSU_PEER_RX_SEC, TSU_PEER_RX_NSEC, PCS_CONTROL,
      PCS_STATUS, RESERVED4[2], PCS_AN_ADV, PCS_AN_LP_BASE, PCS_AN_EXP,
      PCS_AN_NP_TX, PCS_AN_LP_NP, RESERVED5[6], PCS_AN_EXT_STATUS, RESERVED6[8],
      TX_PAUSE_QUANTUM1, TX_PAUSE_QUANTUM2, TX_PAUSE_QUANTUM3, RESERVED7,
      RX_LPI, RX_LPI_TIME, TX_LPI, TX_LPI_TIME, DESIGNCFG_DEBUG1,
      DESIGNCFG_DEBUG2, DESIGNCFG_DEBUG3, DESIGNCFG_DEBUG4, DESIGNCFG_DEBUG5,
      DESIGNCFG_DEBUG6, DESIGNCFG_DEBUG7, DESIGNCFG_DEBUG8, DESIGNCFG_DEBUG9,
      DESIGNCFG_DEBUG10, RESERVED8[22], SPEC_ADD5_BOTTOM, SPEC_ADD5_TOP,
      RESERVED9[60], SPEC_ADD36_BOTTOM, SPEC_ADD36_TOP, INT_Q1_STATUS,
      INT_Q2_STATUS, INT_Q3_STATUS, RESERVED10[11], INT_Q15_STATUS, RESERVED11,
      TRANSMIT_Q1_PTR, TRANSMIT_Q2_PTR, TRANSMIT_Q3_PTR, RESERVED12[11],
      TRANSMIT_Q15_PTR, RESERVED13, RECEIVE_Q1_PTR, RECEIVE_Q2_PTR,
      RECEIVE_Q3_PTR, RESERVED14[3], RECEIVE_Q7_PTR, RESERVED15,
      DMA_RXBUF_SIZE_Q1, DMA_RXBUF_SIZE_Q2, DMA_RXBUF_SIZE_Q3, RESERVED16[3],
      DMA_RXBUF_SIZE_Q7, CBS_CONTROL, CBS_IDLESLOPE_Q_A, CBS_IDLESLOPE_Q_B,
      UPPER_TX_Q_BASE_ADDR, TX_BD_CONTROL, RX_BD_CONTROL, UPPER_RX_Q_BASE_ADDR,
      RESERVED17[2], HIDDEN_REG0, HIDDEN_REG1, HIDDEN_REG2, HIDDEN_REG3,
      RESERVED18[2], HIDDEN_REG4, HIDDEN_REG5;
};

#define ETH0 ((struct ETH_Type *) 0x40490000)

#define ETH_PKT_SIZE 1536  // Max frame size
#define ETH_DESC_CNT 4     // Descriptors count
#define ETH_DS 2           // Descriptor size (words)

static uint8_t s_rxbuf[ETH_DESC_CNT][ETH_PKT_SIZE] MG_ETH_RAM;
static uint8_t s_txbuf[ETH_DESC_CNT][ETH_PKT_SIZE] MG_ETH_RAM;
static uint32_t s_rxdesc[ETH_DESC_CNT][ETH_DS] MG_ETH_RAM MG_8BYTE_ALIGNED;
static uint32_t s_txdesc[ETH_DESC_CNT][ETH_DS] MG_ETH_RAM MG_8BYTE_ALIGNED;
static uint8_t s_txno;  // Current TX descriptor
static uint8_t s_rxno;  // Current RX descriptor

static struct mg_tcpip_if *s_ifp;  // MIP interface
enum { MG_PHY_ADDR = 0, MG_PHYREG_BCR = 0, MG_PHYREG_BSR = 1 };

static uint16_t eth_read_phy(uint8_t addr, uint8_t reg) {
  // WRITE1, READ OPERATION, PHY, REG, WRITE10
  ETH0->PHY_MANAGEMENT = MG_BIT(30) | MG_BIT(29) | ((addr & 0xf) << 24) |
                         ((reg & 0x1f) << 18) | MG_BIT(17);
  while ((ETH0->NETWORK_STATUS & MG_BIT(2)) == 0) (void) 0;
  return ETH0->PHY_MANAGEMENT & 0xffff;
}

static void eth_write_phy(uint8_t addr, uint8_t reg, uint16_t val) {
  ETH0->PHY_MANAGEMENT = MG_BIT(30) | MG_BIT(28) | ((addr & 0xf) << 24) |
                         ((reg & 0x1f) << 18) | MG_BIT(17) | val;
  while ((ETH0->NETWORK_STATUS & MG_BIT(2)) == 0) (void) 0;
}

static uint32_t get_clock_rate(struct mg_tcpip_driver_xmc7_data *d) {
  // see ETH0 -> NETWORK_CONFIG register
  (void) d;
  return 3;
}

static bool mg_tcpip_driver_xmc7_init(struct mg_tcpip_if *ifp) {
  struct mg_tcpip_driver_xmc7_data *d =
      (struct mg_tcpip_driver_xmc7_data *) ifp->driver_data;
  s_ifp = ifp;

  // enable controller, set RGMII mode
  ETH0->CTL = MG_BIT(31) | (4 << 8) | 2;

  uint32_t cr = get_clock_rate(d);
  // set NSP change, ignore RX FCS, data bus width, clock rate
  // frame length 1536, full duplex, speed
  ETH0->NETWORK_CONFIG = MG_BIT(29) | MG_BIT(26) | MG_BIT(21) |
                         ((cr & 7) << 18) | MG_BIT(8) | MG_BIT(1) | MG_BIT(0);

  // config DMA settings: Force TX burst, Discard on Error, set RX buffer size
  // to 1536, TX_PBUF_SIZE, RX_PBUF_SIZE, AMBA_BURST_LENGTH
  ETH0->DMA_CONFIG =
      MG_BIT(26) | MG_BIT(24) | (0x18 << 16) | MG_BIT(10) | (3 << 8) | 4;

  // initialize descriptors
  for (int i = 0; i < ETH_DESC_CNT; i++) {
    s_rxdesc[i][0] = (uint32_t) s_rxbuf[i];
    if (i == ETH_DESC_CNT - 1) {
      s_rxdesc[i][0] |= MG_BIT(1);  // mark last descriptor
    }

    s_txdesc[i][0] = (uint32_t) s_txbuf[i];
    s_txdesc[i][1] = MG_BIT(31);  // OWN descriptor
    if (i == ETH_DESC_CNT - 1) {
      s_txdesc[i][1] |= MG_BIT(30);  // mark last descriptor
    }
  }
  ETH0->RECEIVE_Q_PTR = (uint32_t) s_rxdesc;
  ETH0->TRANSMIT_Q_PTR = (uint32_t) s_txdesc;

  // disable other queues
  ETH0->TRANSMIT_Q2_PTR = 1;
  ETH0->TRANSMIT_Q1_PTR = 1;
  ETH0->RECEIVE_Q2_PTR = 1;
  ETH0->RECEIVE_Q1_PTR = 1;

  // enable interrupts (RX complete)
  ETH0->INT_ENABLE = MG_BIT(1);

  // set MAC address
  ETH0->SPEC_ADD1_BOTTOM =
      ifp->mac[3] << 24 | ifp->mac[2] << 16 | ifp->mac[1] << 8 | ifp->mac[0];
  ETH0->SPEC_ADD1_TOP = ifp->mac[5] << 8 | ifp->mac[4];

  // enable MDIO, TX, RX
  ETH0->NETWORK_CONTROL = MG_BIT(4) | MG_BIT(3) | MG_BIT(2);

  // start transmission
  ETH0->NETWORK_CONTROL |= MG_BIT(9);

  // init phy
  struct mg_phy phy = {eth_read_phy, eth_write_phy};
  mg_phy_init(&phy, d->phy_addr, MG_PHY_CLOCKS_MAC);

  (void) d;
  return true;
}

static size_t mg_tcpip_driver_xmc7_tx(const void *buf, size_t len,
                                      struct mg_tcpip_if *ifp) {
  if (len > sizeof(s_txbuf[s_txno])) {
    MG_ERROR(("Frame too big, %ld", (long) len));
    len = 0;  // Frame is too big
  } else if (((s_txdesc[s_txno][1] & MG_BIT(31)) == 0)) {
    ifp->nerr++;
    MG_ERROR(("No free descriptors"));
    len = 0;  // All descriptors are busy, fail
  } else {
    memcpy(s_txbuf[s_txno], buf, len);
    s_txdesc[s_txno][1] = (s_txno == ETH_DESC_CNT - 1 ? MG_BIT(30) : 0) |
                          MG_BIT(15) | len;  // Last buffer and length

    ETH0->NETWORK_CONTROL |= MG_BIT(9);  // enable transmission
    if (++s_txno >= ETH_DESC_CNT) s_txno = 0;
  }

  MG_DSB();
  ETH0->TRANSMIT_STATUS = ETH0->TRANSMIT_STATUS;
  ETH0->NETWORK_CONTROL |= MG_BIT(9);  // enable transmission

  return len;
}

static void mg_tcpip_driver_xmc7_update_hash_table(struct mg_tcpip_if *ifp) {
  // TODO(): read database, rebuild hash table
  // set multicast MAC address
  ETH0->SPEC_ADD2_BOTTOM = mcast_addr[3] << 24 | mcast_addr[2] << 16 |
                           mcast_addr[1] << 8 | mcast_addr[0];
  ETH0->SPEC_ADD2_TOP = mcast_addr[5] << 8 | mcast_addr[4];
  (void) ifp;
}

static bool mg_tcpip_driver_xmc7_poll(struct mg_tcpip_if *ifp, bool s1) {
  if (ifp->update_mac_hash_table) {
    mg_tcpip_driver_xmc7_update_hash_table(ifp);
    ifp->update_mac_hash_table = false;
  }
  if (!s1) return false;
  struct mg_tcpip_driver_xmc7_data *d =
      (struct mg_tcpip_driver_xmc7_data *) ifp->driver_data;
  uint8_t speed = MG_PHY_SPEED_10M;
  bool up = false, full_duplex = false;
  struct mg_phy phy = {eth_read_phy, eth_write_phy};
  up = mg_phy_up(&phy, d->phy_addr, &full_duplex, &speed);
  if ((ifp->state == MG_TCPIP_STATE_DOWN) && up) {  // link state just went up
    // tmp = reg with flags set to the most likely situation: 100M full-duplex
    // if(link is slow or half) set flags otherwise
    // reg = tmp
    uint32_t netconf = ETH0->NETWORK_CONFIG;
    MG_SET_BITS(netconf, MG_BIT(10),
                MG_BIT(1) | MG_BIT(0));  // 100M, Full-duplex
    uint32_t ctl = ETH0->CTL;
    MG_SET_BITS(ctl, 0xFF00, 4 << 8);  // /5 for 25M clock
    if (speed == MG_PHY_SPEED_1000M) {
      netconf |= MG_BIT(10);        // 1000M
      MG_SET_BITS(ctl, 0xFF00, 0);  // /1 for 125M clock TODO() IS THIS NEEDED ?
    } else if (speed == MG_PHY_SPEED_10M) {
      netconf &= ~MG_BIT(0);         // 10M
      MG_SET_BITS(ctl, 0xFF00, 49);  // /50 for 2.5M clock
    }
    if (full_duplex == false) netconf &= ~MG_BIT(1);  // Half-duplex
    ETH0->NETWORK_CONFIG = netconf;  // IRQ handler does not fiddle with these
    ETH0->CTL = ctl;
    MG_DEBUG(("Link is %uM %s-duplex",
              speed == MG_PHY_SPEED_10M
                  ? 10
                  : (speed == MG_PHY_SPEED_100M ? 100 : 1000),
              full_duplex ? "full" : "half"));
  }
  return up;
}

void ETH_IRQHandler(void) {
  uint32_t irq_status = ETH0->INT_STATUS;
  if (irq_status & MG_BIT(1)) {
    for (uint8_t i = 0; i < 10; i++) {  // read as they arrive, but not forever
      if ((s_rxdesc[s_rxno][0] & MG_BIT(0)) == 0) break;
      size_t len = s_rxdesc[s_rxno][1] & (MG_BIT(13) - 1);
      mg_tcpip_qwrite(s_rxbuf[s_rxno], len, s_ifp);
      s_rxdesc[s_rxno][0] &= ~MG_BIT(0);  // OWN bit: handle control to DMA
      if (++s_rxno >= ETH_DESC_CNT) s_rxno = 0;
    }
  }

  ETH0->INT_STATUS = irq_status;
}

struct mg_tcpip_driver mg_tcpip_driver_xmc7 = {mg_tcpip_driver_xmc7_init,
                                               mg_tcpip_driver_xmc7_tx, NULL,
                                               mg_tcpip_driver_xmc7_poll};
#endif