From 0299efb76103faaa0ed5045034da3d715b2904d0 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 14 Oct 2026 09:16:40 +0200 Subject: [PATCH] Escape characters below 0x60 from a table straight into the write buffer When the bulk scanner in dump_escaped_impl() stops at a control character, a quote or a backslash, that byte used to go through the UTF-8 decoder and a switch that wrote the escape byte by byte into string_buffer, which was then copied into write_buffer at the next run. A byte below 0x60 at a character boundary is a complete code point, so it now skips the decoder: its escape sequence comes from a 96-entry table of 8-byte rows (sequence padded with zeros, length in the last byte), and all 8 bytes are copied straight into write_buffer with one room check. The check sits inside the existing character-boundary block, so it runs once per scanner stop and not once per byte; testing every byte cost about 6% on twitter.json with ensure_ascii. The switch in the decoder's accept branch lost its cases for characters below 0x60, which can no longer reach it. Output is byte-identical. The string-buffer boundary tests now use \u-escaped non-ASCII characters, since simple escapes no longer reach string_buffer, and a new test case checks every ASCII character at every position against a reference escaper. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/output/serializer.hpp | 245 ++++++++++++------ single_include/nlohmann/json.hpp | 245 ++++++++++++------ tests/src/unit-serialization.cpp | 179 +++++++++++-- 3 files changed, 489 insertions(+), 180 deletions(-) diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 64c4fcf03..22bee26a7 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -772,6 +772,33 @@ class serializer break; } } + + // A byte below 0x60 is a complete code point and needs no + // UTF-8 decoding. The scanner stops only at a control + // character, a quote, or a backslash in that range, so look up + // its escape sequence and write all 8 bytes of the table entry + // straight into the write buffer; only the length stored in the + // last byte of the entry is kept. + if (data[i] < 0x60) + { + // emit any bytes still pending in string_buffer first to + // preserve output order + if (bytes != 0) + { + put_buffer(string_buffer, bytes); + bytes = 0; + } + const auto& escape = ascii_escapes()[data[i]]; + if (JSON_HEDLEY_UNLIKELY(write_buffer.size() - write_buffer_pos < escape.size())) + { + flush(); + } + std::memcpy(write_buffer.data() + write_buffer_pos, escape.data(), escape.size()); + write_buffer_pos += static_cast(escape[7]); + bytes_after_last_accept = 0; + undumped_chars = 0; + continue; + } } const auto byte = static_cast(s[i]); @@ -780,92 +807,39 @@ class serializer { case UTF8_ACCEPT: // decode found a new code point { - switch (codepoint) + // the code point is at least 0x60 (smaller ones are handled + // above) and is not printable ASCII (which the scanner + // copies): it is DEL (0x7F) under EnsureAscii or a non-ASCII + // character + if (EnsureAscii) { - case 0x08: // backspace + JSON_ASSERT(codepoint >= 0x7F); + if (error_handler == error_handler_t::keep) { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'b'; - break; + // this character was buffered as raw bytes below + // in case it turned out to be part of an + // ill-formed sequence (which is kept as is); now + // that it decoded to a well-formed code point, + // undo that and \u-escape it like any other + // character instead + bytes = bytes_after_last_accept; } - - case 0x09: // horizontal tab + if (codepoint <= 0xFFFF) { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 't'; - break; + write_u_escape(bytes, static_cast(codepoint)); } - - case 0x0A: // newline + else { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'n'; - break; - } - - case 0x0C: // formfeed - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'f'; - break; - } - - case 0x0D: // carriage return - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'r'; - break; - } - - case 0x22: // quotation mark - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '"'; - break; - } - - case 0x5C: // reverse solidus - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '\\'; - break; - } - - default: - { - // escape control characters (0x00..0x1F) or, if - // EnsureAscii parameter is used, non-ASCII characters - if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F))) - { - if (EnsureAscii && error_handler == error_handler_t::keep) - { - // this character was buffered as raw bytes - // below in case it turned out to be part of - // an ill-formed sequence (which is kept as - // is); now that it decoded to a well-formed - // code point, undo that and \u-escape it - // like any other character instead - bytes = bytes_after_last_accept; - } - if (codepoint <= 0xFFFF) - { - write_u_escape(bytes, static_cast(codepoint)); - } - else - { - write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); - write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); - } - } - else - { - // copy byte to buffer (all previous bytes - // been copied have in default case above) - string_buffer[bytes++] = s[i]; - } - break; + write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); + write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); } } + else + { + // copy byte to buffer (all previous bytes of the + // sequence have been copied in the default case below) + string_buffer[bytes++] = s[i]; + } // write buffer and reset index; there must be 13 bytes // left, as this is the maximal number of bytes to be @@ -1253,6 +1227,121 @@ class serializer } } + /*! + @brief the JSON form of each character below 0x60 + + Entry c holds the escape sequence of character c - the short form for + \\b, \\t, \\n, \\f, \\r, \\" and \\\\, \\u00xx for the other control characters - + or c itself if it needs no escaping, padded with zeros to 7 bytes. The last + byte is the length of the sequence. The 8 bytes of an entry are copied at + once, so writing an escape needs neither a branch nor a byte-wise loop. + */ + static const std::array, 0x60>& ascii_escapes() noexcept + { + static const std::array, 0x60> escapes = + { + { + {{'\\', 'u', '0', '0', '0', '0', 0, 6}}, // 0x00 + {{'\\', 'u', '0', '0', '0', '1', 0, 6}}, // 0x01 + {{'\\', 'u', '0', '0', '0', '2', 0, 6}}, // 0x02 + {{'\\', 'u', '0', '0', '0', '3', 0, 6}}, // 0x03 + {{'\\', 'u', '0', '0', '0', '4', 0, 6}}, // 0x04 + {{'\\', 'u', '0', '0', '0', '5', 0, 6}}, // 0x05 + {{'\\', 'u', '0', '0', '0', '6', 0, 6}}, // 0x06 + {{'\\', 'u', '0', '0', '0', '7', 0, 6}}, // 0x07 + {{'\\', 'b', 0, 0, 0, 0, 0, 2}}, // 0x08 + {{'\\', 't', 0, 0, 0, 0, 0, 2}}, // 0x09 + {{'\\', 'n', 0, 0, 0, 0, 0, 2}}, // 0x0A + {{'\\', 'u', '0', '0', '0', 'b', 0, 6}}, // 0x0B + {{'\\', 'f', 0, 0, 0, 0, 0, 2}}, // 0x0C + {{'\\', 'r', 0, 0, 0, 0, 0, 2}}, // 0x0D + {{'\\', 'u', '0', '0', '0', 'e', 0, 6}}, // 0x0E + {{'\\', 'u', '0', '0', '0', 'f', 0, 6}}, // 0x0F + {{'\\', 'u', '0', '0', '1', '0', 0, 6}}, // 0x10 + {{'\\', 'u', '0', '0', '1', '1', 0, 6}}, // 0x11 + {{'\\', 'u', '0', '0', '1', '2', 0, 6}}, // 0x12 + {{'\\', 'u', '0', '0', '1', '3', 0, 6}}, // 0x13 + {{'\\', 'u', '0', '0', '1', '4', 0, 6}}, // 0x14 + {{'\\', 'u', '0', '0', '1', '5', 0, 6}}, // 0x15 + {{'\\', 'u', '0', '0', '1', '6', 0, 6}}, // 0x16 + {{'\\', 'u', '0', '0', '1', '7', 0, 6}}, // 0x17 + {{'\\', 'u', '0', '0', '1', '8', 0, 6}}, // 0x18 + {{'\\', 'u', '0', '0', '1', '9', 0, 6}}, // 0x19 + {{'\\', 'u', '0', '0', '1', 'a', 0, 6}}, // 0x1A + {{'\\', 'u', '0', '0', '1', 'b', 0, 6}}, // 0x1B + {{'\\', 'u', '0', '0', '1', 'c', 0, 6}}, // 0x1C + {{'\\', 'u', '0', '0', '1', 'd', 0, 6}}, // 0x1D + {{'\\', 'u', '0', '0', '1', 'e', 0, 6}}, // 0x1E + {{'\\', 'u', '0', '0', '1', 'f', 0, 6}}, // 0x1F + {{' ', 0, 0, 0, 0, 0, 0, 1}}, // 0x20 + {{'!', 0, 0, 0, 0, 0, 0, 1}}, // 0x21 + {{'\\', '"', 0, 0, 0, 0, 0, 2}}, // 0x22 + {{'#', 0, 0, 0, 0, 0, 0, 1}}, // 0x23 + {{'$', 0, 0, 0, 0, 0, 0, 1}}, // 0x24 + {{'%', 0, 0, 0, 0, 0, 0, 1}}, // 0x25 + {{'&', 0, 0, 0, 0, 0, 0, 1}}, // 0x26 + {{'\'', 0, 0, 0, 0, 0, 0, 1}}, // 0x27 + {{'(', 0, 0, 0, 0, 0, 0, 1}}, // 0x28 + {{')', 0, 0, 0, 0, 0, 0, 1}}, // 0x29 + {{'*', 0, 0, 0, 0, 0, 0, 1}}, // 0x2A + {{'+', 0, 0, 0, 0, 0, 0, 1}}, // 0x2B + {{',', 0, 0, 0, 0, 0, 0, 1}}, // 0x2C + {{'-', 0, 0, 0, 0, 0, 0, 1}}, // 0x2D + {{'.', 0, 0, 0, 0, 0, 0, 1}}, // 0x2E + {{'/', 0, 0, 0, 0, 0, 0, 1}}, // 0x2F + {{'0', 0, 0, 0, 0, 0, 0, 1}}, // 0x30 + {{'1', 0, 0, 0, 0, 0, 0, 1}}, // 0x31 + {{'2', 0, 0, 0, 0, 0, 0, 1}}, // 0x32 + {{'3', 0, 0, 0, 0, 0, 0, 1}}, // 0x33 + {{'4', 0, 0, 0, 0, 0, 0, 1}}, // 0x34 + {{'5', 0, 0, 0, 0, 0, 0, 1}}, // 0x35 + {{'6', 0, 0, 0, 0, 0, 0, 1}}, // 0x36 + {{'7', 0, 0, 0, 0, 0, 0, 1}}, // 0x37 + {{'8', 0, 0, 0, 0, 0, 0, 1}}, // 0x38 + {{'9', 0, 0, 0, 0, 0, 0, 1}}, // 0x39 + {{':', 0, 0, 0, 0, 0, 0, 1}}, // 0x3A + {{';', 0, 0, 0, 0, 0, 0, 1}}, // 0x3B + {{'<', 0, 0, 0, 0, 0, 0, 1}}, // 0x3C + {{'=', 0, 0, 0, 0, 0, 0, 1}}, // 0x3D + {{'>', 0, 0, 0, 0, 0, 0, 1}}, // 0x3E + {{'?', 0, 0, 0, 0, 0, 0, 1}}, // 0x3F + {{'@', 0, 0, 0, 0, 0, 0, 1}}, // 0x40 + {{'A', 0, 0, 0, 0, 0, 0, 1}}, // 0x41 + {{'B', 0, 0, 0, 0, 0, 0, 1}}, // 0x42 + {{'C', 0, 0, 0, 0, 0, 0, 1}}, // 0x43 + {{'D', 0, 0, 0, 0, 0, 0, 1}}, // 0x44 + {{'E', 0, 0, 0, 0, 0, 0, 1}}, // 0x45 + {{'F', 0, 0, 0, 0, 0, 0, 1}}, // 0x46 + {{'G', 0, 0, 0, 0, 0, 0, 1}}, // 0x47 + {{'H', 0, 0, 0, 0, 0, 0, 1}}, // 0x48 + {{'I', 0, 0, 0, 0, 0, 0, 1}}, // 0x49 + {{'J', 0, 0, 0, 0, 0, 0, 1}}, // 0x4A + {{'K', 0, 0, 0, 0, 0, 0, 1}}, // 0x4B + {{'L', 0, 0, 0, 0, 0, 0, 1}}, // 0x4C + {{'M', 0, 0, 0, 0, 0, 0, 1}}, // 0x4D + {{'N', 0, 0, 0, 0, 0, 0, 1}}, // 0x4E + {{'O', 0, 0, 0, 0, 0, 0, 1}}, // 0x4F + {{'P', 0, 0, 0, 0, 0, 0, 1}}, // 0x50 + {{'Q', 0, 0, 0, 0, 0, 0, 1}}, // 0x51 + {{'R', 0, 0, 0, 0, 0, 0, 1}}, // 0x52 + {{'S', 0, 0, 0, 0, 0, 0, 1}}, // 0x53 + {{'T', 0, 0, 0, 0, 0, 0, 1}}, // 0x54 + {{'U', 0, 0, 0, 0, 0, 0, 1}}, // 0x55 + {{'V', 0, 0, 0, 0, 0, 0, 1}}, // 0x56 + {{'W', 0, 0, 0, 0, 0, 0, 1}}, // 0x57 + {{'X', 0, 0, 0, 0, 0, 0, 1}}, // 0x58 + {{'Y', 0, 0, 0, 0, 0, 0, 1}}, // 0x59 + {{'Z', 0, 0, 0, 0, 0, 0, 1}}, // 0x5A + {{'[', 0, 0, 0, 0, 0, 0, 1}}, // 0x5B + {{'\\', '\\', 0, 0, 0, 0, 0, 2}}, // 0x5C + {{']', 0, 0, 0, 0, 0, 0, 1}}, // 0x5D + {{'^', 0, 0, 0, 0, 0, 0, 1}}, // 0x5E + {{'_', 0, 0, 0, 0, 0, 0, 1}}, // 0x5F + } + }; + return escapes; + } + /*! * @brief write a lowercase "\\uXXXX" escape sequence into @a string_buffer * diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 16c87bf36..3dce32c76 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -27087,6 +27087,33 @@ class serializer break; } } + + // A byte below 0x60 is a complete code point and needs no + // UTF-8 decoding. The scanner stops only at a control + // character, a quote, or a backslash in that range, so look up + // its escape sequence and write all 8 bytes of the table entry + // straight into the write buffer; only the length stored in the + // last byte of the entry is kept. + if (data[i] < 0x60) + { + // emit any bytes still pending in string_buffer first to + // preserve output order + if (bytes != 0) + { + put_buffer(string_buffer, bytes); + bytes = 0; + } + const auto& escape = ascii_escapes()[data[i]]; + if (JSON_HEDLEY_UNLIKELY(write_buffer.size() - write_buffer_pos < escape.size())) + { + flush(); + } + std::memcpy(write_buffer.data() + write_buffer_pos, escape.data(), escape.size()); + write_buffer_pos += static_cast(escape[7]); + bytes_after_last_accept = 0; + undumped_chars = 0; + continue; + } } const auto byte = static_cast(s[i]); @@ -27095,92 +27122,39 @@ class serializer { case UTF8_ACCEPT: // decode found a new code point { - switch (codepoint) + // the code point is at least 0x60 (smaller ones are handled + // above) and is not printable ASCII (which the scanner + // copies): it is DEL (0x7F) under EnsureAscii or a non-ASCII + // character + if (EnsureAscii) { - case 0x08: // backspace + JSON_ASSERT(codepoint >= 0x7F); + if (error_handler == error_handler_t::keep) { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'b'; - break; + // this character was buffered as raw bytes below + // in case it turned out to be part of an + // ill-formed sequence (which is kept as is); now + // that it decoded to a well-formed code point, + // undo that and \u-escape it like any other + // character instead + bytes = bytes_after_last_accept; } - - case 0x09: // horizontal tab + if (codepoint <= 0xFFFF) { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 't'; - break; + write_u_escape(bytes, static_cast(codepoint)); } - - case 0x0A: // newline + else { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'n'; - break; - } - - case 0x0C: // formfeed - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'f'; - break; - } - - case 0x0D: // carriage return - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'r'; - break; - } - - case 0x22: // quotation mark - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '"'; - break; - } - - case 0x5C: // reverse solidus - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '\\'; - break; - } - - default: - { - // escape control characters (0x00..0x1F) or, if - // EnsureAscii parameter is used, non-ASCII characters - if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F))) - { - if (EnsureAscii && error_handler == error_handler_t::keep) - { - // this character was buffered as raw bytes - // below in case it turned out to be part of - // an ill-formed sequence (which is kept as - // is); now that it decoded to a well-formed - // code point, undo that and \u-escape it - // like any other character instead - bytes = bytes_after_last_accept; - } - if (codepoint <= 0xFFFF) - { - write_u_escape(bytes, static_cast(codepoint)); - } - else - { - write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); - write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); - } - } - else - { - // copy byte to buffer (all previous bytes - // been copied have in default case above) - string_buffer[bytes++] = s[i]; - } - break; + write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); + write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); } } + else + { + // copy byte to buffer (all previous bytes of the + // sequence have been copied in the default case below) + string_buffer[bytes++] = s[i]; + } // write buffer and reset index; there must be 13 bytes // left, as this is the maximal number of bytes to be @@ -27568,6 +27542,121 @@ class serializer } } + /*! + @brief the JSON form of each character below 0x60 + + Entry c holds the escape sequence of character c - the short form for + \\b, \\t, \\n, \\f, \\r, \\" and \\\\, \\u00xx for the other control characters - + or c itself if it needs no escaping, padded with zeros to 7 bytes. The last + byte is the length of the sequence. The 8 bytes of an entry are copied at + once, so writing an escape needs neither a branch nor a byte-wise loop. + */ + static const std::array, 0x60>& ascii_escapes() noexcept + { + static const std::array, 0x60> escapes = + { + { + {{'\\', 'u', '0', '0', '0', '0', 0, 6}}, // 0x00 + {{'\\', 'u', '0', '0', '0', '1', 0, 6}}, // 0x01 + {{'\\', 'u', '0', '0', '0', '2', 0, 6}}, // 0x02 + {{'\\', 'u', '0', '0', '0', '3', 0, 6}}, // 0x03 + {{'\\', 'u', '0', '0', '0', '4', 0, 6}}, // 0x04 + {{'\\', 'u', '0', '0', '0', '5', 0, 6}}, // 0x05 + {{'\\', 'u', '0', '0', '0', '6', 0, 6}}, // 0x06 + {{'\\', 'u', '0', '0', '0', '7', 0, 6}}, // 0x07 + {{'\\', 'b', 0, 0, 0, 0, 0, 2}}, // 0x08 + {{'\\', 't', 0, 0, 0, 0, 0, 2}}, // 0x09 + {{'\\', 'n', 0, 0, 0, 0, 0, 2}}, // 0x0A + {{'\\', 'u', '0', '0', '0', 'b', 0, 6}}, // 0x0B + {{'\\', 'f', 0, 0, 0, 0, 0, 2}}, // 0x0C + {{'\\', 'r', 0, 0, 0, 0, 0, 2}}, // 0x0D + {{'\\', 'u', '0', '0', '0', 'e', 0, 6}}, // 0x0E + {{'\\', 'u', '0', '0', '0', 'f', 0, 6}}, // 0x0F + {{'\\', 'u', '0', '0', '1', '0', 0, 6}}, // 0x10 + {{'\\', 'u', '0', '0', '1', '1', 0, 6}}, // 0x11 + {{'\\', 'u', '0', '0', '1', '2', 0, 6}}, // 0x12 + {{'\\', 'u', '0', '0', '1', '3', 0, 6}}, // 0x13 + {{'\\', 'u', '0', '0', '1', '4', 0, 6}}, // 0x14 + {{'\\', 'u', '0', '0', '1', '5', 0, 6}}, // 0x15 + {{'\\', 'u', '0', '0', '1', '6', 0, 6}}, // 0x16 + {{'\\', 'u', '0', '0', '1', '7', 0, 6}}, // 0x17 + {{'\\', 'u', '0', '0', '1', '8', 0, 6}}, // 0x18 + {{'\\', 'u', '0', '0', '1', '9', 0, 6}}, // 0x19 + {{'\\', 'u', '0', '0', '1', 'a', 0, 6}}, // 0x1A + {{'\\', 'u', '0', '0', '1', 'b', 0, 6}}, // 0x1B + {{'\\', 'u', '0', '0', '1', 'c', 0, 6}}, // 0x1C + {{'\\', 'u', '0', '0', '1', 'd', 0, 6}}, // 0x1D + {{'\\', 'u', '0', '0', '1', 'e', 0, 6}}, // 0x1E + {{'\\', 'u', '0', '0', '1', 'f', 0, 6}}, // 0x1F + {{' ', 0, 0, 0, 0, 0, 0, 1}}, // 0x20 + {{'!', 0, 0, 0, 0, 0, 0, 1}}, // 0x21 + {{'\\', '"', 0, 0, 0, 0, 0, 2}}, // 0x22 + {{'#', 0, 0, 0, 0, 0, 0, 1}}, // 0x23 + {{'$', 0, 0, 0, 0, 0, 0, 1}}, // 0x24 + {{'%', 0, 0, 0, 0, 0, 0, 1}}, // 0x25 + {{'&', 0, 0, 0, 0, 0, 0, 1}}, // 0x26 + {{'\'', 0, 0, 0, 0, 0, 0, 1}}, // 0x27 + {{'(', 0, 0, 0, 0, 0, 0, 1}}, // 0x28 + {{')', 0, 0, 0, 0, 0, 0, 1}}, // 0x29 + {{'*', 0, 0, 0, 0, 0, 0, 1}}, // 0x2A + {{'+', 0, 0, 0, 0, 0, 0, 1}}, // 0x2B + {{',', 0, 0, 0, 0, 0, 0, 1}}, // 0x2C + {{'-', 0, 0, 0, 0, 0, 0, 1}}, // 0x2D + {{'.', 0, 0, 0, 0, 0, 0, 1}}, // 0x2E + {{'/', 0, 0, 0, 0, 0, 0, 1}}, // 0x2F + {{'0', 0, 0, 0, 0, 0, 0, 1}}, // 0x30 + {{'1', 0, 0, 0, 0, 0, 0, 1}}, // 0x31 + {{'2', 0, 0, 0, 0, 0, 0, 1}}, // 0x32 + {{'3', 0, 0, 0, 0, 0, 0, 1}}, // 0x33 + {{'4', 0, 0, 0, 0, 0, 0, 1}}, // 0x34 + {{'5', 0, 0, 0, 0, 0, 0, 1}}, // 0x35 + {{'6', 0, 0, 0, 0, 0, 0, 1}}, // 0x36 + {{'7', 0, 0, 0, 0, 0, 0, 1}}, // 0x37 + {{'8', 0, 0, 0, 0, 0, 0, 1}}, // 0x38 + {{'9', 0, 0, 0, 0, 0, 0, 1}}, // 0x39 + {{':', 0, 0, 0, 0, 0, 0, 1}}, // 0x3A + {{';', 0, 0, 0, 0, 0, 0, 1}}, // 0x3B + {{'<', 0, 0, 0, 0, 0, 0, 1}}, // 0x3C + {{'=', 0, 0, 0, 0, 0, 0, 1}}, // 0x3D + {{'>', 0, 0, 0, 0, 0, 0, 1}}, // 0x3E + {{'?', 0, 0, 0, 0, 0, 0, 1}}, // 0x3F + {{'@', 0, 0, 0, 0, 0, 0, 1}}, // 0x40 + {{'A', 0, 0, 0, 0, 0, 0, 1}}, // 0x41 + {{'B', 0, 0, 0, 0, 0, 0, 1}}, // 0x42 + {{'C', 0, 0, 0, 0, 0, 0, 1}}, // 0x43 + {{'D', 0, 0, 0, 0, 0, 0, 1}}, // 0x44 + {{'E', 0, 0, 0, 0, 0, 0, 1}}, // 0x45 + {{'F', 0, 0, 0, 0, 0, 0, 1}}, // 0x46 + {{'G', 0, 0, 0, 0, 0, 0, 1}}, // 0x47 + {{'H', 0, 0, 0, 0, 0, 0, 1}}, // 0x48 + {{'I', 0, 0, 0, 0, 0, 0, 1}}, // 0x49 + {{'J', 0, 0, 0, 0, 0, 0, 1}}, // 0x4A + {{'K', 0, 0, 0, 0, 0, 0, 1}}, // 0x4B + {{'L', 0, 0, 0, 0, 0, 0, 1}}, // 0x4C + {{'M', 0, 0, 0, 0, 0, 0, 1}}, // 0x4D + {{'N', 0, 0, 0, 0, 0, 0, 1}}, // 0x4E + {{'O', 0, 0, 0, 0, 0, 0, 1}}, // 0x4F + {{'P', 0, 0, 0, 0, 0, 0, 1}}, // 0x50 + {{'Q', 0, 0, 0, 0, 0, 0, 1}}, // 0x51 + {{'R', 0, 0, 0, 0, 0, 0, 1}}, // 0x52 + {{'S', 0, 0, 0, 0, 0, 0, 1}}, // 0x53 + {{'T', 0, 0, 0, 0, 0, 0, 1}}, // 0x54 + {{'U', 0, 0, 0, 0, 0, 0, 1}}, // 0x55 + {{'V', 0, 0, 0, 0, 0, 0, 1}}, // 0x56 + {{'W', 0, 0, 0, 0, 0, 0, 1}}, // 0x57 + {{'X', 0, 0, 0, 0, 0, 0, 1}}, // 0x58 + {{'Y', 0, 0, 0, 0, 0, 0, 1}}, // 0x59 + {{'Z', 0, 0, 0, 0, 0, 0, 1}}, // 0x5A + {{'[', 0, 0, 0, 0, 0, 0, 1}}, // 0x5B + {{'\\', '\\', 0, 0, 0, 0, 0, 2}}, // 0x5C + {{']', 0, 0, 0, 0, 0, 0, 1}}, // 0x5D + {{'^', 0, 0, 0, 0, 0, 0, 1}}, // 0x5E + {{'_', 0, 0, 0, 0, 0, 0, 1}}, // 0x5F + } + }; + return escapes; + } + /*! * @brief write a lowercase "\\uXXXX" escape sequence into @a string_buffer * diff --git a/tests/src/unit-serialization.cpp b/tests/src/unit-serialization.cpp index c56c4f5c1..5dd76beb8 100644 --- a/tests/src/unit-serialization.cpp +++ b/tests/src/unit-serialization.cpp @@ -923,66 +923,68 @@ TEST_CASE("serialization boundary values for the string buffer") // mid-string once fewer than 13 bytes remain (`string_buffer.size() - bytes // < 13`), 13 being one more than the most a single code point can ever // write at once (a surrogate pair: two back-to-back "\uXXXX" escapes, 12 - // bytes). Every write into string_buffer that this check protects happens - // in steps of 2 (a simple "\\x" escape) or 6 (one "\uXXXX" unit), so - // `bytes` only ever takes even values at the point the check runs - the - // tightest values actually reachable are therefore 498 (512 - 498 == 14, - // one simple escape away from the threshold) and 500 (512 - 500 == 12, - // where the flush fires immediately and resets bytes to 0). + // bytes). Characters below 0x60 (control characters, '"' and '\\') are + // escaped straight into the write buffer and never reach string_buffer, so + // under ensure_ascii it fills in steps of 6 (one "\uXXXX" unit for a + // character below U+10000) or 12 (a surrogate pair). With 2-byte characters + // such as U+00E4, `bytes` takes multiples of 6 at the point the check runs; + // the tightest values reachable are therefore 498 (512 - 498 == 14, after + // 83 characters) and 504 (512 - 504 == 8, after 84 characters, where the + // flush fires immediately and resets bytes to 0). - SECTION("a run of 2-byte escapes lands bytes on, and one step past, the flush threshold") + SECTION("a run of 6-byte escapes lands bytes on, and one step past, the flush threshold") { for (const int count : { - 249, 250, 251 + 83, 84, 85 }) { CAPTURE(count) - const json j = std::string(static_cast(count), '\n'); + std::string umlauts; std::string expected = "\""; for (int i = 0; i < count; ++i) { - expected += "\\n"; + umlauts += "\xC3\xA4"; + expected += "\\u00e4"; } expected += '"'; - CHECK(j.dump() == expected); + CHECK(json(umlauts).dump(-1, ' ', true) == expected); } } - SECTION("an ASCII prefix leaves the tightest reachable margin before a 12-byte surrogate pair") + SECTION("a prefix leaves the tightest reachable margin before a 12-byte surrogate pair") { // U+1F600 (the "\xF0\x9F\x98\x80" UTF-8 bytes) is dumped under // ensure_ascii as the 12-byte surrogate pair "\ud83d\ude00"; that // write happens in a single step with no intermediate flush check, so // it is the write most exposed by an off-by-one in the "< 13" guard. - // A prefix of 249 newlines leaves exactly 14 bytes of headroom + // A prefix of 83 U+00E4 characters leaves exactly 14 bytes of headroom // (512 - 498), the smallest margin the guard ever actually allows - // into a new code point; 250 newlines instead trigger the guard's own + // into a new code point; 84 characters instead trigger the guard's own // flush first, so the emoji starts from a freshly emptied (512-byte) - // buffer, and 251 repeats that with one more escape already past the + // buffer, and 85 repeats that with one more escape already past the // reset. Together they cover the margin the guard allows landing on, // one step before, and one step after - all must still produce the // identical, correct escapes. for (const int prefix_count : { - 249, 250, 251 + 83, 84, 85 }) { CAPTURE(prefix_count) - const std::string prefix(static_cast(prefix_count), '\n'); - const std::string emoji = "\xF0\x9F\x98\x80"; - const json j = prefix + emoji; - + std::string prefix; std::string expected_prefix; for (int i = 0; i < prefix_count; ++i) { - expected_prefix += "\\n"; + prefix += "\xC3\xA4"; + expected_prefix += "\\u00e4"; } + const std::string emoji = "\xF0\x9F\x98\x80"; + const json j = prefix + emoji; - // newline escaping does not depend on ensure_ascii: only the - // emoji differs (raw UTF-8 bytes vs. a \u-escaped surrogate pair) + // without ensure_ascii, the whole string is copied unchanged std::string expected_raw = "\""; - expected_raw += expected_prefix; + expected_raw += prefix; expected_raw += emoji; expected_raw += '"'; std::string expected_ascii = "\""; @@ -1027,3 +1029,132 @@ TEST_CASE("serialization boundary values for the string buffer") } } } + +namespace +{ +// reference escaper for ASCII strings, written independently of the table in +// serializer::ascii_escapes() +std::string reference_escape_ascii(const std::string& s, const bool ensure_ascii) +{ + std::string result = "\""; + for (const char c : s) + { + const auto byte = static_cast(c); + switch (byte) + { + case '\b': + result += "\\b"; + break; + case '\t': + result += "\\t"; + break; + case '\n': + result += "\\n"; + break; + case '\f': + result += "\\f"; + break; + case '\r': + result += "\\r"; + break; + case '"': + result += "\\\""; + break; + case '\\': + result += "\\\\"; + break; + default: + if (byte < 0x20 || (ensure_ascii && byte == 0x7F)) + { + // byte < 0x80, so the first two hex digits are 0 + const char* const hex = "0123456789abcdef"; + result += "\\u00"; + result += hex[byte >> 4U]; + result += hex[byte & 0x0FU]; + } + else + { + result += c; + } + break; + } + } + result += '"'; + return result; +} +} // namespace + +TEST_CASE("characters below 0x60 are escaped from a table straight into the write buffer") +{ + SECTION("every ASCII character at every position") + { + // lengths up to 17 cover the 8-byte strides of the bulk scanner on + // both sides of the character, so the scanner stops at it from every + // offset + for (std::size_t length = 1; length <= 17; ++length) + { + for (std::size_t pos = 0; pos < length; ++pos) + { + for (unsigned int c = 0; c < 0x80; ++c) + { + std::string s(length, 'a'); + s[pos] = static_cast(c); + CAPTURE(length) + CAPTURE(pos) + CAPTURE(c) + const json j = s; + CHECK(j.dump(-1, ' ', false) == reference_escape_ascii(s, false)); + CHECK(j.dump(-1, ' ', true) == reference_escape_ascii(s, true)); + } + } + } + } + + SECTION("runs of escaped characters") + { + std::string s; + for (unsigned int c = 0; c < 0x20; ++c) + { + s += static_cast(c); + } + s += R"("\"\)"; + CHECK(json(s).dump() == reference_escape_ascii(s, false)); + CHECK(json(s).dump(-1, ' ', true) == reference_escape_ascii(s, true)); + } + + SECTION("escapes straddle the end of the write buffer") + { + // the 8-byte store of a table entry needs room for all 8 bytes, even + // when the escape itself is shorter; a prefix of k plain characters + // shifts the 6-byte escapes across every offset of the 1024-byte + // write buffer + for (std::size_t k = 0; k < 8; ++k) + { + CAPTURE(k) + const std::string s = std::string(k, 'a') + std::string(400, '\x01'); + CHECK(json(s).dump() == reference_escape_ascii(s, false)); + CHECK(json::array({s, s}).dump() == "[" + reference_escape_ascii(s, false) + "," + reference_escape_ascii(s, false) + "]"); + } + } + + SECTION("pending \\u escapes of non-ASCII characters are written first") + { + // under ensure_ascii, U+00E4 is \u-escaped into the string buffer; + // a following escape from the table must not overtake it + const json j = "\xC3\xA4\n\xC3\xA4\"\xF0\x9F\x98\x80\\"; + CHECK(j.dump(-1, ' ', true) == R"("\u00e4\n\u00e4\"\ud83d\ude00\\")"); + CHECK(j.dump(-1, ' ', false) == "\"\xC3\xA4\\n\xC3\xA4\\\"\xF0\x9F\x98\x80\\\\\""); + } + + SECTION("escapes after invalid UTF-8 for every error handler") + { + const json j = "a\xFF\n\xC3\""; + CHECK_THROWS_WITH_AS(utils::ignore_return_value(j.dump()), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0xFF", json::type_error&); + CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == R"("a\n\"")"); + CHECK(j.dump(-1, ' ', true, json::error_handler_t::ignore) == R"("a\n\"")"); + CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"a\xEF\xBF\xBD\\n\xEF\xBF\xBD\\\"\""); + CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == R"("a\ufffd\n\ufffd\"")"); + CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"a\xFF\\n\xC3\\\"\""); + CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"a\xFF\\n\xC3\\\"\""); + } +}