From 7c26e99ac53dbe745a30c41d52ecd9fe42ab0059 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 21 Oct 2026 20:01:38 +0200 Subject: [PATCH] Copy ASCII strings in the view's dump with ensure_ascii (#5879) With ensure_ascii, the view's dump wrote every string through the escaping path, which decodes UTF-8 byte by byte, even strings of the source that are plain ASCII. A string of the source without escape sequences has no quote, backslash, or control character, so ensure_ascii leaves it unchanged exactly when it has no byte >= 0x7F. The dump now checks that eight bytes at a time (one add and one or per word) and copies such strings like the dump without ensure_ascii. The compact walk (dump_compact) now also serves ensure_ascii, through a separate out-of-line instantiation, so the walk without ensure_ascii keeps its speed. Parsing is unchanged. A flag set by the parser would let the dump skip the check and was about twice as fast for compact output, but it costs every parse of string-heavy input 1-2% with GCC. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/view/serializer.hpp | 101 +++++++++++++++++--- single_include/nlohmann/json_view.hpp | 101 +++++++++++++++++--- tests/src/unit-json_view_dump.cpp | 71 ++++++++++++++ 3 files changed, 249 insertions(+), 24 deletions(-) diff --git a/include/nlohmann/detail/view/serializer.hpp b/include/nlohmann/detail/view/serializer.hpp index 5d1074820..922ca58a1 100644 --- a/include/nlohmann/detail/view/serializer.hpp +++ b/include/nlohmann/detail/view/serializer.hpp @@ -171,6 +171,34 @@ inline std::size_t plain_output_run(const unsigned char* s, std::size_t n) noexc return n; } +/// Whether dump() with ensure_ascii writes the string at s unchanged, for a +/// string of the source without escape sequences (which has no quote, +/// backslash, or control character): no byte is DEL (0x7F) or above. Adding 1 +/// to each byte sets its high bit exactly for 0x7F, and a carry into the next +/// byte only comes from 0xFF, whose own high bit is set anyway. +inline bool is_ascii_without_delete(const char* s, std::size_t n) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + const std::uint64_t v = read_eight_bytes(s + i); + if (((v | (v + ones)) & high) != 0) + { + return false; + } + } + for (; i < n; ++i) + { + if (static_cast(s[i]) >= 0x7F) + { + return false; + } + } + return true; +} + /// A stack that starts in a buffer of the caller (a local array) and moves to /// the heap (a vector of the caller) only when that is full, so that dumps of /// shallow documents need no allocation. The top is a pointer, as in @@ -286,15 +314,19 @@ class view_serializer void dump(const node* root) { - if (!m_style.pretty && !m_style.ensure_ascii) + if (!m_style.pretty) { - if (m_style.source_numbers) + if (m_style.ensure_ascii) { - dump_compact(root); + dump_compact_ascii(root); + } + else if (m_style.source_numbers) + { + dump_compact(root); } else { - dump_compact(root); + dump_compact(root); } return; } @@ -400,6 +432,41 @@ class view_serializer m_out.put('"'); } + /// dump_compact() with ensure_ascii, out of line: the walk without it is + /// faster when it is not merged with this one into dump() + NLOHMANN_VIEW_NOINLINE void dump_compact_ascii(const node* root) + { + if (m_style.source_numbers) + { + dump_compact(root); + } + else + { + dump_compact(root); + } + } + + /// whether dump_compact() copies a string or key from the source: one of + /// the source, and with ensure_ascii one without bytes >= 0x7F (an overload + /// each, so that the walk without ensure_ascii is compiled exactly as + /// before: with GCC, a condition that only folds away later changed its + /// block layout and made it 4-6% slower) + static bool copied(const node& n, const char* /*src*/, std::false_type /*ensure_ascii*/) noexcept + { + return (n.flags & node_flags::storage) == 0; + } + + static bool copied(const node& n, const char* src, std::true_type /*ensure_ascii*/) noexcept + { + return (n.flags & node_flags::storage) == 0 && is_ascii_without_delete(src + n.off, n.len); + } + + /// a string with ensure_ascii (dump_compact()), out of line + NLOHMANN_VIEW_NOINLINE void write_ascii_string(const node& n) + { + write_string(n); + } + /// the copies of dump_compact() that are not fixed-size moves (long /// strings, or near the end of the source); out of line, so that the /// compiler does not merge the fixed-size moves into this call @@ -417,16 +484,18 @@ class view_serializer } /*! - @brief the compact output without ensure_ascii (the default dump()) + @brief the compact output (the default dump(), or with ensure_ascii) The same walk as dump(), with the write position in a local variable (stores through char pointers would otherwise force a reload of the buffer's members after each one), and with strings and number tokens of the source copied by fixed-size moves of 32 bytes where the source has that many bytes left, instead of a library call per token. The buffer - keeps 64 bytes of slack for the overshoot. + keeps 64 bytes of slack for the overshoot. With ensure_ascii (Ascii), a + string of the source is copied only if it has no byte >= 0x7F (see + is_ascii_without_delete()); any other string is written by write_string(). */ - template + template void dump_compact(const node* root) { struct frame @@ -478,7 +547,14 @@ class view_serializer const auto escaped = [&](const node & n) { m_out.set_cursor(w); - write_decoded(n); + if (enabled(Ascii)) + { + write_ascii_string(n); + } + else + { + write_decoded(n); + } w = m_out.cursor(); lim = m_out.limit(); }; @@ -501,7 +577,7 @@ class view_serializer switch (static_cast(n->kind)) { case value_t::string: - if ((n->flags & node_flags::storage) == 0) + if (copied(*n, src, std::integral_constant())) { room(n->len + 2); *w++ = '"'; @@ -632,7 +708,7 @@ class view_serializer if (cur.object) { const node& key = *at; - if ((key.flags & node_flags::storage) == 0) + if (copied(key, src, std::integral_constant())) { room(key.len + 3); *w++ = '"'; @@ -843,9 +919,10 @@ class view_serializer { const char* const s = m_doc.str(n); m_out.put('"'); - if ((n.flags & node_flags::storage) == 0 && !m_style.ensure_ascii) + if ((n.flags & node_flags::storage) == 0 && (!m_style.ensure_ascii || is_ascii_without_delete(s, n.len))) { - // a string of the source without escape sequences has nothing to escape + // a string of the source without escape sequences has nothing to + // escape (with ensure_ascii, if it is printable ASCII) m_out.put(s, n.len); } else if (m_style.ensure_ascii) diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index a2829e3ea..66f5b0bbd 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -6237,6 +6237,34 @@ inline std::size_t plain_output_run(const unsigned char* s, std::size_t n) noexc return n; } +/// Whether dump() with ensure_ascii writes the string at s unchanged, for a +/// string of the source without escape sequences (which has no quote, +/// backslash, or control character): no byte is DEL (0x7F) or above. Adding 1 +/// to each byte sets its high bit exactly for 0x7F, and a carry into the next +/// byte only comes from 0xFF, whose own high bit is set anyway. +inline bool is_ascii_without_delete(const char* s, std::size_t n) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + const std::uint64_t v = read_eight_bytes(s + i); + if (((v | (v + ones)) & high) != 0) + { + return false; + } + } + for (; i < n; ++i) + { + if (static_cast(s[i]) >= 0x7F) + { + return false; + } + } + return true; +} + /// A stack that starts in a buffer of the caller (a local array) and moves to /// the heap (a vector of the caller) only when that is full, so that dumps of /// shallow documents need no allocation. The top is a pointer, as in @@ -6352,15 +6380,19 @@ class view_serializer void dump(const node* root) { - if (!m_style.pretty && !m_style.ensure_ascii) + if (!m_style.pretty) { - if (m_style.source_numbers) + if (m_style.ensure_ascii) { - dump_compact(root); + dump_compact_ascii(root); + } + else if (m_style.source_numbers) + { + dump_compact(root); } else { - dump_compact(root); + dump_compact(root); } return; } @@ -6466,6 +6498,41 @@ class view_serializer m_out.put('"'); } + /// dump_compact() with ensure_ascii, out of line: the walk without it is + /// faster when it is not merged with this one into dump() + NLOHMANN_VIEW_NOINLINE void dump_compact_ascii(const node* root) + { + if (m_style.source_numbers) + { + dump_compact(root); + } + else + { + dump_compact(root); + } + } + + /// whether dump_compact() copies a string or key from the source: one of + /// the source, and with ensure_ascii one without bytes >= 0x7F (an overload + /// each, so that the walk without ensure_ascii is compiled exactly as + /// before: with GCC, a condition that only folds away later changed its + /// block layout and made it 4-6% slower) + static bool copied(const node& n, const char* /*src*/, std::false_type /*ensure_ascii*/) noexcept + { + return (n.flags & node_flags::storage) == 0; + } + + static bool copied(const node& n, const char* src, std::true_type /*ensure_ascii*/) noexcept + { + return (n.flags & node_flags::storage) == 0 && is_ascii_without_delete(src + n.off, n.len); + } + + /// a string with ensure_ascii (dump_compact()), out of line + NLOHMANN_VIEW_NOINLINE void write_ascii_string(const node& n) + { + write_string(n); + } + /// the copies of dump_compact() that are not fixed-size moves (long /// strings, or near the end of the source); out of line, so that the /// compiler does not merge the fixed-size moves into this call @@ -6483,16 +6550,18 @@ class view_serializer } /*! - @brief the compact output without ensure_ascii (the default dump()) + @brief the compact output (the default dump(), or with ensure_ascii) The same walk as dump(), with the write position in a local variable (stores through char pointers would otherwise force a reload of the buffer's members after each one), and with strings and number tokens of the source copied by fixed-size moves of 32 bytes where the source has that many bytes left, instead of a library call per token. The buffer - keeps 64 bytes of slack for the overshoot. + keeps 64 bytes of slack for the overshoot. With ensure_ascii (Ascii), a + string of the source is copied only if it has no byte >= 0x7F (see + is_ascii_without_delete()); any other string is written by write_string(). */ - template + template void dump_compact(const node* root) { struct frame @@ -6544,7 +6613,14 @@ class view_serializer const auto escaped = [&](const node & n) { m_out.set_cursor(w); - write_decoded(n); + if (enabled(Ascii)) + { + write_ascii_string(n); + } + else + { + write_decoded(n); + } w = m_out.cursor(); lim = m_out.limit(); }; @@ -6567,7 +6643,7 @@ class view_serializer switch (static_cast(n->kind)) { case value_t::string: - if ((n->flags & node_flags::storage) == 0) + if (copied(*n, src, std::integral_constant())) { room(n->len + 2); *w++ = '"'; @@ -6698,7 +6774,7 @@ class view_serializer if (cur.object) { const node& key = *at; - if ((key.flags & node_flags::storage) == 0) + if (copied(key, src, std::integral_constant())) { room(key.len + 3); *w++ = '"'; @@ -6909,9 +6985,10 @@ class view_serializer { const char* const s = m_doc.str(n); m_out.put('"'); - if ((n.flags & node_flags::storage) == 0 && !m_style.ensure_ascii) + if ((n.flags & node_flags::storage) == 0 && (!m_style.ensure_ascii || is_ascii_without_delete(s, n.len))) { - // a string of the source without escape sequences has nothing to escape + // a string of the source without escape sequences has nothing to + // escape (with ensure_ascii, if it is printable ASCII) m_out.put(s, n.len); } else if (m_style.ensure_ascii) diff --git a/tests/src/unit-json_view_dump.cpp b/tests/src/unit-json_view_dump.cpp index ea6e4e9cc..3b9da6b89 100644 --- a/tests/src/unit-json_view_dump.cpp +++ b/tests/src/unit-json_view_dump.cpp @@ -12,6 +12,7 @@ using nlohmann::json; using nlohmann::ordered_json; using nlohmann::json_document; +using nlohmann::ordered_json_editable_document; using nlohmann::json_view; using nlohmann::ordered_json_document; using nlohmann::ordered_json_view; @@ -219,6 +220,76 @@ TEST_CASE("json_view dump") CHECK(d.root()["a"].dump(-1, ' ', true).size() == 2 + (5000 * 6)); } + SECTION("ensure_ascii with strings of the source, decoded strings, and edited strings") + { + // plain ASCII, non-ASCII, escaped (decoded), and mixed strings, as values and as keys; + // the second text has a DEL, which ensure_ascii writes as an escape + for (const char* const del : + {"", "\x7F" + }) + { + const std::string text = std::string("{\"plain\":\"abc\",\"caf\xC3\xA9\":\"\xC3\xA9\xE2\x82\xAC\xF0\x9F\x98\x80\",") + + "\"esc\\n\\\"\":\"a\\tb\\u00e4\\ud83d\\ude00\",\"del\":\"a" + del + "b\",\"\":\"\"," // NOLINT(modernize-raw-string-literal) + + "\"list\":[\"x\",\"\xC3\xBC\",\"\\\\\",\"long ASCII string with more than thirty-two bytes in it\"," + + "\"long string with more than thirty-two bytes and one non-ASCII byte \xC3\xB6\"]," + + "\"n\":[1,2.5,-0,true,null]}"; + const ordered_json j = ordered_json::parse(text); + const ordered_json_document d = ordered_json_document::parse(text); + CHECK(d.root().dump(-1, ' ', true) == j.dump(-1, ' ', true)); + CHECK(d.root().dump(4, ' ', true) == j.dump(4, ' ', true)); + CHECK(d.root().dump() == j.dump()); + CHECK(d.root()["list"].dump(-1, ' ', true) == j["list"].dump(-1, ' ', true)); + CHECK(d.root()["del"].dump(-1, ' ', true) == j["del"].dump(-1, ' ', true)); + + // strings that an edit wrote (the source strings that they replace were plain) + ordered_json_editable_document e = ordered_json_editable_document::parse(text); + ordered_json k = j; + e.set(e.root()["plain"], std::string("\xC3\xA4gypten")); + k["plain"] = "\xC3\xA4gypten"; + e.set(e.root()["list"][0], std::string("ascii after edit")); + k["list"][0] = "ascii after edit"; + e.set(e.root()["list"][1], std::string("\xC3\xA4")); + k["list"][1] = "\xC3\xA4"; + e.set(e.root()["del"], std::string("tab\there")); + k["del"] = "tab\there"; + CHECK(e.root().dump(-1, ' ', true) == k.dump(-1, ' ', true)); + CHECK(e.root().dump(2, ' ', true) == k.dump(2, ' ', true)); + CHECK(e.root().dump() == k.dump()); + + // strings that were edited from or to ASCII text + e.set(e.root()["list"][2], std::string("\xC3\xA4")); + k["list"][2] = "\xC3\xA4"; + e.set(e.root()["list"][4], std::string("plain again")); + k["list"][4] = "plain again"; + CHECK(e.root().dump(-1, ' ', true) == k.dump(-1, ' ', true)); + } + } + + SECTION("ensure_ascii with DEL or a non-ASCII character at every position") + { + // the check whether a string of the source has no byte >= 0x7F reads + // eight bytes at a time and then the rest one by one + for (std::size_t length = 1; length <= 20; ++length) + { + for (std::size_t pos = 0; pos < length; ++pos) + { + for (const char* const c : + { + "\x7F", "\x7E", "\xC3\xA4", "\xC3\xBF" // NOLINT(modernize-raw-string-literal) + }) + { + const std::string s = std::string(pos, 'a') + c + std::string(length - pos - 1, 'b'); + std::string text = "[\""; + text.append(s).append("\",{\"").append(s).append("\":1}]"); + const json_document d = json_document::parse(text); + const json j = json::parse(text); + CHECK(d.root().dump(-1, ' ', true) == j.dump(-1, ' ', true)); + CHECK(d.root().dump(1, ' ', true) == j.dump(1, ' ', true)); + } + } + } + } + SECTION("streams and discarded views") { const json_document d = json_document::parse(R"({"a": [1, 2]})");