From ae30ba55c3d6708182b03b5fa4c5145e3ce003ca Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 16 Oct 2026 22:00:00 +0200 Subject: [PATCH] Write strings and keys of up to 32 characters in one go with vectors The previous change handles plain strings of up to 16 characters with two overlapping 8-byte words. With SSE2 or NEON, two overlapping 16-byte vectors do the same for 16 to 32 characters: one test for a quote, backslash, control character, DEL or non-ASCII byte over both, and two stores. Targets without either keep the limit of 16. Apple M1, Apple clang, retired instructions per dump(), on top of the previous changes (it replaces the table of #5878 for these strings): keys and values of 1..40 characters 7.07M -> 4.94M (-30 %) twitter 4.15M -> 3.30M (-21 %) paths 9.09M -> 8.08M (-11 %) citm_catalog 5.76M -> 5.53M (-4 %) jeopardy 461M -> 451M (-2 %) Linux x86-64 (callgrind), gcc 13 / clang 18: twitter -11 % / -9 %, keys and values of 1..40 characters -19 % / -17 %, paths -7 % / -7 %. Documents without strings in that range are unchanged. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/output/serializer.hpp | 83 ++++++++++++++++--- single_include/nlohmann/json.hpp | 83 ++++++++++++++++--- tests/src/unit-serialization.cpp | 9 +- 3 files changed, 145 insertions(+), 30 deletions(-) diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index d4f0d863b..cbde53cc4 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -620,7 +620,7 @@ class serializer JSON_HEDLEY_NEVER_INLINE void dump_string_value(const string_t& s) { #if JSON_DUMP_SHORT_STRINGS - if (s.size() <= 16 && put_short_quoted(s, "\"")) + if (s.size() <= short_string_max && put_short_quoted(s, "\"")) { return; } @@ -756,6 +756,9 @@ class serializer } #if JSON_DUMP_SHORT_STRINGS + /// the longest string put_short_quoted() takes + static constexpr std::size_t short_string_max = JSON_SCAN_VECTOR ? 32 : 16; + /// the high bit of every byte of @a v that is not printable ASCII or is a /// quote, a backslash, or DEL (zero if the bytes can be copied as they are) static std::uint64_t swar_not_plain(const std::uint64_t v) noexcept @@ -786,17 +789,18 @@ class serializer /*! @brief write `"`, @a s and @a tail if @a s is short and needs no escaping - A string of up to 16 printable ASCII characters (no quote, backslash, or DEL) - is read with two overlapping loads and written with two overlapping stores, - instead of scanning it and calling memcpy with a length that is only known - at run time. Returns false, having changed nothing, if @a s does not - qualify or the write buffer lacks the room. + A string of up to 16 (32 with vector instructions) printable ASCII + characters (no quote, backslash, or DEL) is read with two overlapping loads + and written with two overlapping stores, instead of scanning it and calling + memcpy with a length that is only known at run time. Returns false, having + changed nothing, if @a s does not qualify or the write buffer lacks the + room. */ template bool put_short_quoted(const string_t& s, const char (&tail)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) { const std::size_t n = s.size(); - if (n > 16 || write_buffer_pos + n + N > write_buffer.size()) + if (n > short_string_max || write_buffer_pos + n + N > write_buffer.size()) { return false; } @@ -806,14 +810,26 @@ class serializer if (n >= 8) { - const auto a = load_word(p); - const auto b = load_word(p + n - 8); - if ((swar_not_plain(a) | swar_not_plain(b)) != 0) +#if JSON_SCAN_VECTOR + if (n >= 16) { - return false; + if (!copy_plain_vectors(d, p, n)) + { + return false; + } + } + else +#endif + { + const auto a = load_word(p); + const auto b = load_word(p + n - 8); + if ((swar_not_plain(a) | swar_not_plain(b)) != 0) + { + return false; + } + store_word(d, a); + store_word(d + n - 8, b); } - store_word(d, a); - store_word(d + n - 8, b); } else if (n >= 4) { @@ -857,6 +873,47 @@ class serializer write_buffer_pos += n + N; return true; } + +#if JSON_SCAN_VECTOR + /// copy the 16 to 32 bytes [@a p, @a p + @a n) to @a d with two overlapping + /// vectors, unless one of them is a quote, a backslash, a control + /// character, DEL or not ASCII + static bool copy_plain_vectors(char* d, const char* p, const std::size_t n) noexcept + { +#if JSON_SCAN_NEON + const uint8x16_t a = vld1q_u8(reinterpret_cast(p)); + const uint8x16_t b = vld1q_u8(reinterpret_cast(p + n - 16)); + const uint8x16_t special = vorrq_u8(vorrq_u8(vorrq_u8(vceqq_u8(a, vdupq_n_u8('"')), vceqq_u8(a, vdupq_n_u8('\\'))), + vorrq_u8(vceqq_u8(b, vdupq_n_u8('"')), vceqq_u8(b, vdupq_n_u8('\\')))), + vorrq_u8(vorrq_u8(vcltq_s8(vreinterpretq_s8_u8(a), vdupq_n_s8(0x20)), vcltq_s8(vreinterpretq_s8_u8(b), vdupq_n_s8(0x20))), + vorrq_u8(vceqq_u8(a, vdupq_n_u8(0x7F)), vceqq_u8(b, vdupq_n_u8(0x7F))))); + if (vmaxvq_u8(special) != 0) + { + return false; + } + vst1q_u8(reinterpret_cast(d), a); + vst1q_u8(reinterpret_cast(d + n - 16), b); +#else + const __m128i a = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + const __m128i b = _mm_loadu_si128(static_cast(static_cast(p + n - 16))); // NOLINT(bugprone-casting-through-void) + const __m128i quote = _mm_set1_epi8('"'); + const __m128i backslash = _mm_set1_epi8('\\'); + const __m128i space = _mm_set1_epi8(0x20); + const __m128i del = _mm_set1_epi8(0x7F); + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(a, quote), _mm_cmpeq_epi8(a, backslash)), + _mm_or_si128(_mm_cmpeq_epi8(b, quote), _mm_cmpeq_epi8(b, backslash))), + _mm_or_si128(_mm_or_si128(_mm_cmplt_epi8(a, space), _mm_cmplt_epi8(b, space)), + _mm_or_si128(_mm_cmpeq_epi8(a, del), _mm_cmpeq_epi8(b, del)))); + if (_mm_movemask_epi8(special) != 0) + { + return false; + } + _mm_storeu_si128(static_cast<__m128i*>(static_cast(d)), a); // NOLINT(bugprone-casting-through-void) + _mm_storeu_si128(static_cast<__m128i*>(static_cast(d + n - 16)), b); // NOLINT(bugprone-casting-through-void) +#endif + return true; + } +#endif #endif JSON_PRIVATE_UNLESS_TESTED: diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index acff3d7b4..e37ec2e60 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -30764,7 +30764,7 @@ class serializer JSON_HEDLEY_NEVER_INLINE void dump_string_value(const string_t& s) { #if JSON_DUMP_SHORT_STRINGS - if (s.size() <= 16 && put_short_quoted(s, "\"")) + if (s.size() <= short_string_max && put_short_quoted(s, "\"")) { return; } @@ -30900,6 +30900,9 @@ class serializer } #if JSON_DUMP_SHORT_STRINGS + /// the longest string put_short_quoted() takes + static constexpr std::size_t short_string_max = JSON_SCAN_VECTOR ? 32 : 16; + /// the high bit of every byte of @a v that is not printable ASCII or is a /// quote, a backslash, or DEL (zero if the bytes can be copied as they are) static std::uint64_t swar_not_plain(const std::uint64_t v) noexcept @@ -30930,17 +30933,18 @@ class serializer /*! @brief write `"`, @a s and @a tail if @a s is short and needs no escaping - A string of up to 16 printable ASCII characters (no quote, backslash, or DEL) - is read with two overlapping loads and written with two overlapping stores, - instead of scanning it and calling memcpy with a length that is only known - at run time. Returns false, having changed nothing, if @a s does not - qualify or the write buffer lacks the room. + A string of up to 16 (32 with vector instructions) printable ASCII + characters (no quote, backslash, or DEL) is read with two overlapping loads + and written with two overlapping stores, instead of scanning it and calling + memcpy with a length that is only known at run time. Returns false, having + changed nothing, if @a s does not qualify or the write buffer lacks the + room. */ template bool put_short_quoted(const string_t& s, const char (&tail)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) { const std::size_t n = s.size(); - if (n > 16 || write_buffer_pos + n + N > write_buffer.size()) + if (n > short_string_max || write_buffer_pos + n + N > write_buffer.size()) { return false; } @@ -30950,14 +30954,26 @@ class serializer if (n >= 8) { - const auto a = load_word(p); - const auto b = load_word(p + n - 8); - if ((swar_not_plain(a) | swar_not_plain(b)) != 0) +#if JSON_SCAN_VECTOR + if (n >= 16) { - return false; + if (!copy_plain_vectors(d, p, n)) + { + return false; + } + } + else +#endif + { + const auto a = load_word(p); + const auto b = load_word(p + n - 8); + if ((swar_not_plain(a) | swar_not_plain(b)) != 0) + { + return false; + } + store_word(d, a); + store_word(d + n - 8, b); } - store_word(d, a); - store_word(d + n - 8, b); } else if (n >= 4) { @@ -31001,6 +31017,47 @@ class serializer write_buffer_pos += n + N; return true; } + +#if JSON_SCAN_VECTOR + /// copy the 16 to 32 bytes [@a p, @a p + @a n) to @a d with two overlapping + /// vectors, unless one of them is a quote, a backslash, a control + /// character, DEL or not ASCII + static bool copy_plain_vectors(char* d, const char* p, const std::size_t n) noexcept + { +#if JSON_SCAN_NEON + const uint8x16_t a = vld1q_u8(reinterpret_cast(p)); + const uint8x16_t b = vld1q_u8(reinterpret_cast(p + n - 16)); + const uint8x16_t special = vorrq_u8(vorrq_u8(vorrq_u8(vceqq_u8(a, vdupq_n_u8('"')), vceqq_u8(a, vdupq_n_u8('\\'))), + vorrq_u8(vceqq_u8(b, vdupq_n_u8('"')), vceqq_u8(b, vdupq_n_u8('\\')))), + vorrq_u8(vorrq_u8(vcltq_s8(vreinterpretq_s8_u8(a), vdupq_n_s8(0x20)), vcltq_s8(vreinterpretq_s8_u8(b), vdupq_n_s8(0x20))), + vorrq_u8(vceqq_u8(a, vdupq_n_u8(0x7F)), vceqq_u8(b, vdupq_n_u8(0x7F))))); + if (vmaxvq_u8(special) != 0) + { + return false; + } + vst1q_u8(reinterpret_cast(d), a); + vst1q_u8(reinterpret_cast(d + n - 16), b); +#else + const __m128i a = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + const __m128i b = _mm_loadu_si128(static_cast(static_cast(p + n - 16))); // NOLINT(bugprone-casting-through-void) + const __m128i quote = _mm_set1_epi8('"'); + const __m128i backslash = _mm_set1_epi8('\\'); + const __m128i space = _mm_set1_epi8(0x20); + const __m128i del = _mm_set1_epi8(0x7F); + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(a, quote), _mm_cmpeq_epi8(a, backslash)), + _mm_or_si128(_mm_cmpeq_epi8(b, quote), _mm_cmpeq_epi8(b, backslash))), + _mm_or_si128(_mm_or_si128(_mm_cmplt_epi8(a, space), _mm_cmplt_epi8(b, space)), + _mm_or_si128(_mm_cmpeq_epi8(a, del), _mm_cmpeq_epi8(b, del)))); + if (_mm_movemask_epi8(special) != 0) + { + return false; + } + _mm_storeu_si128(static_cast<__m128i*>(static_cast(d)), a); // NOLINT(bugprone-casting-through-void) + _mm_storeu_si128(static_cast<__m128i*>(static_cast(d + n - 16)), b); // NOLINT(bugprone-casting-through-void) +#endif + return true; + } +#endif #endif JSON_PRIVATE_UNLESS_TESTED: diff --git a/tests/src/unit-serialization.cpp b/tests/src/unit-serialization.cpp index 6164e5fdb..1173b851b 100644 --- a/tests/src/unit-serialization.cpp +++ b/tests/src/unit-serialization.cpp @@ -1583,9 +1583,10 @@ TEST_CASE("invalid UTF-8 at every position is handled like the reference") TEST_CASE("short strings are written like the reference, as values and as keys") { - // strings of up to 16 characters that need no escaping are written as a - // whole (together with the quotes and, for a key, the colon); everything - // else, including a string that is merely short, takes the general path. + // strings of up to 16 characters (32 with vector instructions) that need no + // escaping are written as a whole (together with the quotes and, for a key, + // the colon); everything else, including a string that is merely short, + // takes the general path. // Every length around the edges of the size classes of the fast path is // combined with each kind of byte that must send a string to the general // path, at each position, and compared with the reference for values and, @@ -1595,7 +1596,7 @@ TEST_CASE("short strings are written like the reference, as values and as keys") "\"", "\\", "\n", "\x1f", "\x7f", "\xc3\xa9", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xff", "\x80", "\xc3" }; - for (std::size_t length = 0; length <= 20; ++length) + for (std::size_t length = 0; length <= 36; ++length) { const std::string plain(length, 'k'); check_dump_against_reference(plain, false);