json_view: skip the digits of a fraction 16 bytes at a time (SSE2)

The digits after the decimal point were skipped by skip_digits(), 16 compare-and-branch
steps in a row: about 3.5 instructions per digit, and an exit branch that is
mispredicted when the lengths of the fractions vary. On x86 with SSE2, the
fraction is now skipped by skip_fraction_digits(): 16 bytes are compared at
once and the end of the run is taken from the bit mask, which is about a third
of the instructions for a fraction of 12 digits. The last bytes of the input
are skipped one at a time as before. Where there is no SSE2 (AArch64, other
targets, JSON_NO_SIMD) skip_fraction_digits() is skip_digits(): nothing changes
there. Integer digits keep skip_digits(), whose exit is predictable.

GCC unrolls the loop over the last bytes in every inlined copy of the new
function, which made the parse loop 4 KiB larger and parse() of documents with
hardly any fractions about 4 % slower. For GCC that loop is therefore out of
line (skip_digits_tail()); Clang is faster with it inline.

json_document::parse(), A/B in one binary, 6 code layouts, x86-64 Haswell,
time after / before (GCC 16.2 / Clang 23):

- numbers.json (10,001 floats): 0.715 / 0.680
- canada.json: 0.778 / 0.790
- mesh.json: 0.908 / 0.883
- marine_ik.json: 0.937 / 0.926
- mesh.pretty.json: 0.923 / 0.960

Files with few or no fractions and files with fractions of a fixed length
(20,000 prices with two decimals, 60,000 coordinates with six): between 0.98 and
1.014 on both compilers, except citm_catalog.json with GCC (1.05: the
instruction count is +0.3 %, and that row varies by +-7 % between builds of
unrelated variants on this machine).

New test: fractions of every length from 0 to 40 digits followed by every kind of
byte that ends a number or lies next to the digits in ASCII, in several
positions, and cut in every place near the end of the text, against json::parse()
(the length of the number is part of what is compared), for all option
combinations and with and without a terminating NUL. Checked on x86-64 with GCC 16.2,
Clang 23 and Apple clang under Rosetta, and with a deliberate off-by-one at the 16th byte.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/output/serializer.hpp

@@ -1977,23 +1977,6 @@ class serializer
             return;
         }
 
-        // 17 to 20 digits: the first one to four, then twice eight
-        if (magnitude_fits_64_bits(abs_value, std::integral_constant<bool, magnitude_has_at_most_64_bits>()))
-        {
-            const auto value = static_cast<std::uint64_t>(abs_value);
-            const std::uint64_t top = value / 10000000000000000u; // 1 to 1844
-            const std::uint64_t low16 = value - (top * 10000000000000000u);
-            const std::uint64_t middle = low16 / 100000000u;
-            const std::uint64_t first = dtoa_impl::eight_digit_bytes(top);
-            const auto leading = static_cast<unsigned>(count_leading_zeros(first) / 8); // (first is not 0)
-            char* const p = buffer_ptr + n_chars;
-            dtoa_impl::store_msb_first(p, (first << (8 * leading)) + 0x3030303030303030u);
-            dtoa_impl::store_msb_first(p + 8 - leading, dtoa_impl::eight_digit_bytes(middle) + 0x3030303030303030u);
-            dtoa_impl::store_msb_first(p + 16 - leading, dtoa_impl::eight_digit_bytes(low16 - (middle * 100000000u)) + 0x3030303030303030u);
-            write_buffer_pos += n_chars + 24 - leading;
-            return;
-        }
-
         n_chars += count_digits(abs_value);
 
         // spare 1 byte for '\0'
@@ -2139,23 +2122,6 @@ class serializer
         }
     }
 
-    /// whether every number_magnitude_t fits 64 bits
-    static constexpr bool magnitude_has_at_most_64_bits = std::numeric_limits<number_magnitude_t>::is_specialized
-        && std::numeric_limits<number_magnitude_t>::digits <= 64;
-
-    /// whether @a x fits 64 bits: always if number_magnitude_t has at most
-    /// 64 bits (selected by type, as the comparison would be reported as
-    /// always true), else compared
-    static bool magnitude_fits_64_bits(const number_magnitude_t& /*unused*/, std::true_type /*unused*/) noexcept
-    {
-        return true;
-    }
-
-    static bool magnitude_fits_64_bits(const number_magnitude_t& x, std::false_type /*unused*/) noexcept
-    {
-        return x <= 18446744073709551615u;
-    }
-
     /*
      * Helper function for dump_integer
      *

include/nlohmann/detail/view/builder.hpp

@@ -343,21 +343,6 @@ class builder
                 }
                 arena_ensure(w, w_end, 4);
                 w = put_utf8(w, cp);
-                // A chain of unicode escapes ("\u3042\u3044\u3046", as Python's json.dumps writes non-ASCII text) is
-                // decoded here, without going round the loop above for each of them. Anything else (a surrogate, a
-                // bad digit, fewer than six bytes) leaves p at the backslash: the loop above handles it, and reports
-                // the same error at the same position as it would without this loop.
-                while (end - p >= 6 && p[0] == '\\' && p[1] == 'u')
-                {
-                    const int next = hex_codepoint(p + 2);
-                    if (next < 0 || (next & 0xF800) == 0xD800)
-                    {
-                        break;
-                    }
-                    p += 6;
-                    arena_ensure(w, w_end, 4);
-                    w = put_utf8(w, next);
-                }
             }
             else if (NLOHMANN_VIEW_LIKELY(d < 128 && simple_escape[d] != 0))
             {
@@ -1097,7 +1082,7 @@ indent_done:
             {
                 ++p;
                 const unsigned char* const f0 = p;
-                p = skip_digits(p, e);
+                p = skip_fraction_digits(p, e);
                 if (NLOHMANN_VIEW_UNLIKELY(p == f0))
                 {
                     return fail(error_code::number_after_dot);

include/nlohmann/detail/view/scan.hpp

@@ -198,6 +198,57 @@ inline const unsigned char* find_block_comment_end(const unsigned char* p, const
     return p;
 }
 
+/// The digits of a fraction, whose number varies from one number to the next: skip_digits() with 16 bytes compared at
+/// once and the end of the run taken from a bit mask (SSE2), instead of a compare and a branch for each digit. That
+/// is about a third of the instructions for a fraction of 12 digits, and no branch that is mispredicted when the
+/// lengths vary. Where there is no SSE2, skip_digits() is used.
+#if NLOHMANN_VIEW_SSE2
+// The last bytes of the input, one at a time. GCC unrolls this loop in every inlined copy of the function below, which
+// makes the parse loop 4 KiB larger and slows down documents that have hardly any fractions; out of line, it does
+// not. Clang does better with the loop inline (a call in the parse loop costs it more than the unrolling).
+#if defined(__GNUC__) && !defined(__clang__)
+inline NLOHMANN_VIEW_NOINLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept
+{
+    while (p != e && is_digit(*p))
+    {
+        ++p;
+    }
+    return p;
+}
+#else
+NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept
+{
+    while (p != e && is_digit(*p))
+    {
+        ++p;
+    }
+    return p;
+}
+#endif
+
+NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept
+{
+    while (e - p >= 16)
+    {
+        const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(p))); // NOLINT(bugprone-casting-through-void)
+        // a digit is above '/' and below ':' (signed compares: a byte with the top bit set is negative, and neither)
+        const __m128i digits = _mm_and_si128(_mm_cmpgt_epi8(in, _mm_set1_epi8('/')), _mm_cmplt_epi8(in, _mm_set1_epi8(':')));
+        const auto not_digit = static_cast<std::uint64_t>(~static_cast<unsigned>(_mm_movemask_epi8(digits)) & 0xFFFFu);
+        if (not_digit != 0)
+        {
+            return p + count_trailing_zeros(not_digit);
+        }
+        p += 16;
+    }
+    return skip_digits_tail(p, e);
+}
+#else
+NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept
+{
+    return skip_digits(p, e);
+}
+#endif
+
 /// powers of ten up to 10^19 as integers
 inline std::uint64_t int_pow10(unsigned k) noexcept
 {

single_include/nlohmann/json.hpp

@@ -36224,23 +36224,6 @@ class serializer
             return;
         }
 
-        // 17 to 20 digits: the first one to four, then twice eight
-        if (magnitude_fits_64_bits(abs_value, std::integral_constant<bool, magnitude_has_at_most_64_bits>()))
-        {
-            const auto value = static_cast<std::uint64_t>(abs_value);
-            const std::uint64_t top = value / 10000000000000000u; // 1 to 1844
-            const std::uint64_t low16 = value - (top * 10000000000000000u);
-            const std::uint64_t middle = low16 / 100000000u;
-            const std::uint64_t first = dtoa_impl::eight_digit_bytes(top);
-            const auto leading = static_cast<unsigned>(count_leading_zeros(first) / 8); // (first is not 0)
-            char* const p = buffer_ptr + n_chars;
-            dtoa_impl::store_msb_first(p, (first << (8 * leading)) + 0x3030303030303030u);
-            dtoa_impl::store_msb_first(p + 8 - leading, dtoa_impl::eight_digit_bytes(middle) + 0x3030303030303030u);
-            dtoa_impl::store_msb_first(p + 16 - leading, dtoa_impl::eight_digit_bytes(low16 - (middle * 100000000u)) + 0x3030303030303030u);
-            write_buffer_pos += n_chars + 24 - leading;
-            return;
-        }
-
         n_chars += count_digits(abs_value);
 
         // spare 1 byte for '\0'
@@ -36386,23 +36369,6 @@ class serializer
         }
     }
 
-    /// whether every number_magnitude_t fits 64 bits
-    static constexpr bool magnitude_has_at_most_64_bits = std::numeric_limits<number_magnitude_t>::is_specialized
-        && std::numeric_limits<number_magnitude_t>::digits <= 64;
-
-    /// whether @a x fits 64 bits: always if number_magnitude_t has at most
-    /// 64 bits (selected by type, as the comparison would be reported as
-    /// always true), else compared
-    static bool magnitude_fits_64_bits(const number_magnitude_t& /*unused*/, std::true_type /*unused*/) noexcept
-    {
-        return true;
-    }
-
-    static bool magnitude_fits_64_bits(const number_magnitude_t& x, std::false_type /*unused*/) noexcept
-    {
-        return x <= 18446744073709551615u;
-    }
-
     /*
      * Helper function for dump_integer
      *

single_include/nlohmann/json_view.hpp

@@ -1218,6 +1218,57 @@ inline const unsigned char* find_block_comment_end(const unsigned char* p, const
     return p;
 }
 
+/// The digits of a fraction, whose number varies from one number to the next: skip_digits() with 16 bytes compared at
+/// once and the end of the run taken from a bit mask (SSE2), instead of a compare and a branch for each digit. That
+/// is about a third of the instructions for a fraction of 12 digits, and no branch that is mispredicted when the
+/// lengths vary. Where there is no SSE2, skip_digits() is used.
+#if NLOHMANN_VIEW_SSE2
+// The last bytes of the input, one at a time. GCC unrolls this loop in every inlined copy of the function below, which
+// makes the parse loop 4 KiB larger and slows down documents that have hardly any fractions; out of line, it does
+// not. Clang does better with the loop inline (a call in the parse loop costs it more than the unrolling).
+#if defined(__GNUC__) && !defined(__clang__)
+inline NLOHMANN_VIEW_NOINLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept
+{
+    while (p != e && is_digit(*p))
+    {
+        ++p;
+    }
+    return p;
+}
+#else
+NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept
+{
+    while (p != e && is_digit(*p))
+    {
+        ++p;
+    }
+    return p;
+}
+#endif
+
+NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept
+{
+    while (e - p >= 16)
+    {
+        const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(p))); // NOLINT(bugprone-casting-through-void)
+        // a digit is above '/' and below ':' (signed compares: a byte with the top bit set is negative, and neither)
+        const __m128i digits = _mm_and_si128(_mm_cmpgt_epi8(in, _mm_set1_epi8('/')), _mm_cmplt_epi8(in, _mm_set1_epi8(':')));
+        const auto not_digit = static_cast<std::uint64_t>(~static_cast<unsigned>(_mm_movemask_epi8(digits)) & 0xFFFFu);
+        if (not_digit != 0)
+        {
+            return p + count_trailing_zeros(not_digit);
+        }
+        p += 16;
+    }
+    return skip_digits_tail(p, e);
+}
+#else
+NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept
+{
+    return skip_digits(p, e);
+}
+#endif
+
 /// powers of ten up to 10^19 as integers
 inline std::uint64_t int_pow10(unsigned k) noexcept
 {
@@ -1585,21 +1636,6 @@ class builder
                 }
                 arena_ensure(w, w_end, 4);
                 w = put_utf8(w, cp);
-                // A chain of unicode escapes ("\u3042\u3044\u3046", as Python's json.dumps writes non-ASCII text) is
-                // decoded here, without going round the loop above for each of them. Anything else (a surrogate, a
-                // bad digit, fewer than six bytes) leaves p at the backslash: the loop above handles it, and reports
-                // the same error at the same position as it would without this loop.
-                while (end - p >= 6 && p[0] == '\\' && p[1] == 'u')
-                {
-                    const int next = hex_codepoint(p + 2);
-                    if (next < 0 || (next & 0xF800) == 0xD800)
-                    {
-                        break;
-                    }
-                    p += 6;
-                    arena_ensure(w, w_end, 4);
-                    w = put_utf8(w, next);
-                }
             }
             else if (NLOHMANN_VIEW_LIKELY(d < 128 && simple_escape[d] != 0))
             {
@@ -2339,7 +2375,7 @@ indent_done:
             {
                 ++p;
                 const unsigned char* const f0 = p;
-                p = skip_digits(p, e);
+                p = skip_fraction_digits(p, e);
                 if (NLOHMANN_VIEW_UNLIKELY(p == f0))
                 {
                     return fail(error_code::number_after_dot);

tests/src/unit-json_view_builder.cpp

@@ -523,6 +523,47 @@ TEST_CASE("json_view builder: strings across vector blocks")
     }
 }
 
+TEST_CASE("json_view builder: fraction digits")
+{
+    // The digits of a fraction are skipped 16 bytes at a time where there are
+    // that many left (SSE2), and one at a time after that. Fractions of every
+    // length from 0 to 40 digits, followed by every kind of byte that ends a
+    // number or lies next to the digits in ASCII ('/' and ':'), as values, as
+    // keys' neighbours, and at the very end of the text (a vector without a
+    // terminating NUL). The length of the number is part of the value that
+    // the test compares with json::parse().
+    const std::string followers = ",]}/:.eE+-x \n\t\"09";
+    std::mt19937 rng(2028); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible
+    for (std::size_t digits = 0; digits <= 40; ++digits)
+    {
+        for (int i = 0; i < 3; ++i)
+        {
+            std::string fraction;
+            for (std::size_t k = 0; k < digits; ++k)
+            {
+                fraction += static_cast<char>('0' + (rng() % 10));
+            }
+            const std::string number = std::string(rng() % 2 == 0 ? "-" : "") + std::to_string(rng() % 100000) + "." + fraction;
+            check_same(number);
+            check_same("[" + number + "]");
+            check_same("[" + number + "," + number + "]");
+            check_same("{\"a\":" + number + "}");
+            for (const char follower : followers)
+            {
+                check_same(number + follower);
+                check_same("[" + number + follower + "]");
+                check_same("[" + number + follower + std::string(rng() % 20, '1'));
+            }
+            // everything near the end of the text: the number cut in every place
+            for (std::size_t cut = 1; cut <= 18 && cut <= number.size(); ++cut)
+            {
+                check_same(number.substr(0, number.size() - cut));
+                check_same("[" + number.substr(0, number.size() - cut));
+            }
+        }
+    }
+}
+
 TEST_CASE("json_view builder: escapes between runs of text")
 {
     // The text between escapes is copied 16 bytes at a time where there are
@@ -575,59 +616,6 @@ TEST_CASE("json_view builder: escapes between runs of text")
     }
 }
 
-TEST_CASE("json_view builder: chains of unicode escapes")
-{
-    // Consecutive \uXXXX escapes are decoded in a loop of their own. Chains of
-    // every length from 1 to 40 of escapes that give one, two, and three
-    // bytes of UTF-8, in upper and lower case, followed by what ends a chain:
-    // the end of the string, text, another escape, UTF-8, and everything that
-    // is not a valid escape (a bad digit at every position of the last
-    // escape, a lone surrogate, a surrogate pair, a cut in every place).
-    const std::array<const char*, 8> chain = {{"\\u0041", "\\u00e4", "\\u00E4", "\\u20ac", "\\u3042", "\\uFFFF", "\\u0000", "\\u07ff"}};
-    const std::array<const char*, 12> ends = {{"", "abc", "\\n", "\\u00e9", "\\ud83d\\ude00", "\\ud83d", "\\ude00", "\\ud83d\\u0041", "\\u12G4", "\\u12", "\xc3\xa4", "\\"}};
-    std::mt19937 rng(2027); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible
-    for (std::size_t length = 1; length <= 40; ++length)
-    {
-        for (int i = 0; i < 20; ++i)
-        {
-            std::string content = std::string(rng() % 20, 'a');
-            for (std::size_t k = 0; k < length; ++k)
-            {
-                content += chain[rng() % chain.size()];
-            }
-            const std::string whole = content;
-            for (const char* end : ends)
-            {
-                check_string(whole + end);
-                check_string(whole + end + std::string(rng() % 19, 'z'));
-            }
-            // a bad digit at every position of every escape near the end of the chain
-            for (std::size_t back = 1; back <= 6 && back <= length; ++back)
-            {
-                for (std::size_t digit = 2; digit < 6; ++digit)
-                {
-                    std::string broken = whole;
-                    broken[broken.size() - (6 * back) + digit] = "gG\"\x01 z"[rng() % 6];
-                    check_string(broken);
-                }
-            }
-            // the text cut in every place of the last three escapes
-            for (std::size_t cut = 0; cut <= 18 && cut <= whole.size(); ++cut)
-            {
-                check_string(whole.substr(0, whole.size() - cut));
-            }
-        }
-    }
-    // a chain longer than the first growth of the arena, in a key and in a value
-    std::string long_chain;
-    for (int i = 0; i < 3000; ++i)
-    {
-        long_chain += chain[static_cast<std::size_t>(i) % chain.size()];
-    }
-    check_string(long_chain);
-    check_string(long_chain + "\\ud83d\\ude00" + long_chain);
-}
-
 TEST_CASE("json_view node integer bits")
 {
     using nlohmann::detail::view::integer_bits;

tests/src/unit-json_view_from_json.cpp

@@ -443,30 +443,6 @@ TEST_CASE("json_view from_json")
         CHECK(d.root().materialize().get<point>().x == 3);
     }
 
-    SECTION("escaped keys: members are found by their decoded names")
-    {
-        // keys are decoded before they are compared, as by parse(); a lookup
-        // that compares the raw bytes of the source would miss these members
-        const json_document d = json_document::parse(R"([{"\u0078": 1, "y": 2}, {"V\u0061lue": 3, "Te\u0078t": "t"}, {"\u0062": "escaped"}, {"x": 1, "\u0078": 3, "y": 2}])");
-        const json j = d.root().materialize();
-        CHECK(d.root()[0].get<point>().x == 1);
-        CHECK(d.root()[0].get<point>().y == 2);
-        CHECK((d.root()[0].get<point>() == j[0].get<point>()));
-        CHECK(d.root()[1].get<with_names>().value == 3);
-        CHECK(d.root()[1].get<with_names>().text == "t");
-        CHECK(d.root()[2].get<with_defaults>().b == "escaped");
-        CHECK(d.root()[2].get<with_defaults>().a == 1);
-        // a duplicate key written once plain and once escaped: still the first member
-        CHECK(d.root()[3].get<point>().x == 1);
-        CHECK(j[3].get<point>().x == 3);
-#if !defined(JSON_NOEXCEPTION) && !JSON_DIAGNOSTICS && !JSON_DIAGNOSTIC_POSITIONS
-        // an escaped backslash is part of the decoded key: "x\\" is not "x"
-        const json_document e = json_document::parse(R"({"x\\": 1, "y": 2})");
-        CHECK(exception_of_call([&] { static_cast<void>(e.root().get<point>()); }) == exception_of_call([&] { static_cast<void>(e.root().materialize().get<point>()); }));
-        CHECK(exception_of_call([&] { static_cast<void>(e.root().get<point>()); }) == "[json.exception.out_of_range.403] key 'x' not found");
-#endif
-    }
-
     SECTION("editable documents and ordered_json")
     {
         json_editable_document e = json_editable_document::parse(R"({"x": 1, "y": 2})");

tests/src/unit-serialization.cpp

@@ -23,7 +23,6 @@ using nlohmann::json;
 #include <cstdint>
 #include <sstream>
 #include <iomanip>
-#include <limits>
 #include <string>
 #include <vector>
 
@@ -249,51 +248,6 @@ TEST_CASE_TEMPLATE("serialization for extreme integer values", T, int32_t, uint3
     }
 }
 
-TEST_CASE("integers of every digit count are written like std::to_string")
-{
-    // dump_integer() writes up to 16 digits in two groups of eight and 17 to
-    // 20 digits as one to four digits and two groups of eight
-    std::vector<std::uint64_t> values;
-    std::uint64_t power = 1;
-    for (int digits = 1; digits <= 20; ++digits)
-    {
-        values.push_back(power);                     // 1 followed by zeros
-        values.push_back(power + 1);
-        values.push_back((power * 2) - 1);
-        if (digits < 20)
-        {
-            values.push_back((power * 10) - 1);      // all 9s
-            values.push_back((power * 10) - power);  // 9 followed by zeros
-            power *= 10;
-        }
-    }
-    values.push_back((std::numeric_limits<std::uint64_t>::max)());
-    values.push_back(static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()));
-    values.push_back(static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()) + 1);
-    std::uint64_t state = 0x9E3779B97F4A7C15u;
-    for (int i = 0; i < 20000; ++i)
-    {
-        state = (state * 6364136223846793005u) + 1442695040888963407u;
-        values.push_back(state >> (state % 61));     // every bit length
-    }
-
-    for (const auto v : values)
-    {
-        CAPTURE(v)
-        CHECK(json(v).dump() == std::to_string(v));
-        if (v <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()))
-        {
-            const auto s = static_cast<std::int64_t>(v);
-            CHECK(json(s).dump() == std::to_string(s));
-            CHECK(json(-s).dump() == std::to_string(-s));
-        }
-    }
-    CHECK(json((std::numeric_limits<std::int64_t>::min)()).dump() == "-9223372036854775808");
-
-    // in a container, so the digits are followed by other output
-    CHECK(json::array({std::uint64_t{18446744073709551615u}, std::int64_t{-1234567890123456789}, 1}).dump() == "[18446744073709551615,-1234567890123456789,1]");
-}
-
 TEST_CASE("dump with binary values")
 {
     auto binary = json::binary({1, 2, 3, 4});