Write integers of 17 to 20 digits eight digits at a time in dump() (#5964)

#5634 made dump() write integers below 10^16 eight digits at a time; larger
ones still took the two-digits-per-division loop and count_digits(). Random
64-bit integers almost all have 17 to 20 digits (99.9 % of signed_ints.json
and unsigned_ints.json), so Dump/signed_ints and Dump/unsigned_ints gained
only 1.4x over 3.12.0, against 2.3x for small integers.

A 64-bit magnitude is now written as its first one to four digits and two
groups of eight. Wider magnitudes (__int128 and class types, #5805) keep the
loop: number types of at most 64 bits are selected by type, so no comparison
is reported as always true; for others the value is compared. The sign
handling of #1447/#5815 (remove_sign() into number_magnitude_t) is unchanged.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/output/serializer.hpp

@@ -1977,6 +1977,23 @@ class serializer
             return;
         }
 
+        // 17 to 20 digits: the first one to four, then twice eight
+        if (magnitude_fits_64_bits(abs_value, std::integral_constant<bool, magnitude_has_at_most_64_bits>()))
+        {
+            const auto value = static_cast<std::uint64_t>(abs_value);
+            const std::uint64_t top = value / 10000000000000000u; // 1 to 1844
+            const std::uint64_t low16 = value - (top * 10000000000000000u);
+            const std::uint64_t middle = low16 / 100000000u;
+            const std::uint64_t first = dtoa_impl::eight_digit_bytes(top);
+            const auto leading = static_cast<unsigned>(count_leading_zeros(first) / 8); // (first is not 0)
+            char* const p = buffer_ptr + n_chars;
+            dtoa_impl::store_msb_first(p, (first << (8 * leading)) + 0x3030303030303030u);
+            dtoa_impl::store_msb_first(p + 8 - leading, dtoa_impl::eight_digit_bytes(middle) + 0x3030303030303030u);
+            dtoa_impl::store_msb_first(p + 16 - leading, dtoa_impl::eight_digit_bytes(low16 - (middle * 100000000u)) + 0x3030303030303030u);
+            write_buffer_pos += n_chars + 24 - leading;
+            return;
+        }
+
         n_chars += count_digits(abs_value);
 
         // spare 1 byte for '\0'
@@ -2122,6 +2139,23 @@ class serializer
         }
     }
 
+    /// whether every number_magnitude_t fits 64 bits
+    static constexpr bool magnitude_has_at_most_64_bits = std::numeric_limits<number_magnitude_t>::is_specialized
+        && std::numeric_limits<number_magnitude_t>::digits <= 64;
+
+    /// whether @a x fits 64 bits: always if number_magnitude_t has at most
+    /// 64 bits (selected by type, as the comparison would be reported as
+    /// always true), else compared
+    static bool magnitude_fits_64_bits(const number_magnitude_t& /*unused*/, std::true_type /*unused*/) noexcept
+    {
+        return true;
+    }
+
+    static bool magnitude_fits_64_bits(const number_magnitude_t& x, std::false_type /*unused*/) noexcept
+    {
+        return x <= 18446744073709551615u;
+    }
+
     /*
      * Helper function for dump_integer
      *

include/nlohmann/detail/view/builder.hpp

@@ -343,6 +343,21 @@ class builder
                 }
                 arena_ensure(w, w_end, 4);
                 w = put_utf8(w, cp);
+                // A chain of unicode escapes ("\u3042\u3044\u3046", as Python's json.dumps writes non-ASCII text) is
+                // decoded here, without going round the loop above for each of them. Anything else (a surrogate, a
+                // bad digit, fewer than six bytes) leaves p at the backslash: the loop above handles it, and reports
+                // the same error at the same position as it would without this loop.
+                while (end - p >= 6 && p[0] == '\\' && p[1] == 'u')
+                {
+                    const int next = hex_codepoint(p + 2);
+                    if (next < 0 || (next & 0xF800) == 0xD800)
+                    {
+                        break;
+                    }
+                    p += 6;
+                    arena_ensure(w, w_end, 4);
+                    w = put_utf8(w, next);
+                }
             }
             else if (NLOHMANN_VIEW_LIKELY(d < 128 && simple_escape[d] != 0))
             {
@@ -1082,7 +1097,7 @@ indent_done:
             {
                 ++p;
                 const unsigned char* const f0 = p;
-                p = skip_fraction_digits(p, e);
+                p = skip_digits(p, e);
                 if (NLOHMANN_VIEW_UNLIKELY(p == f0))
                 {
                     return fail(error_code::number_after_dot);

include/nlohmann/detail/view/scan.hpp

@@ -198,57 +198,6 @@ inline const unsigned char* find_block_comment_end(const unsigned char* p, const
     return p;
 }
 
-/// The digits of a fraction, whose number varies from one number to the next: skip_digits() with 16 bytes compared at
-/// once and the end of the run taken from a bit mask (SSE2), instead of a compare and a branch for each digit. That
-/// is about a third of the instructions for a fraction of 12 digits, and no branch that is mispredicted when the
-/// lengths vary. Where there is no SSE2, skip_digits() is used.
-#if NLOHMANN_VIEW_SSE2
-// The last bytes of the input, one at a time. GCC unrolls this loop in every inlined copy of the function below, which
-// makes the parse loop 4 KiB larger and slows down documents that have hardly any fractions; out of line, it does
-// not. Clang does better with the loop inline (a call in the parse loop costs it more than the unrolling).
-#if defined(__GNUC__) && !defined(__clang__)
-inline NLOHMANN_VIEW_NOINLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept
-{
-    while (p != e && is_digit(*p))
-    {
-        ++p;
-    }
-    return p;
-}
-#else
-NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept
-{
-    while (p != e && is_digit(*p))
-    {
-        ++p;
-    }
-    return p;
-}
-#endif
-
-NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept
-{
-    while (e - p >= 16)
-    {
-        const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(p))); // NOLINT(bugprone-casting-through-void)
-        // a digit is above '/' and below ':' (signed compares: a byte with the top bit set is negative, and neither)
-        const __m128i digits = _mm_and_si128(_mm_cmpgt_epi8(in, _mm_set1_epi8('/')), _mm_cmplt_epi8(in, _mm_set1_epi8(':')));
-        const auto not_digit = static_cast<std::uint64_t>(~static_cast<unsigned>(_mm_movemask_epi8(digits)) & 0xFFFFu);
-        if (not_digit != 0)
-        {
-            return p + count_trailing_zeros(not_digit);
-        }
-        p += 16;
-    }
-    return skip_digits_tail(p, e);
-}
-#else
-NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept
-{
-    return skip_digits(p, e);
-}
-#endif
-
 /// powers of ten up to 10^19 as integers
 inline std::uint64_t int_pow10(unsigned k) noexcept
 {

single_include/nlohmann/json.hpp

@@ -36224,6 +36224,23 @@ class serializer
             return;
         }
 
+        // 17 to 20 digits: the first one to four, then twice eight
+        if (magnitude_fits_64_bits(abs_value, std::integral_constant<bool, magnitude_has_at_most_64_bits>()))
+        {
+            const auto value = static_cast<std::uint64_t>(abs_value);
+            const std::uint64_t top = value / 10000000000000000u; // 1 to 1844
+            const std::uint64_t low16 = value - (top * 10000000000000000u);
+            const std::uint64_t middle = low16 / 100000000u;
+            const std::uint64_t first = dtoa_impl::eight_digit_bytes(top);
+            const auto leading = static_cast<unsigned>(count_leading_zeros(first) / 8); // (first is not 0)
+            char* const p = buffer_ptr + n_chars;
+            dtoa_impl::store_msb_first(p, (first << (8 * leading)) + 0x3030303030303030u);
+            dtoa_impl::store_msb_first(p + 8 - leading, dtoa_impl::eight_digit_bytes(middle) + 0x3030303030303030u);
+            dtoa_impl::store_msb_first(p + 16 - leading, dtoa_impl::eight_digit_bytes(low16 - (middle * 100000000u)) + 0x3030303030303030u);
+            write_buffer_pos += n_chars + 24 - leading;
+            return;
+        }
+
         n_chars += count_digits(abs_value);
 
         // spare 1 byte for '\0'
@@ -36369,6 +36386,23 @@ class serializer
         }
     }
 
+    /// whether every number_magnitude_t fits 64 bits
+    static constexpr bool magnitude_has_at_most_64_bits = std::numeric_limits<number_magnitude_t>::is_specialized
+        && std::numeric_limits<number_magnitude_t>::digits <= 64;
+
+    /// whether @a x fits 64 bits: always if number_magnitude_t has at most
+    /// 64 bits (selected by type, as the comparison would be reported as
+    /// always true), else compared
+    static bool magnitude_fits_64_bits(const number_magnitude_t& /*unused*/, std::true_type /*unused*/) noexcept
+    {
+        return true;
+    }
+
+    static bool magnitude_fits_64_bits(const number_magnitude_t& x, std::false_type /*unused*/) noexcept
+    {
+        return x <= 18446744073709551615u;
+    }
+
     /*
      * Helper function for dump_integer
      *

single_include/nlohmann/json_view.hpp

@@ -1218,57 +1218,6 @@ inline const unsigned char* find_block_comment_end(const unsigned char* p, const
     return p;
 }
 
-/// The digits of a fraction, whose number varies from one number to the next: skip_digits() with 16 bytes compared at
-/// once and the end of the run taken from a bit mask (SSE2), instead of a compare and a branch for each digit. That
-/// is about a third of the instructions for a fraction of 12 digits, and no branch that is mispredicted when the
-/// lengths vary. Where there is no SSE2, skip_digits() is used.
-#if NLOHMANN_VIEW_SSE2
-// The last bytes of the input, one at a time. GCC unrolls this loop in every inlined copy of the function below, which
-// makes the parse loop 4 KiB larger and slows down documents that have hardly any fractions; out of line, it does
-// not. Clang does better with the loop inline (a call in the parse loop costs it more than the unrolling).
-#if defined(__GNUC__) && !defined(__clang__)
-inline NLOHMANN_VIEW_NOINLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept
-{
-    while (p != e && is_digit(*p))
-    {
-        ++p;
-    }
-    return p;
-}
-#else
-NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept
-{
-    while (p != e && is_digit(*p))
-    {
-        ++p;
-    }
-    return p;
-}
-#endif
-
-NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept
-{
-    while (e - p >= 16)
-    {
-        const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(p))); // NOLINT(bugprone-casting-through-void)
-        // a digit is above '/' and below ':' (signed compares: a byte with the top bit set is negative, and neither)
-        const __m128i digits = _mm_and_si128(_mm_cmpgt_epi8(in, _mm_set1_epi8('/')), _mm_cmplt_epi8(in, _mm_set1_epi8(':')));
-        const auto not_digit = static_cast<std::uint64_t>(~static_cast<unsigned>(_mm_movemask_epi8(digits)) & 0xFFFFu);
-        if (not_digit != 0)
-        {
-            return p + count_trailing_zeros(not_digit);
-        }
-        p += 16;
-    }
-    return skip_digits_tail(p, e);
-}
-#else
-NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept
-{
-    return skip_digits(p, e);
-}
-#endif
-
 /// powers of ten up to 10^19 as integers
 inline std::uint64_t int_pow10(unsigned k) noexcept
 {
@@ -1636,6 +1585,21 @@ class builder
                 }
                 arena_ensure(w, w_end, 4);
                 w = put_utf8(w, cp);
+                // A chain of unicode escapes ("\u3042\u3044\u3046", as Python's json.dumps writes non-ASCII text) is
+                // decoded here, without going round the loop above for each of them. Anything else (a surrogate, a
+                // bad digit, fewer than six bytes) leaves p at the backslash: the loop above handles it, and reports
+                // the same error at the same position as it would without this loop.
+                while (end - p >= 6 && p[0] == '\\' && p[1] == 'u')
+                {
+                    const int next = hex_codepoint(p + 2);
+                    if (next < 0 || (next & 0xF800) == 0xD800)
+                    {
+                        break;
+                    }
+                    p += 6;
+                    arena_ensure(w, w_end, 4);
+                    w = put_utf8(w, next);
+                }
             }
             else if (NLOHMANN_VIEW_LIKELY(d < 128 && simple_escape[d] != 0))
             {
@@ -2375,7 +2339,7 @@ indent_done:
             {
                 ++p;
                 const unsigned char* const f0 = p;
-                p = skip_fraction_digits(p, e);
+                p = skip_digits(p, e);
                 if (NLOHMANN_VIEW_UNLIKELY(p == f0))
                 {
                     return fail(error_code::number_after_dot);

tests/src/unit-json_view_builder.cpp

@@ -523,47 +523,6 @@ TEST_CASE("json_view builder: strings across vector blocks")
     }
 }
 
-TEST_CASE("json_view builder: fraction digits")
-{
-    // The digits of a fraction are skipped 16 bytes at a time where there are
-    // that many left (SSE2), and one at a time after that. Fractions of every
-    // length from 0 to 40 digits, followed by every kind of byte that ends a
-    // number or lies next to the digits in ASCII ('/' and ':'), as values, as
-    // keys' neighbours, and at the very end of the text (a vector without a
-    // terminating NUL). The length of the number is part of the value that
-    // the test compares with json::parse().
-    const std::string followers = ",]}/:.eE+-x \n\t\"09";
-    std::mt19937 rng(2028); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible
-    for (std::size_t digits = 0; digits <= 40; ++digits)
-    {
-        for (int i = 0; i < 3; ++i)
-        {
-            std::string fraction;
-            for (std::size_t k = 0; k < digits; ++k)
-            {
-                fraction += static_cast<char>('0' + (rng() % 10));
-            }
-            const std::string number = std::string(rng() % 2 == 0 ? "-" : "") + std::to_string(rng() % 100000) + "." + fraction;
-            check_same(number);
-            check_same("[" + number + "]");
-            check_same("[" + number + "," + number + "]");
-            check_same("{\"a\":" + number + "}");
-            for (const char follower : followers)
-            {
-                check_same(number + follower);
-                check_same("[" + number + follower + "]");
-                check_same("[" + number + follower + std::string(rng() % 20, '1'));
-            }
-            // everything near the end of the text: the number cut in every place
-            for (std::size_t cut = 1; cut <= 18 && cut <= number.size(); ++cut)
-            {
-                check_same(number.substr(0, number.size() - cut));
-                check_same("[" + number.substr(0, number.size() - cut));
-            }
-        }
-    }
-}
-
 TEST_CASE("json_view builder: escapes between runs of text")
 {
     // The text between escapes is copied 16 bytes at a time where there are
@@ -616,6 +575,59 @@ TEST_CASE("json_view builder: escapes between runs of text")
     }
 }
 
+TEST_CASE("json_view builder: chains of unicode escapes")
+{
+    // Consecutive \uXXXX escapes are decoded in a loop of their own. Chains of
+    // every length from 1 to 40 of escapes that give one, two, and three
+    // bytes of UTF-8, in upper and lower case, followed by what ends a chain:
+    // the end of the string, text, another escape, UTF-8, and everything that
+    // is not a valid escape (a bad digit at every position of the last
+    // escape, a lone surrogate, a surrogate pair, a cut in every place).
+    const std::array<const char*, 8> chain = {{"\\u0041", "\\u00e4", "\\u00E4", "\\u20ac", "\\u3042", "\\uFFFF", "\\u0000", "\\u07ff"}};
+    const std::array<const char*, 12> ends = {{"", "abc", "\\n", "\\u00e9", "\\ud83d\\ude00", "\\ud83d", "\\ude00", "\\ud83d\\u0041", "\\u12G4", "\\u12", "\xc3\xa4", "\\"}};
+    std::mt19937 rng(2027); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible
+    for (std::size_t length = 1; length <= 40; ++length)
+    {
+        for (int i = 0; i < 20; ++i)
+        {
+            std::string content = std::string(rng() % 20, 'a');
+            for (std::size_t k = 0; k < length; ++k)
+            {
+                content += chain[rng() % chain.size()];
+            }
+            const std::string whole = content;
+            for (const char* end : ends)
+            {
+                check_string(whole + end);
+                check_string(whole + end + std::string(rng() % 19, 'z'));
+            }
+            // a bad digit at every position of every escape near the end of the chain
+            for (std::size_t back = 1; back <= 6 && back <= length; ++back)
+            {
+                for (std::size_t digit = 2; digit < 6; ++digit)
+                {
+                    std::string broken = whole;
+                    broken[broken.size() - (6 * back) + digit] = "gG\"\x01 z"[rng() % 6];
+                    check_string(broken);
+                }
+            }
+            // the text cut in every place of the last three escapes
+            for (std::size_t cut = 0; cut <= 18 && cut <= whole.size(); ++cut)
+            {
+                check_string(whole.substr(0, whole.size() - cut));
+            }
+        }
+    }
+    // a chain longer than the first growth of the arena, in a key and in a value
+    std::string long_chain;
+    for (int i = 0; i < 3000; ++i)
+    {
+        long_chain += chain[static_cast<std::size_t>(i) % chain.size()];
+    }
+    check_string(long_chain);
+    check_string(long_chain + "\\ud83d\\ude00" + long_chain);
+}
+
 TEST_CASE("json_view node integer bits")
 {
     using nlohmann::detail::view::integer_bits;

tests/src/unit-json_view_from_json.cpp

@@ -443,6 +443,30 @@ TEST_CASE("json_view from_json")
         CHECK(d.root().materialize().get<point>().x == 3);
     }
 
+    SECTION("escaped keys: members are found by their decoded names")
+    {
+        // keys are decoded before they are compared, as by parse(); a lookup
+        // that compares the raw bytes of the source would miss these members
+        const json_document d = json_document::parse(R"([{"\u0078": 1, "y": 2}, {"V\u0061lue": 3, "Te\u0078t": "t"}, {"\u0062": "escaped"}, {"x": 1, "\u0078": 3, "y": 2}])");
+        const json j = d.root().materialize();
+        CHECK(d.root()[0].get<point>().x == 1);
+        CHECK(d.root()[0].get<point>().y == 2);
+        CHECK((d.root()[0].get<point>() == j[0].get<point>()));
+        CHECK(d.root()[1].get<with_names>().value == 3);
+        CHECK(d.root()[1].get<with_names>().text == "t");
+        CHECK(d.root()[2].get<with_defaults>().b == "escaped");
+        CHECK(d.root()[2].get<with_defaults>().a == 1);
+        // a duplicate key written once plain and once escaped: still the first member
+        CHECK(d.root()[3].get<point>().x == 1);
+        CHECK(j[3].get<point>().x == 3);
+#if !defined(JSON_NOEXCEPTION) && !JSON_DIAGNOSTICS && !JSON_DIAGNOSTIC_POSITIONS
+        // an escaped backslash is part of the decoded key: "x\\" is not "x"
+        const json_document e = json_document::parse(R"({"x\\": 1, "y": 2})");
+        CHECK(exception_of_call([&] { static_cast<void>(e.root().get<point>()); }) == exception_of_call([&] { static_cast<void>(e.root().materialize().get<point>()); }));
+        CHECK(exception_of_call([&] { static_cast<void>(e.root().get<point>()); }) == "[json.exception.out_of_range.403] key 'x' not found");
+#endif
+    }
+
     SECTION("editable documents and ordered_json")
     {
         json_editable_document e = json_editable_document::parse(R"({"x": 1, "y": 2})");

tests/src/unit-serialization.cpp

@@ -23,6 +23,7 @@ using nlohmann::json;
 #include <cstdint>
 #include <sstream>
 #include <iomanip>
+#include <limits>
 #include <string>
 #include <vector>
 
@@ -248,6 +249,51 @@ TEST_CASE_TEMPLATE("serialization for extreme integer values", T, int32_t, uint3
     }
 }
 
+TEST_CASE("integers of every digit count are written like std::to_string")
+{
+    // dump_integer() writes up to 16 digits in two groups of eight and 17 to
+    // 20 digits as one to four digits and two groups of eight
+    std::vector<std::uint64_t> values;
+    std::uint64_t power = 1;
+    for (int digits = 1; digits <= 20; ++digits)
+    {
+        values.push_back(power);                     // 1 followed by zeros
+        values.push_back(power + 1);
+        values.push_back((power * 2) - 1);
+        if (digits < 20)
+        {
+            values.push_back((power * 10) - 1);      // all 9s
+            values.push_back((power * 10) - power);  // 9 followed by zeros
+            power *= 10;
+        }
+    }
+    values.push_back((std::numeric_limits<std::uint64_t>::max)());
+    values.push_back(static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()));
+    values.push_back(static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()) + 1);
+    std::uint64_t state = 0x9E3779B97F4A7C15u;
+    for (int i = 0; i < 20000; ++i)
+    {
+        state = (state * 6364136223846793005u) + 1442695040888963407u;
+        values.push_back(state >> (state % 61));     // every bit length
+    }
+
+    for (const auto v : values)
+    {
+        CAPTURE(v)
+        CHECK(json(v).dump() == std::to_string(v));
+        if (v <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()))
+        {
+            const auto s = static_cast<std::int64_t>(v);
+            CHECK(json(s).dump() == std::to_string(s));
+            CHECK(json(-s).dump() == std::to_string(-s));
+        }
+    }
+    CHECK(json((std::numeric_limits<std::int64_t>::min)()).dump() == "-9223372036854775808");
+
+    // in a container, so the digits are followed by other output
+    CHECK(json::array({std::uint64_t{18446744073709551615u}, std::int64_t{-1234567890123456789}, 1}).dump() == "[18446744073709551615,-1234567890123456789,1]");
+}
+
 TEST_CASE("dump with binary values")
 {
     auto binary = json::binary({1, 2, 3, 4});