Write integers of 17 to 20 digits eight digits at a time in dump() (#5964)

#5634 made dump() write integers below 10^16 eight digits at a time; larger
ones still took the two-digits-per-division loop and count_digits(). Random
64-bit integers almost all have 17 to 20 digits (99.9 % of signed_ints.json
and unsigned_ints.json), so Dump/signed_ints and Dump/unsigned_ints gained
only 1.4x over 3.12.0, against 2.3x for small integers.

A 64-bit magnitude is now written as its first one to four digits and two
groups of eight. Wider magnitudes (__int128 and class types, #5805) keep the
loop: number types of at most 64 bits are selected by type, so no comparison
is reported as always true; for others the value is compared. The sign
handling of #1447/#5815 (remove_sign() into number_magnitude_t) is unchanged.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/output/binary_writer.hpp

@@ -2088,8 +2088,7 @@ class binary_writer
         /// arrays: the first index with one more digit; only maintained by
         /// @ref calc_bson_sizes
         std::uint64_t index_boundary = 10;
-        /// @ref calc_bson_sizes: where its size goes in the table; @ref
-        /// write_bson_document_single_pass: the position of its size field
+        /// @ref calc_bson_sizes only: where its size goes in the table
         std::size_t size_slot;
         /// @ref calc_bson_sizes only: the size of its entries seen so far
         std::size_t entries_size = 0;
@@ -2340,197 +2339,6 @@ class binary_writer
         }
     }
 
-    template<typename SinkType>
-    using sink_written_t = decltype(std::declval<const SinkType&>().written());
-
-    /*!
-    @brief whether BSON documents can be written in a single pass: the sink
-           can overwrite what it has written (output_vector_sink), and every
-           integer fits the 64 bits BSON has, so that writing an array of
-           numbers needs no range check
-    */
-    using bson_single_pass_sink = std::integral_constant < bool,
-          is_detected<sink_written_t, OutputSinkType>::value
-          && std::numeric_limits<typename BasicJsonType::number_integer_t>::is_specialized
-          && std::numeric_limits<typename BasicJsonType::number_integer_t>::digits <= std::numeric_limits<std::int64_t>::digits
-          && std::numeric_limits<typename BasicJsonType::number_unsigned_t>::is_specialized
-          && std::numeric_limits<typename BasicJsonType::number_unsigned_t>::digits <= std::numeric_limits<std::uint64_t>::digits >;
-
-    /// write a placeholder for the size of an object or array whose entries
-    /// follow; @return its position, for @ref patch_bson_size
-    std::size_t write_bson_size_placeholder()
-    {
-        const std::size_t pos = oa.written();
-        write_number<std::int32_t>(0, true);
-        return pos;
-    }
-
-    /// end the object or array whose size is at @a pos and write its size
-    /// there: everything from that field to the terminating null byte
-    /// @return false (and the size is not written) if it does not fit
-    bool close_bson_container(const std::size_t pos)
-    {
-        oa.write_character(to_char_type(0x00));
-        const std::size_t size = oa.written() - pos;
-        if (JSON_HEDLEY_UNLIKELY(!value_in_range_of<std::int32_t>(size)))
-        {
-            return false;
-        }
-        std::array<CharType, sizeof(std::int32_t)> bytes{};
-        store_little_endian<std::int32_t>(bytes.data(), static_cast<std::int32_t>(size));
-        oa.overwrite(pos, bytes.data(), bytes.size());
-        return true;
-    }
-
-    /*!
-    @brief check the value @a j of a BSON entry as @ref calc_bson_value_size
-           does, for @ref write_bson_document_single_pass
-    @return at least the size of the value
-    A number, boolean, or null needs no check there (its integers have at most
-    64 bits) and has at most 8 characters; a discarded value is rejected with
-    the same exception when it is written.
-    */
-    std::size_t check_bson_value(const BasicJsonType& j)
-    {
-        return (j.is_string() || j.is_binary()) ? calc_bson_value_size(j) : 8;
-    }
-
-    /// whether an entry of @a entry_size characters keeps the document that
-    /// begins at @a start, closed by @a open_containers null bytes, within
-    /// the 32-bit size of BSON
-    bool bson_entry_fits(const std::size_t start, const std::size_t entry_size, const std::size_t open_containers) const noexcept
-    {
-        const std::size_t limit = static_cast<std::size_t>((std::numeric_limits<std::int32_t>::max)());
-        const std::size_t written = oa.written() - start;
-        return written <= limit && entry_size <= limit - written && open_containers <= limit - written - entry_size;
-    }
-
-    /*!
-    @brief Serializes the JSON object @a document as a BSON document in a
-           single pass, writing the size of each object and array once it is
-           complete
-
-    Checks every key and value as @ref calc_bson_sizes does, in the same
-    order, so the same exception is thrown; the output written up to then is
-    discarded with the vector it was written to.
-
-    @return false as soon as the document or a string, binary value, object,
-            or array in it would be larger than the 32-bit BSON sizes allow;
-            nothing large has been written then, and the caller writes the
-            document in two passes, which reports that error
-    */
-    template < typename SinkType = OutputSinkType,
-               typename std::enable_if < is_detected<sink_written_t, SinkType>::value, int >::type = 0 >
-    bool write_bson_document_single_pass(const BasicJsonType& document)
-    {
-        const std::size_t start = oa.written();
-        // the object or array whose entries are being written (with the
-        // position of its size), and the ones it is in
-        bson_frame current(&document, write_bson_size_placeholder());
-        std::vector<bson_frame> parents;
-
-        while (true)
-        {
-            const BasicJsonType* nested = nullptr;
-            std::size_t nested_size_pos = 0;
-            if (current.value->is_object())
-            {
-                const auto& object = *current.value->m_data.m_value.object;
-                while (nested == nullptr && current.member != object.cend())
-                {
-                    const auto& el = *current.member;
-                    ++current.member;
-                    const std::size_t header_size = calc_bson_entry_header_size(el.first, el.second);
-                    const std::size_t value_size = el.second.is_structured() ? 0 : check_bson_value(el.second);
-                    if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, header_size + value_size, parents.size() + 1)))
-                    {
-                        return false;
-                    }
-                    if (el.second.is_structured())
-                    {
-                        write_bson_entry_header(el.first, el.second.is_object() ? 0x03 : 0x04);
-                        nested_size_pos = write_bson_size_placeholder();
-                        nested = &el.second;
-                    }
-                    else
-                    {
-                        write_bson_value(el.first, el.second);
-                    }
-                }
-            }
-            else
-            {
-                const auto& array = *current.value->m_data.m_value.array;
-                while (nested == nullptr && current.index < array.size())
-                {
-                    if (is_bson_fixed_size(array[current.index]))
-                    {
-                        write_bson_array_fixed_entries(current);
-                        if (current.index == array.size())
-                        {
-                            break;
-                        }
-                    }
-
-                    // a string, binary value, object or array
-                    const BasicJsonType& el = array[current.index];
-                    const string_t index_name(current.index_name.data(), current.index_name_size);
-                    current.next_index();
-                    const std::size_t value_size = el.is_structured() ? 0 : check_bson_value(el);
-                    if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, 2 + index_name.size() + value_size, parents.size() + 1)))
-                    {
-                        return false;
-                    }
-                    if (el.is_structured())
-                    {
-                        write_bson_entry_header(index_name, el.is_object() ? 0x03 : 0x04);
-                        nested_size_pos = write_bson_size_placeholder();
-                        nested = &el;
-                    }
-                    else
-                    {
-                        write_bson_value(index_name, el);
-                    }
-                }
-            }
-
-            if (nested != nullptr)
-            {
-                if (nested->is_array())
-                {
-                    // an array of booleans, numbers and null is written without a frame on the stack
-                    bson_frame leaf(nested, nested_size_pos);
-                    write_bson_array_fixed_entries(leaf);
-                    if (leaf.index == nested->m_data.m_value.array->size())
-                    {
-                        if (JSON_HEDLEY_UNLIKELY(!close_bson_container(nested_size_pos)))
-                        {
-                            return false;
-                        }
-                        continue;
-                    }
-                    parents.push_back(std::move(current));
-                    current = std::move(leaf);
-                    continue;
-                }
-                parents.push_back(std::move(current));
-                current = bson_frame(nested, nested_size_pos);
-                continue;
-            }
-
-            if (JSON_HEDLEY_UNLIKELY(!close_bson_container(current.size_slot)))
-            {
-                return false;
-            }
-            if (parents.empty())
-            {
-                return true;
-            }
-            current = std::move(parents.back());
-            parents.pop_back();
-        }
-    }
-
     /*!
     @brief Serializes the JSON object @a document as a BSON document
 
@@ -2542,39 +2350,6 @@ class binary_writer
     @pre       document.type() == value_t::object
     */
     void write_bson_document(const BasicJsonType& document)
-    {
-        write_bson_document(document, bson_single_pass_sink());
-    }
-
-    /// the sink can overwrite: one pass, unless the error handler replaces or
-    /// drops ill-formed UTF-8, which changes the sizes the checks compute
-    void write_bson_document(const BasicJsonType& document, std::true_type /*single_pass*/)
-    {
-        const std::size_t start = oa.written();
-        if ((error_handler == error_handler_t::keep || error_handler == error_handler_t::strict)
-                && write_bson_document_single_pass(document))
-        {
-            return;
-        }
-        // a length does not fit: start over, so that it is reported as the
-        // two passes report it, before anything is written
-        oa.rewind(start);
-        write_bson_document_two_pass(document);
-    }
-
-    void write_bson_document(const BasicJsonType& document, std::false_type /*single_pass*/)
-    {
-        write_bson_document_two_pass(document);
-    }
-
-    /*!
-    @brief Serializes the JSON object @a document as a BSON document: the
-           sizes of all objects and arrays first, then the entries
-
-    @param[in] document  the JSON object to serialize
-    @pre       document.type() == value_t::object
-    */
-    void write_bson_document_two_pass(const BasicJsonType& document)
     {
         std::vector<std::size_t> nested_sizes;
         const std::size_t document_size = calc_bson_sizes(document, nested_sizes);

include/nlohmann/detail/output/output_adapters.hpp

@@ -227,29 +227,6 @@ class output_vector_sink
     void flush() noexcept
     {}
 
-    /// the number of characters written so far, the position of the next one
-    std::size_t written() const noexcept
-    {
-        return position();
-    }
-
-    /// @brief replace @a length characters written before, from position @a pos
-    ///
-    /// Lets a writer put a length in front of data it has written since, such
-    /// as the size of a BSON document, without computing it in advance.
-    void overwrite(const std::size_t pos, const CharType* s, const std::size_t length) noexcept
-    {
-        JSON_ASSERT(pos + length <= position());
-        std::memcpy(v->data() + pos, s, length * sizeof(CharType));
-    }
-
-    /// drop the characters written from position @a pos on
-    void rewind(const std::size_t pos) noexcept
-    {
-        JSON_ASSERT(pos <= position());
-        cursor = v->data() + pos;
-    }
-
   private:
     /// number of characters of *v written so far
     std::size_t position() const noexcept

include/nlohmann/detail/output/serializer.hpp

@@ -1977,6 +1977,23 @@ class serializer
             return;
         }
 
+        // 17 to 20 digits: the first one to four, then twice eight
+        if (magnitude_fits_64_bits(abs_value, std::integral_constant<bool, magnitude_has_at_most_64_bits>()))
+        {
+            const auto value = static_cast<std::uint64_t>(abs_value);
+            const std::uint64_t top = value / 10000000000000000u; // 1 to 1844
+            const std::uint64_t low16 = value - (top * 10000000000000000u);
+            const std::uint64_t middle = low16 / 100000000u;
+            const std::uint64_t first = dtoa_impl::eight_digit_bytes(top);
+            const auto leading = static_cast<unsigned>(count_leading_zeros(first) / 8); // (first is not 0)
+            char* const p = buffer_ptr + n_chars;
+            dtoa_impl::store_msb_first(p, (first << (8 * leading)) + 0x3030303030303030u);
+            dtoa_impl::store_msb_first(p + 8 - leading, dtoa_impl::eight_digit_bytes(middle) + 0x3030303030303030u);
+            dtoa_impl::store_msb_first(p + 16 - leading, dtoa_impl::eight_digit_bytes(low16 - (middle * 100000000u)) + 0x3030303030303030u);
+            write_buffer_pos += n_chars + 24 - leading;
+            return;
+        }
+
         n_chars += count_digits(abs_value);
 
         // spare 1 byte for '\0'
@@ -2122,6 +2139,23 @@ class serializer
         }
     }
 
+    /// whether every number_magnitude_t fits 64 bits
+    static constexpr bool magnitude_has_at_most_64_bits = std::numeric_limits<number_magnitude_t>::is_specialized
+        && std::numeric_limits<number_magnitude_t>::digits <= 64;
+
+    /// whether @a x fits 64 bits: always if number_magnitude_t has at most
+    /// 64 bits (selected by type, as the comparison would be reported as
+    /// always true), else compared
+    static bool magnitude_fits_64_bits(const number_magnitude_t& /*unused*/, std::true_type /*unused*/) noexcept
+    {
+        return true;
+    }
+
+    static bool magnitude_fits_64_bits(const number_magnitude_t& x, std::false_type /*unused*/) noexcept
+    {
+        return x <= 18446744073709551615u;
+    }
+
     /*
      * Helper function for dump_integer
      *

include/nlohmann/detail/view/builder.hpp

@@ -343,6 +343,21 @@ class builder
                 }
                 arena_ensure(w, w_end, 4);
                 w = put_utf8(w, cp);
+                // A chain of unicode escapes ("\u3042\u3044\u3046", as Python's json.dumps writes non-ASCII text) is
+                // decoded here, without going round the loop above for each of them. Anything else (a surrogate, a
+                // bad digit, fewer than six bytes) leaves p at the backslash: the loop above handles it, and reports
+                // the same error at the same position as it would without this loop.
+                while (end - p >= 6 && p[0] == '\\' && p[1] == 'u')
+                {
+                    const int next = hex_codepoint(p + 2);
+                    if (next < 0 || (next & 0xF800) == 0xD800)
+                    {
+                        break;
+                    }
+                    p += 6;
+                    arena_ensure(w, w_end, 4);
+                    w = put_utf8(w, next);
+                }
             }
             else if (NLOHMANN_VIEW_LIKELY(d < 128 && simple_escape[d] != 0))
             {

single_include/nlohmann/json.hpp

@@ -26870,29 +26870,6 @@ class output_vector_sink
     void flush() noexcept
     {}
 
-    /// the number of characters written so far, the position of the next one
-    std::size_t written() const noexcept
-    {
-        return position();
-    }
-
-    /// @brief replace @a length characters written before, from position @a pos
-    ///
-    /// Lets a writer put a length in front of data it has written since, such
-    /// as the size of a BSON document, without computing it in advance.
-    void overwrite(const std::size_t pos, const CharType* s, const std::size_t length) noexcept
-    {
-        JSON_ASSERT(pos + length <= position());
-        std::memcpy(v->data() + pos, s, length * sizeof(CharType));
-    }
-
-    /// drop the characters written from position @a pos on
-    void rewind(const std::size_t pos) noexcept
-    {
-        JSON_ASSERT(pos <= position());
-        cursor = v->data() + pos;
-    }
-
   private:
     /// number of characters of *v written so far
     std::size_t position() const noexcept
@@ -29337,8 +29314,7 @@ class binary_writer
         /// arrays: the first index with one more digit; only maintained by
         /// @ref calc_bson_sizes
         std::uint64_t index_boundary = 10;
-        /// @ref calc_bson_sizes: where its size goes in the table; @ref
-        /// write_bson_document_single_pass: the position of its size field
+        /// @ref calc_bson_sizes only: where its size goes in the table
         std::size_t size_slot;
         /// @ref calc_bson_sizes only: the size of its entries seen so far
         std::size_t entries_size = 0;
@@ -29589,197 +29565,6 @@ class binary_writer
         }
     }
 
-    template<typename SinkType>
-    using sink_written_t = decltype(std::declval<const SinkType&>().written());
-
-    /*!
-    @brief whether BSON documents can be written in a single pass: the sink
-           can overwrite what it has written (output_vector_sink), and every
-           integer fits the 64 bits BSON has, so that writing an array of
-           numbers needs no range check
-    */
-    using bson_single_pass_sink = std::integral_constant < bool,
-          is_detected<sink_written_t, OutputSinkType>::value
-          && std::numeric_limits<typename BasicJsonType::number_integer_t>::is_specialized
-          && std::numeric_limits<typename BasicJsonType::number_integer_t>::digits <= std::numeric_limits<std::int64_t>::digits
-          && std::numeric_limits<typename BasicJsonType::number_unsigned_t>::is_specialized
-          && std::numeric_limits<typename BasicJsonType::number_unsigned_t>::digits <= std::numeric_limits<std::uint64_t>::digits >;
-
-    /// write a placeholder for the size of an object or array whose entries
-    /// follow; @return its position, for @ref patch_bson_size
-    std::size_t write_bson_size_placeholder()
-    {
-        const std::size_t pos = oa.written();
-        write_number<std::int32_t>(0, true);
-        return pos;
-    }
-
-    /// end the object or array whose size is at @a pos and write its size
-    /// there: everything from that field to the terminating null byte
-    /// @return false (and the size is not written) if it does not fit
-    bool close_bson_container(const std::size_t pos)
-    {
-        oa.write_character(to_char_type(0x00));
-        const std::size_t size = oa.written() - pos;
-        if (JSON_HEDLEY_UNLIKELY(!value_in_range_of<std::int32_t>(size)))
-        {
-            return false;
-        }
-        std::array<CharType, sizeof(std::int32_t)> bytes{};
-        store_little_endian<std::int32_t>(bytes.data(), static_cast<std::int32_t>(size));
-        oa.overwrite(pos, bytes.data(), bytes.size());
-        return true;
-    }
-
-    /*!
-    @brief check the value @a j of a BSON entry as @ref calc_bson_value_size
-           does, for @ref write_bson_document_single_pass
-    @return at least the size of the value
-    A number, boolean, or null needs no check there (its integers have at most
-    64 bits) and has at most 8 characters; a discarded value is rejected with
-    the same exception when it is written.
-    */
-    std::size_t check_bson_value(const BasicJsonType& j)
-    {
-        return (j.is_string() || j.is_binary()) ? calc_bson_value_size(j) : 8;
-    }
-
-    /// whether an entry of @a entry_size characters keeps the document that
-    /// begins at @a start, closed by @a open_containers null bytes, within
-    /// the 32-bit size of BSON
-    bool bson_entry_fits(const std::size_t start, const std::size_t entry_size, const std::size_t open_containers) const noexcept
-    {
-        const std::size_t limit = static_cast<std::size_t>((std::numeric_limits<std::int32_t>::max)());
-        const std::size_t written = oa.written() - start;
-        return written <= limit && entry_size <= limit - written && open_containers <= limit - written - entry_size;
-    }
-
-    /*!
-    @brief Serializes the JSON object @a document as a BSON document in a
-           single pass, writing the size of each object and array once it is
-           complete
-
-    Checks every key and value as @ref calc_bson_sizes does, in the same
-    order, so the same exception is thrown; the output written up to then is
-    discarded with the vector it was written to.
-
-    @return false as soon as the document or a string, binary value, object,
-            or array in it would be larger than the 32-bit BSON sizes allow;
-            nothing large has been written then, and the caller writes the
-            document in two passes, which reports that error
-    */
-    template < typename SinkType = OutputSinkType,
-               typename std::enable_if < is_detected<sink_written_t, SinkType>::value, int >::type = 0 >
-    bool write_bson_document_single_pass(const BasicJsonType& document)
-    {
-        const std::size_t start = oa.written();
-        // the object or array whose entries are being written (with the
-        // position of its size), and the ones it is in
-        bson_frame current(&document, write_bson_size_placeholder());
-        std::vector<bson_frame> parents;
-
-        while (true)
-        {
-            const BasicJsonType* nested = nullptr;
-            std::size_t nested_size_pos = 0;
-            if (current.value->is_object())
-            {
-                const auto& object = *current.value->m_data.m_value.object;
-                while (nested == nullptr && current.member != object.cend())
-                {
-                    const auto& el = *current.member;
-                    ++current.member;
-                    const std::size_t header_size = calc_bson_entry_header_size(el.first, el.second);
-                    const std::size_t value_size = el.second.is_structured() ? 0 : check_bson_value(el.second);
-                    if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, header_size + value_size, parents.size() + 1)))
-                    {
-                        return false;
-                    }
-                    if (el.second.is_structured())
-                    {
-                        write_bson_entry_header(el.first, el.second.is_object() ? 0x03 : 0x04);
-                        nested_size_pos = write_bson_size_placeholder();
-                        nested = &el.second;
-                    }
-                    else
-                    {
-                        write_bson_value(el.first, el.second);
-                    }
-                }
-            }
-            else
-            {
-                const auto& array = *current.value->m_data.m_value.array;
-                while (nested == nullptr && current.index < array.size())
-                {
-                    if (is_bson_fixed_size(array[current.index]))
-                    {
-                        write_bson_array_fixed_entries(current);
-                        if (current.index == array.size())
-                        {
-                            break;
-                        }
-                    }
-
-                    // a string, binary value, object or array
-                    const BasicJsonType& el = array[current.index];
-                    const string_t index_name(current.index_name.data(), current.index_name_size);
-                    current.next_index();
-                    const std::size_t value_size = el.is_structured() ? 0 : check_bson_value(el);
-                    if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, 2 + index_name.size() + value_size, parents.size() + 1)))
-                    {
-                        return false;
-                    }
-                    if (el.is_structured())
-                    {
-                        write_bson_entry_header(index_name, el.is_object() ? 0x03 : 0x04);
-                        nested_size_pos = write_bson_size_placeholder();
-                        nested = &el;
-                    }
-                    else
-                    {
-                        write_bson_value(index_name, el);
-                    }
-                }
-            }
-
-            if (nested != nullptr)
-            {
-                if (nested->is_array())
-                {
-                    // an array of booleans, numbers and null is written without a frame on the stack
-                    bson_frame leaf(nested, nested_size_pos);
-                    write_bson_array_fixed_entries(leaf);
-                    if (leaf.index == nested->m_data.m_value.array->size())
-                    {
-                        if (JSON_HEDLEY_UNLIKELY(!close_bson_container(nested_size_pos)))
-                        {
-                            return false;
-                        }
-                        continue;
-                    }
-                    parents.push_back(std::move(current));
-                    current = std::move(leaf);
-                    continue;
-                }
-                parents.push_back(std::move(current));
-                current = bson_frame(nested, nested_size_pos);
-                continue;
-            }
-
-            if (JSON_HEDLEY_UNLIKELY(!close_bson_container(current.size_slot)))
-            {
-                return false;
-            }
-            if (parents.empty())
-            {
-                return true;
-            }
-            current = std::move(parents.back());
-            parents.pop_back();
-        }
-    }
-
     /*!
     @brief Serializes the JSON object @a document as a BSON document
 
@@ -29791,39 +29576,6 @@ class binary_writer
     @pre       document.type() == value_t::object
     */
     void write_bson_document(const BasicJsonType& document)
-    {
-        write_bson_document(document, bson_single_pass_sink());
-    }
-
-    /// the sink can overwrite: one pass, unless the error handler replaces or
-    /// drops ill-formed UTF-8, which changes the sizes the checks compute
-    void write_bson_document(const BasicJsonType& document, std::true_type /*single_pass*/)
-    {
-        const std::size_t start = oa.written();
-        if ((error_handler == error_handler_t::keep || error_handler == error_handler_t::strict)
-                && write_bson_document_single_pass(document))
-        {
-            return;
-        }
-        // a length does not fit: start over, so that it is reported as the
-        // two passes report it, before anything is written
-        oa.rewind(start);
-        write_bson_document_two_pass(document);
-    }
-
-    void write_bson_document(const BasicJsonType& document, std::false_type /*single_pass*/)
-    {
-        write_bson_document_two_pass(document);
-    }
-
-    /*!
-    @brief Serializes the JSON object @a document as a BSON document: the
-           sizes of all objects and arrays first, then the entries
-
-    @param[in] document  the JSON object to serialize
-    @pre       document.type() == value_t::object
-    */
-    void write_bson_document_two_pass(const BasicJsonType& document)
     {
         std::vector<std::size_t> nested_sizes;
         const std::size_t document_size = calc_bson_sizes(document, nested_sizes);
@@ -36472,6 +36224,23 @@ class serializer
             return;
         }
 
+        // 17 to 20 digits: the first one to four, then twice eight
+        if (magnitude_fits_64_bits(abs_value, std::integral_constant<bool, magnitude_has_at_most_64_bits>()))
+        {
+            const auto value = static_cast<std::uint64_t>(abs_value);
+            const std::uint64_t top = value / 10000000000000000u; // 1 to 1844
+            const std::uint64_t low16 = value - (top * 10000000000000000u);
+            const std::uint64_t middle = low16 / 100000000u;
+            const std::uint64_t first = dtoa_impl::eight_digit_bytes(top);
+            const auto leading = static_cast<unsigned>(count_leading_zeros(first) / 8); // (first is not 0)
+            char* const p = buffer_ptr + n_chars;
+            dtoa_impl::store_msb_first(p, (first << (8 * leading)) + 0x3030303030303030u);
+            dtoa_impl::store_msb_first(p + 8 - leading, dtoa_impl::eight_digit_bytes(middle) + 0x3030303030303030u);
+            dtoa_impl::store_msb_first(p + 16 - leading, dtoa_impl::eight_digit_bytes(low16 - (middle * 100000000u)) + 0x3030303030303030u);
+            write_buffer_pos += n_chars + 24 - leading;
+            return;
+        }
+
         n_chars += count_digits(abs_value);
 
         // spare 1 byte for '\0'
@@ -36617,6 +36386,23 @@ class serializer
         }
     }
 
+    /// whether every number_magnitude_t fits 64 bits
+    static constexpr bool magnitude_has_at_most_64_bits = std::numeric_limits<number_magnitude_t>::is_specialized
+        && std::numeric_limits<number_magnitude_t>::digits <= 64;
+
+    /// whether @a x fits 64 bits: always if number_magnitude_t has at most
+    /// 64 bits (selected by type, as the comparison would be reported as
+    /// always true), else compared
+    static bool magnitude_fits_64_bits(const number_magnitude_t& /*unused*/, std::true_type /*unused*/) noexcept
+    {
+        return true;
+    }
+
+    static bool magnitude_fits_64_bits(const number_magnitude_t& x, std::false_type /*unused*/) noexcept
+    {
+        return x <= 18446744073709551615u;
+    }
+
     /*
      * Helper function for dump_integer
      *

single_include/nlohmann/json_view.hpp

@@ -1585,6 +1585,21 @@ class builder
                 }
                 arena_ensure(w, w_end, 4);
                 w = put_utf8(w, cp);
+                // A chain of unicode escapes ("\u3042\u3044\u3046", as Python's json.dumps writes non-ASCII text) is
+                // decoded here, without going round the loop above for each of them. Anything else (a surrogate, a
+                // bad digit, fewer than six bytes) leaves p at the backslash: the loop above handles it, and reports
+                // the same error at the same position as it would without this loop.
+                while (end - p >= 6 && p[0] == '\\' && p[1] == 'u')
+                {
+                    const int next = hex_codepoint(p + 2);
+                    if (next < 0 || (next & 0xF800) == 0xD800)
+                    {
+                        break;
+                    }
+                    p += 6;
+                    arena_ensure(w, w_end, 4);
+                    w = put_utf8(w, next);
+                }
             }
             else if (NLOHMANN_VIEW_LIKELY(d < 128 && simple_escape[d] != 0))
             {

tests/src/unit-bson.cpp

@@ -2267,132 +2267,3 @@ TEST_CASE("BSON large strings and binaries (chunked reader)")
         }
     }
 }
-
-TEST_CASE("BSON written in one pass and in two passes")
-{
-    // to_bson() into a returned vector writes each size once the object or
-    // array is complete; through an output adapter, all sizes are computed
-    // first. Both must produce the same bytes and throw the same exceptions.
-    const auto two_pass = [](const json & j, const json::error_handler_t handler)
-    {
-        std::vector<std::uint8_t> result;
-        json::to_bson(j, result, handler);
-        return result;
-    };
-
-    const auto check_same = [&](const json & j, const json::error_handler_t handler)
-    {
-        CAPTURE(j.dump(-1, ' ', true, json::error_handler_t::replace))
-        CAPTURE(static_cast<int>(handler))
-#if !defined(JSON_NOEXCEPTION)
-        std::string one_pass_error;
-        std::string two_pass_error;
-        std::vector<std::uint8_t> one;
-        std::vector<std::uint8_t> two;
-        try
-        {
-            one = json::to_bson(j, handler);
-        }
-        catch (const json::exception& e)
-        {
-            one_pass_error = e.what();
-        }
-        try
-        {
-            two = two_pass(j, handler);
-        }
-        catch (const json::exception& e)
-        {
-            two_pass_error = e.what();
-        }
-        CHECK(one_pass_error == two_pass_error);
-        CHECK(one == two);
-#else
-        CHECK(json::to_bson(j, handler) == two_pass(j, handler));
-#endif
-    };
-
-    const json::error_handler_t handlers[] = {json::error_handler_t::keep, json::error_handler_t::strict,
-                                              json::error_handler_t::replace, json::error_handler_t::ignore
-                                             };
-
-    SECTION("corpus")
-    {
-        for (const auto& value : utils::round_trip_corpus::values())
-        {
-            const json document = value.is_object() ? value : json{{"value", value}};
-            for (const auto handler : handlers)
-            {
-                check_same(document, handler);
-            }
-        }
-    }
-
-    SECTION("nesting and arrays of every length class")
-    {
-        json document = json::object();
-        document["empty object"] = json::object();
-        document["empty array"] = json::array();
-        for (const std::size_t n :
-                {
-                    std::size_t(1), std::size_t(9), std::size_t(10), std::size_t(11), std::size_t(99), std::size_t(100), std::size_t(101), std::size_t(1000)
-                })
-        {
-            json numbers = json::array();
-            json mixed = json::array();
-            for (std::size_t i = 0; i < n; ++i)
-            {
-                numbers.push_back(i % 3 == 0 ? json(static_cast<double>(i) / 3) : json(static_cast<std::int64_t>(i) * 1000003));
-                mixed.push_back(i % 4 == 0 ? json("s" + std::to_string(i)) : i % 4 == 1 ? json::array({i, json::object({{"k", i}})}) : i % 4 == 2 ? json(nullptr) : json(i % 2 == 0));
-            }
-            document["numbers " + std::to_string(n)] = numbers;
-            document["mixed " + std::to_string(n)] = mixed;
-            document["nested " + std::to_string(n)] = json::object({{"a", numbers}, {"b", json::array({mixed, json::object()})}});
-        }
-        document["binary"] = json::binary({1, 2, 3}, 0x80);
-        document["binary old"] = json::binary({1, 2, 3}, 0x02);
-        for (const auto handler : handlers)
-        {
-            check_same(document, handler);
-        }
-    }
-
-    SECTION("errors at every position")
-    {
-        // an entry that cannot be written, at each place among valid ones
-        const std::string bad_utf8 = "\xC3\x28";
-        const json bad_entries[] =
-        {
-            json::object({{std::string("a\0b", 3), 1}}),
-            json::object({{"s", bad_utf8}}),
-            json::object({{bad_utf8, 1}}),
-            json::array({1, 2, bad_utf8}),
-            json::object({{"b", json::binary({1}, 0x100)}}),
-            json::object({{"d", json(json::value_t::discarded)}}),
-            json::array({json(json::value_t::discarded)}),
-        };
-        for (const auto& bad : bad_entries)
-        {
-            for (std::size_t pos = 0; pos < 4; ++pos)
-            {
-                json document = json::object();
-                json list = json::array();
-                for (std::size_t i = 0; i < 4; ++i)
-                {
-                    if (i == pos)
-                    {
-                        document["x" + std::to_string(i)] = bad;
-                        list.push_back(bad);
-                    }
-                    document["n" + std::to_string(i)] = json::array({i, "str", json::object({{"k", i}})});
-                    list.push_back(i);
-                }
-                document["list"] = list;
-                for (const auto handler : handlers)
-                {
-                    check_same(document, handler);
-                }
-            }
-        }
-    }
-}

tests/src/unit-json_view_builder.cpp

@@ -575,6 +575,59 @@ TEST_CASE("json_view builder: escapes between runs of text")
     }
 }
 
+TEST_CASE("json_view builder: chains of unicode escapes")
+{
+    // Consecutive \uXXXX escapes are decoded in a loop of their own. Chains of
+    // every length from 1 to 40 of escapes that give one, two, and three
+    // bytes of UTF-8, in upper and lower case, followed by what ends a chain:
+    // the end of the string, text, another escape, UTF-8, and everything that
+    // is not a valid escape (a bad digit at every position of the last
+    // escape, a lone surrogate, a surrogate pair, a cut in every place).
+    const std::array<const char*, 8> chain = {{"\\u0041", "\\u00e4", "\\u00E4", "\\u20ac", "\\u3042", "\\uFFFF", "\\u0000", "\\u07ff"}};
+    const std::array<const char*, 12> ends = {{"", "abc", "\\n", "\\u00e9", "\\ud83d\\ude00", "\\ud83d", "\\ude00", "\\ud83d\\u0041", "\\u12G4", "\\u12", "\xc3\xa4", "\\"}};
+    std::mt19937 rng(2027); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible
+    for (std::size_t length = 1; length <= 40; ++length)
+    {
+        for (int i = 0; i < 20; ++i)
+        {
+            std::string content = std::string(rng() % 20, 'a');
+            for (std::size_t k = 0; k < length; ++k)
+            {
+                content += chain[rng() % chain.size()];
+            }
+            const std::string whole = content;
+            for (const char* end : ends)
+            {
+                check_string(whole + end);
+                check_string(whole + end + std::string(rng() % 19, 'z'));
+            }
+            // a bad digit at every position of every escape near the end of the chain
+            for (std::size_t back = 1; back <= 6 && back <= length; ++back)
+            {
+                for (std::size_t digit = 2; digit < 6; ++digit)
+                {
+                    std::string broken = whole;
+                    broken[broken.size() - (6 * back) + digit] = "gG\"\x01 z"[rng() % 6];
+                    check_string(broken);
+                }
+            }
+            // the text cut in every place of the last three escapes
+            for (std::size_t cut = 0; cut <= 18 && cut <= whole.size(); ++cut)
+            {
+                check_string(whole.substr(0, whole.size() - cut));
+            }
+        }
+    }
+    // a chain longer than the first growth of the arena, in a key and in a value
+    std::string long_chain;
+    for (int i = 0; i < 3000; ++i)
+    {
+        long_chain += chain[static_cast<std::size_t>(i) % chain.size()];
+    }
+    check_string(long_chain);
+    check_string(long_chain + "\\ud83d\\ude00" + long_chain);
+}
+
 TEST_CASE("json_view node integer bits")
 {
     using nlohmann::detail::view::integer_bits;

tests/src/unit-json_view_from_json.cpp

@@ -443,6 +443,30 @@ TEST_CASE("json_view from_json")
         CHECK(d.root().materialize().get<point>().x == 3);
     }
 
+    SECTION("escaped keys: members are found by their decoded names")
+    {
+        // keys are decoded before they are compared, as by parse(); a lookup
+        // that compares the raw bytes of the source would miss these members
+        const json_document d = json_document::parse(R"([{"\u0078": 1, "y": 2}, {"V\u0061lue": 3, "Te\u0078t": "t"}, {"\u0062": "escaped"}, {"x": 1, "\u0078": 3, "y": 2}])");
+        const json j = d.root().materialize();
+        CHECK(d.root()[0].get<point>().x == 1);
+        CHECK(d.root()[0].get<point>().y == 2);
+        CHECK((d.root()[0].get<point>() == j[0].get<point>()));
+        CHECK(d.root()[1].get<with_names>().value == 3);
+        CHECK(d.root()[1].get<with_names>().text == "t");
+        CHECK(d.root()[2].get<with_defaults>().b == "escaped");
+        CHECK(d.root()[2].get<with_defaults>().a == 1);
+        // a duplicate key written once plain and once escaped: still the first member
+        CHECK(d.root()[3].get<point>().x == 1);
+        CHECK(j[3].get<point>().x == 3);
+#if !defined(JSON_NOEXCEPTION) && !JSON_DIAGNOSTICS && !JSON_DIAGNOSTIC_POSITIONS
+        // an escaped backslash is part of the decoded key: "x\\" is not "x"
+        const json_document e = json_document::parse(R"({"x\\": 1, "y": 2})");
+        CHECK(exception_of_call([&] { static_cast<void>(e.root().get<point>()); }) == exception_of_call([&] { static_cast<void>(e.root().materialize().get<point>()); }));
+        CHECK(exception_of_call([&] { static_cast<void>(e.root().get<point>()); }) == "[json.exception.out_of_range.403] key 'x' not found");
+#endif
+    }
+
     SECTION("editable documents and ordered_json")
     {
         json_editable_document e = json_editable_document::parse(R"({"x": 1, "y": 2})");

tests/src/unit-serialization.cpp

@@ -23,6 +23,7 @@ using nlohmann::json;
 #include <cstdint>
 #include <sstream>
 #include <iomanip>
+#include <limits>
 #include <string>
 #include <vector>
 
@@ -248,6 +249,51 @@ TEST_CASE_TEMPLATE("serialization for extreme integer values", T, int32_t, uint3
     }
 }
 
+TEST_CASE("integers of every digit count are written like std::to_string")
+{
+    // dump_integer() writes up to 16 digits in two groups of eight and 17 to
+    // 20 digits as one to four digits and two groups of eight
+    std::vector<std::uint64_t> values;
+    std::uint64_t power = 1;
+    for (int digits = 1; digits <= 20; ++digits)
+    {
+        values.push_back(power);                     // 1 followed by zeros
+        values.push_back(power + 1);
+        values.push_back((power * 2) - 1);
+        if (digits < 20)
+        {
+            values.push_back((power * 10) - 1);      // all 9s
+            values.push_back((power * 10) - power);  // 9 followed by zeros
+            power *= 10;
+        }
+    }
+    values.push_back((std::numeric_limits<std::uint64_t>::max)());
+    values.push_back(static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()));
+    values.push_back(static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()) + 1);
+    std::uint64_t state = 0x9E3779B97F4A7C15u;
+    for (int i = 0; i < 20000; ++i)
+    {
+        state = (state * 6364136223846793005u) + 1442695040888963407u;
+        values.push_back(state >> (state % 61));     // every bit length
+    }
+
+    for (const auto v : values)
+    {
+        CAPTURE(v)
+        CHECK(json(v).dump() == std::to_string(v));
+        if (v <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()))
+        {
+            const auto s = static_cast<std::int64_t>(v);
+            CHECK(json(s).dump() == std::to_string(s));
+            CHECK(json(-s).dump() == std::to_string(-s));
+        }
+    }
+    CHECK(json((std::numeric_limits<std::int64_t>::min)()).dump() == "-9223372036854775808");
+
+    // in a container, so the digits are followed by other output
+    CHECK(json::array({std::uint64_t{18446744073709551615u}, std::int64_t{-1234567890123456789}, 1}).dump() == "[18446744073709551615,-1234567890123456789,1]");
+}
+
 TEST_CASE("dump with binary values")
 {
     auto binary = json::binary({1, 2, 3, 4});