Binary readers and writers: faster UTF-8 checks of strings

With error_handler_t::strict (and replace/ignore), every string a binary reader reads is checked for valid UTF-8, and every string a binary writer writes. The readers used the byte-at-a-time decoder; the writers skipped ASCII eight bytes at a time. - New valid_utf8_string() decides strings of up to 16 ASCII bytes, most keys and many values, with one or two word tests, and checks longer strings 16 bytes at a time with the NEON/SSSE3 UTF-8 check the lexer uses (vector_utf8_valid_blocks: the same lookup, but quotes and control characters are ordinary characters). Other targets and JSON_NO_SIMD use valid_utf8_prefix() as before. - The readers check strings with it (for one-byte character types) in an out-of-line function, so that reading without a check does not pay for its code. - The writers' check_utf8() uses it, and the CBOR and MessagePack writers write a checked string of up to max_text_in_reservation characters in one reservation, as they write unchecked ones. Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-31 13:00:39 +00:00 · 2026-10-31 09:45:14 +01:00
9 changed files with 463 additions and 691 deletions
@@ -4293,6 +4293,19 @@ class binary_reader
        return get_bytes(len, "string", result);
    }

+    /// whether @a s is valid UTF-8: ASCII eight bytes at a time, the rest
+    /// sequence by sequence (as the binary writers check strings)
+    static bool is_valid_utf8_string(const string_t& s, std::true_type /*is_byte*/) noexcept
+    {
+        return valid_utf8_string(reinterpret_cast<const unsigned char*>(s.data()), s.size());
+    }
+
+    /// whether @a s, of a character type wider than a byte, is valid UTF-8
+    static bool is_valid_utf8_string(const string_t& s, std::false_type /*is_byte*/) noexcept
+    {
+        return is_valid_utf8(s);
+    }
+
    /*!
    @brief validate a decoded text string (value or object key) against @ref error_handler

@@ -4312,7 +4325,16 @@ class binary_reader
    */
    bool check_string_utf8(string_t& result, const char* context)
    {
-        if (error_handler == error_handler_t::keep || is_valid_utf8(result))
+        return error_handler == error_handler_t::keep || check_string_utf8_handler(result, context);
+    }
+
+    /// @ref check_string_utf8 for an error handler other than @ref
+    /// error_handler_t::keep; out of line, so that reading strings without a
+    /// check does not pay for the code of the check
+    JSON_HEDLEY_NEVER_INLINE
+    bool check_string_utf8_handler(string_t& result, const char* context)
+    {
+        if (is_valid_utf8_string(result, std::integral_constant<bool, sizeof(typename string_t::value_type) == 1>()))
        {
            return true;
        }
@@ -322,6 +322,78 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons
    block_out = block;
    return false;
 }
+
+/*!
+@brief skip 16-byte blocks of well-formed UTF-8 at p (a character boundary)
+
+Like vector_utf8_run, but quotes, backslashes, and control characters are
+ordinary characters. Returns the start of the first block in which the check
+found an error (which may belong to a sequence begun in the block before) or
+where fewer than 16 bytes are left; a sequence may be cut at that point, so the
+caller resumes at its start, one sequence at a time.
+*/
+JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vector_utf8_valid_blocks(const unsigned char* block, const unsigned char* e) noexcept
+{
+    using lookup = utf8_lookup4<>;
+#if JSON_SCAN_NEON
+    const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high.data());
+    const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low.data());
+    const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high.data());
+    uint8x16_t prev = vdupq_n_u8(0);
+    while (e - block >= 16)
+    {
+        const uint8x16_t in = vld1q_u8(block);
+        if (vmaxvq_u8(vorrq_u8(in, prev)) < 0x80)
+        {
+            // ASCII here, and no sequence from the block before to finish
+            prev = in;
+            block += 16;
+            continue;
+        }
+        const uint8x16_t prev1 = vextq_u8(prev, in, 15);
+        const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4)));
+        const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80)));
+        const uint8x16_t err = veorq_u8(vandq_u8(must23, vdupq_n_u8(0x80)), sc);
+        if (vmaxvq_u8(err) != 0)
+        {
+            break;
+        }
+        prev = in;
+        block += 16;
+    }
+#else
+    const __m128i t1h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_high.data()))); // NOLINT(bugprone-casting-through-void)
+    const __m128i t1l = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_low.data()))); // NOLINT(bugprone-casting-through-void)
+    const __m128i t2h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_2_high.data()))); // NOLINT(bugprone-casting-through-void)
+    const __m128i nibble = _mm_set1_epi8(0x0F);
+    const __m128i zero = _mm_setzero_si128();
+    __m128i prev = zero;
+    while (e - block >= 16)
+    {
+        const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(block))); // NOLINT(bugprone-casting-through-void)
+        if (_mm_movemask_epi8(_mm_or_si128(in, prev)) == 0)
+        {
+            prev = in;
+            block += 16;
+            continue;
+        }
+        const __m128i prev1 = _mm_alignr_epi8(in, prev, 15);
+        const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)),
+                                         _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))),
+                                         _mm_shuffle_epi8(t2h, _mm_and_si128(_mm_srli_epi16(in, 4), nibble)));
+        const __m128i must23 = _mm_or_si128(_mm_subs_epu8(_mm_alignr_epi8(in, prev, 14), _mm_set1_epi8(0xE0 - 0x80)),
+                                            _mm_subs_epu8(_mm_alignr_epi8(in, prev, 13), _mm_set1_epi8(0xF0 - 0x80)));
+        const __m128i err = _mm_xor_si128(_mm_and_si128(must23, _mm_set1_epi8(static_cast<char>(-128))), sc);
+        if (_mm_movemask_epi8(_mm_cmpeq_epi8(err, zero)) != 0xFFFF)
+        {
+            break;
+        }
+        prev = in;
+        block += 16;
+    }
+#endif
+    return block;
+}
 #endif

 }  // namespace detail
@@ -236,6 +236,64 @@ inline std::size_t valid_utf8_prefix(const unsigned char* data, std::size_t n) n
    return pos;
 }

+/*!
+@brief whether [data, data+n) is valid UTF-8, for checking whole strings
+
+The same answer as valid_utf8_prefix(data, n) == n. Short ASCII strings, the
+most common keys and values, are decided in one test, and long strings are
+checked 16 bytes at a time where the vector check is available.
+*/
+inline bool valid_utf8_string(const unsigned char* data, std::size_t n) noexcept
+{
+    constexpr std::uint64_t high = 0x8080808080808080ull;
+    if (n <= 16)
+    {
+        // two overlapping words cover 8 to 16 bytes, one partial word fewer
+        std::uint64_t first = 0;
+        std::uint64_t last = 0;
+        if (n >= 8)
+        {
+            std::memcpy(&first, data, sizeof(first));
+            std::memcpy(&last, data + n - 8, sizeof(last));
+        }
+        else if (n != 0)
+        {
+            std::memcpy(&first, data, n);
+        }
+        if (((first | last) & high) == 0)
+        {
+            return true;
+        }
+    }
+#if JSON_SCAN_VECTOR_UTF8
+    else if (n >= 32
+#if JSON_SCAN_SSSE3_DISPATCH
+             && cpu_has_ssse3()
+#endif
+            )
+    {
+        const unsigned char* block = vector_utf8_valid_blocks(data, data + n);
+        // the rest from the start of a sequence that may be cut at block
+        for (int i = 1; i <= 3 && block - i >= data; ++i)
+        {
+            const unsigned char c = block[-i];
+            if (c < 0x80)
+            {
+                break;
+            }
+            if (c >= 0xC0)
+            {
+                block -= i;
+                break;
+            }
+        }
+        const auto checked = static_cast<std::size_t>(block - data);
+        return checked + valid_utf8_prefix(block, n - checked) == n;
+    }
+#endif
+    return valid_utf8_prefix(data, n) == n;
+}
+
 #if JSON_SCAN_VECTOR_UTF8
 /// the end of scan_string_vector from block, where the vector loop stopped
 /// (ill-formed UTF-8, or fewer than 16 bytes left): one byte or sequence at a
@@ -2088,8 +2088,7 @@ class binary_writer
        /// arrays: the first index with one more digit; only maintained by
        /// @ref calc_bson_sizes
        std::uint64_t index_boundary = 10;
-        /// @ref calc_bson_sizes: where its size goes in the table; @ref
-        /// write_bson_document_single_pass: the position of its size field
+        /// @ref calc_bson_sizes only: where its size goes in the table
        std::size_t size_slot;
        /// @ref calc_bson_sizes only: the size of its entries seen so far
        std::size_t entries_size = 0;
@@ -2340,197 +2339,6 @@ class binary_writer
        }
    }

-    template<typename SinkType>
-    using sink_written_t = decltype(std::declval<const SinkType&>().written());
-
-    /*!
-    @brief whether BSON documents can be written in a single pass: the sink
-           can overwrite what it has written (output_vector_sink), and every
-           integer fits the 64 bits BSON has, so that writing an array of
-           numbers needs no range check
-    */
-    using bson_single_pass_sink = std::integral_constant < bool,
-          is_detected<sink_written_t, OutputSinkType>::value
-          && std::numeric_limits<typename BasicJsonType::number_integer_t>::is_specialized
-          && std::numeric_limits<typename BasicJsonType::number_integer_t>::digits <= std::numeric_limits<std::int64_t>::digits
-          && std::numeric_limits<typename BasicJsonType::number_unsigned_t>::is_specialized
-          && std::numeric_limits<typename BasicJsonType::number_unsigned_t>::digits <= std::numeric_limits<std::uint64_t>::digits >;
-
-    /// write a placeholder for the size of an object or array whose entries
-    /// follow; @return its position, for @ref patch_bson_size
-    std::size_t write_bson_size_placeholder()
-    {
-        const std::size_t pos = oa.written();
-        write_number<std::int32_t>(0, true);
-        return pos;
-    }
-
-    /// end the object or array whose size is at @a pos and write its size
-    /// there: everything from that field to the terminating null byte
-    /// @return false (and the size is not written) if it does not fit
-    bool close_bson_container(const std::size_t pos)
-    {
-        oa.write_character(to_char_type(0x00));
-        const std::size_t size = oa.written() - pos;
-        if (JSON_HEDLEY_UNLIKELY(!value_in_range_of<std::int32_t>(size)))
-        {
-            return false;
-        }
-        std::array<CharType, sizeof(std::int32_t)> bytes{};
-        store_little_endian<std::int32_t>(bytes.data(), static_cast<std::int32_t>(size));
-        oa.overwrite(pos, bytes.data(), bytes.size());
-        return true;
-    }
-
-    /*!
-    @brief check the value @a j of a BSON entry as @ref calc_bson_value_size
-           does, for @ref write_bson_document_single_pass
-    @return at least the size of the value
-    A number, boolean, or null needs no check there (its integers have at most
-    64 bits) and has at most 8 characters; a discarded value is rejected with
-    the same exception when it is written.
-    */
-    std::size_t check_bson_value(const BasicJsonType& j)
-    {
-        return (j.is_string() || j.is_binary()) ? calc_bson_value_size(j) : 8;
-    }
-
-    /// whether an entry of @a entry_size characters keeps the document that
-    /// begins at @a start, closed by @a open_containers null bytes, within
-    /// the 32-bit size of BSON
-    bool bson_entry_fits(const std::size_t start, const std::size_t entry_size, const std::size_t open_containers) const noexcept
-    {
-        const std::size_t limit = static_cast<std::size_t>((std::numeric_limits<std::int32_t>::max)());
-        const std::size_t written = oa.written() - start;
-        return written <= limit && entry_size <= limit - written && open_containers <= limit - written - entry_size;
-    }
-
-    /*!
-    @brief Serializes the JSON object @a document as a BSON document in a
-           single pass, writing the size of each object and array once it is
-           complete
-
-    Checks every key and value as @ref calc_bson_sizes does, in the same
-    order, so the same exception is thrown; the output written up to then is
-    discarded with the vector it was written to.
-
-    @return false as soon as the document or a string, binary value, object,
-            or array in it would be larger than the 32-bit BSON sizes allow;
-            nothing large has been written then, and the caller writes the
-            document in two passes, which reports that error
-    */
-    template < typename SinkType = OutputSinkType,
-               typename std::enable_if < is_detected<sink_written_t, SinkType>::value, int >::type = 0 >
-    bool write_bson_document_single_pass(const BasicJsonType& document)
-    {
-        const std::size_t start = oa.written();
-        // the object or array whose entries are being written (with the
-        // position of its size), and the ones it is in
-        bson_frame current(&document, write_bson_size_placeholder());
-        std::vector<bson_frame> parents;
-
-        while (true)
-        {
-            const BasicJsonType* nested = nullptr;
-            std::size_t nested_size_pos = 0;
-            if (current.value->is_object())
-            {
-                const auto& object = *current.value->m_data.m_value.object;
-                while (nested == nullptr && current.member != object.cend())
-                {
-                    const auto& el = *current.member;
-                    ++current.member;
-                    const std::size_t header_size = calc_bson_entry_header_size(el.first, el.second);
-                    const std::size_t value_size = el.second.is_structured() ? 0 : check_bson_value(el.second);
-                    if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, header_size + value_size, parents.size() + 1)))
-                    {
-                        return false;
-                    }
-                    if (el.second.is_structured())
-                    {
-                        write_bson_entry_header(el.first, el.second.is_object() ? 0x03 : 0x04);
-                        nested_size_pos = write_bson_size_placeholder();
-                        nested = &el.second;
-                    }
-                    else
-                    {
-                        write_bson_value(el.first, el.second);
-                    }
-                }
-            }
-            else
-            {
-                const auto& array = *current.value->m_data.m_value.array;
-                while (nested == nullptr && current.index < array.size())
-                {
-                    if (is_bson_fixed_size(array[current.index]))
-                    {
-                        write_bson_array_fixed_entries(current);
-                        if (current.index == array.size())
-                        {
-                            break;
-                        }
-                    }
-
-                    // a string, binary value, object or array
-                    const BasicJsonType& el = array[current.index];
-                    const string_t index_name(current.index_name.data(), current.index_name_size);
-                    current.next_index();
-                    const std::size_t value_size = el.is_structured() ? 0 : check_bson_value(el);
-                    if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, 2 + index_name.size() + value_size, parents.size() + 1)))
-                    {
-                        return false;
-                    }
-                    if (el.is_structured())
-                    {
-                        write_bson_entry_header(index_name, el.is_object() ? 0x03 : 0x04);
-                        nested_size_pos = write_bson_size_placeholder();
-                        nested = &el;
-                    }
-                    else
-                    {
-                        write_bson_value(index_name, el);
-                    }
-                }
-            }
-
-            if (nested != nullptr)
-            {
-                if (nested->is_array())
-                {
-                    // an array of booleans, numbers and null is written without a frame on the stack
-                    bson_frame leaf(nested, nested_size_pos);
-                    write_bson_array_fixed_entries(leaf);
-                    if (leaf.index == nested->m_data.m_value.array->size())
-                    {
-                        if (JSON_HEDLEY_UNLIKELY(!close_bson_container(nested_size_pos)))
-                        {
-                            return false;
-                        }
-                        continue;
-                    }
-                    parents.push_back(std::move(current));
-                    current = std::move(leaf);
-                    continue;
-                }
-                parents.push_back(std::move(current));
-                current = bson_frame(nested, nested_size_pos);
-                continue;
-            }
-
-            if (JSON_HEDLEY_UNLIKELY(!close_bson_container(current.size_slot)))
-            {
-                return false;
-            }
-            if (parents.empty())
-            {
-                return true;
-            }
-            current = std::move(parents.back());
-            parents.pop_back();
-        }
-    }
-
    /*!
    @brief Serializes the JSON object @a document as a BSON document

@@ -2542,39 +2350,6 @@ class binary_writer
    @pre       document.type() == value_t::object
    */
    void write_bson_document(const BasicJsonType& document)
-    {
-        write_bson_document(document, bson_single_pass_sink());
-    }
-
-    /// the sink can overwrite: one pass, unless the error handler replaces or
-    /// drops ill-formed UTF-8, which changes the sizes the checks compute
-    void write_bson_document(const BasicJsonType& document, std::true_type /*single_pass*/)
-    {
-        const std::size_t start = oa.written();
-        if ((error_handler == error_handler_t::keep || error_handler == error_handler_t::strict)
-                && write_bson_document_single_pass(document))
-        {
-            return;
-        }
-        // a length does not fit: start over, so that it is reported as the
-        // two passes report it, before anything is written
-        oa.rewind(start);
-        write_bson_document_two_pass(document);
-    }
-
-    void write_bson_document(const BasicJsonType& document, std::false_type /*single_pass*/)
-    {
-        write_bson_document_two_pass(document);
-    }
-
-    /*!
-    @brief Serializes the JSON object @a document as a BSON document: the
-           sizes of all objects and arrays first, then the entries
-
-    @param[in] document  the JSON object to serialize
-    @pre       document.type() == value_t::object
-    */
-    void write_bson_document_two_pass(const BasicJsonType& document)
    {
        std::vector<std::size_t> nested_sizes;
        const std::size_t document_size = calc_bson_sizes(document, nested_sizes);
@@ -3305,6 +3080,25 @@ class binary_writer
        }
    }

+    /// write the head and the characters of @a value, at most
+    /// max_text_in_reservation of them, in one reservation
+    JSON_HEDLEY_ALWAYS_INLINE void write_cbor_string_reserved(const string_t& value)
+    {
+        CharType* const out = oa.reserve(value.size() + 2);
+        CharType* p = out;
+        if (value.size() <= 0x17)
+        {
+            *p++ = to_char_type(static_cast<std::uint8_t>(0x60 + value.size()));
+        }
+        else
+        {
+            *p++ = to_char_type(0x78);
+            *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+        }
+        copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+        oa.commit(static_cast<std::size_t>(p - out) + value.size());
+    }
+
    /*!
    @brief write a CBOR text string

@@ -3326,19 +3120,7 @@ class binary_writer
            // the string is written as it is: head and characters in one reservation
            if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
            {
-                CharType* const out = oa.reserve(value.size() + 2);
-                CharType* p = out;
-                if (value.size() <= 0x17)
-                {
-                    *p++ = to_char_type(static_cast<std::uint8_t>(0x60 + value.size()));
-                }
-                else
-                {
-                    *p++ = to_char_type(0x78);
-                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
-                }
-                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
-                oa.commit(static_cast<std::size_t>(p - out) + value.size());
+                write_cbor_string_reserved(value);
                return;
            }
        }
@@ -3352,6 +3134,13 @@ class binary_writer
    JSON_HEDLEY_NEVER_INLINE
    void write_cbor_checked_string(const string_t& value, const BasicJsonType& context)
    {
+        if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation)
+        {
+            check_utf8(value, context);
+            write_cbor_string_reserved(value);
+            return;
+        }
+
        string_t storage;
        const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);

@@ -3366,6 +3155,27 @@ class binary_writer
    // MsgPack //
    /////////////

+    /// write the head and the characters of @a value, at most
+    /// max_text_in_reservation of them, in one reservation
+    JSON_HEDLEY_ALWAYS_INLINE void write_msgpack_string_reserved(const string_t& value)
+    {
+        CharType* const out = oa.reserve(value.size() + 2);
+        CharType* p = out;
+        if (value.size() <= 31)
+        {
+            // fixstr
+            *p++ = to_char_type(static_cast<std::uint8_t>(0xA0 | value.size()));
+        }
+        else
+        {
+            // str 8
+            *p++ = to_char_type(0xD9);
+            *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+        }
+        copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+        oa.commit(static_cast<std::size_t>(p - out) + value.size());
+    }
+
    /*!
    @brief write a MessagePack str

@@ -3383,21 +3193,7 @@ class binary_writer
            // the string is written as it is: head and characters in one reservation
            if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
            {
-                CharType* const out = oa.reserve(value.size() + 2);
-                CharType* p = out;
-                if (value.size() <= 31)
-                {
-                    // fixstr
-                    *p++ = to_char_type(static_cast<std::uint8_t>(0xA0 | value.size()));
-                }
-                else
-                {
-                    // str 8
-                    *p++ = to_char_type(0xD9);
-                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
-                }
-                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
-                oa.commit(static_cast<std::size_t>(p - out) + value.size());
+                write_msgpack_string_reserved(value);
                return;
            }
        }
@@ -3410,6 +3206,13 @@ class binary_writer
    JSON_HEDLEY_NEVER_INLINE
    void write_msgpack_checked_string(const string_t& value, const BasicJsonType& context)
    {
+        if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation)
+        {
+            check_utf8(value, context);
+            write_msgpack_string_reserved(value);
+            return;
+        }
+
        string_t storage;
        const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);

@@ -5019,10 +4822,9 @@ class binary_writer
    static void check_utf8(const string_t& s, const BasicJsonType& context)
    {
        const auto* data = reinterpret_cast<const unsigned char*>(s.data());
-        const std::size_t valid = valid_utf8_prefix(data, s.size());
-        if (JSON_HEDLEY_UNLIKELY(valid != s.size()))
+        if (JSON_HEDLEY_UNLIKELY(!valid_utf8_string(data, s.size())))
        {
-            throw_invalid_utf8(data, valid, context);
+            throw_invalid_utf8(data, valid_utf8_prefix(data, s.size()), context);
        }
    }

@@ -227,29 +227,6 @@ class output_vector_sink
    void flush() noexcept
    {}

-    /// the number of characters written so far, the position of the next one
-    std::size_t written() const noexcept
-    {
-        return position();
-    }
-
-    /// @brief replace @a length characters written before, from position @a pos
-    ///
-    /// Lets a writer put a length in front of data it has written since, such
-    /// as the size of a BSON document, without computing it in advance.
-    void overwrite(const std::size_t pos, const CharType* s, const std::size_t length) noexcept
-    {
-        JSON_ASSERT(pos + length <= position());
-        std::memcpy(v->data() + pos, s, length * sizeof(CharType));
-    }
-
-    /// drop the characters written from position @a pos on
-    void rewind(const std::size_t pos) noexcept
-    {
-        JSON_ASSERT(pos <= position());
-        cursor = v->data() + pos;
-    }
-
  private:
    /// number of characters of *v written so far
    std::size_t position() const noexcept
@@ -12377,6 +12377,78 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons
    block_out = block;
    return false;
 }
+
+/*!
+@brief skip 16-byte blocks of well-formed UTF-8 at p (a character boundary)
+
+Like vector_utf8_run, but quotes, backslashes, and control characters are
+ordinary characters. Returns the start of the first block in which the check
+found an error (which may belong to a sequence begun in the block before) or
+where fewer than 16 bytes are left; a sequence may be cut at that point, so the
+caller resumes at its start, one sequence at a time.
+*/
+JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vector_utf8_valid_blocks(const unsigned char* block, const unsigned char* e) noexcept
+{
+    using lookup = utf8_lookup4<>;
+#if JSON_SCAN_NEON
+    const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high.data());
+    const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low.data());
+    const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high.data());
+    uint8x16_t prev = vdupq_n_u8(0);
+    while (e - block >= 16)
+    {
+        const uint8x16_t in = vld1q_u8(block);
+        if (vmaxvq_u8(vorrq_u8(in, prev)) < 0x80)
+        {
+            // ASCII here, and no sequence from the block before to finish
+            prev = in;
+            block += 16;
+            continue;
+        }
+        const uint8x16_t prev1 = vextq_u8(prev, in, 15);
+        const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4)));
+        const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80)));
+        const uint8x16_t err = veorq_u8(vandq_u8(must23, vdupq_n_u8(0x80)), sc);
+        if (vmaxvq_u8(err) != 0)
+        {
+            break;
+        }
+        prev = in;
+        block += 16;
+    }
+#else
+    const __m128i t1h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_high.data()))); // NOLINT(bugprone-casting-through-void)
+    const __m128i t1l = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_low.data()))); // NOLINT(bugprone-casting-through-void)
+    const __m128i t2h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_2_high.data()))); // NOLINT(bugprone-casting-through-void)
+    const __m128i nibble = _mm_set1_epi8(0x0F);
+    const __m128i zero = _mm_setzero_si128();
+    __m128i prev = zero;
+    while (e - block >= 16)
+    {
+        const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(block))); // NOLINT(bugprone-casting-through-void)
+        if (_mm_movemask_epi8(_mm_or_si128(in, prev)) == 0)
+        {
+            prev = in;
+            block += 16;
+            continue;
+        }
+        const __m128i prev1 = _mm_alignr_epi8(in, prev, 15);
+        const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)),
+                                         _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))),
+                                         _mm_shuffle_epi8(t2h, _mm_and_si128(_mm_srli_epi16(in, 4), nibble)));
+        const __m128i must23 = _mm_or_si128(_mm_subs_epu8(_mm_alignr_epi8(in, prev, 14), _mm_set1_epi8(0xE0 - 0x80)),
+                                            _mm_subs_epu8(_mm_alignr_epi8(in, prev, 13), _mm_set1_epi8(0xF0 - 0x80)));
+        const __m128i err = _mm_xor_si128(_mm_and_si128(must23, _mm_set1_epi8(static_cast<char>(-128))), sc);
+        if (_mm_movemask_epi8(_mm_cmpeq_epi8(err, zero)) != 0xFFFF)
+        {
+            break;
+        }
+        prev = in;
+        block += 16;
+    }
+#endif
+    return block;
+}
 #endif

 }  // namespace detail
@@ -12604,6 +12676,64 @@ inline std::size_t valid_utf8_prefix(const unsigned char* data, std::size_t n) n
    return pos;
 }

+/*!
+@brief whether [data, data+n) is valid UTF-8, for checking whole strings
+
+The same answer as valid_utf8_prefix(data, n) == n. Short ASCII strings, the
+most common keys and values, are decided in one test, and long strings are
+checked 16 bytes at a time where the vector check is available.
+*/
+inline bool valid_utf8_string(const unsigned char* data, std::size_t n) noexcept
+{
+    constexpr std::uint64_t high = 0x8080808080808080ull;
+    if (n <= 16)
+    {
+        // two overlapping words cover 8 to 16 bytes, one partial word fewer
+        std::uint64_t first = 0;
+        std::uint64_t last = 0;
+        if (n >= 8)
+        {
+            std::memcpy(&first, data, sizeof(first));
+            std::memcpy(&last, data + n - 8, sizeof(last));
+        }
+        else if (n != 0)
+        {
+            std::memcpy(&first, data, n);
+        }
+        if (((first | last) & high) == 0)
+        {
+            return true;
+        }
+    }
+#if JSON_SCAN_VECTOR_UTF8
+    else if (n >= 32
+#if JSON_SCAN_SSSE3_DISPATCH
+             && cpu_has_ssse3()
+#endif
+            )
+    {
+        const unsigned char* block = vector_utf8_valid_blocks(data, data + n);
+        // the rest from the start of a sequence that may be cut at block
+        for (int i = 1; i <= 3 && block - i >= data; ++i)
+        {
+            const unsigned char c = block[-i];
+            if (c < 0x80)
+            {
+                break;
+            }
+            if (c >= 0xC0)
+            {
+                block -= i;
+                break;
+            }
+        }
+        const auto checked = static_cast<std::size_t>(block - data);
+        return checked + valid_utf8_prefix(block, n - checked) == n;
+    }
+#endif
+    return valid_utf8_prefix(data, n) == n;
+}
+
 #if JSON_SCAN_VECTOR_UTF8
 /// the end of scan_string_vector from block, where the vector loop stopped
 /// (ill-formed UTF-8, or fewer than 16 bytes left): one byte or sequence at a
@@ -22069,6 +22199,19 @@ class binary_reader
        return get_bytes(len, "string", result);
    }

+    /// whether @a s is valid UTF-8: ASCII eight bytes at a time, the rest
+    /// sequence by sequence (as the binary writers check strings)
+    static bool is_valid_utf8_string(const string_t& s, std::true_type /*is_byte*/) noexcept
+    {
+        return valid_utf8_string(reinterpret_cast<const unsigned char*>(s.data()), s.size());
+    }
+
+    /// whether @a s, of a character type wider than a byte, is valid UTF-8
+    static bool is_valid_utf8_string(const string_t& s, std::false_type /*is_byte*/) noexcept
+    {
+        return is_valid_utf8(s);
+    }
+
    /*!
    @brief validate a decoded text string (value or object key) against @ref error_handler

@@ -22088,7 +22231,16 @@ class binary_reader
    */
    bool check_string_utf8(string_t& result, const char* context)
    {
-        if (error_handler == error_handler_t::keep || is_valid_utf8(result))
+        return error_handler == error_handler_t::keep || check_string_utf8_handler(result, context);
+    }
+
+    /// @ref check_string_utf8 for an error handler other than @ref
+    /// error_handler_t::keep; out of line, so that reading strings without a
+    /// check does not pay for the code of the check
+    JSON_HEDLEY_NEVER_INLINE
+    bool check_string_utf8_handler(string_t& result, const char* context)
+    {
+        if (is_valid_utf8_string(result, std::integral_constant<bool, sizeof(typename string_t::value_type) == 1>()))
        {
            return true;
        }
@@ -26870,29 +27022,6 @@ class output_vector_sink
    void flush() noexcept
    {}

-    /// the number of characters written so far, the position of the next one
-    std::size_t written() const noexcept
-    {
-        return position();
-    }
-
-    /// @brief replace @a length characters written before, from position @a pos
-    ///
-    /// Lets a writer put a length in front of data it has written since, such
-    /// as the size of a BSON document, without computing it in advance.
-    void overwrite(const std::size_t pos, const CharType* s, const std::size_t length) noexcept
-    {
-        JSON_ASSERT(pos + length <= position());
-        std::memcpy(v->data() + pos, s, length * sizeof(CharType));
-    }
-
-    /// drop the characters written from position @a pos on
-    void rewind(const std::size_t pos) noexcept
-    {
-        JSON_ASSERT(pos <= position());
-        cursor = v->data() + pos;
-    }
-
  private:
    /// number of characters of *v written so far
    std::size_t position() const noexcept
@@ -29337,8 +29466,7 @@ class binary_writer
        /// arrays: the first index with one more digit; only maintained by
        /// @ref calc_bson_sizes
        std::uint64_t index_boundary = 10;
-        /// @ref calc_bson_sizes: where its size goes in the table; @ref
-        /// write_bson_document_single_pass: the position of its size field
+        /// @ref calc_bson_sizes only: where its size goes in the table
        std::size_t size_slot;
        /// @ref calc_bson_sizes only: the size of its entries seen so far
        std::size_t entries_size = 0;
@@ -29589,197 +29717,6 @@ class binary_writer
        }
    }

-    template<typename SinkType>
-    using sink_written_t = decltype(std::declval<const SinkType&>().written());
-
-    /*!
-    @brief whether BSON documents can be written in a single pass: the sink
-           can overwrite what it has written (output_vector_sink), and every
-           integer fits the 64 bits BSON has, so that writing an array of
-           numbers needs no range check
-    */
-    using bson_single_pass_sink = std::integral_constant < bool,
-          is_detected<sink_written_t, OutputSinkType>::value
-          && std::numeric_limits<typename BasicJsonType::number_integer_t>::is_specialized
-          && std::numeric_limits<typename BasicJsonType::number_integer_t>::digits <= std::numeric_limits<std::int64_t>::digits
-          && std::numeric_limits<typename BasicJsonType::number_unsigned_t>::is_specialized
-          && std::numeric_limits<typename BasicJsonType::number_unsigned_t>::digits <= std::numeric_limits<std::uint64_t>::digits >;
-
-    /// write a placeholder for the size of an object or array whose entries
-    /// follow; @return its position, for @ref patch_bson_size
-    std::size_t write_bson_size_placeholder()
-    {
-        const std::size_t pos = oa.written();
-        write_number<std::int32_t>(0, true);
-        return pos;
-    }
-
-    /// end the object or array whose size is at @a pos and write its size
-    /// there: everything from that field to the terminating null byte
-    /// @return false (and the size is not written) if it does not fit
-    bool close_bson_container(const std::size_t pos)
-    {
-        oa.write_character(to_char_type(0x00));
-        const std::size_t size = oa.written() - pos;
-        if (JSON_HEDLEY_UNLIKELY(!value_in_range_of<std::int32_t>(size)))
-        {
-            return false;
-        }
-        std::array<CharType, sizeof(std::int32_t)> bytes{};
-        store_little_endian<std::int32_t>(bytes.data(), static_cast<std::int32_t>(size));
-        oa.overwrite(pos, bytes.data(), bytes.size());
-        return true;
-    }
-
-    /*!
-    @brief check the value @a j of a BSON entry as @ref calc_bson_value_size
-           does, for @ref write_bson_document_single_pass
-    @return at least the size of the value
-    A number, boolean, or null needs no check there (its integers have at most
-    64 bits) and has at most 8 characters; a discarded value is rejected with
-    the same exception when it is written.
-    */
-    std::size_t check_bson_value(const BasicJsonType& j)
-    {
-        return (j.is_string() || j.is_binary()) ? calc_bson_value_size(j) : 8;
-    }
-
-    /// whether an entry of @a entry_size characters keeps the document that
-    /// begins at @a start, closed by @a open_containers null bytes, within
-    /// the 32-bit size of BSON
-    bool bson_entry_fits(const std::size_t start, const std::size_t entry_size, const std::size_t open_containers) const noexcept
-    {
-        const std::size_t limit = static_cast<std::size_t>((std::numeric_limits<std::int32_t>::max)());
-        const std::size_t written = oa.written() - start;
-        return written <= limit && entry_size <= limit - written && open_containers <= limit - written - entry_size;
-    }
-
-    /*!
-    @brief Serializes the JSON object @a document as a BSON document in a
-           single pass, writing the size of each object and array once it is
-           complete
-
-    Checks every key and value as @ref calc_bson_sizes does, in the same
-    order, so the same exception is thrown; the output written up to then is
-    discarded with the vector it was written to.
-
-    @return false as soon as the document or a string, binary value, object,
-            or array in it would be larger than the 32-bit BSON sizes allow;
-            nothing large has been written then, and the caller writes the
-            document in two passes, which reports that error
-    */
-    template < typename SinkType = OutputSinkType,
-               typename std::enable_if < is_detected<sink_written_t, SinkType>::value, int >::type = 0 >
-    bool write_bson_document_single_pass(const BasicJsonType& document)
-    {
-        const std::size_t start = oa.written();
-        // the object or array whose entries are being written (with the
-        // position of its size), and the ones it is in
-        bson_frame current(&document, write_bson_size_placeholder());
-        std::vector<bson_frame> parents;
-
-        while (true)
-        {
-            const BasicJsonType* nested = nullptr;
-            std::size_t nested_size_pos = 0;
-            if (current.value->is_object())
-            {
-                const auto& object = *current.value->m_data.m_value.object;
-                while (nested == nullptr && current.member != object.cend())
-                {
-                    const auto& el = *current.member;
-                    ++current.member;
-                    const std::size_t header_size = calc_bson_entry_header_size(el.first, el.second);
-                    const std::size_t value_size = el.second.is_structured() ? 0 : check_bson_value(el.second);
-                    if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, header_size + value_size, parents.size() + 1)))
-                    {
-                        return false;
-                    }
-                    if (el.second.is_structured())
-                    {
-                        write_bson_entry_header(el.first, el.second.is_object() ? 0x03 : 0x04);
-                        nested_size_pos = write_bson_size_placeholder();
-                        nested = &el.second;
-                    }
-                    else
-                    {
-                        write_bson_value(el.first, el.second);
-                    }
-                }
-            }
-            else
-            {
-                const auto& array = *current.value->m_data.m_value.array;
-                while (nested == nullptr && current.index < array.size())
-                {
-                    if (is_bson_fixed_size(array[current.index]))
-                    {
-                        write_bson_array_fixed_entries(current);
-                        if (current.index == array.size())
-                        {
-                            break;
-                        }
-                    }
-
-                    // a string, binary value, object or array
-                    const BasicJsonType& el = array[current.index];
-                    const string_t index_name(current.index_name.data(), current.index_name_size);
-                    current.next_index();
-                    const std::size_t value_size = el.is_structured() ? 0 : check_bson_value(el);
-                    if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, 2 + index_name.size() + value_size, parents.size() + 1)))
-                    {
-                        return false;
-                    }
-                    if (el.is_structured())
-                    {
-                        write_bson_entry_header(index_name, el.is_object() ? 0x03 : 0x04);
-                        nested_size_pos = write_bson_size_placeholder();
-                        nested = &el;
-                    }
-                    else
-                    {
-                        write_bson_value(index_name, el);
-                    }
-                }
-            }
-
-            if (nested != nullptr)
-            {
-                if (nested->is_array())
-                {
-                    // an array of booleans, numbers and null is written without a frame on the stack
-                    bson_frame leaf(nested, nested_size_pos);
-                    write_bson_array_fixed_entries(leaf);
-                    if (leaf.index == nested->m_data.m_value.array->size())
-                    {
-                        if (JSON_HEDLEY_UNLIKELY(!close_bson_container(nested_size_pos)))
-                        {
-                            return false;
-                        }
-                        continue;
-                    }
-                    parents.push_back(std::move(current));
-                    current = std::move(leaf);
-                    continue;
-                }
-                parents.push_back(std::move(current));
-                current = bson_frame(nested, nested_size_pos);
-                continue;
-            }
-
-            if (JSON_HEDLEY_UNLIKELY(!close_bson_container(current.size_slot)))
-            {
-                return false;
-            }
-            if (parents.empty())
-            {
-                return true;
-            }
-            current = std::move(parents.back());
-            parents.pop_back();
-        }
-    }
-
    /*!
    @brief Serializes the JSON object @a document as a BSON document

@@ -29791,39 +29728,6 @@ class binary_writer
    @pre       document.type() == value_t::object
    */
    void write_bson_document(const BasicJsonType& document)
-    {
-        write_bson_document(document, bson_single_pass_sink());
-    }
-
-    /// the sink can overwrite: one pass, unless the error handler replaces or
-    /// drops ill-formed UTF-8, which changes the sizes the checks compute
-    void write_bson_document(const BasicJsonType& document, std::true_type /*single_pass*/)
-    {
-        const std::size_t start = oa.written();
-        if ((error_handler == error_handler_t::keep || error_handler == error_handler_t::strict)
-                && write_bson_document_single_pass(document))
-        {
-            return;
-        }
-        // a length does not fit: start over, so that it is reported as the
-        // two passes report it, before anything is written
-        oa.rewind(start);
-        write_bson_document_two_pass(document);
-    }
-
-    void write_bson_document(const BasicJsonType& document, std::false_type /*single_pass*/)
-    {
-        write_bson_document_two_pass(document);
-    }
-
-    /*!
-    @brief Serializes the JSON object @a document as a BSON document: the
-           sizes of all objects and arrays first, then the entries
-
-    @param[in] document  the JSON object to serialize
-    @pre       document.type() == value_t::object
-    */
-    void write_bson_document_two_pass(const BasicJsonType& document)
    {
        std::vector<std::size_t> nested_sizes;
        const std::size_t document_size = calc_bson_sizes(document, nested_sizes);
@@ -30554,6 +30458,25 @@ class binary_writer
        }
    }

+    /// write the head and the characters of @a value, at most
+    /// max_text_in_reservation of them, in one reservation
+    JSON_HEDLEY_ALWAYS_INLINE void write_cbor_string_reserved(const string_t& value)
+    {
+        CharType* const out = oa.reserve(value.size() + 2);
+        CharType* p = out;
+        if (value.size() <= 0x17)
+        {
+            *p++ = to_char_type(static_cast<std::uint8_t>(0x60 + value.size()));
+        }
+        else
+        {
+            *p++ = to_char_type(0x78);
+            *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+        }
+        copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+        oa.commit(static_cast<std::size_t>(p - out) + value.size());
+    }
+
    /*!
    @brief write a CBOR text string

@@ -30575,19 +30498,7 @@ class binary_writer
            // the string is written as it is: head and characters in one reservation
            if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
            {
-                CharType* const out = oa.reserve(value.size() + 2);
-                CharType* p = out;
-                if (value.size() <= 0x17)
-                {
-                    *p++ = to_char_type(static_cast<std::uint8_t>(0x60 + value.size()));
-                }
-                else
-                {
-                    *p++ = to_char_type(0x78);
-                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
-                }
-                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
-                oa.commit(static_cast<std::size_t>(p - out) + value.size());
+                write_cbor_string_reserved(value);
                return;
            }
        }
@@ -30601,6 +30512,13 @@ class binary_writer
    JSON_HEDLEY_NEVER_INLINE
    void write_cbor_checked_string(const string_t& value, const BasicJsonType& context)
    {
+        if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation)
+        {
+            check_utf8(value, context);
+            write_cbor_string_reserved(value);
+            return;
+        }
+
        string_t storage;
        const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);

@@ -30615,6 +30533,27 @@ class binary_writer
    // MsgPack //
    /////////////

+    /// write the head and the characters of @a value, at most
+    /// max_text_in_reservation of them, in one reservation
+    JSON_HEDLEY_ALWAYS_INLINE void write_msgpack_string_reserved(const string_t& value)
+    {
+        CharType* const out = oa.reserve(value.size() + 2);
+        CharType* p = out;
+        if (value.size() <= 31)
+        {
+            // fixstr
+            *p++ = to_char_type(static_cast<std::uint8_t>(0xA0 | value.size()));
+        }
+        else
+        {
+            // str 8
+            *p++ = to_char_type(0xD9);
+            *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+        }
+        copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+        oa.commit(static_cast<std::size_t>(p - out) + value.size());
+    }
+
    /*!
    @brief write a MessagePack str

@@ -30632,21 +30571,7 @@ class binary_writer
            // the string is written as it is: head and characters in one reservation
            if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
            {
-                CharType* const out = oa.reserve(value.size() + 2);
-                CharType* p = out;
-                if (value.size() <= 31)
-                {
-                    // fixstr
-                    *p++ = to_char_type(static_cast<std::uint8_t>(0xA0 | value.size()));
-                }
-                else
-                {
-                    // str 8
-                    *p++ = to_char_type(0xD9);
-                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
-                }
-                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
-                oa.commit(static_cast<std::size_t>(p - out) + value.size());
+                write_msgpack_string_reserved(value);
                return;
            }
        }
@@ -30659,6 +30584,13 @@ class binary_writer
    JSON_HEDLEY_NEVER_INLINE
    void write_msgpack_checked_string(const string_t& value, const BasicJsonType& context)
    {
+        if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation)
+        {
+            check_utf8(value, context);
+            write_msgpack_string_reserved(value);
+            return;
+        }
+
        string_t storage;
        const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);

@@ -32268,10 +32200,9 @@ class binary_writer
    static void check_utf8(const string_t& s, const BasicJsonType& context)
    {
        const auto* data = reinterpret_cast<const unsigned char*>(s.data());
-        const std::size_t valid = valid_utf8_prefix(data, s.size());
-        if (JSON_HEDLEY_UNLIKELY(valid != s.size()))
+        if (JSON_HEDLEY_UNLIKELY(!valid_utf8_string(data, s.size())))
        {
-            throw_invalid_utf8(data, valid, context);
+            throw_invalid_utf8(data, valid_utf8_prefix(data, s.size()), context);
        }
    }

@@ -2267,132 +2267,3 @@ TEST_CASE("BSON large strings and binaries (chunked reader)")
        }
    }
 }
-
-TEST_CASE("BSON written in one pass and in two passes")
-{
-    // to_bson() into a returned vector writes each size once the object or
-    // array is complete; through an output adapter, all sizes are computed
-    // first. Both must produce the same bytes and throw the same exceptions.
-    const auto two_pass = [](const json & j, const json::error_handler_t handler)
-    {
-        std::vector<std::uint8_t> result;
-        json::to_bson(j, result, handler);
-        return result;
-    };
-
-    const auto check_same = [&](const json & j, const json::error_handler_t handler)
-    {
-        CAPTURE(j.dump(-1, ' ', true, json::error_handler_t::replace))
-        CAPTURE(static_cast<int>(handler))
-#if !defined(JSON_NOEXCEPTION)
-        std::string one_pass_error;
-        std::string two_pass_error;
-        std::vector<std::uint8_t> one;
-        std::vector<std::uint8_t> two;
-        try
-        {
-            one = json::to_bson(j, handler);
-        }
-        catch (const json::exception& e)
-        {
-            one_pass_error = e.what();
-        }
-        try
-        {
-            two = two_pass(j, handler);
-        }
-        catch (const json::exception& e)
-        {
-            two_pass_error = e.what();
-        }
-        CHECK(one_pass_error == two_pass_error);
-        CHECK(one == two);
-#else
-        CHECK(json::to_bson(j, handler) == two_pass(j, handler));
-#endif
-    };
-
-    const json::error_handler_t handlers[] = {json::error_handler_t::keep, json::error_handler_t::strict,
-                                              json::error_handler_t::replace, json::error_handler_t::ignore
-                                             };
-
-    SECTION("corpus")
-    {
-        for (const auto& value : utils::round_trip_corpus::values())
-        {
-            const json document = value.is_object() ? value : json{{"value", value}};
-            for (const auto handler : handlers)
-            {
-                check_same(document, handler);
-            }
-        }
-    }
-
-    SECTION("nesting and arrays of every length class")
-    {
-        json document = json::object();
-        document["empty object"] = json::object();
-        document["empty array"] = json::array();
-        for (const std::size_t n :
-                {
-                    std::size_t(1), std::size_t(9), std::size_t(10), std::size_t(11), std::size_t(99), std::size_t(100), std::size_t(101), std::size_t(1000)
-                })
-        {
-            json numbers = json::array();
-            json mixed = json::array();
-            for (std::size_t i = 0; i < n; ++i)
-            {
-                numbers.push_back(i % 3 == 0 ? json(static_cast<double>(i) / 3) : json(static_cast<std::int64_t>(i) * 1000003));
-                mixed.push_back(i % 4 == 0 ? json("s" + std::to_string(i)) : i % 4 == 1 ? json::array({i, json::object({{"k", i}})}) : i % 4 == 2 ? json(nullptr) : json(i % 2 == 0));
-            }
-            document["numbers " + std::to_string(n)] = numbers;
-            document["mixed " + std::to_string(n)] = mixed;
-            document["nested " + std::to_string(n)] = json::object({{"a", numbers}, {"b", json::array({mixed, json::object()})}});
-        }
-        document["binary"] = json::binary({1, 2, 3}, 0x80);
-        document["binary old"] = json::binary({1, 2, 3}, 0x02);
-        for (const auto handler : handlers)
-        {
-            check_same(document, handler);
-        }
-    }
-
-    SECTION("errors at every position")
-    {
-        // an entry that cannot be written, at each place among valid ones
-        const std::string bad_utf8 = "\xC3\x28";
-        const json bad_entries[] =
-        {
-            json::object({{std::string("a\0b", 3), 1}}),
-            json::object({{"s", bad_utf8}}),
-            json::object({{bad_utf8, 1}}),
-            json::array({1, 2, bad_utf8}),
-            json::object({{"b", json::binary({1}, 0x100)}}),
-            json::object({{"d", json(json::value_t::discarded)}}),
-            json::array({json(json::value_t::discarded)}),
-        };
-        for (const auto& bad : bad_entries)
-        {
-            for (std::size_t pos = 0; pos < 4; ++pos)
-            {
-                json document = json::object();
-                json list = json::array();
-                for (std::size_t i = 0; i < 4; ++i)
-                {
-                    if (i == pos)
-                    {
-                        document["x" + std::to_string(i)] = bad;
-                        list.push_back(bad);
-                    }
-                    document["n" + std::to_string(i)] = json::array({i, "str", json::object({{"k", i}})});
-                    list.push_back(i);
-                }
-                document["list"] = list;
-                for (const auto handler : handlers)
-                {
-                    check_same(document, handler);
-                }
-            }
-        }
-    }
-}
@@ -403,6 +403,7 @@ TEST_CASE("guard page")
                    CHECK(nlohmann::detail::find_string_special(p, n) <= n);
                    CHECK(nlohmann::detail::find_ascii_copyable_run(p, n) <= n);
                    CHECK(nlohmann::detail::valid_utf8_prefix(p, n) <= n);
+                    CHECK(nlohmann::detail::valid_utf8_string(p, n) == (nlohmann::detail::valid_utf8_prefix(p, n) == n));
 #if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17)
                    CHECK(nlohmann::detail::find_string_delimiter(p, n) <= n);
 #endif
@@ -13,6 +13,7 @@ using nlohmann::json;

 #include <algorithm>
 #include <fstream>
+#include <random>
 #include <string>
 #include "make_test_data_available.hpp"
 #include "test_utils.hpp"
@@ -1631,3 +1632,40 @@ TEST_CASE("Unicode RFC 3629")
 }

 DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+TEST_CASE("UTF-8 check of whole strings")
+{
+    // valid_utf8_string() decides short strings with a word test and long ones
+    // 16 bytes at a time where a vector check is available; it must agree with
+    // the byte-wise check for every mix of ASCII, sequences of each length,
+    // ill-formed and truncated sequences, at every length and offset
+    const char* const pieces[] =
+    {
+        "a", "\x7F", "\xC3\xA9", "\xE2\x82\xAC", "\xE0\xA0\x80", "\xEF\xBF\xBF", "\xF0\x9F\x98\x80", "\xF4\x8F\xBF\xBF",
+        "\x80", "\xC0\xAF", "\xC3", "\xE2\x82", "\xED\xA0\x80", "\xF0\x9F", "\xF4\x90\x80\x80", "\xFF"
+    };
+    // std::mt19937's sequence is fixed by the standard; no distribution is used
+    std::mt19937 rng(5);
+    std::size_t valid = 0;
+    for (int i = 0; i < 100000; ++i)
+    {
+        std::string s;
+        const std::size_t length = rng() % 100;
+        const std::uint32_t ascii = rng() % 101;
+        while (s.size() < length)
+        {
+            // mostly well-formed pieces (the first eight), so that long valid
+            // strings are common
+            s += (rng() % 100 < ascii) ? std::string("x") : std::string(pieces[rng() % (rng() % 8 == 0 ? 16 : 8)]);
+        }
+        const auto* data = reinterpret_cast<const unsigned char*>(s.data());
+        const bool expected = nlohmann::detail::is_valid_utf8(s);
+        valid += expected ? 1 : 0;
+        CAPTURE(s)
+        CHECK(nlohmann::detail::valid_utf8_string(data, s.size()) == expected);
+        CHECK((nlohmann::detail::valid_utf8_prefix(data, s.size()) == s.size()) == expected);
+    }
+    // both outcomes are common enough to be tested
+    CHECK(valid > 20000);
+    CHECK(valid < 80000);
+}