Binary readers and writers: faster UTF-8 checks of strings

With error_handler_t::strict (and replace/ignore), every string a binary reader reads is checked for valid UTF-8, and every string a binary writer writes. The readers used the byte-at-a-time decoder; the writers skipped ASCII eight bytes at a time. - New valid_utf8_string() decides strings of up to 16 ASCII bytes, most keys and many values, with one or two word tests, and checks longer strings 16 bytes at a time with the NEON/SSSE3 UTF-8 check the lexer uses (vector_utf8_valid_blocks: the same lookup, but quotes and control characters are ordinary characters). Other targets and JSON_NO_SIMD use valid_utf8_prefix() as before. - The readers check strings with it (for one-byte character types) in an out-of-line function, so that reading without a check does not pay for its code. - The writers' check_utf8() uses it, and the CBOR and MessagePack writers write a checked string of up to max_text_in_reservation characters in one reservation, as they write unchecked ones. Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-31 13:00:39 +00:00 · 2026-10-31 09:45:14 +01:00
11 changed files with 551 additions and 282 deletions
@@ -4293,6 +4293,19 @@ class binary_reader
        return get_bytes(len, "string", result);
    }

+    /// whether @a s is valid UTF-8: ASCII eight bytes at a time, the rest
+    /// sequence by sequence (as the binary writers check strings)
+    static bool is_valid_utf8_string(const string_t& s, std::true_type /*is_byte*/) noexcept
+    {
+        return valid_utf8_string(reinterpret_cast<const unsigned char*>(s.data()), s.size());
+    }
+
+    /// whether @a s, of a character type wider than a byte, is valid UTF-8
+    static bool is_valid_utf8_string(const string_t& s, std::false_type /*is_byte*/) noexcept
+    {
+        return is_valid_utf8(s);
+    }
+
    /*!
    @brief validate a decoded text string (value or object key) against @ref error_handler

@@ -4312,7 +4325,16 @@ class binary_reader
    */
    bool check_string_utf8(string_t& result, const char* context)
    {
-        if (error_handler == error_handler_t::keep || is_valid_utf8(result))
+        return error_handler == error_handler_t::keep || check_string_utf8_handler(result, context);
+    }
+
+    /// @ref check_string_utf8 for an error handler other than @ref
+    /// error_handler_t::keep; out of line, so that reading strings without a
+    /// check does not pay for the code of the check
+    JSON_HEDLEY_NEVER_INLINE
+    bool check_string_utf8_handler(string_t& result, const char* context)
+    {
+        if (is_valid_utf8_string(result, std::integral_constant<bool, sizeof(typename string_t::value_type) == 1>()))
        {
            return true;
        }
@@ -268,10 +268,10 @@ class lexer : public lexer_base<BasicJsonType>
  public:
    using token_type = typename lexer_base<BasicJsonType>::token_type;

-    explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_values_ = false) noexcept
+    explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept
        : ia(std::move(adapter))
        , ignore_comments(ignore_comments_)
-        , discard_values(discard_values_)
+        , discard_number_values(discard_number_values_)
    {
        // the token string of an error before the first token is everything
        // read so far
@@ -388,8 +388,6 @@ class lexer : public lexer_base<BasicJsonType>
    checks if it is inside the range. If a violation was detected, set up an
    error message and return false. Otherwise, return true.

-    @param[in] keep    whether the bytes are added to token_buffer (false for
-                       accept(), see scan())
    @param[in] ranges  list of integers; interpreted as list of pairs of
                       inclusive lower and upper bound, respectively

@@ -398,18 +396,17 @@ class lexer : public lexer_base<BasicJsonType>

    @return true if and only if no range violation was detected
    */
-    template<bool Keep>
-    bool next_byte_in_range(std::integral_constant<bool, Keep> keep, std::initializer_list<char_int_type> ranges)
+    bool next_byte_in_range(std::initializer_list<char_int_type> ranges)
    {
        JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6);
-        add_string_char(keep, current);
+        add(current);

        for (auto range = ranges.begin(); range != ranges.end(); ++range)
        {
            get();
            if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions)
            {
-                add_string_char(keep, current);
+                add(current);
            }
            else
            {
@@ -422,16 +419,14 @@ class lexer : public lexer_base<BasicJsonType>
    }

    /// contiguous input: bulk-append the run of ordinary characters and complete
-    /// well-formed UTF-8 sequences starting at the current read position (only
-    /// skip it if @a keep is false), then
+    /// well-formed UTF-8 sequences starting at the current read position, then
    /// read the first byte that needs individual handling (the closing quote, an
    /// escape, a control character, or an ill-formed UTF-8 byte) like get().
    /// That byte is taken from the buffer that was just scanned. Through get(),
    /// Apple Clang merged this read with the other get() calls of the scan
    /// function, which load the read position again right after it was stored
    /// (3% of json::accept() on jeopardy.json on Apple M1).
-    template<bool Keep>
-    char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant<bool, Keep> keep)
+    char_int_type get_after_string_run(std::true_type /*bulk*/)
    {
        // a pending unget must be consumed through the normal path first
        if (next_unget)
@@ -449,7 +444,7 @@ class lexer : public lexer_base<BasicJsonType>
        const std::size_t pos = lexer_string_bulk_run(data, remaining);
        if (pos != 0)
        {
-            append_string_run(keep, data, pos);
+            token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), pos);
            bulk_consume(reinterpret_cast<const char*>(data), pos);
        }
        if (pos == remaining)
@@ -493,8 +488,7 @@ class lexer : public lexer_base<BasicJsonType>
    }

    /// streaming input: no bulk fast path
-    template<bool Keep>
-    char_int_type get_after_string_run(std::false_type /*bulk*/, std::integral_constant<bool, Keep> /*keep*/)
+    char_int_type get_after_string_run(std::false_type /*bulk*/)
    {
        return get();
    }
@@ -508,18 +502,13 @@ class lexer : public lexer_base<BasicJsonType>
    may contain \0 bytes), and token_buffer.size() is the number of bytes in the
    string.

-    @param[in] keep  whether the string is copied into token_buffer; if false
-                     (accept(), see scan()), it is only checked, with the same
-                     errors, and token_buffer stays empty
-
    @return token_type::value_string if string could be successfully scanned,
            token_type::parse_error otherwise

    @note In case of errors, variable error_message contains a textual
          description.
    */
-    template<bool Keep>
-    token_type scan_string(std::integral_constant<bool, Keep> keep)
+    token_type scan_string()
    {
        // reset token_buffer (ignore opening quote)
        reset();
@@ -531,7 +520,7 @@ class lexer : public lexer_base<BasicJsonType>
        {
            // bulk-consume ordinary characters from contiguous input, then
            // get the next character, the one that ends the run
-            switch (get_after_string_run(std::integral_constant<bool, bulk_scan>(), keep))
+            switch (get_after_string_run(std::integral_constant<bool, bulk_scan>()))
            {
                // end of file while parsing the string
                case char_traits<char_type>::eof():
@@ -553,35 +542,35 @@ class lexer : public lexer_base<BasicJsonType>
                    {
                        // quotation mark
                        case '\"':
-                            add_string_char(keep, '\"');
+                            add('\"');
                            break;
                        // reverse solidus
                        case '\\':
-                            add_string_char(keep, '\\');
+                            add('\\');
                            break;
                        // solidus
                        case '/':
-                            add_string_char(keep, '/');
+                            add('/');
                            break;
                        // backspace
                        case 'b':
-                            add_string_char(keep, '\b');
+                            add('\b');
                            break;
                        // form feed
                        case 'f':
-                            add_string_char(keep, '\f');
+                            add('\f');
                            break;
                        // line feed
                        case 'n':
-                            add_string_char(keep, '\n');
+                            add('\n');
                            break;
                        // carriage return
                        case 'r':
-                            add_string_char(keep, '\r');
+                            add('\r');
                            break;
                        // tab
                        case 't':
-                            add_string_char(keep, '\t');
+                            add('\t');
                            break;

                        // unicode escapes
@@ -649,7 +638,10 @@ class lexer : public lexer_base<BasicJsonType>
                            JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF);

                            // translate codepoint into bytes
-                            add_codepoint(keep, codepoint);
+                            encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte)
+                            {
+                                add(static_cast<char_int_type>(byte));
+                            });

                            break;
                        }
@@ -832,7 +824,7 @@ class lexer : public lexer_base<BasicJsonType>
                case 0x7E:
                case 0x7F:
                {
-                    add_string_char(keep, current);
+                    add(current);
                    break;
                }

@@ -868,7 +860,7 @@ class lexer : public lexer_base<BasicJsonType>
                case 0xDE:
                case 0xDF:
                {
-                    if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range(keep, {0x80, 0xBF})))
+                    if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF})))
                    {
                        return token_type::parse_error;
                    }
@@ -878,7 +870,7 @@ class lexer : public lexer_base<BasicJsonType>
                // U+0800..U+0FFF: bytes E0 A0..BF 80..BF
                case 0xE0:
                {
-                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0xA0, 0xBF, 0x80, 0xBF}))))
+                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF}))))
                    {
                        return token_type::parse_error;
                    }
@@ -902,7 +894,7 @@ class lexer : public lexer_base<BasicJsonType>
                case 0xEE:
                case 0xEF:
                {
-                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF}))))
+                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF}))))
                    {
                        return token_type::parse_error;
                    }
@@ -912,7 +904,7 @@ class lexer : public lexer_base<BasicJsonType>
                // U+D000..U+D7FF: bytes ED 80..9F 80..BF
                case 0xED:
                {
-                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x9F, 0x80, 0xBF}))))
+                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF}))))
                    {
                        return token_type::parse_error;
                    }
@@ -922,7 +914,7 @@ class lexer : public lexer_base<BasicJsonType>
                // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
                case 0xF0:
                {
-                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
+                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
                    {
                        return token_type::parse_error;
                    }
@@ -934,7 +926,7 @@ class lexer : public lexer_base<BasicJsonType>
                case 0xF2:
                case 0xF3:
                {
-                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
+                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
                    {
                        return token_type::parse_error;
                    }
@@ -944,7 +936,7 @@ class lexer : public lexer_base<BasicJsonType>
                // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
                case 0xF4:
                {
-                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
+                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
                    {
                        return token_type::parse_error;
                    }
@@ -1435,7 +1427,7 @@ scan_number_done:
    token_type convert_number(token_type number_type, std::size_t mantissa_end)
    {
        // accept() only needs to know whether the input is valid, so it sets
-        // discard_values (see json.hpp), and an integer token whose
+        // discard_number_values (see json.hpp), and an integer token whose
        // digit count shows that it fits is reported without calling
        // convert_integer(). A number with up to 18 digits always fits into
        // both std::uint64_t and std::int64_t (18 nines is about 1e18, below
@@ -1450,7 +1442,7 @@ scan_number_done:
        // json_sax_acceptor, whose number callbacks discard their argument and
        // return true, and the parser rejects neither integers nor finite
        // floats. value_unsigned/value_integer are left unset here, so a caller
-        // that reads the converted value must not set discard_values.
+        // that reads the converted value must not set discard_number_values.
        //
        // On contiguous input, scan_number_bulk_contiguous() converts integer
        // tokens itself and does not pass them to this function, unless
@@ -1458,7 +1450,7 @@ scan_number_done:
        // reached for input without bulk access (e.g. streams), with
        // JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous()
        // falls back to scan_number().
-        if (discard_values)
+        if (discard_number_values)
        {
            constexpr std::size_t safe_digit_count = 18;
            if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count)
@@ -1709,7 +1701,7 @@ scan_number_done:
        // w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a
        // value that underflows is a finite number as well. Only a token that
        // may overflow has to be converted to find out.
-        if (JSON_HEDLEY_UNLIKELY(discard_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
+        if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
        {
            value_float = 0;
            return true;
@@ -2041,34 +2033,6 @@ scan_number_done:
        token_buffer.push_back(static_cast<typename string_t::value_type>(c));
    }

-    /// add a character of a string token; accept() checks the string but does
-    /// not keep its value (@a keep is false)
-    void add_string_char(std::true_type /*keep*/, char_int_type c)
-    {
-        token_buffer.push_back(static_cast<typename string_t::value_type>(c));
-    }
-
-    static void add_string_char(std::false_type /*keep*/, char_int_type /*c*/) noexcept {}
-
-    /// add the UTF-8 bytes of a code point from a \\u escape to a string token
-    void add_codepoint(std::true_type /*keep*/, int codepoint)
-    {
-        encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte)
-        {
-            add(static_cast<char_int_type>(byte));
-        });
-    }
-
-    static void add_codepoint(std::false_type /*keep*/, int /*codepoint*/) noexcept {}
-
-    /// append a run of ordinary string characters (see get_after_string_run())
-    void append_string_run(std::true_type /*keep*/, const unsigned char* data, std::size_t n)
-    {
-        token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), n);
-    }
-
-    static void append_string_run(std::false_type /*keep*/, const unsigned char* /*data*/, std::size_t /*n*/) noexcept {}
-
  public:
    /////////////////////
    // value getters
@@ -2463,11 +2427,7 @@ scan_number_done:

            // string
            case '\"':
-                // accept() (discard_values) checks strings without copying them
-                // into token_buffer. The choice is made once per string: a
-                // check inside the string loop made number-heavy documents up
-                // to 3 % slower (mesh.json, Apple M1)
-                return discard_values ? scan_string(std::false_type {}) : scan_string(std::true_type {});
+                return scan_string();

            // number
            case '-':
@@ -2552,13 +2512,11 @@ scan_number_done:
    std::size_t decimal_point_position = std::string::npos;

    /// whether the caller only needs the token types and never looks at the
-    /// values of number and string tokens; set only by accept(), which parses
-    /// through json_sax_acceptor. When set, convert_number() skips converting
-    /// integer tokens whose digit count guarantees that they fit into 64 bits
-    /// (see there), floats are converted only when they may overflow (see
-    /// convert_accumulated_float()), and strings are checked without being
-    /// copied into token_buffer (see scan())
-    const bool discard_values = false;
+    /// converted numeric values; set only by accept(), which parses through
+    /// json_sax_acceptor. When set, convert_number() skips converting integer
+    /// tokens whose digit count guarantees that they fit into 64 bits (see
+    /// there)
+    const bool discard_number_values = false;
 };

 }  // namespace detail
@@ -89,9 +89,9 @@ class parser
                    const bool allow_exceptions_ = true,
                    const bool ignore_comments = false,
                    const bool ignore_trailing_commas_ = false,
-                    const bool discard_values_ = false)
+                    const bool discard_number_values_ = false)
        : callback(std::move(cb))
-        , m_lexer(std::move(adapter), ignore_comments, discard_values_)
+        , m_lexer(std::move(adapter), ignore_comments, discard_number_values_)
        , allow_exceptions(allow_exceptions_)
        , ignore_trailing_commas(ignore_trailing_commas_)
    {
@@ -322,6 +322,78 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons
    block_out = block;
    return false;
 }
+
+/*!
+@brief skip 16-byte blocks of well-formed UTF-8 at p (a character boundary)
+
+Like vector_utf8_run, but quotes, backslashes, and control characters are
+ordinary characters. Returns the start of the first block in which the check
+found an error (which may belong to a sequence begun in the block before) or
+where fewer than 16 bytes are left; a sequence may be cut at that point, so the
+caller resumes at its start, one sequence at a time.
+*/
+JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vector_utf8_valid_blocks(const unsigned char* block, const unsigned char* e) noexcept
+{
+    using lookup = utf8_lookup4<>;
+#if JSON_SCAN_NEON
+    const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high.data());
+    const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low.data());
+    const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high.data());
+    uint8x16_t prev = vdupq_n_u8(0);
+    while (e - block >= 16)
+    {
+        const uint8x16_t in = vld1q_u8(block);
+        if (vmaxvq_u8(vorrq_u8(in, prev)) < 0x80)
+        {
+            // ASCII here, and no sequence from the block before to finish
+            prev = in;
+            block += 16;
+            continue;
+        }
+        const uint8x16_t prev1 = vextq_u8(prev, in, 15);
+        const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4)));
+        const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80)));
+        const uint8x16_t err = veorq_u8(vandq_u8(must23, vdupq_n_u8(0x80)), sc);
+        if (vmaxvq_u8(err) != 0)
+        {
+            break;
+        }
+        prev = in;
+        block += 16;
+    }
+#else
+    const __m128i t1h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_high.data()))); // NOLINT(bugprone-casting-through-void)
+    const __m128i t1l = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_low.data()))); // NOLINT(bugprone-casting-through-void)
+    const __m128i t2h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_2_high.data()))); // NOLINT(bugprone-casting-through-void)
+    const __m128i nibble = _mm_set1_epi8(0x0F);
+    const __m128i zero = _mm_setzero_si128();
+    __m128i prev = zero;
+    while (e - block >= 16)
+    {
+        const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(block))); // NOLINT(bugprone-casting-through-void)
+        if (_mm_movemask_epi8(_mm_or_si128(in, prev)) == 0)
+        {
+            prev = in;
+            block += 16;
+            continue;
+        }
+        const __m128i prev1 = _mm_alignr_epi8(in, prev, 15);
+        const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)),
+                                         _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))),
+                                         _mm_shuffle_epi8(t2h, _mm_and_si128(_mm_srli_epi16(in, 4), nibble)));
+        const __m128i must23 = _mm_or_si128(_mm_subs_epu8(_mm_alignr_epi8(in, prev, 14), _mm_set1_epi8(0xE0 - 0x80)),
+                                            _mm_subs_epu8(_mm_alignr_epi8(in, prev, 13), _mm_set1_epi8(0xF0 - 0x80)));
+        const __m128i err = _mm_xor_si128(_mm_and_si128(must23, _mm_set1_epi8(static_cast<char>(-128))), sc);
+        if (_mm_movemask_epi8(_mm_cmpeq_epi8(err, zero)) != 0xFFFF)
+        {
+            break;
+        }
+        prev = in;
+        block += 16;
+    }
+#endif
+    return block;
+}
 #endif

 }  // namespace detail
@@ -236,6 +236,64 @@ inline std::size_t valid_utf8_prefix(const unsigned char* data, std::size_t n) n
    return pos;
 }

+/*!
+@brief whether [data, data+n) is valid UTF-8, for checking whole strings
+
+The same answer as valid_utf8_prefix(data, n) == n. Short ASCII strings, the
+most common keys and values, are decided in one test, and long strings are
+checked 16 bytes at a time where the vector check is available.
+*/
+inline bool valid_utf8_string(const unsigned char* data, std::size_t n) noexcept
+{
+    constexpr std::uint64_t high = 0x8080808080808080ull;
+    if (n <= 16)
+    {
+        // two overlapping words cover 8 to 16 bytes, one partial word fewer
+        std::uint64_t first = 0;
+        std::uint64_t last = 0;
+        if (n >= 8)
+        {
+            std::memcpy(&first, data, sizeof(first));
+            std::memcpy(&last, data + n - 8, sizeof(last));
+        }
+        else if (n != 0)
+        {
+            std::memcpy(&first, data, n);
+        }
+        if (((first | last) & high) == 0)
+        {
+            return true;
+        }
+    }
+#if JSON_SCAN_VECTOR_UTF8
+    else if (n >= 32
+#if JSON_SCAN_SSSE3_DISPATCH
+             && cpu_has_ssse3()
+#endif
+            )
+    {
+        const unsigned char* block = vector_utf8_valid_blocks(data, data + n);
+        // the rest from the start of a sequence that may be cut at block
+        for (int i = 1; i <= 3 && block - i >= data; ++i)
+        {
+            const unsigned char c = block[-i];
+            if (c < 0x80)
+            {
+                break;
+            }
+            if (c >= 0xC0)
+            {
+                block -= i;
+                break;
+            }
+        }
+        const auto checked = static_cast<std::size_t>(block - data);
+        return checked + valid_utf8_prefix(block, n - checked) == n;
+    }
+#endif
+    return valid_utf8_prefix(data, n) == n;
+}
+
 #if JSON_SCAN_VECTOR_UTF8
 /// the end of scan_string_vector from block, where the vector loop stopped
 /// (ill-formed UTF-8, or fewer than 16 bytes left): one byte or sequence at a
@@ -3080,6 +3080,25 @@ class binary_writer
        }
    }

+    /// write the head and the characters of @a value, at most
+    /// max_text_in_reservation of them, in one reservation
+    JSON_HEDLEY_ALWAYS_INLINE void write_cbor_string_reserved(const string_t& value)
+    {
+        CharType* const out = oa.reserve(value.size() + 2);
+        CharType* p = out;
+        if (value.size() <= 0x17)
+        {
+            *p++ = to_char_type(static_cast<std::uint8_t>(0x60 + value.size()));
+        }
+        else
+        {
+            *p++ = to_char_type(0x78);
+            *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+        }
+        copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+        oa.commit(static_cast<std::size_t>(p - out) + value.size());
+    }
+
    /*!
    @brief write a CBOR text string

@@ -3101,19 +3120,7 @@ class binary_writer
            // the string is written as it is: head and characters in one reservation
            if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
            {
-                CharType* const out = oa.reserve(value.size() + 2);
-                CharType* p = out;
-                if (value.size() <= 0x17)
-                {
-                    *p++ = to_char_type(static_cast<std::uint8_t>(0x60 + value.size()));
-                }
-                else
-                {
-                    *p++ = to_char_type(0x78);
-                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
-                }
-                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
-                oa.commit(static_cast<std::size_t>(p - out) + value.size());
+                write_cbor_string_reserved(value);
                return;
            }
        }
@@ -3127,6 +3134,13 @@ class binary_writer
    JSON_HEDLEY_NEVER_INLINE
    void write_cbor_checked_string(const string_t& value, const BasicJsonType& context)
    {
+        if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation)
+        {
+            check_utf8(value, context);
+            write_cbor_string_reserved(value);
+            return;
+        }
+
        string_t storage;
        const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);

@@ -3141,6 +3155,27 @@ class binary_writer
    // MsgPack //
    /////////////

+    /// write the head and the characters of @a value, at most
+    /// max_text_in_reservation of them, in one reservation
+    JSON_HEDLEY_ALWAYS_INLINE void write_msgpack_string_reserved(const string_t& value)
+    {
+        CharType* const out = oa.reserve(value.size() + 2);
+        CharType* p = out;
+        if (value.size() <= 31)
+        {
+            // fixstr
+            *p++ = to_char_type(static_cast<std::uint8_t>(0xA0 | value.size()));
+        }
+        else
+        {
+            // str 8
+            *p++ = to_char_type(0xD9);
+            *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+        }
+        copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+        oa.commit(static_cast<std::size_t>(p - out) + value.size());
+    }
+
    /*!
    @brief write a MessagePack str

@@ -3158,21 +3193,7 @@ class binary_writer
            // the string is written as it is: head and characters in one reservation
            if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
            {
-                CharType* const out = oa.reserve(value.size() + 2);
-                CharType* p = out;
-                if (value.size() <= 31)
-                {
-                    // fixstr
-                    *p++ = to_char_type(static_cast<std::uint8_t>(0xA0 | value.size()));
-                }
-                else
-                {
-                    // str 8
-                    *p++ = to_char_type(0xD9);
-                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
-                }
-                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
-                oa.commit(static_cast<std::size_t>(p - out) + value.size());
+                write_msgpack_string_reserved(value);
                return;
            }
        }
@@ -3185,6 +3206,13 @@ class binary_writer
    JSON_HEDLEY_NEVER_INLINE
    void write_msgpack_checked_string(const string_t& value, const BasicJsonType& context)
    {
+        if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation)
+        {
+            check_utf8(value, context);
+            write_msgpack_string_reserved(value);
+            return;
+        }
+
        string_t storage;
        const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);

@@ -4794,10 +4822,9 @@ class binary_writer
    static void check_utf8(const string_t& s, const BasicJsonType& context)
    {
        const auto* data = reinterpret_cast<const unsigned char*>(s.data());
-        const std::size_t valid = valid_utf8_prefix(data, s.size());
-        if (JSON_HEDLEY_UNLIKELY(valid != s.size()))
+        if (JSON_HEDLEY_UNLIKELY(!valid_utf8_string(data, s.size())))
        {
-            throw_invalid_utf8(data, valid, context);
+            throw_invalid_utf8(data, valid_utf8_prefix(data, s.size()), context);
        }
    }

@@ -184,11 +184,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
        const bool allow_exceptions = true,
        const bool ignore_comments = false,
        const bool ignore_trailing_commas = false,
-        const bool discard_values = false
+        const bool discard_number_values = false
    )
    {
        return ::nlohmann::detail::parser<basic_json, InputAdapterType>(std::move(adapter),
-               std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_values);
+               std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values);
    }

  private:
@@ -12377,6 +12377,78 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons
    block_out = block;
    return false;
 }
+
+/*!
+@brief skip 16-byte blocks of well-formed UTF-8 at p (a character boundary)
+
+Like vector_utf8_run, but quotes, backslashes, and control characters are
+ordinary characters. Returns the start of the first block in which the check
+found an error (which may belong to a sequence begun in the block before) or
+where fewer than 16 bytes are left; a sequence may be cut at that point, so the
+caller resumes at its start, one sequence at a time.
+*/
+JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vector_utf8_valid_blocks(const unsigned char* block, const unsigned char* e) noexcept
+{
+    using lookup = utf8_lookup4<>;
+#if JSON_SCAN_NEON
+    const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high.data());
+    const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low.data());
+    const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high.data());
+    uint8x16_t prev = vdupq_n_u8(0);
+    while (e - block >= 16)
+    {
+        const uint8x16_t in = vld1q_u8(block);
+        if (vmaxvq_u8(vorrq_u8(in, prev)) < 0x80)
+        {
+            // ASCII here, and no sequence from the block before to finish
+            prev = in;
+            block += 16;
+            continue;
+        }
+        const uint8x16_t prev1 = vextq_u8(prev, in, 15);
+        const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4)));
+        const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80)));
+        const uint8x16_t err = veorq_u8(vandq_u8(must23, vdupq_n_u8(0x80)), sc);
+        if (vmaxvq_u8(err) != 0)
+        {
+            break;
+        }
+        prev = in;
+        block += 16;
+    }
+#else
+    const __m128i t1h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_high.data()))); // NOLINT(bugprone-casting-through-void)
+    const __m128i t1l = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_low.data()))); // NOLINT(bugprone-casting-through-void)
+    const __m128i t2h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_2_high.data()))); // NOLINT(bugprone-casting-through-void)
+    const __m128i nibble = _mm_set1_epi8(0x0F);
+    const __m128i zero = _mm_setzero_si128();
+    __m128i prev = zero;
+    while (e - block >= 16)
+    {
+        const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(block))); // NOLINT(bugprone-casting-through-void)
+        if (_mm_movemask_epi8(_mm_or_si128(in, prev)) == 0)
+        {
+            prev = in;
+            block += 16;
+            continue;
+        }
+        const __m128i prev1 = _mm_alignr_epi8(in, prev, 15);
+        const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)),
+                                         _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))),
+                                         _mm_shuffle_epi8(t2h, _mm_and_si128(_mm_srli_epi16(in, 4), nibble)));
+        const __m128i must23 = _mm_or_si128(_mm_subs_epu8(_mm_alignr_epi8(in, prev, 14), _mm_set1_epi8(0xE0 - 0x80)),
+                                            _mm_subs_epu8(_mm_alignr_epi8(in, prev, 13), _mm_set1_epi8(0xF0 - 0x80)));
+        const __m128i err = _mm_xor_si128(_mm_and_si128(must23, _mm_set1_epi8(static_cast<char>(-128))), sc);
+        if (_mm_movemask_epi8(_mm_cmpeq_epi8(err, zero)) != 0xFFFF)
+        {
+            break;
+        }
+        prev = in;
+        block += 16;
+    }
+#endif
+    return block;
+}
 #endif

 }  // namespace detail
@@ -12604,6 +12676,64 @@ inline std::size_t valid_utf8_prefix(const unsigned char* data, std::size_t n) n
    return pos;
 }

+/*!
+@brief whether [data, data+n) is valid UTF-8, for checking whole strings
+
+The same answer as valid_utf8_prefix(data, n) == n. Short ASCII strings, the
+most common keys and values, are decided in one test, and long strings are
+checked 16 bytes at a time where the vector check is available.
+*/
+inline bool valid_utf8_string(const unsigned char* data, std::size_t n) noexcept
+{
+    constexpr std::uint64_t high = 0x8080808080808080ull;
+    if (n <= 16)
+    {
+        // two overlapping words cover 8 to 16 bytes, one partial word fewer
+        std::uint64_t first = 0;
+        std::uint64_t last = 0;
+        if (n >= 8)
+        {
+            std::memcpy(&first, data, sizeof(first));
+            std::memcpy(&last, data + n - 8, sizeof(last));
+        }
+        else if (n != 0)
+        {
+            std::memcpy(&first, data, n);
+        }
+        if (((first | last) & high) == 0)
+        {
+            return true;
+        }
+    }
+#if JSON_SCAN_VECTOR_UTF8
+    else if (n >= 32
+#if JSON_SCAN_SSSE3_DISPATCH
+             && cpu_has_ssse3()
+#endif
+            )
+    {
+        const unsigned char* block = vector_utf8_valid_blocks(data, data + n);
+        // the rest from the start of a sequence that may be cut at block
+        for (int i = 1; i <= 3 && block - i >= data; ++i)
+        {
+            const unsigned char c = block[-i];
+            if (c < 0x80)
+            {
+                break;
+            }
+            if (c >= 0xC0)
+            {
+                block -= i;
+                break;
+            }
+        }
+        const auto checked = static_cast<std::size_t>(block - data);
+        return checked + valid_utf8_prefix(block, n - checked) == n;
+    }
+#endif
+    return valid_utf8_prefix(data, n) == n;
+}
+
 #if JSON_SCAN_VECTOR_UTF8
 /// the end of scan_string_vector from block, where the vector loop stopped
 /// (ill-formed UTF-8, or fewer than 16 bytes left): one byte or sequence at a
@@ -13124,10 +13254,10 @@ class lexer : public lexer_base<BasicJsonType>
  public:
    using token_type = typename lexer_base<BasicJsonType>::token_type;

-    explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_values_ = false) noexcept
+    explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept
        : ia(std::move(adapter))
        , ignore_comments(ignore_comments_)
-        , discard_values(discard_values_)
+        , discard_number_values(discard_number_values_)
    {
        // the token string of an error before the first token is everything
        // read so far
@@ -13244,8 +13374,6 @@ class lexer : public lexer_base<BasicJsonType>
    checks if it is inside the range. If a violation was detected, set up an
    error message and return false. Otherwise, return true.

-    @param[in] keep    whether the bytes are added to token_buffer (false for
-                       accept(), see scan())
    @param[in] ranges  list of integers; interpreted as list of pairs of
                       inclusive lower and upper bound, respectively

@@ -13254,18 +13382,17 @@ class lexer : public lexer_base<BasicJsonType>

    @return true if and only if no range violation was detected
    */
-    template<bool Keep>
-    bool next_byte_in_range(std::integral_constant<bool, Keep> keep, std::initializer_list<char_int_type> ranges)
+    bool next_byte_in_range(std::initializer_list<char_int_type> ranges)
    {
        JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6);
-        add_string_char(keep, current);
+        add(current);

        for (auto range = ranges.begin(); range != ranges.end(); ++range)
        {
            get();
            if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions)
            {
-                add_string_char(keep, current);
+                add(current);
            }
            else
            {
@@ -13278,16 +13405,14 @@ class lexer : public lexer_base<BasicJsonType>
    }

    /// contiguous input: bulk-append the run of ordinary characters and complete
-    /// well-formed UTF-8 sequences starting at the current read position (only
-    /// skip it if @a keep is false), then
+    /// well-formed UTF-8 sequences starting at the current read position, then
    /// read the first byte that needs individual handling (the closing quote, an
    /// escape, a control character, or an ill-formed UTF-8 byte) like get().
    /// That byte is taken from the buffer that was just scanned. Through get(),
    /// Apple Clang merged this read with the other get() calls of the scan
    /// function, which load the read position again right after it was stored
    /// (3% of json::accept() on jeopardy.json on Apple M1).
-    template<bool Keep>
-    char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant<bool, Keep> keep)
+    char_int_type get_after_string_run(std::true_type /*bulk*/)
    {
        // a pending unget must be consumed through the normal path first
        if (next_unget)
@@ -13305,7 +13430,7 @@ class lexer : public lexer_base<BasicJsonType>
        const std::size_t pos = lexer_string_bulk_run(data, remaining);
        if (pos != 0)
        {
-            append_string_run(keep, data, pos);
+            token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), pos);
            bulk_consume(reinterpret_cast<const char*>(data), pos);
        }
        if (pos == remaining)
@@ -13349,8 +13474,7 @@ class lexer : public lexer_base<BasicJsonType>
    }

    /// streaming input: no bulk fast path
-    template<bool Keep>
-    char_int_type get_after_string_run(std::false_type /*bulk*/, std::integral_constant<bool, Keep> /*keep*/)
+    char_int_type get_after_string_run(std::false_type /*bulk*/)
    {
        return get();
    }
@@ -13364,18 +13488,13 @@ class lexer : public lexer_base<BasicJsonType>
    may contain \0 bytes), and token_buffer.size() is the number of bytes in the
    string.

-    @param[in] keep  whether the string is copied into token_buffer; if false
-                     (accept(), see scan()), it is only checked, with the same
-                     errors, and token_buffer stays empty
-
    @return token_type::value_string if string could be successfully scanned,
            token_type::parse_error otherwise

    @note In case of errors, variable error_message contains a textual
          description.
    */
-    template<bool Keep>
-    token_type scan_string(std::integral_constant<bool, Keep> keep)
+    token_type scan_string()
    {
        // reset token_buffer (ignore opening quote)
        reset();
@@ -13387,7 +13506,7 @@ class lexer : public lexer_base<BasicJsonType>
        {
            // bulk-consume ordinary characters from contiguous input, then
            // get the next character, the one that ends the run
-            switch (get_after_string_run(std::integral_constant<bool, bulk_scan>(), keep))
+            switch (get_after_string_run(std::integral_constant<bool, bulk_scan>()))
            {
                // end of file while parsing the string
                case char_traits<char_type>::eof():
@@ -13409,35 +13528,35 @@ class lexer : public lexer_base<BasicJsonType>
                    {
                        // quotation mark
                        case '\"':
-                            add_string_char(keep, '\"');
+                            add('\"');
                            break;
                        // reverse solidus
                        case '\\':
-                            add_string_char(keep, '\\');
+                            add('\\');
                            break;
                        // solidus
                        case '/':
-                            add_string_char(keep, '/');
+                            add('/');
                            break;
                        // backspace
                        case 'b':
-                            add_string_char(keep, '\b');
+                            add('\b');
                            break;
                        // form feed
                        case 'f':
-                            add_string_char(keep, '\f');
+                            add('\f');
                            break;
                        // line feed
                        case 'n':
-                            add_string_char(keep, '\n');
+                            add('\n');
                            break;
                        // carriage return
                        case 'r':
-                            add_string_char(keep, '\r');
+                            add('\r');
                            break;
                        // tab
                        case 't':
-                            add_string_char(keep, '\t');
+                            add('\t');
                            break;

                        // unicode escapes
@@ -13505,7 +13624,10 @@ class lexer : public lexer_base<BasicJsonType>
                            JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF);

                            // translate codepoint into bytes
-                            add_codepoint(keep, codepoint);
+                            encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte)
+                            {
+                                add(static_cast<char_int_type>(byte));
+                            });

                            break;
                        }
@@ -13688,7 +13810,7 @@ class lexer : public lexer_base<BasicJsonType>
                case 0x7E:
                case 0x7F:
                {
-                    add_string_char(keep, current);
+                    add(current);
                    break;
                }

@@ -13724,7 +13846,7 @@ class lexer : public lexer_base<BasicJsonType>
                case 0xDE:
                case 0xDF:
                {
-                    if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range(keep, {0x80, 0xBF})))
+                    if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF})))
                    {
                        return token_type::parse_error;
                    }
@@ -13734,7 +13856,7 @@ class lexer : public lexer_base<BasicJsonType>
                // U+0800..U+0FFF: bytes E0 A0..BF 80..BF
                case 0xE0:
                {
-                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0xA0, 0xBF, 0x80, 0xBF}))))
+                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF}))))
                    {
                        return token_type::parse_error;
                    }
@@ -13758,7 +13880,7 @@ class lexer : public lexer_base<BasicJsonType>
                case 0xEE:
                case 0xEF:
                {
-                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF}))))
+                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF}))))
                    {
                        return token_type::parse_error;
                    }
@@ -13768,7 +13890,7 @@ class lexer : public lexer_base<BasicJsonType>
                // U+D000..U+D7FF: bytes ED 80..9F 80..BF
                case 0xED:
                {
-                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x9F, 0x80, 0xBF}))))
+                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF}))))
                    {
                        return token_type::parse_error;
                    }
@@ -13778,7 +13900,7 @@ class lexer : public lexer_base<BasicJsonType>
                // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
                case 0xF0:
                {
-                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
+                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
                    {
                        return token_type::parse_error;
                    }
@@ -13790,7 +13912,7 @@ class lexer : public lexer_base<BasicJsonType>
                case 0xF2:
                case 0xF3:
                {
-                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
+                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
                    {
                        return token_type::parse_error;
                    }
@@ -13800,7 +13922,7 @@ class lexer : public lexer_base<BasicJsonType>
                // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
                case 0xF4:
                {
-                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
+                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
                    {
                        return token_type::parse_error;
                    }
@@ -14291,7 +14413,7 @@ scan_number_done:
    token_type convert_number(token_type number_type, std::size_t mantissa_end)
    {
        // accept() only needs to know whether the input is valid, so it sets
-        // discard_values (see json.hpp), and an integer token whose
+        // discard_number_values (see json.hpp), and an integer token whose
        // digit count shows that it fits is reported without calling
        // convert_integer(). A number with up to 18 digits always fits into
        // both std::uint64_t and std::int64_t (18 nines is about 1e18, below
@@ -14306,7 +14428,7 @@ scan_number_done:
        // json_sax_acceptor, whose number callbacks discard their argument and
        // return true, and the parser rejects neither integers nor finite
        // floats. value_unsigned/value_integer are left unset here, so a caller
-        // that reads the converted value must not set discard_values.
+        // that reads the converted value must not set discard_number_values.
        //
        // On contiguous input, scan_number_bulk_contiguous() converts integer
        // tokens itself and does not pass them to this function, unless
@@ -14314,7 +14436,7 @@ scan_number_done:
        // reached for input without bulk access (e.g. streams), with
        // JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous()
        // falls back to scan_number().
-        if (discard_values)
+        if (discard_number_values)
        {
            constexpr std::size_t safe_digit_count = 18;
            if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count)
@@ -14565,7 +14687,7 @@ scan_number_done:
        // w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a
        // value that underflows is a finite number as well. Only a token that
        // may overflow has to be converted to find out.
-        if (JSON_HEDLEY_UNLIKELY(discard_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
+        if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
        {
            value_float = 0;
            return true;
@@ -14897,34 +15019,6 @@ scan_number_done:
        token_buffer.push_back(static_cast<typename string_t::value_type>(c));
    }

-    /// add a character of a string token; accept() checks the string but does
-    /// not keep its value (@a keep is false)
-    void add_string_char(std::true_type /*keep*/, char_int_type c)
-    {
-        token_buffer.push_back(static_cast<typename string_t::value_type>(c));
-    }
-
-    static void add_string_char(std::false_type /*keep*/, char_int_type /*c*/) noexcept {}
-
-    /// add the UTF-8 bytes of a code point from a \\u escape to a string token
-    void add_codepoint(std::true_type /*keep*/, int codepoint)
-    {
-        encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte)
-        {
-            add(static_cast<char_int_type>(byte));
-        });
-    }
-
-    static void add_codepoint(std::false_type /*keep*/, int /*codepoint*/) noexcept {}
-
-    /// append a run of ordinary string characters (see get_after_string_run())
-    void append_string_run(std::true_type /*keep*/, const unsigned char* data, std::size_t n)
-    {
-        token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), n);
-    }
-
-    static void append_string_run(std::false_type /*keep*/, const unsigned char* /*data*/, std::size_t /*n*/) noexcept {}
-
  public:
    /////////////////////
    // value getters
@@ -15319,11 +15413,7 @@ scan_number_done:

            // string
            case '\"':
-                // accept() (discard_values) checks strings without copying them
-                // into token_buffer. The choice is made once per string: a
-                // check inside the string loop made number-heavy documents up
-                // to 3 % slower (mesh.json, Apple M1)
-                return discard_values ? scan_string(std::false_type {}) : scan_string(std::true_type {});
+                return scan_string();

            // number
            case '-':
@@ -15408,13 +15498,11 @@ scan_number_done:
    std::size_t decimal_point_position = std::string::npos;

    /// whether the caller only needs the token types and never looks at the
-    /// values of number and string tokens; set only by accept(), which parses
-    /// through json_sax_acceptor. When set, convert_number() skips converting
-    /// integer tokens whose digit count guarantees that they fit into 64 bits
-    /// (see there), floats are converted only when they may overflow (see
-    /// convert_accumulated_float()), and strings are checked without being
-    /// copied into token_buffer (see scan())
-    const bool discard_values = false;
+    /// converted numeric values; set only by accept(), which parses through
+    /// json_sax_acceptor. When set, convert_number() skips converting integer
+    /// tokens whose digit count guarantees that they fit into 64 bits (see
+    /// there)
+    const bool discard_number_values = false;
 };

 }  // namespace detail
@@ -22111,6 +22199,19 @@ class binary_reader
        return get_bytes(len, "string", result);
    }

+    /// whether @a s is valid UTF-8: ASCII eight bytes at a time, the rest
+    /// sequence by sequence (as the binary writers check strings)
+    static bool is_valid_utf8_string(const string_t& s, std::true_type /*is_byte*/) noexcept
+    {
+        return valid_utf8_string(reinterpret_cast<const unsigned char*>(s.data()), s.size());
+    }
+
+    /// whether @a s, of a character type wider than a byte, is valid UTF-8
+    static bool is_valid_utf8_string(const string_t& s, std::false_type /*is_byte*/) noexcept
+    {
+        return is_valid_utf8(s);
+    }
+
    /*!
    @brief validate a decoded text string (value or object key) against @ref error_handler

@@ -22130,7 +22231,16 @@ class binary_reader
    */
    bool check_string_utf8(string_t& result, const char* context)
    {
-        if (error_handler == error_handler_t::keep || is_valid_utf8(result))
+        return error_handler == error_handler_t::keep || check_string_utf8_handler(result, context);
+    }
+
+    /// @ref check_string_utf8 for an error handler other than @ref
+    /// error_handler_t::keep; out of line, so that reading strings without a
+    /// check does not pay for the code of the check
+    JSON_HEDLEY_NEVER_INLINE
+    bool check_string_utf8_handler(string_t& result, const char* context)
+    {
+        if (is_valid_utf8_string(result, std::integral_constant<bool, sizeof(typename string_t::value_type) == 1>()))
        {
            return true;
        }
@@ -22623,9 +22733,9 @@ class parser
                    const bool allow_exceptions_ = true,
                    const bool ignore_comments = false,
                    const bool ignore_trailing_commas_ = false,
-                    const bool discard_values_ = false)
+                    const bool discard_number_values_ = false)
        : callback(std::move(cb))
-        , m_lexer(std::move(adapter), ignore_comments, discard_values_)
+        , m_lexer(std::move(adapter), ignore_comments, discard_number_values_)
        , allow_exceptions(allow_exceptions_)
        , ignore_trailing_commas(ignore_trailing_commas_)
    {
@@ -30348,6 +30458,25 @@ class binary_writer
        }
    }

+    /// write the head and the characters of @a value, at most
+    /// max_text_in_reservation of them, in one reservation
+    JSON_HEDLEY_ALWAYS_INLINE void write_cbor_string_reserved(const string_t& value)
+    {
+        CharType* const out = oa.reserve(value.size() + 2);
+        CharType* p = out;
+        if (value.size() <= 0x17)
+        {
+            *p++ = to_char_type(static_cast<std::uint8_t>(0x60 + value.size()));
+        }
+        else
+        {
+            *p++ = to_char_type(0x78);
+            *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+        }
+        copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+        oa.commit(static_cast<std::size_t>(p - out) + value.size());
+    }
+
    /*!
    @brief write a CBOR text string

@@ -30369,19 +30498,7 @@ class binary_writer
            // the string is written as it is: head and characters in one reservation
            if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
            {
-                CharType* const out = oa.reserve(value.size() + 2);
-                CharType* p = out;
-                if (value.size() <= 0x17)
-                {
-                    *p++ = to_char_type(static_cast<std::uint8_t>(0x60 + value.size()));
-                }
-                else
-                {
-                    *p++ = to_char_type(0x78);
-                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
-                }
-                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
-                oa.commit(static_cast<std::size_t>(p - out) + value.size());
+                write_cbor_string_reserved(value);
                return;
            }
        }
@@ -30395,6 +30512,13 @@ class binary_writer
    JSON_HEDLEY_NEVER_INLINE
    void write_cbor_checked_string(const string_t& value, const BasicJsonType& context)
    {
+        if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation)
+        {
+            check_utf8(value, context);
+            write_cbor_string_reserved(value);
+            return;
+        }
+
        string_t storage;
        const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);

@@ -30409,6 +30533,27 @@ class binary_writer
    // MsgPack //
    /////////////

+    /// write the head and the characters of @a value, at most
+    /// max_text_in_reservation of them, in one reservation
+    JSON_HEDLEY_ALWAYS_INLINE void write_msgpack_string_reserved(const string_t& value)
+    {
+        CharType* const out = oa.reserve(value.size() + 2);
+        CharType* p = out;
+        if (value.size() <= 31)
+        {
+            // fixstr
+            *p++ = to_char_type(static_cast<std::uint8_t>(0xA0 | value.size()));
+        }
+        else
+        {
+            // str 8
+            *p++ = to_char_type(0xD9);
+            *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+        }
+        copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+        oa.commit(static_cast<std::size_t>(p - out) + value.size());
+    }
+
    /*!
    @brief write a MessagePack str

@@ -30426,21 +30571,7 @@ class binary_writer
            // the string is written as it is: head and characters in one reservation
            if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
            {
-                CharType* const out = oa.reserve(value.size() + 2);
-                CharType* p = out;
-                if (value.size() <= 31)
-                {
-                    // fixstr
-                    *p++ = to_char_type(static_cast<std::uint8_t>(0xA0 | value.size()));
-                }
-                else
-                {
-                    // str 8
-                    *p++ = to_char_type(0xD9);
-                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
-                }
-                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
-                oa.commit(static_cast<std::size_t>(p - out) + value.size());
+                write_msgpack_string_reserved(value);
                return;
            }
        }
@@ -30453,6 +30584,13 @@ class binary_writer
    JSON_HEDLEY_NEVER_INLINE
    void write_msgpack_checked_string(const string_t& value, const BasicJsonType& context)
    {
+        if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation)
+        {
+            check_utf8(value, context);
+            write_msgpack_string_reserved(value);
+            return;
+        }
+
        string_t storage;
        const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);

@@ -32062,10 +32200,9 @@ class binary_writer
    static void check_utf8(const string_t& s, const BasicJsonType& context)
    {
        const auto* data = reinterpret_cast<const unsigned char*>(s.data());
-        const std::size_t valid = valid_utf8_prefix(data, s.size());
-        if (JSON_HEDLEY_UNLIKELY(valid != s.size()))
+        if (JSON_HEDLEY_UNLIKELY(!valid_utf8_string(data, s.size())))
        {
-            throw_invalid_utf8(data, valid, context);
+            throw_invalid_utf8(data, valid_utf8_prefix(data, s.size()), context);
        }
    }

@@ -37022,11 +37159,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
        const bool allow_exceptions = true,
        const bool ignore_comments = false,
        const bool ignore_trailing_commas = false,
-        const bool discard_values = false
+        const bool discard_number_values = false
                                 )
    {
        return ::nlohmann::detail::parser<basic_json, InputAdapterType>(std::move(adapter),
-            std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_values);
+            std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values);
    }

  private:
@@ -2982,50 +2982,6 @@ TEST_CASE("accept() agrees with parse() on floats around the overflow limit")
    }
 }

-TEST_CASE("accept() agrees with parse() on strings, which it checks without copying them")
-{
-    // accept() scans strings and object keys with a separate instantiation of
-    // the string scanner that does not copy them into the token buffer; it
-    // must accept and reject exactly what parse() does
-    const std::vector<std::string> pieces =
-    {
-        "", "abc", "\\\"\\\\\\/\\b\\f\\n\\r\\t", "\\u0041\\u00e4\\u20AC", "\\ud83d\\ude00", "\\uD83D\\uDE00",
-        "\\ud800", "\\udc00", "\\ud800\\u0041", "\\ud800x", "\\udbff\\udfff", "\\u12", "\\u12g4", "\\x", "\\",
-        "\xc3\xa4", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xf4\x8f\xbf\xbf", "\xc0\xaf", "\xed\xa0\x80", "\xf4\x90\x80\x80",
-        "\xff", "\xe2\x82", "\xc3", "\x01", "\x1f", "\x7f", "\n", std::string(1, '\0')
-    };
-
-    for (std::size_t run = 0; run <= 70; ++run)
-    {
-        for (const std::string& piece : pieces)
-        {
-            // a run of ordinary characters puts the piece at every offset of the bulk scanners
-            const std::string str = std::string(run, 'a') + piece;
-            for (const std::string& text :
-                    {
-                        "\"" + str + "\"", "[\"" + str + "\"]", "{\"" + str + "\":1}", "{\"k\":\"" + str + "\",\"" + str + "\":[\"" + str + "\"]}",
-                        "[\"" + str
-                    })
-            {
-                CAPTURE(text)
-                for (int options = 0; options < 4; ++options)
-                {
-                    const bool ignore_comments = (options & 1) != 0;
-                    const bool ignore_trailing_commas = (options & 2) != 0;
-                    const bool parses = !json::parse(text, nullptr, false, ignore_comments, ignore_trailing_commas).is_discarded();
-                    CHECK(json::accept(text, ignore_comments, ignore_trailing_commas) == parses);
-
-                    std::istringstream stream(text);
-                    CHECK(json::accept(stream, ignore_comments, ignore_trailing_commas) == parses);
-
-                    const std::list<char> chars(text.begin(), text.end());
-                    CHECK(json::accept(chars.begin(), chars.end(), ignore_comments, ignore_trailing_commas) == parses);
-                }
-            }
-        }
-    }
-}
-
 namespace
 {
 // a DOM parser that reports parse errors without throwing, as in the sax_no_exception example
@@ -403,6 +403,7 @@ TEST_CASE("guard page")
                    CHECK(nlohmann::detail::find_string_special(p, n) <= n);
                    CHECK(nlohmann::detail::find_ascii_copyable_run(p, n) <= n);
                    CHECK(nlohmann::detail::valid_utf8_prefix(p, n) <= n);
+                    CHECK(nlohmann::detail::valid_utf8_string(p, n) == (nlohmann::detail::valid_utf8_prefix(p, n) == n));
 #if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17)
                    CHECK(nlohmann::detail::find_string_delimiter(p, n) <= n);
 #endif
@@ -13,6 +13,7 @@ using nlohmann::json;

 #include <algorithm>
 #include <fstream>
+#include <random>
 #include <string>
 #include "make_test_data_available.hpp"
 #include "test_utils.hpp"
@@ -1631,3 +1632,40 @@ TEST_CASE("Unicode RFC 3629")
 }

 DOCTEST_CLANG_SUPPRESS_WARNING_POP
+
+TEST_CASE("UTF-8 check of whole strings")
+{
+    // valid_utf8_string() decides short strings with a word test and long ones
+    // 16 bytes at a time where a vector check is available; it must agree with
+    // the byte-wise check for every mix of ASCII, sequences of each length,
+    // ill-formed and truncated sequences, at every length and offset
+    const char* const pieces[] =
+    {
+        "a", "\x7F", "\xC3\xA9", "\xE2\x82\xAC", "\xE0\xA0\x80", "\xEF\xBF\xBF", "\xF0\x9F\x98\x80", "\xF4\x8F\xBF\xBF",
+        "\x80", "\xC0\xAF", "\xC3", "\xE2\x82", "\xED\xA0\x80", "\xF0\x9F", "\xF4\x90\x80\x80", "\xFF"
+    };
+    // std::mt19937's sequence is fixed by the standard; no distribution is used
+    std::mt19937 rng(5);
+    std::size_t valid = 0;
+    for (int i = 0; i < 100000; ++i)
+    {
+        std::string s;
+        const std::size_t length = rng() % 100;
+        const std::uint32_t ascii = rng() % 101;
+        while (s.size() < length)
+        {
+            // mostly well-formed pieces (the first eight), so that long valid
+            // strings are common
+            s += (rng() % 100 < ascii) ? std::string("x") : std::string(pieces[rng() % (rng() % 8 == 0 ? 16 : 8)]);
+        }
+        const auto* data = reinterpret_cast<const unsigned char*>(s.data());
+        const bool expected = nlohmann::detail::is_valid_utf8(s);
+        valid += expected ? 1 : 0;
+        CAPTURE(s)
+        CHECK(nlohmann::detail::valid_utf8_string(data, s.size()) == expected);
+        CHECK((nlohmann::detail::valid_utf8_prefix(data, s.size()) == s.size()) == expected);
+    }
+    // both outcomes are common enough to be tested
+    CHECK(valid > 20000);
+    CHECK(valid < 80000);
+}