Fix UBSan implicit-conversion in BSON stream tests

std::string(first, last) over std::uint8_t iterators converts each byte
to char implicitly, which -fsanitize=implicit-conversion reports as soon
as a byte is >= 0x80 (the document size byte of the enlarged test input,
and 0xFF in the new array element name test). Build the string from a
const char* and a size instead.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01N5NasCxh8jKXBaa6HBx9Xk
Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/binary_reader.hpp

@@ -465,15 +465,17 @@ class binary_reader
             }
 
             const std::size_t element_type_parse_position = chars_read;
+            // an array's elements are named "0", "1", ... in the wire format,
+            // and those names are not passed on: they are skipped unless
+            // error_handler_t::strict must reject ill-formed UTF-8 in them
+            const bool is_object = container_stack.back().is_object;
             key.clear();
-            if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))
+            if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key, is_object || error_handler == error_handler_t::strict)))
             {
                 return false;
             }
 
-            // an array's elements are named "0", "1", ... in the wire format,
+            if (is_object && !sax->key(key))
-            // and those names are not passed on
-            if (container_stack.back().is_object && !sax->key(key))
             {
                 return false;
             }
@@ -489,14 +491,18 @@ class binary_reader
     @brief Parses a C-style string from the BSON input.
     @param[in,out] result  A reference to the string variable where the read
                             string is to be stored.
+    @param[in] store  whether the characters are needed; if not (the names of
+                      an array's elements), they are skipped and @a result may
+                      be left unchanged, and they are not checked against
+                      @ref error_handler
     @return `true` if the \\x00-byte indicating the end of the string was
              encountered before the EOF; false` indicates an unexpected EOF.
     */
-    bool get_bson_cstr(string_t& result)
+    bool get_bson_cstr(string_t& result, const bool store = true)
     {
-        if (get_bson_cstr_bulk(result, std::integral_constant<bool, bulk_scan> {}))
+        if (get_bson_cstr_bulk(result, store, std::integral_constant<bool, bulk_scan> {}))
         {
-            return check_string_utf8(result, "key");
+            return !store || check_string_utf8(result, "key");
         }
 
         auto out = std::back_inserter(result);
@@ -509,7 +515,7 @@ class binary_reader
             }
             if (current == 0x00)
             {
-                return check_string_utf8(result, "key");
+                return !store || check_string_utf8(result, "key");
             }
             *out++ = static_cast<typename string_t::value_type>(current);
         }
@@ -519,10 +525,11 @@ class binary_reader
     @brief read a C-style string from contiguous input in one step
 
     @param[in,out] result  the string to append to
+    @param[in] store  whether to append the characters to @a result
     @return whether the string was read; if the input has no \\x00-byte, nothing
             is read, and @ref get_bson_cstr reports the end of the input
     */
-    bool get_bson_cstr_bulk(string_t& result, std::true_type /*bulk*/)
+    bool get_bson_cstr_bulk(string_t& result, const bool store, std::true_type /*bulk*/)
     {
         const std::size_t remaining = ia.bulk_remaining();
         if (remaining == 0)
@@ -530,18 +537,15 @@ class binary_reader
             return false;
         }
         const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
-        // a plain loop rather than std::memchr: most keys are short (array
+        const std::size_t length = find_zero_byte(data, remaining);
-        // indices are keys, too), and the call would cost more than it saves
-        std::size_t length = 0;
-        while (length < remaining && data[length] != 0x00)
-        {
-            ++length;
-        }
         if (length == remaining)
         {
             return false;
         }
+        if (store)
+        {
             result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
+        }
         // consume the string and its \x00-byte, as the byte-wise path does
         ia.bulk_skip(length + 1);
         chars_read += length + 1;
@@ -550,7 +554,7 @@ class binary_reader
     }
 
     /// input that is not contiguous: C-style strings are read byte by byte
-    bool get_bson_cstr_bulk(string_t& /*result*/, std::false_type /*bulk*/) const noexcept
+    bool get_bson_cstr_bulk(string_t& /*result*/, const bool /*store*/, std::false_type /*bulk*/) const noexcept
     {
         return false;
     }
@@ -4293,19 +4297,6 @@ class binary_reader
         return get_bytes(len, "string", result);
     }
 
-    /// whether @a s is valid UTF-8: ASCII eight bytes at a time, the rest
-    /// sequence by sequence (as the binary writers check strings)
-    static bool is_valid_utf8_string(const string_t& s, std::true_type /*is_byte*/) noexcept
-    {
-        return valid_utf8_string(reinterpret_cast<const unsigned char*>(s.data()), s.size());
-    }
-
-    /// whether @a s, of a character type wider than a byte, is valid UTF-8
-    static bool is_valid_utf8_string(const string_t& s, std::false_type /*is_byte*/) noexcept
-    {
-        return is_valid_utf8(s);
-    }
-
     /*!
     @brief validate a decoded text string (value or object key) against @ref error_handler
 
@@ -4325,16 +4316,7 @@ class binary_reader
     */
     bool check_string_utf8(string_t& result, const char* context)
     {
-        return error_handler == error_handler_t::keep || check_string_utf8_handler(result, context);
+        if (error_handler == error_handler_t::keep || is_valid_utf8(result))
-    }
-
-    /// @ref check_string_utf8 for an error handler other than @ref
-    /// error_handler_t::keep; out of line, so that reading strings without a
-    /// check does not pay for the code of the check
-    JSON_HEDLEY_NEVER_INLINE
-    bool check_string_utf8_handler(string_t& result, const char* context)
-    {
-        if (is_valid_utf8_string(result, std::integral_constant<bool, sizeof(typename string_t::value_type) == 1>()))
         {
             return true;
         }

include/nlohmann/detail/input/simd_scan.hpp

@@ -322,78 +322,6 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons
     block_out = block;
     return false;
 }
-
-/*!
-@brief skip 16-byte blocks of well-formed UTF-8 at p (a character boundary)
-
-Like vector_utf8_run, but quotes, backslashes, and control characters are
-ordinary characters. Returns the start of the first block in which the check
-found an error (which may belong to a sequence begun in the block before) or
-where fewer than 16 bytes are left; a sequence may be cut at that point, so the
-caller resumes at its start, one sequence at a time.
-*/
-JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vector_utf8_valid_blocks(const unsigned char* block, const unsigned char* e) noexcept
-{
-    using lookup = utf8_lookup4<>;
-#if JSON_SCAN_NEON
-    const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high.data());
-    const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low.data());
-    const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high.data());
-    uint8x16_t prev = vdupq_n_u8(0);
-    while (e - block >= 16)
-    {
-        const uint8x16_t in = vld1q_u8(block);
-        if (vmaxvq_u8(vorrq_u8(in, prev)) < 0x80)
-        {
-            // ASCII here, and no sequence from the block before to finish
-            prev = in;
-            block += 16;
-            continue;
-        }
-        const uint8x16_t prev1 = vextq_u8(prev, in, 15);
-        const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4)));
-        const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80)));
-        const uint8x16_t err = veorq_u8(vandq_u8(must23, vdupq_n_u8(0x80)), sc);
-        if (vmaxvq_u8(err) != 0)
-        {
-            break;
-        }
-        prev = in;
-        block += 16;
-    }
-#else
-    const __m128i t1h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_high.data()))); // NOLINT(bugprone-casting-through-void)
-    const __m128i t1l = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_low.data()))); // NOLINT(bugprone-casting-through-void)
-    const __m128i t2h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_2_high.data()))); // NOLINT(bugprone-casting-through-void)
-    const __m128i nibble = _mm_set1_epi8(0x0F);
-    const __m128i zero = _mm_setzero_si128();
-    __m128i prev = zero;
-    while (e - block >= 16)
-    {
-        const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(block))); // NOLINT(bugprone-casting-through-void)
-        if (_mm_movemask_epi8(_mm_or_si128(in, prev)) == 0)
-        {
-            prev = in;
-            block += 16;
-            continue;
-        }
-        const __m128i prev1 = _mm_alignr_epi8(in, prev, 15);
-        const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)),
-                                         _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))),
-                                         _mm_shuffle_epi8(t2h, _mm_and_si128(_mm_srli_epi16(in, 4), nibble)));
-        const __m128i must23 = _mm_or_si128(_mm_subs_epu8(_mm_alignr_epi8(in, prev, 14), _mm_set1_epi8(0xE0 - 0x80)),
-                                            _mm_subs_epu8(_mm_alignr_epi8(in, prev, 13), _mm_set1_epi8(0xF0 - 0x80)));
-        const __m128i err = _mm_xor_si128(_mm_and_si128(must23, _mm_set1_epi8(static_cast<char>(-128))), sc);
-        if (_mm_movemask_epi8(_mm_cmpeq_epi8(err, zero)) != 0xFFFF)
-        {
-            break;
-        }
-        prev = in;
-        block += 16;
-    }
-#endif
-    return block;
-}
 #endif
 
 }  // namespace detail

include/nlohmann/detail/input/string_scan.hpp

@@ -90,6 +90,34 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t find_string_special(const unsigned char* d
     return n;
 }
 
+// return the index of the first 0x00 byte in [data, data+n), or n if there is
+// none; scans 8 bytes at a time (BSON's C-style strings)
+JSON_HEDLEY_ALWAYS_INLINE std::size_t find_zero_byte(const unsigned char* data, std::size_t n) noexcept
+{
+    constexpr std::uint64_t ones = 0x0101010101010101ull;
+    constexpr std::uint64_t high = 0x8080808080808080ull;
+    std::size_t i = 0;
+    for (; i + 8 <= n; i += 8)
+    {
+        const std::uint64_t v = read_eight_bytes(data + i);
+        const std::uint64_t zero = (v - ones) & ~v & high;
+        if (zero != 0)
+        {
+            // the lowest flagged byte is the first zero byte (see
+            // find_string_special())
+            return i + (static_cast<std::size_t>(count_trailing_zeros(zero)) / 8);
+        }
+    }
+    for (; i < n; ++i)
+    {
+        if (data[i] == 0x00)
+        {
+            return i;
+        }
+    }
+    return n;
+}
+
 // classify a byte as one the serializer must NOT copy verbatim when
 // ensure_ascii is requested: the closing quote, an escape, a control character
 // (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else -
@@ -236,64 +264,6 @@ inline std::size_t valid_utf8_prefix(const unsigned char* data, std::size_t n) n
     return pos;
 }
 
-/*!
-@brief whether [data, data+n) is valid UTF-8, for checking whole strings
-
-The same answer as valid_utf8_prefix(data, n) == n. Short ASCII strings, the
-most common keys and values, are decided in one test, and long strings are
-checked 16 bytes at a time where the vector check is available.
-*/
-inline bool valid_utf8_string(const unsigned char* data, std::size_t n) noexcept
-{
-    constexpr std::uint64_t high = 0x8080808080808080ull;
-    if (n <= 16)
-    {
-        // two overlapping words cover 8 to 16 bytes, one partial word fewer
-        std::uint64_t first = 0;
-        std::uint64_t last = 0;
-        if (n >= 8)
-        {
-            std::memcpy(&first, data, sizeof(first));
-            std::memcpy(&last, data + n - 8, sizeof(last));
-        }
-        else if (n != 0)
-        {
-            std::memcpy(&first, data, n);
-        }
-        if (((first | last) & high) == 0)
-        {
-            return true;
-        }
-    }
-#if JSON_SCAN_VECTOR_UTF8
-    else if (n >= 32
-#if JSON_SCAN_SSSE3_DISPATCH
-             && cpu_has_ssse3()
-#endif
-            )
-    {
-        const unsigned char* block = vector_utf8_valid_blocks(data, data + n);
-        // the rest from the start of a sequence that may be cut at block
-        for (int i = 1; i <= 3 && block - i >= data; ++i)
-        {
-            const unsigned char c = block[-i];
-            if (c < 0x80)
-            {
-                break;
-            }
-            if (c >= 0xC0)
-            {
-                block -= i;
-                break;
-            }
-        }
-        const auto checked = static_cast<std::size_t>(block - data);
-        return checked + valid_utf8_prefix(block, n - checked) == n;
-    }
-#endif
-    return valid_utf8_prefix(data, n) == n;
-}
-
 #if JSON_SCAN_VECTOR_UTF8
 /// the end of scan_string_vector from block, where the vector loop stopped
 /// (ill-formed UTF-8, or fewer than 16 bytes left): one byte or sequence at a

include/nlohmann/detail/output/binary_writer.hpp

@@ -3080,25 +3080,6 @@ class binary_writer
         }
     }
 
-    /// write the head and the characters of @a value, at most
-    /// max_text_in_reservation of them, in one reservation
-    JSON_HEDLEY_ALWAYS_INLINE void write_cbor_string_reserved(const string_t& value)
-    {
-        CharType* const out = oa.reserve(value.size() + 2);
-        CharType* p = out;
-        if (value.size() <= 0x17)
-        {
-            *p++ = to_char_type(static_cast<std::uint8_t>(0x60 + value.size()));
-        }
-        else
-        {
-            *p++ = to_char_type(0x78);
-            *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
-        }
-        copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
-        oa.commit(static_cast<std::size_t>(p - out) + value.size());
-    }
-
     /*!
     @brief write a CBOR text string
 
@@ -3120,7 +3101,19 @@ class binary_writer
             // the string is written as it is: head and characters in one reservation
             if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
             {
-                write_cbor_string_reserved(value);
+                CharType* const out = oa.reserve(value.size() + 2);
+                CharType* p = out;
+                if (value.size() <= 0x17)
+                {
+                    *p++ = to_char_type(static_cast<std::uint8_t>(0x60 + value.size()));
+                }
+                else
+                {
+                    *p++ = to_char_type(0x78);
+                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+                }
+                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+                oa.commit(static_cast<std::size_t>(p - out) + value.size());
                 return;
             }
         }
@@ -3134,13 +3127,6 @@ class binary_writer
     JSON_HEDLEY_NEVER_INLINE
     void write_cbor_checked_string(const string_t& value, const BasicJsonType& context)
     {
-        if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation)
-        {
-            check_utf8(value, context);
-            write_cbor_string_reserved(value);
-            return;
-        }
-
         string_t storage;
         const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);
 
@@ -3155,27 +3141,6 @@ class binary_writer
     // MsgPack //
     /////////////
 
-    /// write the head and the characters of @a value, at most
-    /// max_text_in_reservation of them, in one reservation
-    JSON_HEDLEY_ALWAYS_INLINE void write_msgpack_string_reserved(const string_t& value)
-    {
-        CharType* const out = oa.reserve(value.size() + 2);
-        CharType* p = out;
-        if (value.size() <= 31)
-        {
-            // fixstr
-            *p++ = to_char_type(static_cast<std::uint8_t>(0xA0 | value.size()));
-        }
-        else
-        {
-            // str 8
-            *p++ = to_char_type(0xD9);
-            *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
-        }
-        copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
-        oa.commit(static_cast<std::size_t>(p - out) + value.size());
-    }
-
     /*!
     @brief write a MessagePack str
 
@@ -3193,7 +3158,21 @@ class binary_writer
             // the string is written as it is: head and characters in one reservation
             if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
             {
-                write_msgpack_string_reserved(value);
+                CharType* const out = oa.reserve(value.size() + 2);
+                CharType* p = out;
+                if (value.size() <= 31)
+                {
+                    // fixstr
+                    *p++ = to_char_type(static_cast<std::uint8_t>(0xA0 | value.size()));
+                }
+                else
+                {
+                    // str 8
+                    *p++ = to_char_type(0xD9);
+                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+                }
+                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+                oa.commit(static_cast<std::size_t>(p - out) + value.size());
                 return;
             }
         }
@@ -3206,13 +3185,6 @@ class binary_writer
     JSON_HEDLEY_NEVER_INLINE
     void write_msgpack_checked_string(const string_t& value, const BasicJsonType& context)
     {
-        if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation)
-        {
-            check_utf8(value, context);
-            write_msgpack_string_reserved(value);
-            return;
-        }
-
         string_t storage;
         const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);
 
@@ -4822,9 +4794,10 @@ class binary_writer
     static void check_utf8(const string_t& s, const BasicJsonType& context)
     {
         const auto* data = reinterpret_cast<const unsigned char*>(s.data());
-        if (JSON_HEDLEY_UNLIKELY(!valid_utf8_string(data, s.size())))
+        const std::size_t valid = valid_utf8_prefix(data, s.size());
+        if (JSON_HEDLEY_UNLIKELY(valid != s.size()))
         {
-            throw_invalid_utf8(data, valid_utf8_prefix(data, s.size()), context);
+            throw_invalid_utf8(data, valid, context);
         }
     }
 

single_include/nlohmann/json.hpp

@@ -12377,78 +12377,6 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons
     block_out = block;
     return false;
 }
-
-/*!
-@brief skip 16-byte blocks of well-formed UTF-8 at p (a character boundary)
-
-Like vector_utf8_run, but quotes, backslashes, and control characters are
-ordinary characters. Returns the start of the first block in which the check
-found an error (which may belong to a sequence begun in the block before) or
-where fewer than 16 bytes are left; a sequence may be cut at that point, so the
-caller resumes at its start, one sequence at a time.
-*/
-JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vector_utf8_valid_blocks(const unsigned char* block, const unsigned char* e) noexcept
-{
-    using lookup = utf8_lookup4<>;
-#if JSON_SCAN_NEON
-    const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high.data());
-    const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low.data());
-    const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high.data());
-    uint8x16_t prev = vdupq_n_u8(0);
-    while (e - block >= 16)
-    {
-        const uint8x16_t in = vld1q_u8(block);
-        if (vmaxvq_u8(vorrq_u8(in, prev)) < 0x80)
-        {
-            // ASCII here, and no sequence from the block before to finish
-            prev = in;
-            block += 16;
-            continue;
-        }
-        const uint8x16_t prev1 = vextq_u8(prev, in, 15);
-        const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4)));
-        const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80)));
-        const uint8x16_t err = veorq_u8(vandq_u8(must23, vdupq_n_u8(0x80)), sc);
-        if (vmaxvq_u8(err) != 0)
-        {
-            break;
-        }
-        prev = in;
-        block += 16;
-    }
-#else
-    const __m128i t1h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_high.data()))); // NOLINT(bugprone-casting-through-void)
-    const __m128i t1l = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_low.data()))); // NOLINT(bugprone-casting-through-void)
-    const __m128i t2h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_2_high.data()))); // NOLINT(bugprone-casting-through-void)
-    const __m128i nibble = _mm_set1_epi8(0x0F);
-    const __m128i zero = _mm_setzero_si128();
-    __m128i prev = zero;
-    while (e - block >= 16)
-    {
-        const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(block))); // NOLINT(bugprone-casting-through-void)
-        if (_mm_movemask_epi8(_mm_or_si128(in, prev)) == 0)
-        {
-            prev = in;
-            block += 16;
-            continue;
-        }
-        const __m128i prev1 = _mm_alignr_epi8(in, prev, 15);
-        const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)),
-                                         _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))),
-                                         _mm_shuffle_epi8(t2h, _mm_and_si128(_mm_srli_epi16(in, 4), nibble)));
-        const __m128i must23 = _mm_or_si128(_mm_subs_epu8(_mm_alignr_epi8(in, prev, 14), _mm_set1_epi8(0xE0 - 0x80)),
-                                            _mm_subs_epu8(_mm_alignr_epi8(in, prev, 13), _mm_set1_epi8(0xF0 - 0x80)));
-        const __m128i err = _mm_xor_si128(_mm_and_si128(must23, _mm_set1_epi8(static_cast<char>(-128))), sc);
-        if (_mm_movemask_epi8(_mm_cmpeq_epi8(err, zero)) != 0xFFFF)
-        {
-            break;
-        }
-        prev = in;
-        block += 16;
-    }
-#endif
-    return block;
-}
 #endif
 
 }  // namespace detail
@@ -12530,6 +12458,34 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t find_string_special(const unsigned char* d
     return n;
 }
 
+// return the index of the first 0x00 byte in [data, data+n), or n if there is
+// none; scans 8 bytes at a time (BSON's C-style strings)
+JSON_HEDLEY_ALWAYS_INLINE std::size_t find_zero_byte(const unsigned char* data, std::size_t n) noexcept
+{
+    constexpr std::uint64_t ones = 0x0101010101010101ull;
+    constexpr std::uint64_t high = 0x8080808080808080ull;
+    std::size_t i = 0;
+    for (; i + 8 <= n; i += 8)
+    {
+        const std::uint64_t v = read_eight_bytes(data + i);
+        const std::uint64_t zero = (v - ones) & ~v & high;
+        if (zero != 0)
+        {
+            // the lowest flagged byte is the first zero byte (see
+            // find_string_special())
+            return i + (static_cast<std::size_t>(count_trailing_zeros(zero)) / 8);
+        }
+    }
+    for (; i < n; ++i)
+    {
+        if (data[i] == 0x00)
+        {
+            return i;
+        }
+    }
+    return n;
+}
+
 // classify a byte as one the serializer must NOT copy verbatim when
 // ensure_ascii is requested: the closing quote, an escape, a control character
 // (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else -
@@ -12676,64 +12632,6 @@ inline std::size_t valid_utf8_prefix(const unsigned char* data, std::size_t n) n
     return pos;
 }
 
-/*!
-@brief whether [data, data+n) is valid UTF-8, for checking whole strings
-
-The same answer as valid_utf8_prefix(data, n) == n. Short ASCII strings, the
-most common keys and values, are decided in one test, and long strings are
-checked 16 bytes at a time where the vector check is available.
-*/
-inline bool valid_utf8_string(const unsigned char* data, std::size_t n) noexcept
-{
-    constexpr std::uint64_t high = 0x8080808080808080ull;
-    if (n <= 16)
-    {
-        // two overlapping words cover 8 to 16 bytes, one partial word fewer
-        std::uint64_t first = 0;
-        std::uint64_t last = 0;
-        if (n >= 8)
-        {
-            std::memcpy(&first, data, sizeof(first));
-            std::memcpy(&last, data + n - 8, sizeof(last));
-        }
-        else if (n != 0)
-        {
-            std::memcpy(&first, data, n);
-        }
-        if (((first | last) & high) == 0)
-        {
-            return true;
-        }
-    }
-#if JSON_SCAN_VECTOR_UTF8
-    else if (n >= 32
-#if JSON_SCAN_SSSE3_DISPATCH
-             && cpu_has_ssse3()
-#endif
-            )
-    {
-        const unsigned char* block = vector_utf8_valid_blocks(data, data + n);
-        // the rest from the start of a sequence that may be cut at block
-        for (int i = 1; i <= 3 && block - i >= data; ++i)
-        {
-            const unsigned char c = block[-i];
-            if (c < 0x80)
-            {
-                break;
-            }
-            if (c >= 0xC0)
-            {
-                block -= i;
-                break;
-            }
-        }
-        const auto checked = static_cast<std::size_t>(block - data);
-        return checked + valid_utf8_prefix(block, n - checked) == n;
-    }
-#endif
-    return valid_utf8_prefix(data, n) == n;
-}
-
 #if JSON_SCAN_VECTOR_UTF8
 /// the end of scan_string_vector from block, where the vector loop stopped
 /// (ill-formed UTF-8, or fewer than 16 bytes left): one byte or sequence at a
@@ -18371,15 +18269,17 @@ class binary_reader
             }
 
             const std::size_t element_type_parse_position = chars_read;
+            // an array's elements are named "0", "1", ... in the wire format,
+            // and those names are not passed on: they are skipped unless
+            // error_handler_t::strict must reject ill-formed UTF-8 in them
+            const bool is_object = container_stack.back().is_object;
             key.clear();
-            if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))
+            if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key, is_object || error_handler == error_handler_t::strict)))
             {
                 return false;
             }
 
-            // an array's elements are named "0", "1", ... in the wire format,
+            if (is_object && !sax->key(key))
-            // and those names are not passed on
-            if (container_stack.back().is_object && !sax->key(key))
             {
                 return false;
             }
@@ -18395,14 +18295,18 @@ class binary_reader
     @brief Parses a C-style string from the BSON input.
     @param[in,out] result  A reference to the string variable where the read
                             string is to be stored.
+    @param[in] store  whether the characters are needed; if not (the names of
+                      an array's elements), they are skipped and @a result may
+                      be left unchanged, and they are not checked against
+                      @ref error_handler
     @return `true` if the \\x00-byte indicating the end of the string was
              encountered before the EOF; false` indicates an unexpected EOF.
     */
-    bool get_bson_cstr(string_t& result)
+    bool get_bson_cstr(string_t& result, const bool store = true)
     {
-        if (get_bson_cstr_bulk(result, std::integral_constant<bool, bulk_scan> {}))
+        if (get_bson_cstr_bulk(result, store, std::integral_constant<bool, bulk_scan> {}))
         {
-            return check_string_utf8(result, "key");
+            return !store || check_string_utf8(result, "key");
         }
 
         auto out = std::back_inserter(result);
@@ -18415,7 +18319,7 @@ class binary_reader
             }
             if (current == 0x00)
             {
-                return check_string_utf8(result, "key");
+                return !store || check_string_utf8(result, "key");
             }
             *out++ = static_cast<typename string_t::value_type>(current);
         }
@@ -18425,10 +18329,11 @@ class binary_reader
     @brief read a C-style string from contiguous input in one step
 
     @param[in,out] result  the string to append to
+    @param[in] store  whether to append the characters to @a result
     @return whether the string was read; if the input has no \\x00-byte, nothing
             is read, and @ref get_bson_cstr reports the end of the input
     */
-    bool get_bson_cstr_bulk(string_t& result, std::true_type /*bulk*/)
+    bool get_bson_cstr_bulk(string_t& result, const bool store, std::true_type /*bulk*/)
     {
         const std::size_t remaining = ia.bulk_remaining();
         if (remaining == 0)
@@ -18436,18 +18341,15 @@ class binary_reader
             return false;
         }
         const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
-        // a plain loop rather than std::memchr: most keys are short (array
+        const std::size_t length = find_zero_byte(data, remaining);
-        // indices are keys, too), and the call would cost more than it saves
-        std::size_t length = 0;
-        while (length < remaining && data[length] != 0x00)
-        {
-            ++length;
-        }
         if (length == remaining)
         {
             return false;
         }
+        if (store)
+        {
             result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
+        }
         // consume the string and its \x00-byte, as the byte-wise path does
         ia.bulk_skip(length + 1);
         chars_read += length + 1;
@@ -18456,7 +18358,7 @@ class binary_reader
     }
 
     /// input that is not contiguous: C-style strings are read byte by byte
-    bool get_bson_cstr_bulk(string_t& /*result*/, std::false_type /*bulk*/) const noexcept
+    bool get_bson_cstr_bulk(string_t& /*result*/, const bool /*store*/, std::false_type /*bulk*/) const noexcept
     {
         return false;
     }
@@ -22199,19 +22101,6 @@ class binary_reader
         return get_bytes(len, "string", result);
     }
 
-    /// whether @a s is valid UTF-8: ASCII eight bytes at a time, the rest
-    /// sequence by sequence (as the binary writers check strings)
-    static bool is_valid_utf8_string(const string_t& s, std::true_type /*is_byte*/) noexcept
-    {
-        return valid_utf8_string(reinterpret_cast<const unsigned char*>(s.data()), s.size());
-    }
-
-    /// whether @a s, of a character type wider than a byte, is valid UTF-8
-    static bool is_valid_utf8_string(const string_t& s, std::false_type /*is_byte*/) noexcept
-    {
-        return is_valid_utf8(s);
-    }
-
     /*!
     @brief validate a decoded text string (value or object key) against @ref error_handler
 
@@ -22231,16 +22120,7 @@ class binary_reader
     */
     bool check_string_utf8(string_t& result, const char* context)
     {
-        return error_handler == error_handler_t::keep || check_string_utf8_handler(result, context);
+        if (error_handler == error_handler_t::keep || is_valid_utf8(result))
-    }
-
-    /// @ref check_string_utf8 for an error handler other than @ref
-    /// error_handler_t::keep; out of line, so that reading strings without a
-    /// check does not pay for the code of the check
-    JSON_HEDLEY_NEVER_INLINE
-    bool check_string_utf8_handler(string_t& result, const char* context)
-    {
-        if (is_valid_utf8_string(result, std::integral_constant<bool, sizeof(typename string_t::value_type) == 1>()))
         {
             return true;
         }
@@ -30458,25 +30338,6 @@ class binary_writer
         }
     }
 
-    /// write the head and the characters of @a value, at most
-    /// max_text_in_reservation of them, in one reservation
-    JSON_HEDLEY_ALWAYS_INLINE void write_cbor_string_reserved(const string_t& value)
-    {
-        CharType* const out = oa.reserve(value.size() + 2);
-        CharType* p = out;
-        if (value.size() <= 0x17)
-        {
-            *p++ = to_char_type(static_cast<std::uint8_t>(0x60 + value.size()));
-        }
-        else
-        {
-            *p++ = to_char_type(0x78);
-            *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
-        }
-        copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
-        oa.commit(static_cast<std::size_t>(p - out) + value.size());
-    }
-
     /*!
     @brief write a CBOR text string
 
@@ -30498,7 +30359,19 @@ class binary_writer
             // the string is written as it is: head and characters in one reservation
             if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
             {
-                write_cbor_string_reserved(value);
+                CharType* const out = oa.reserve(value.size() + 2);
+                CharType* p = out;
+                if (value.size() <= 0x17)
+                {
+                    *p++ = to_char_type(static_cast<std::uint8_t>(0x60 + value.size()));
+                }
+                else
+                {
+                    *p++ = to_char_type(0x78);
+                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+                }
+                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+                oa.commit(static_cast<std::size_t>(p - out) + value.size());
                 return;
             }
         }
@@ -30512,13 +30385,6 @@ class binary_writer
     JSON_HEDLEY_NEVER_INLINE
     void write_cbor_checked_string(const string_t& value, const BasicJsonType& context)
     {
-        if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation)
-        {
-            check_utf8(value, context);
-            write_cbor_string_reserved(value);
-            return;
-        }
-
         string_t storage;
         const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);
 
@@ -30533,27 +30399,6 @@ class binary_writer
     // MsgPack //
     /////////////
 
-    /// write the head and the characters of @a value, at most
-    /// max_text_in_reservation of them, in one reservation
-    JSON_HEDLEY_ALWAYS_INLINE void write_msgpack_string_reserved(const string_t& value)
-    {
-        CharType* const out = oa.reserve(value.size() + 2);
-        CharType* p = out;
-        if (value.size() <= 31)
-        {
-            // fixstr
-            *p++ = to_char_type(static_cast<std::uint8_t>(0xA0 | value.size()));
-        }
-        else
-        {
-            // str 8
-            *p++ = to_char_type(0xD9);
-            *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
-        }
-        copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
-        oa.commit(static_cast<std::size_t>(p - out) + value.size());
-    }
-
     /*!
     @brief write a MessagePack str
 
@@ -30571,7 +30416,21 @@ class binary_writer
             // the string is written as it is: head and characters in one reservation
             if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
             {
-                write_msgpack_string_reserved(value);
+                CharType* const out = oa.reserve(value.size() + 2);
+                CharType* p = out;
+                if (value.size() <= 31)
+                {
+                    // fixstr
+                    *p++ = to_char_type(static_cast<std::uint8_t>(0xA0 | value.size()));
+                }
+                else
+                {
+                    // str 8
+                    *p++ = to_char_type(0xD9);
+                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+                }
+                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+                oa.commit(static_cast<std::size_t>(p - out) + value.size());
                 return;
             }
         }
@@ -30584,13 +30443,6 @@ class binary_writer
     JSON_HEDLEY_NEVER_INLINE
     void write_msgpack_checked_string(const string_t& value, const BasicJsonType& context)
     {
-        if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation)
-        {
-            check_utf8(value, context);
-            write_msgpack_string_reserved(value);
-            return;
-        }
-
         string_t storage;
         const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);
 
@@ -32200,9 +32052,10 @@ class binary_writer
     static void check_utf8(const string_t& s, const BasicJsonType& context)
     {
         const auto* data = reinterpret_cast<const unsigned char*>(s.data());
-        if (JSON_HEDLEY_UNLIKELY(!valid_utf8_string(data, s.size())))
+        const std::size_t valid = valid_utf8_prefix(data, s.size());
+        if (JSON_HEDLEY_UNLIKELY(valid != s.size()))
         {
-            throw_invalid_utf8(data, valid_utf8_prefix(data, s.size()), context);
+            throw_invalid_utf8(data, valid, context);
         }
     }
 

tests/src/unit-bson.cpp

@@ -1164,7 +1164,13 @@ TEST_CASE("BSON keys from contiguous and stream input")
     // contiguous input reads a key up to its \x00-byte in one step, a stream
     // reads it byte by byte; both must give the same value or error for the
     // complete document and for every truncation of it
-    const json j = {{"", true}, {"k", {1, 2, 3}}, {std::string(40, 'x'), {{"nested key", "value"}}}};
+    json j = {{"", true}, {"k", {1, 2, 3}}, {std::string(40, 'x'), {{"nested key", "value"}}}};
+    // keys and array element names ("0" to "11") that end at every position of
+    // an 8-byte word, which contiguous input scans at a time
+    for (std::size_t length = 1; length <= 17; ++length)
+    {
+        j[std::string(length, static_cast<char>('a' + length))] = json::array({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, {11}});
+    }
     const std::vector<std::uint8_t> bson = json::to_bson(j);
     CHECK(json::from_bson(bson) == j);
 
@@ -1184,7 +1190,7 @@ TEST_CASE("BSON keys from contiguous and stream input")
         }
         try
         {
-            std::istringstream stream(std::string(input.begin(), input.end()));
+            std::istringstream stream(std::string(reinterpret_cast<const char*>(input.data()), input.size()));
             from_stream = json::from_bson(stream).dump();
         }
         catch (const json::parse_error& e)
@@ -1194,6 +1200,33 @@ TEST_CASE("BSON keys from contiguous and stream input")
         CHECK(from_vector == from_stream);
     }
 }
+
+TEST_CASE("BSON array element names")
+{
+    // an array's element names are not part of the value, so they are not
+    // passed on; error_handler_t::strict still rejects ill-formed UTF-8 in them
+    const std::vector<std::uint8_t> bson =
+    {
+        0x1D, 0x00, 0x00, 0x00, // size (little endian)
+        0x04, 'a', 0x00,        // entry: array "a"
+        0x15, 0x00, 0x00, 0x00, // size (little endian)
+        0x08, 0xFF, 0x00, 0x01, // entry: boolean named "\xFF", true
+        0x08, '1', '2', '3', '4', '5', '6', '7', '8', '9', 0x00, 0x00, // entry: boolean named "123456789", false
+        0x00,                   // end of the array
+        0x00                    // end of the document
+    };
+    const json expected = {{"a", {true, false}}};
+
+    CHECK(json::from_bson(bson) == expected);
+    CHECK(json::from_bson(bson, true, true, json::error_handler_t::replace) == expected);
+    CHECK(json::from_bson(bson, true, true, json::error_handler_t::ignore) == expected);
+
+    json _;
+    CHECK_THROWS_WITH_AS(_ = json::from_bson(bson, true, true, json::error_handler_t::strict), "[json.exception.parse_error.113] parse error at byte 14: syntax error while parsing BSON key: invalid string: ill-formed UTF-8 byte", json::parse_error&);
+
+    std::istringstream stream(std::string(reinterpret_cast<const char*>(bson.data()), bson.size()));
+    CHECK(json::from_bson(stream) == expected);
+}
 #endif
 
 TEST_CASE("Negative size of binary value")

tests/src/unit-guard-page.cpp

@@ -403,7 +403,6 @@ TEST_CASE("guard page")
                     CHECK(nlohmann::detail::find_string_special(p, n) <= n);
                     CHECK(nlohmann::detail::find_ascii_copyable_run(p, n) <= n);
                     CHECK(nlohmann::detail::valid_utf8_prefix(p, n) <= n);
-                    CHECK(nlohmann::detail::valid_utf8_string(p, n) == (nlohmann::detail::valid_utf8_prefix(p, n) == n));
 #if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17)
                     CHECK(nlohmann::detail::find_string_delimiter(p, n) <= n);
 #endif

tests/src/unit-unicode.cpp

@@ -13,7 +13,6 @@ using nlohmann::json;
 
 #include <algorithm>
 #include <fstream>
-#include <random>
 #include <string>
 #include "make_test_data_available.hpp"
 #include "test_utils.hpp"
@@ -1632,40 +1631,3 @@ TEST_CASE("Unicode RFC 3629")
 }
 
 DOCTEST_CLANG_SUPPRESS_WARNING_POP
-
-TEST_CASE("UTF-8 check of whole strings")
-{
-    // valid_utf8_string() decides short strings with a word test and long ones
-    // 16 bytes at a time where a vector check is available; it must agree with
-    // the byte-wise check for every mix of ASCII, sequences of each length,
-    // ill-formed and truncated sequences, at every length and offset
-    const char* const pieces[] = // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
-    {
-        "a", "\x7F", "\xC3\xA9", "\xE2\x82\xAC", "\xE0\xA0\x80", "\xEF\xBF\xBF", "\xF0\x9F\x98\x80", "\xF4\x8F\xBF\xBF",
-        "\x80", "\xC0\xAF", "\xC3", "\xE2\x82", "\xED\xA0\x80", "\xF0\x9F", "\xF4\x90\x80\x80", "\xFF"
-    };
-    // std::mt19937's sequence is fixed by the standard; no distribution is used
-    std::mt19937 rng(5); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed)
-    std::size_t valid = 0;
-    for (int i = 0; i < 100000; ++i)
-    {
-        std::string s;
-        const std::size_t length = rng() % 100;
-        const auto ascii = static_cast<std::uint32_t>(rng() % 101);
-        while (s.size() < length)
-        {
-            // mostly well-formed pieces (the first eight), so that long valid
-            // strings are common
-            s += (rng() % 100 < ascii) ? std::string("x") : std::string(pieces[rng() % (rng() % 8 == 0 ? 16 : 8)]);
-        }
-        const auto* data = reinterpret_cast<const unsigned char*>(s.data());
-        const bool expected = nlohmann::detail::is_valid_utf8(s);
-        valid += expected ? 1 : 0;
-        CAPTURE(s)
-        CHECK(nlohmann::detail::valid_utf8_string(data, s.size()) == expected);
-        CHECK((nlohmann::detail::valid_utf8_prefix(data, s.size()) == s.size()) == expected);
-    }
-    // both outcomes are common enough to be tested
-    CHECK(valid > 20000);
-    CHECK(valid < 80000);
-}