Escape characters below 0x60 from a table straight into the write buffer

When the bulk scanner in dump_escaped_impl() stops at a control
character, a quote or a backslash, that byte used to go through the
UTF-8 decoder and a switch that wrote the escape byte by byte into
string_buffer, which was then copied into write_buffer at the next run.

A byte below 0x60 at a character boundary is a complete code point, so
it now skips the decoder: its escape sequence comes from a 96-entry
table of 8-byte rows (sequence padded with zeros, length in the last
byte), and all 8 bytes are copied straight into write_buffer with one
room check. The check sits inside the existing character-boundary
block, so it runs once per scanner stop and not once per byte; testing
every byte cost about 6% on twitter.json with ensure_ascii.

The switch in the decoder's accept branch lost its cases for
characters below 0x60, which can no longer reach it.

Output is byte-identical. The string-buffer boundary tests now use
\u-escaped non-ASCII characters, since simple escapes no longer reach
string_buffer, and a new test case checks every ASCII character at
every position against a reference escaper.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

cmake/ci.cmake

@@ -610,9 +610,8 @@ foreach(SRC_FILE ${SRC_FILES})
     add_executable(single_${RELATIVE_SRC_FILE} EXCLUDE_FROM_ALL ${PROJECT_BINARY_DIR}/src_single/${RELATIVE_SRC_FILE}.cpp)
     target_include_directories(single_${RELATIVE_SRC_FILE} PRIVATE ${PROJECT_SOURCE_DIR}/include)
     target_compile_features(single_${RELATIVE_SRC_FILE} PRIVATE cxx_std_11)
-    if(RELATIVE_SRC_FILE STREQUAL "json" OR RELATIVE_SRC_FILE STREQUAL "json_literals")
-        # see below: report the diagnostics of json.hpp and json_literals.hpp without --error, so they
-        # do not fail the build
+    if(RELATIVE_SRC_FILE STREQUAL "json")
+        # see below: report the diagnostics of json.hpp without --error, so they do not fail the build
         set_property(TARGET single_${RELATIVE_SRC_FILE} PROPERTY CXX_INCLUDE_WHAT_YOU_USE ${IWYU_TOOL} -Xiwyu --max_line_length=300)
     else()
         set_property(TARGET single_${RELATIVE_SRC_FILE} PROPERTY CXX_INCLUDE_WHAT_YOU_USE "${iwyu_path_and_options}")
@@ -626,18 +625,31 @@ foreach(SRC_FILE ${SRC_FILES})
     # reporting its diagnostics (informational, via CXX_INCLUDE_WHAT_YOU_USE above) but exclude it
     # from the hard gate below so a fresh IWYU/compiler combination does not fail this target on a
     # nondeterministic suggestion for a header that already re-exports everything on purpose.
-    # json_literals.hpp and json.hpp include each other on purpose (json.hpp includes it at its end
-    # unless JSON_NO_AUTOMATIC_UDLS is defined), and IWYU, not following the cycle, suggests replacing
-    # json.hpp with json_fwd.hpp although the literals need the complete basic_json; exclude it, too.
-    if(NOT RELATIVE_SRC_FILE STREQUAL "json" AND NOT RELATIVE_SRC_FILE STREQUAL "json_literals")
+    if(NOT RELATIVE_SRC_FILE STREQUAL "json")
         list(APPEND single_binaries_tus src_single/${RELATIVE_SRC_FILE}.cpp)
     endif()
 endforeach()
 
+# The checks above look at each header on its own. Client code that includes only <nlohmann/json.hpp>
+# must not be told by include analyzers to include another nlohmann header for what it uses
+# (issue #5812): json.hpp marks those includes "IWYU pragma: export", and both IWYU and clang-tidy's
+# misc-include-cleaner (disabled in .clang-tidy for the library itself) check these clients.
+file(GLOB CONSUMER_FILES ${PROJECT_SOURCE_DIR}/tests/iwyu_consumer/*.cpp)
+foreach(CONSUMER_FILE ${CONSUMER_FILES})
+    get_filename_component(CONSUMER_NAME "${CONSUMER_FILE}" NAME_WE)
+    add_executable(iwyu_consumer_${CONSUMER_NAME} EXCLUDE_FROM_ALL ${CONSUMER_FILE})
+    target_include_directories(iwyu_consumer_${CONSUMER_NAME} PRIVATE ${PROJECT_SOURCE_DIR}/include)
+    target_compile_features(iwyu_consumer_${CONSUMER_NAME} PRIVATE cxx_std_11)
+    list(APPEND single_binaries iwyu_consumer_${CONSUMER_NAME})
+    list(APPEND single_binaries_tus ${CONSUMER_FILE})
+endforeach()
+
 if(IWYU_TOOL_PY)
     add_custom_target(ci_single_binaries
         DEPENDS ${single_binaries}
         COMMAND ${IWYU_TOOL_PY} -p ${PROJECT_BINARY_DIR} ${single_binaries_tus} -- ${iwyu_options}
+        COMMAND ${CLANG_TIDY_TOOL} -p ${PROJECT_BINARY_DIR} --checks=-*,misc-include-cleaner --warnings-as-errors=* ${CONSUMER_FILES}
+        VERBATIM
         COMMENT "Check if headers are self-contained"
     )
 else()

docs/mkdocs/docs/api/basic_json/diff.md

@@ -59,6 +59,7 @@ Currently, only `remove`, `add`, and `replace` operations are generated.
 - [patch](patch.md) applies a JSON Patch
 - [patch_inplace](patch_inplace.md) applies a JSON Patch in place
 - [merge_patch](merge_patch.md) applies a JSON Merge Patch
+- [merge_diff](merge_diff.md) creates a diff as a JSON Merge Patch
 
 ## Version history
 

docs/mkdocs/docs/api/basic_json/index.md

@@ -285,6 +285,7 @@ Access to the JSON value
 ### JSON Merge Patch functions
 
 - [**merge_patch**](merge_patch.md) - applies a JSON Merge Patch
+- [**merge_diff**](merge_diff.md) (_static_) - creates a diff as a JSON Merge Patch
 
 ## Static functions
 

docs/mkdocs/docs/api/basic_json/merge_diff.md

@@ -0,0 +1,79 @@
+# <small>nlohmann::basic_json::</small>merge_diff
+
+```cpp
+static basic_json merge_diff(const basic_json& source,
+                             const basic_json& target);
+```
+
+Creates a [JSON Merge Patch](https://tools.ietf.org/html/rfc7396) so that value `source` can be changed into the value
+`target` by calling the [`merge_patch`](merge_patch.md) function. The patch contains only what differs: changed and
+added members with their new values, and removed members with the value `#!json null`. Nested objects are compared
+member by member; any other value that differs, including an array, is replaced as a whole.
+
+For two JSON values `source` and `target`, where `target` contains no object member whose value is `#!json null`, the
+following code always yields `#!cpp true`:
+```cpp
+basic_json patched = source;
+patched.merge_patch(merge_diff(source, target));
+patched == target;
+```
+
+## Parameters
+
+`source` (in)
+:   JSON value to compare from
+
+`target` (in)
+:   JSON value to compare against
+
+## Return value
+
+a JSON Merge Patch to convert the `source` to `target`
+
+## Exception safety
+
+Strong guarantee: `source` and `target` are never modified.
+
+## Complexity
+
+Linear in the total number of values (including nested ones) in `source` and `target`, times the cost of a key-based
+object operation (looking up a key, or adding one to the patch): logarithmic for [`json`](../json.md), and linear for
+[`ordered_json`](../ordered_json.md) (see [`ordered_map` complexity](../ordered_map.md#complexity)). For
+`ordered_json`, diffing two objects with `n` and `m` members therefore takes O((n+m)²).
+
+## Notes
+
+- A merge patch uses `#!json null` to remove a member, so it cannot set a member to `#!json null`. As in RFC 7396, a
+  `#!json null` member of `target` is treated as absent: the patch removes the member from `source` instead of setting
+  it to `#!json null`. A member that is `#!json null` in both `source` and `target` is unchanged and left out of the
+  patch, so `source` keeps it.
+- A merge patch cannot reorder object members, and [`merge_patch`](merge_patch.md) appends added members at the end.
+  For [`ordered_json`](../ordered_json.md), the patched value therefore has the same members as `target`, but not
+  necessarily in the same order, and may compare unequal to it. The patch lists changed and removed members in the
+  order of `source`, followed by added members in the order of `target`.
+
+## Examples
+
+??? example
+
+    The following code shows how a JSON Merge Patch is created as a diff for two JSON values.
+
+    ```cpp
+    --8<-- "examples/merge_diff.cpp"
+    ```
+
+    Output:
+
+    ```json
+    --8<-- "examples/merge_diff.output"
+    ```
+
+## See also
+
+- [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396)
+- [merge_patch](merge_patch.md) applies a JSON Merge Patch
+- [diff](diff.md) creates a diff as a JSON Patch
+
+## Version history
+
+- Added in version 3.13.0.

docs/mkdocs/docs/api/basic_json/merge_patch.md

@@ -67,6 +67,7 @@ returned by `#!cpp (*this)[key]`); it is read as it was when `merge_patch()` was
 
 - [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396)
 - [patch](patch.md) apply a JSON patch
+- [merge_diff](merge_diff.md) creates a diff as a JSON Merge Patch
 
 ## Version history
 

docs/mkdocs/docs/api/basic_json/operator_eq.md

@@ -11,12 +11,10 @@ template<typename ScalarType>
 bool operator==(ScalarType lhs, const const_reference rhs) noexcept(/* see below */); // (2)
 
 // since C++20
-class basic_json {
-    bool operator==(const_reference rhs) const noexcept;              // (1)
+bool operator==(const_reference rhs) const noexcept;             // (1)
 
-    template<typename ScalarType>
-    bool operator==(ScalarType rhs) const noexcept(/* see below */); // (2)
-};
+template<typename ScalarType>
+bool operator==(ScalarType rhs) const noexcept(/* see below */); // (2)
 ```
 
 1. Compares two JSON values for equality according to the following rules:

docs/mkdocs/docs/api/basic_json/operator_spaceship.md

@@ -2,12 +2,10 @@
 
 ```cpp
 // since C++20
-class basic_json {
-    std::partial_ordering operator<=>(const_reference rhs) const noexcept;  // (1)
+std::partial_ordering operator<=>(const_reference rhs) const noexcept;                   // (1)
 
-    template<typename ScalarType>
-    std::partial_ordering operator<=>(const ScalarType rhs) const noexcept(/* see below */); // (2)
-};
+template<typename ScalarType>
+std::partial_ordering operator<=>(const ScalarType rhs) const noexcept(/* see below */); // (2)
 ```
 
 1. 3-way compares two JSON values producing a result of type `std::partial_ordering` according to the following rules:

docs/mkdocs/docs/api/json_pointer/operator_eq.md

@@ -18,13 +18,11 @@ bool operator==(
     const json_pointer<RefStringTypeRhs>& rhs);                     // (2)
 
 // since C++20
-class json_pointer {
-    template<typename RefStringTypeRhs>
-    bool operator==(
-        const json_pointer<RefStringTypeRhs>& rhs) const noexcept;  // (1)
+template<typename RefStringTypeRhs>
+bool operator==(
+    const json_pointer<RefStringTypeRhs>& rhs) const noexcept;      // (1)
 
-    bool operator==(const string_t& rhs) const;                     // (2)
-};
+bool operator==(const string_t& rhs) const;                         // (2)
 ```
 
 1. Compares two JSON pointers for equality by comparing their reference tokens.

docs/mkdocs/docs/api/json_pointer/operator_spaceship.md

@@ -2,10 +2,8 @@
 
 ```cpp
 // since C++20
-class json_pointer {
-    template<typename RefStringTypeRhs>
-    std::strong_ordering operator<=>(const json_pointer<RefStringTypeRhs>& rhs) const noexcept; // *NOPAD*
-};
+template<typename RefStringTypeRhs>
+std::strong_ordering operator<=>(const json_pointer<RefStringTypeRhs>& rhs) const noexcept; // *NOPAD*
 ```
 
 3-way compares two JSON pointers by lexicographically comparing their sequences of reference tokens: corresponding

docs/mkdocs/docs/examples/merge_diff.cpp

@@ -0,0 +1,51 @@
+#include <iostream>
+#include <iomanip>
+#include <nlohmann/json.hpp>
+
+using json = nlohmann::json;
+using namespace nlohmann::literals;
+
+int main()
+{
+    // the source document
+    json source = R"(
+        {
+            "title": "Goodbye!",
+            "author": {
+                "givenName": "John",
+                "familyName": "Doe"
+            },
+            "tags": [
+                "example",
+                "sample"
+            ],
+            "content": "This will be unchanged"
+        }
+    )"_json;
+
+    // the target document
+    json target = R"(
+        {
+            "title": "Hello!",
+            "author": {
+                "givenName": "John"
+            },
+            "tags": [
+                "example"
+            ],
+            "content": "This will be unchanged",
+            "phoneNumber": "+01-123-456-7890"
+        }
+    )"_json;
+
+    // create the patch
+    json patch = json::merge_diff(source, target);
+
+    // roundtrip
+    json patched_source = source;
+    patched_source.merge_patch(patch);
+
+    // output patch and roundtrip result
+    std::cout << std::setw(4) << patch << "\n\n"
+              << std::setw(4) << patched_source << std::endl;
+}

docs/mkdocs/docs/examples/merge_diff.output

@@ -0,0 +1,22 @@
+{
+    "author": {
+        "familyName": null
+    },
+    "phoneNumber": "+01-123-456-7890",
+    "tags": [
+        "example"
+    ],
+    "title": "Hello!"
+}
+
+{
+    "author": {
+        "givenName": "John"
+    },
+    "content": "This will be unchanged",
+    "phoneNumber": "+01-123-456-7890",
+    "tags": [
+        "example"
+    ],
+    "title": "Hello!"
+}

docs/mkdocs/docs/features/merge_patch.md

@@ -9,6 +9,9 @@ syntax that closely mimics the document being modified. Unlike [JSON Patch](json
 express every kind of change (e.g., it cannot reorder array elements or remove a specific array element), but it is
 easier to read and write for object-shaped documents.
 
+The function [`merge_diff`](../api/basic_json/merge_diff.md) creates a merge patch that changes one JSON value into
+another.
+
 !!! tip "Not a general deep merge"
 
     A merge patch is not a general deep merge: a `#!json null` value in the patch deletes the key from the target.
@@ -35,4 +38,5 @@ easier to read and write for object-shaped documents.
 - [JSON Patch and Diff](json_patch.md) - a more expressive alternative that describes a sequence of operations
 - [JSON Pointer](json_pointer.md) - the addressing scheme used by JSON Patch
 - Function [`merge_patch`](../api/basic_json/merge_patch.md)
+- Function [`merge_diff`](../api/basic_json/merge_diff.md)
 - Function [`update`](../api/basic_json/update.md) - merge objects, optionally recursively

docs/mkdocs/docs/home/architecture.md

@@ -185,9 +185,11 @@ The lexer detects two optional extensions at compile time. Only `iterator_input_
 random-access input of single-byte characters:
 
 - `supports_seek`, `get_consumed_count()`, and `copy_consumed_range()` let the lexer reconstruct already consumed input
-  for error messages instead of copying every character it reads.
+  for error messages instead of copying every character it reads. With `get_read_count()` and
+  `count_consumed_lines()`, the lexer also derives the read position from the adapter and counts lines only when an
+  error needs them, instead of counting characters and lines on every read.
 - `supports_bulk_scan`, `bulk_data()`, `bulk_remaining()`, and `bulk_skip()` let the lexer scan strings directly in
-  contiguous memory, several bytes at a time.
+  contiguous memory, several bytes at a time, and skip the spaces that indent a line in 8-byte steps.
 
 The function `input_adapter` picks the right adapter for the argument passed to `parse`, `accept`, `sax_parse`, or the
 `from_*` functions:

docs/mkdocs/mkdocs.yml

@@ -182,6 +182,7 @@ nav:
           - 'json_serializer': api/basic_json/json_serializer.md
           - 'max_size': api/basic_json/max_size.md
           - 'meta': api/basic_json/meta.md
+          - 'merge_diff': api/basic_json/merge_diff.md
           - 'merge_patch': api/basic_json/merge_patch.md
           - 'number_float_t': api/basic_json/number_float_t.md
           - 'number_integer_t': api/basic_json/number_integer_t.md

include/nlohmann/detail/bit_ops.hpp

@@ -42,6 +42,25 @@ inline int count_leading_zeros(std::uint64_t x) noexcept
 #endif
 }
 
+/// number of trailing zero bits of x (x != 0)
+inline int count_trailing_zeros(std::uint64_t x) noexcept
+{
+#if defined(__GNUC__) || defined(__clang__)
+    return __builtin_ctzll(x);
+#else
+    int n = 0;
+    for (int shift = 32; shift != 0; shift >>= 1)
+    {
+        if ((x << (64 - shift)) == 0)
+        {
+            n += shift;
+            x >>= shift;
+        }
+    }
+    return n;
+#endif
+}
+
 /// the 128-bit product of two 64-bit numbers
 struct uint128_parts
 {

include/nlohmann/detail/input/input_adapters.hpp

@@ -8,12 +8,12 @@
 
 #pragma once
 
-#include <algorithm> // min
+#include <algorithm> // count, find, min
 #include <array> // array
 #include <cstddef> // size_t
 #include <cstdint> // uint32_t
 #include <cstring> // strlen
-#include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next
+#include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next, reverse_iterator
 #include <streambuf> // streambuf
 #include <string> // string, char_traits
 #include <type_traits> // enable_if, is_base_of, is_pointer, is_integral, remove_pointer
@@ -297,6 +297,7 @@ class iterator_input_adapter
             return result;
         }
 
+        ++eof_reads;
         return char_traits<char_type>::eof();
     }
 
@@ -306,6 +307,26 @@ class iterator_input_adapter
         return static_cast<std::size_t>(std::distance(begin, current));
     }
 
+    // number of get_character() calls so far, including those that returned
+    // EOF; the lexer derives its position from it (see lexer::lazy_position)
+    std::size_t get_read_count() const
+    {
+        return get_consumed_count() + eof_reads;
+    }
+
+    // number of newlines in the consumed range [0, last_index); @a line_start
+    // is set to the index after the last of them (0 if there is none); only
+    // valid when supports_seek
+    std::size_t count_consumed_lines(std::size_t last_index, std::size_t& line_start) const
+    {
+        const auto to = std::next(begin, static_cast<typename std::iterator_traits<IteratorType>::difference_type>(last_index));
+        const auto newline = char_traits<char_type>::to_char_type('\n');
+        using reverse_it = std::reverse_iterator<IteratorType>;
+        const auto last_newline = std::find(reverse_it(to), reverse_it(begin), newline);
+        line_start = static_cast<std::size_t>(std::distance(begin, last_newline.base()));
+        return static_cast<std::size_t>(std::count(begin, last_newline.base(), newline));
+    }
+
     // append the already-consumed characters in the half-open range
     // [first_index, last_index) to @a out; only valid when supports_seek
     template<typename ContainerType>
@@ -422,6 +443,8 @@ class iterator_input_adapter
     IteratorType begin;
     IteratorType current;
     SentinelType end;
+    /// number of get_character() calls that returned EOF
+    std::size_t eof_reads = 0;
 
     template<typename BaseInputAdapter, size_t T>
     friend struct wide_string_input_helper;

include/nlohmann/detail/input/json_sax.hpp

@@ -217,7 +217,7 @@ struct diagnostic_positions
         {
             // Lexer has read the first character of the container, so
             // subtract 1 from the position to get the correct start position.
-            v.start_position = lexer->get_position() - 1;
+            v.start_position = lexer->get_chars_read_total() - 1;
         }
     }
 
@@ -231,7 +231,7 @@ struct diagnostic_positions
         if (lexer)
         {
             // Lexer's position is past the closing brace or bracket, so set that as the end position.
-            v.end_position = lexer->get_position();
+            v.end_position = lexer->get_chars_read_total();
         }
     }
 
@@ -247,7 +247,7 @@ struct diagnostic_positions
             // Lexer has read past the current field value, so set the end position to the current position.
             // The start position will be set below based on the length of the string representation
             // of the value.
-            v.end_position = lexer->get_position();
+            v.end_position = lexer->get_chars_read_total();
 
             switch (v.type())
             {

include/nlohmann/detail/input/lexer.hpp

@@ -17,6 +17,7 @@
 #include <utility> // move
 #include <vector> // vector
 
+#include <nlohmann/detail/bit_ops.hpp>
 #include <nlohmann/detail/input/input_adapters.hpp>
 #include <nlohmann/detail/input/number_parse.hpp>
 #include <nlohmann/detail/input/position_t.hpp>
@@ -186,6 +187,15 @@ class lexer : public lexer_base<BasicJsonType>
     static constexpr bool lazy_token_string =
         input_adapter_supports_seek<InputAdapterType>(is_detected<detect_supports_seek, InputAdapterType> {});
 
+    /// whether the read position is derived from the input adapter on demand
+    /// instead of being counted on every scanned character. Seekable adapters
+    /// know how much input they consumed and can revisit it, so
+    /// chars_read_total is the adapter's read count (minus a pending unget),
+    /// and line and column are recomputed from the consumed input only when
+    /// get_position() is called (errors). Counting per character kept two
+    /// increments and a newline check in the hot path of every get().
+    static constexpr bool lazy_position = lazy_token_string;
+
     /// whether a simulated unget can be passed on to the input adapter, which
     /// then leaves the character in the input; see
     /// input_adapter_supports_lookahead
@@ -195,7 +205,9 @@ class lexer : public lexer_base<BasicJsonType>
     /// whether string scanning may bulk-consume runs of ordinary characters
     /// directly from a contiguous input buffer (SWAR fast path). This requires
     /// the token to be reconstructible lazily (lazy_token_string), so bypassing
-    /// the per-character capture in get() cannot lose error diagnostics.
+    /// the per-character capture in get() cannot lose error diagnostics. It
+    /// also implies lazy_position, so the bulk paths need not advance the
+    /// position counters.
     static constexpr bool bulk_scan =
         lazy_token_string
         && input_adapter_supports_bulk_scan<InputAdapterType>(is_detected<detect_supports_bulk_scan, InputAdapterType> {});
@@ -331,10 +343,6 @@ class lexer : public lexer_base<BasicJsonType>
         }
         token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), pos);
         ia.bulk_skip(pos);
-        // the run contains no newline (all bytes < 0x20 are treated as special),
-        // so only the flat character counters advance
-        position.chars_read_total += pos;
-        position.chars_read_current_line += pos;
     }
 
     /// streaming input: no bulk fast path
@@ -1567,8 +1575,6 @@ scan_number_done:
             if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized))
             {
                 ia.bulk_skip(len - 1);
-                position.chars_read_total += (len - 1);
-                position.chars_read_current_line += (len - 1);
                 return integer_result;
             }
             // The value does not fit an integer, so this token converts as a
@@ -1585,8 +1591,6 @@ scan_number_done:
         decimal_point_position = dot_index;
 
         ia.bulk_skip(len - 1);
-        position.chars_read_total += (len - 1);
-        position.chars_read_current_line += (len - 1);
 
         return convert_number(number_type, mantissa_end);
     }
@@ -1637,7 +1641,7 @@ scan_number_done:
 
 #if JSON_DIAGNOSTIC_POSITIONS
         // the first character of the token has already been read, hence the -1
-        token_start_position = position.chars_read_total - 1;
+        token_start_position = get_chars_read_total() - 1;
 #endif
 
         note_token_start(std::integral_constant<bool, lazy_token_string> {});
@@ -1690,6 +1694,15 @@ scan_number_done:
     /// per-character position counters (line-count-on-'\n' bookkeeping is
     /// handled afterwards, in track_after_read(), once `current` is known)
     void advance_position() noexcept
+    {
+        advance_position_impl(std::integral_constant<bool, lazy_position> {});
+    }
+
+    /// seekable adapter: the position is derived from the adapter on demand
+    void advance_position_impl(std::true_type /*lazy*/) const noexcept {}
+
+    /// streaming adapter: count the character
+    void advance_position_impl(std::false_type /*lazy*/) noexcept
     {
         ++position.chars_read_total;
         ++position.chars_read_current_line;
@@ -1703,7 +1716,16 @@ scan_number_done:
         // seekable adapters reconstruct the token lazily on error (see
         // get_token_string), so the eager per-character copy is skipped
         capture_char(std::integral_constant<bool, lazy_token_string> {});
+        track_newline(std::integral_constant<bool, lazy_position> {});
+        return current;
+    }
 
+    /// seekable adapter: lines are counted on demand in get_position()
+    void track_newline(std::true_type /*lazy*/) const noexcept {}
+
+    /// streaming adapter: start a new line after reading a newline
+    void track_newline(std::false_type /*lazy*/) noexcept
+    {
         if (current == '\n')
         {
             ++position.lines_read;
@@ -1712,8 +1734,6 @@ scan_number_done:
             chars_read_before_newline = position.chars_read_current_line;
             position.chars_read_current_line = 0;
         }
-
-        return current;
     }
 
     /*!
@@ -1757,12 +1777,22 @@ scan_number_done:
     We implement unget by setting variable next_unget to true. The input is not
     changed - we just simulate ungetting by modifying chars_read_total,
     chars_read_current_line, and token_string. The next call to get() will
-    behave as if the unget character is read again.
+    behave as if the unget character is read again. With lazy_position, the
+    pending next_unget is all get_chars_read_total() needs.
     */
     void unget()
     {
         next_unget = true;
+        unget_position(std::integral_constant<bool, lazy_position> {});
+        uncapture_char(std::integral_constant<bool, lazy_token_string> {});
+    }
 
+    /// seekable adapter: get_chars_read_total() accounts for next_unget
+    void unget_position(std::true_type /*lazy*/) const noexcept {}
+
+    /// streaming adapter: step the counters back over the ungotten character
+    void unget_position(std::false_type /*lazy*/) noexcept
+    {
         --position.chars_read_total;
 
         // in case we "unget" a newline, we have to also decrement the lines_read
@@ -1785,8 +1815,6 @@ scan_number_done:
         {
             --position.chars_read_current_line;
         }
-
-        uncapture_char(std::integral_constant<bool, lazy_token_string> {});
     }
 
     /// adapter without lookahead: nothing to do (see release_lookahead)
@@ -1857,10 +1885,21 @@ scan_number_done:
     // diagnostics
     /////////////////////
 
-    /// return position of last read token
-    constexpr position_t get_position() const noexcept
+    /*!
+    @brief return the number of characters read so far
+
+    Unlike get_position(), this is O(1) for all adapters, so it may be called
+    for every value (JSON_DIAGNOSTIC_POSITIONS).
+    */
+    std::size_t get_chars_read_total() const noexcept
     {
-        return position;
+        return get_chars_read_total_impl(std::integral_constant<bool, lazy_position> {});
+    }
+
+    /// return position of last read token
+    position_t get_position() const
+    {
+        return get_position_impl(std::integral_constant<bool, lazy_position> {});
     }
 
     /*!
@@ -1897,6 +1936,43 @@ scan_number_done:
     }
 #endif
 
+  private:
+    /// seekable adapter: every get() either read from the adapter (including
+    /// reads at EOF, which get() counts as characters as well) or replayed an
+    /// ungotten character, so the count is the adapter's read count minus a
+    /// still pending unget
+    std::size_t get_chars_read_total_impl(std::true_type /*lazy*/) const noexcept
+    {
+        return ia.get_read_count() - (next_unget ? 1u : 0u);
+    }
+
+    /// streaming adapter: the counter is maintained by get() and unget()
+    std::size_t get_chars_read_total_impl(std::false_type /*lazy*/) const noexcept
+    {
+        return position.chars_read_total;
+    }
+
+    /// seekable adapter: count the lines in the consumed input; O(input read
+    /// so far), but only called on errors
+    position_t get_position_impl(std::true_type /*lazy*/) const
+    {
+        position_t result;
+        result.chars_read_total = get_chars_read_total();
+        // reads at EOF count as characters but are not part of the input
+        const std::size_t consumed = (std::min)(result.chars_read_total, ia.get_consumed_count());
+        std::size_t line_start = 0;
+        result.lines_read = ia.count_consumed_lines(consumed, line_start);
+        result.chars_read_current_line = result.chars_read_total - line_start;
+        return result;
+    }
+
+    /// streaming adapter: the position is maintained by get() and unget()
+    position_t get_position_impl(std::false_type /*lazy*/) const noexcept
+    {
+        return position;
+    }
+
+  public:
     /// seekable adapter: rebuild the last read token from the input on demand
     const std::vector<char_type>& collect_token_chars(std::vector<char_type>& out, std::true_type /*lazy*/) const
     {
@@ -2004,15 +2080,57 @@ scan_number_done:
         // into one that reloads the pointer from memory every character
         do
         {
+            if (current == '\n')
+            {
+                skip_indentation(std::integral_constant<bool, bulk_scan> {});
+            }
             get_ignoring_pending_unget();
         }
         while (current_is_whitespace());
     }
 
+    /*!
+    @brief contiguous input: skip the space indentation that follows a newline
+           8 bytes at a time
+
+    Pretty-printed JSON indents every line with a run of spaces, which the loop
+    in skip_whitespace() would otherwise read one character at a time. The
+    spaces are skipped 8 bytes at a time; the last 7 bytes of the input and any
+    other whitespace are left to that loop. Skipping bypasses get(), which is
+    fine because bulk_scan implies lazy_position: the position is derived from
+    the adapter, not counted per character.
+    */
+    void skip_indentation(std::true_type /*bulk*/)
+    {
+        JSON_ASSERT(!next_unget);
+        const std::size_t remaining = ia.bulk_remaining();
+        if (remaining < 8)
+        {
+            return;
+        }
+        const char* const data = reinterpret_cast<const char*>(ia.bulk_data());
+        std::size_t pos = 0;
+        for (; pos + 8 <= remaining; pos += 8)
+        {
+            // little-endian, so the first byte is the lowest
+            const std::uint64_t non_space = read_eight_bytes(data + pos) ^ 0x2020202020202020ull;
+            if (non_space != 0)
+            {
+                // skip the spaces before the first other byte of this word
+                pos += static_cast<std::size_t>(count_trailing_zeros(non_space)) / 8;
+                break;
+            }
+        }
+        ia.bulk_skip(pos);
+    }
+
+    /// streaming input: no bulk fast path
+    void skip_indentation(std::false_type /*bulk*/) const noexcept {}
+
     token_type scan()
     {
         // initially, skip the BOM
-        if (position.chars_read_total == 0 && !skip_bom())
+        if (get_chars_read_total() == 0 && !skip_bom())
         {
             error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given";
             return token_type::parse_error;
@@ -2039,7 +2157,7 @@ scan_number_done:
     token_type scan_expecting(token_type expected_type)
     {
         JSON_ASSERT(expected_type == token_type::name_separator || expected_type == token_type::value_separator);
-        JSON_ASSERT(position.chars_read_total > 0);
+        JSON_ASSERT(get_chars_read_total() > 0);
         const char_int_type expected_char = static_cast<unsigned char>((expected_type == token_type::name_separator) ? ':' : ',');
         skip_whitespace();
         if (JSON_HEDLEY_LIKELY(current == expected_char))

include/nlohmann/detail/input/parser.hpp

@@ -370,7 +370,7 @@ class parser
                     }
                     case token_type::end_of_input:
                     {
-                        if (JSON_HEDLEY_UNLIKELY(m_lexer.get_position().chars_read_total == 1))
+                        if (JSON_HEDLEY_UNLIKELY(m_lexer.get_chars_read_total() == 1))
                         {
                             return syntax_error(*sax, "attempting to parse an empty input; check that your input string or stream contains the expected JSON");
                         }
@@ -520,8 +520,10 @@ class parser
     {
         // MSVC 2015 reports C4100 (unreferenced parameter) when SAX::parse_error is static
         static_cast<void>(sax);
-        return sax.parse_error(m_lexer.get_position(), m_lexer.get_token_string(),
-                               parse_error::create(exception_id::syntax_error, m_lexer.get_position(), message, nullptr));
+        // computing the position is O(input read so far) for seekable input
+        const position_t position = m_lexer.get_position();
+        return sax.parse_error(position, m_lexer.get_token_string(),
+                               parse_error::create(exception_id::syntax_error, position, message, nullptr));
     }
 
     std::string exception_message(const token_type expected, const std::string& context)

include/nlohmann/detail/output/serializer.hpp

@@ -772,6 +772,33 @@ class serializer
                         break;
                     }
                 }
+
+                // A byte below 0x60 is a complete code point and needs no
+                // UTF-8 decoding. The scanner stops only at a control
+                // character, a quote, or a backslash in that range, so look up
+                // its escape sequence and write all 8 bytes of the table entry
+                // straight into the write buffer; only the length stored in the
+                // last byte of the entry is kept.
+                if (data[i] < 0x60)
+                {
+                    // emit any bytes still pending in string_buffer first to
+                    // preserve output order
+                    if (bytes != 0)
+                    {
+                        put_buffer(string_buffer, bytes);
+                        bytes = 0;
+                    }
+                    const auto& escape = ascii_escapes()[data[i]];
+                    if (JSON_HEDLEY_UNLIKELY(write_buffer.size() - write_buffer_pos < escape.size()))
+                    {
+                        flush();
+                    }
+                    std::memcpy(write_buffer.data() + write_buffer_pos, escape.data(), escape.size());
+                    write_buffer_pos += static_cast<std::size_t>(escape[7]);
+                    bytes_after_last_accept = 0;
+                    undumped_chars = 0;
+                    continue;
+                }
             }
 
             const auto byte = static_cast<std::uint8_t>(s[i]);
@@ -780,92 +807,39 @@ class serializer
             {
                 case UTF8_ACCEPT:  // decode found a new code point
                 {
-                    switch (codepoint)
+                    // the code point is at least 0x60 (smaller ones are handled
+                    // above) and is not printable ASCII (which the scanner
+                    // copies): it is DEL (0x7F) under EnsureAscii or a non-ASCII
+                    // character
+                    if (EnsureAscii)
                     {
-                        case 0x08: // backspace
+                        JSON_ASSERT(codepoint >= 0x7F);
+                        if (error_handler == error_handler_t::keep)
                         {
-                            string_buffer[bytes++] = '\\';
-                            string_buffer[bytes++] = 'b';
-                            break;
+                            // this character was buffered as raw bytes below
+                            // in case it turned out to be part of an
+                            // ill-formed sequence (which is kept as is); now
+                            // that it decoded to a well-formed code point,
+                            // undo that and \u-escape it like any other
+                            // character instead
+                            bytes = bytes_after_last_accept;
                         }
-
-                        case 0x09: // horizontal tab
+                        if (codepoint <= 0xFFFF)
                         {
-                            string_buffer[bytes++] = '\\';
-                            string_buffer[bytes++] = 't';
-                            break;
+                            write_u_escape(bytes, static_cast<std::uint16_t>(codepoint));
                         }
-
-                        case 0x0A: // newline
+                        else
                         {
-                            string_buffer[bytes++] = '\\';
-                            string_buffer[bytes++] = 'n';
-                            break;
-                        }
-
-                        case 0x0C: // formfeed
-                        {
-                            string_buffer[bytes++] = '\\';
-                            string_buffer[bytes++] = 'f';
-                            break;
-                        }
-
-                        case 0x0D: // carriage return
-                        {
-                            string_buffer[bytes++] = '\\';
-                            string_buffer[bytes++] = 'r';
-                            break;
-                        }
-
-                        case 0x22: // quotation mark
-                        {
-                            string_buffer[bytes++] = '\\';
-                            string_buffer[bytes++] = '"';
-                            break;
-                        }
-
-                        case 0x5C: // reverse solidus
-                        {
-                            string_buffer[bytes++] = '\\';
-                            string_buffer[bytes++] = '\\';
-                            break;
-                        }
-
-                        default:
-                        {
-                            // escape control characters (0x00..0x1F) or, if
-                            // EnsureAscii parameter is used, non-ASCII characters
-                            if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F)))
-                            {
-                                if (EnsureAscii && error_handler == error_handler_t::keep)
-                                {
-                                    // this character was buffered as raw bytes
-                                    // below in case it turned out to be part of
-                                    // an ill-formed sequence (which is kept as
-                                    // is); now that it decoded to a well-formed
-                                    // code point, undo that and \u-escape it
-                                    // like any other character instead
-                                    bytes = bytes_after_last_accept;
-                                }
-                                if (codepoint <= 0xFFFF)
-                                {
-                                    write_u_escape(bytes, static_cast<std::uint16_t>(codepoint));
-                                }
-                                else
-                                {
-                                    write_u_escape(bytes, static_cast<std::uint16_t>(0xD7C0u + (codepoint >> 10u)));
-                                    write_u_escape(bytes, static_cast<std::uint16_t>(0xDC00u + (codepoint & 0x3FFu)));
-                                }
-                            }
-                            else
-                            {
-                                // copy byte to buffer (all previous bytes
-                                // been copied have in default case above)
-                                string_buffer[bytes++] = s[i];
-                            }
-                            break;
+                            write_u_escape(bytes, static_cast<std::uint16_t>(0xD7C0u + (codepoint >> 10u)));
+                            write_u_escape(bytes, static_cast<std::uint16_t>(0xDC00u + (codepoint & 0x3FFu)));
                         }
                     }
+                    else
+                    {
+                        // copy byte to buffer (all previous bytes of the
+                        // sequence have been copied in the default case below)
+                        string_buffer[bytes++] = s[i];
+                    }
 
                     // write buffer and reset index; there must be 13 bytes
                     // left, as this is the maximal number of bytes to be
@@ -1253,6 +1227,121 @@ class serializer
         }
     }
 
+    /*!
+    @brief the JSON form of each character below 0x60
+
+    Entry c holds the escape sequence of character c - the short form for
+    \\b, \\t, \\n, \\f, \\r, \\" and \\\\, \\u00xx for the other control characters -
+    or c itself if it needs no escaping, padded with zeros to 7 bytes. The last
+    byte is the length of the sequence. The 8 bytes of an entry are copied at
+    once, so writing an escape needs neither a branch nor a byte-wise loop.
+    */
+    static const std::array<std::array<char, 8>, 0x60>& ascii_escapes() noexcept
+    {
+        static const std::array<std::array<char, 8>, 0x60> escapes =
+        {
+            {
+                {{'\\', 'u', '0', '0', '0', '0', 0, 6}}, // 0x00
+                {{'\\', 'u', '0', '0', '0', '1', 0, 6}}, // 0x01
+                {{'\\', 'u', '0', '0', '0', '2', 0, 6}}, // 0x02
+                {{'\\', 'u', '0', '0', '0', '3', 0, 6}}, // 0x03
+                {{'\\', 'u', '0', '0', '0', '4', 0, 6}}, // 0x04
+                {{'\\', 'u', '0', '0', '0', '5', 0, 6}}, // 0x05
+                {{'\\', 'u', '0', '0', '0', '6', 0, 6}}, // 0x06
+                {{'\\', 'u', '0', '0', '0', '7', 0, 6}}, // 0x07
+                {{'\\', 'b', 0, 0, 0, 0, 0, 2}}, // 0x08
+                {{'\\', 't', 0, 0, 0, 0, 0, 2}}, // 0x09
+                {{'\\', 'n', 0, 0, 0, 0, 0, 2}}, // 0x0A
+                {{'\\', 'u', '0', '0', '0', 'b', 0, 6}}, // 0x0B
+                {{'\\', 'f', 0, 0, 0, 0, 0, 2}}, // 0x0C
+                {{'\\', 'r', 0, 0, 0, 0, 0, 2}}, // 0x0D
+                {{'\\', 'u', '0', '0', '0', 'e', 0, 6}}, // 0x0E
+                {{'\\', 'u', '0', '0', '0', 'f', 0, 6}}, // 0x0F
+                {{'\\', 'u', '0', '0', '1', '0', 0, 6}}, // 0x10
+                {{'\\', 'u', '0', '0', '1', '1', 0, 6}}, // 0x11
+                {{'\\', 'u', '0', '0', '1', '2', 0, 6}}, // 0x12
+                {{'\\', 'u', '0', '0', '1', '3', 0, 6}}, // 0x13
+                {{'\\', 'u', '0', '0', '1', '4', 0, 6}}, // 0x14
+                {{'\\', 'u', '0', '0', '1', '5', 0, 6}}, // 0x15
+                {{'\\', 'u', '0', '0', '1', '6', 0, 6}}, // 0x16
+                {{'\\', 'u', '0', '0', '1', '7', 0, 6}}, // 0x17
+                {{'\\', 'u', '0', '0', '1', '8', 0, 6}}, // 0x18
+                {{'\\', 'u', '0', '0', '1', '9', 0, 6}}, // 0x19
+                {{'\\', 'u', '0', '0', '1', 'a', 0, 6}}, // 0x1A
+                {{'\\', 'u', '0', '0', '1', 'b', 0, 6}}, // 0x1B
+                {{'\\', 'u', '0', '0', '1', 'c', 0, 6}}, // 0x1C
+                {{'\\', 'u', '0', '0', '1', 'd', 0, 6}}, // 0x1D
+                {{'\\', 'u', '0', '0', '1', 'e', 0, 6}}, // 0x1E
+                {{'\\', 'u', '0', '0', '1', 'f', 0, 6}}, // 0x1F
+                {{' ', 0, 0, 0, 0, 0, 0, 1}}, // 0x20
+                {{'!', 0, 0, 0, 0, 0, 0, 1}}, // 0x21
+                {{'\\', '"', 0, 0, 0, 0, 0, 2}}, // 0x22
+                {{'#', 0, 0, 0, 0, 0, 0, 1}}, // 0x23
+                {{'$', 0, 0, 0, 0, 0, 0, 1}}, // 0x24
+                {{'%', 0, 0, 0, 0, 0, 0, 1}}, // 0x25
+                {{'&', 0, 0, 0, 0, 0, 0, 1}}, // 0x26
+                {{'\'', 0, 0, 0, 0, 0, 0, 1}}, // 0x27
+                {{'(', 0, 0, 0, 0, 0, 0, 1}}, // 0x28
+                {{')', 0, 0, 0, 0, 0, 0, 1}}, // 0x29
+                {{'*', 0, 0, 0, 0, 0, 0, 1}}, // 0x2A
+                {{'+', 0, 0, 0, 0, 0, 0, 1}}, // 0x2B
+                {{',', 0, 0, 0, 0, 0, 0, 1}}, // 0x2C
+                {{'-', 0, 0, 0, 0, 0, 0, 1}}, // 0x2D
+                {{'.', 0, 0, 0, 0, 0, 0, 1}}, // 0x2E
+                {{'/', 0, 0, 0, 0, 0, 0, 1}}, // 0x2F
+                {{'0', 0, 0, 0, 0, 0, 0, 1}}, // 0x30
+                {{'1', 0, 0, 0, 0, 0, 0, 1}}, // 0x31
+                {{'2', 0, 0, 0, 0, 0, 0, 1}}, // 0x32
+                {{'3', 0, 0, 0, 0, 0, 0, 1}}, // 0x33
+                {{'4', 0, 0, 0, 0, 0, 0, 1}}, // 0x34
+                {{'5', 0, 0, 0, 0, 0, 0, 1}}, // 0x35
+                {{'6', 0, 0, 0, 0, 0, 0, 1}}, // 0x36
+                {{'7', 0, 0, 0, 0, 0, 0, 1}}, // 0x37
+                {{'8', 0, 0, 0, 0, 0, 0, 1}}, // 0x38
+                {{'9', 0, 0, 0, 0, 0, 0, 1}}, // 0x39
+                {{':', 0, 0, 0, 0, 0, 0, 1}}, // 0x3A
+                {{';', 0, 0, 0, 0, 0, 0, 1}}, // 0x3B
+                {{'<', 0, 0, 0, 0, 0, 0, 1}}, // 0x3C
+                {{'=', 0, 0, 0, 0, 0, 0, 1}}, // 0x3D
+                {{'>', 0, 0, 0, 0, 0, 0, 1}}, // 0x3E
+                {{'?', 0, 0, 0, 0, 0, 0, 1}}, // 0x3F
+                {{'@', 0, 0, 0, 0, 0, 0, 1}}, // 0x40
+                {{'A', 0, 0, 0, 0, 0, 0, 1}}, // 0x41
+                {{'B', 0, 0, 0, 0, 0, 0, 1}}, // 0x42
+                {{'C', 0, 0, 0, 0, 0, 0, 1}}, // 0x43
+                {{'D', 0, 0, 0, 0, 0, 0, 1}}, // 0x44
+                {{'E', 0, 0, 0, 0, 0, 0, 1}}, // 0x45
+                {{'F', 0, 0, 0, 0, 0, 0, 1}}, // 0x46
+                {{'G', 0, 0, 0, 0, 0, 0, 1}}, // 0x47
+                {{'H', 0, 0, 0, 0, 0, 0, 1}}, // 0x48
+                {{'I', 0, 0, 0, 0, 0, 0, 1}}, // 0x49
+                {{'J', 0, 0, 0, 0, 0, 0, 1}}, // 0x4A
+                {{'K', 0, 0, 0, 0, 0, 0, 1}}, // 0x4B
+                {{'L', 0, 0, 0, 0, 0, 0, 1}}, // 0x4C
+                {{'M', 0, 0, 0, 0, 0, 0, 1}}, // 0x4D
+                {{'N', 0, 0, 0, 0, 0, 0, 1}}, // 0x4E
+                {{'O', 0, 0, 0, 0, 0, 0, 1}}, // 0x4F
+                {{'P', 0, 0, 0, 0, 0, 0, 1}}, // 0x50
+                {{'Q', 0, 0, 0, 0, 0, 0, 1}}, // 0x51
+                {{'R', 0, 0, 0, 0, 0, 0, 1}}, // 0x52
+                {{'S', 0, 0, 0, 0, 0, 0, 1}}, // 0x53
+                {{'T', 0, 0, 0, 0, 0, 0, 1}}, // 0x54
+                {{'U', 0, 0, 0, 0, 0, 0, 1}}, // 0x55
+                {{'V', 0, 0, 0, 0, 0, 0, 1}}, // 0x56
+                {{'W', 0, 0, 0, 0, 0, 0, 1}}, // 0x57
+                {{'X', 0, 0, 0, 0, 0, 0, 1}}, // 0x58
+                {{'Y', 0, 0, 0, 0, 0, 0, 1}}, // 0x59
+                {{'Z', 0, 0, 0, 0, 0, 0, 1}}, // 0x5A
+                {{'[', 0, 0, 0, 0, 0, 0, 1}}, // 0x5B
+                {{'\\', '\\', 0, 0, 0, 0, 0, 2}}, // 0x5C
+                {{']', 0, 0, 0, 0, 0, 0, 1}}, // 0x5D
+                {{'^', 0, 0, 0, 0, 0, 0, 1}}, // 0x5E
+                {{'_', 0, 0, 0, 0, 0, 0, 1}}, // 0x5F
+            }
+        };
+        return escapes;
+    }
+
     /*!
      * @brief write a lowercase "\\uXXXX" escape sequence into @a string_buffer
      *

include/nlohmann/json.hpp

@@ -42,9 +42,14 @@
 // each of these, not only the forward declarations from json_fwd.hpp, so IWYU's suggestion to
 // drop them (nothing in this file otherwise names the type) would break every downstream
 // translation unit that relies on basic_json<>'s defaults actually being usable.
-#include <nlohmann/adl_serializer.hpp> // IWYU pragma: keep
+// export: client code names what these headers declare (nlohmann::json and ordered_json from
+// json_fwd.hpp, adl_serializer for specializations, json_ref behind brace initialization, the
+// NLOHMANN_DEFINE_* and NLOHMANN_JSON_NAMESPACE_* macros, and the literals at the end of this file), and include analyzers must
+// attribute it to this header, not tell clients to include the internal ones (issue #5812).
+// tests/iwyu_consumer checks this.
+#include <nlohmann/adl_serializer.hpp> // IWYU pragma: export
 #include <nlohmann/byte_container_with_subtype.hpp>
-#include <nlohmann/detail/abi_macros.hpp>
+#include <nlohmann/detail/abi_macros.hpp> // IWYU pragma: export
 #include <nlohmann/detail/conversions/from_json.hpp> // IWYU pragma: keep
 #include <nlohmann/detail/conversions/to_json.hpp> // IWYU pragma: keep
 #include <nlohmann/detail/exceptions.hpp>
@@ -61,8 +66,8 @@
 #include <nlohmann/detail/iterators/primitive_iterator.hpp>
 #include <nlohmann/detail/json_custom_base_class.hpp>
 #include <nlohmann/detail/json_pointer.hpp>
-#include <nlohmann/detail/json_ref.hpp>
-#include <nlohmann/detail/macro_scope.hpp>
+#include <nlohmann/detail/json_ref.hpp> // IWYU pragma: export
+#include <nlohmann/detail/macro_scope.hpp> // IWYU pragma: export
 #include <nlohmann/detail/meta/detected.hpp>
 #include <nlohmann/detail/string_concat.hpp>
 #include <nlohmann/detail/string_escape.hpp>
@@ -74,7 +79,7 @@
 #include <nlohmann/detail/output/serializer.hpp>
 #include <nlohmann/detail/recursion_depth_limit.hpp>
 #include <nlohmann/detail/value_t.hpp>
-#include <nlohmann/json_fwd.hpp>
+#include <nlohmann/json_fwd.hpp> // IWYU pragma: export
 #include <nlohmann/ordered_map.hpp> // IWYU pragma: keep
 #include <nlohmann/thirdparty/hedley/hedley.hpp>
 
@@ -7449,6 +7454,176 @@ public:
     }
 
   public:
+    /// @brief creates a diff as a JSON Merge Patch
+    /// @sa https://json.nlohmann.me/api/basic_json/merge_diff/
+    JSON_HEDLEY_WARN_UNUSED_RESULT
+    static basic_json merge_diff(const basic_json& source, const basic_json& target)
+    {
+        // unless both are objects, the patch is the target itself
+        if (!source.is_object() || !target.is_object())
+        {
+            return target;
+        }
+
+        return merge_diff_recursively(source, target, 0);
+    }
+
+  private:
+    /// @brief two objects @ref merge_diff_iteratively is diffing
+    struct merge_diff_frame
+    {
+        merge_diff_frame(const basic_json* source_, const basic_json* target_)
+            : source(source_), target(target_), member(source_->cbegin()), patch(value_t::object)
+        {}
+
+        // declared for GCC's -Weffc++, as in diff_frame
+        merge_diff_frame(const merge_diff_frame&) = default;
+        merge_diff_frame(merge_diff_frame&&) = default;
+        merge_diff_frame& operator=(const merge_diff_frame&) = default;
+        merge_diff_frame& operator=(merge_diff_frame&&) = default;
+        ~merge_diff_frame() = default;
+
+        /// the objects being diffed
+        const basic_json* source;
+        const basic_json* target;
+        /// the member of source being diffed; advanced once it is done
+        const_iterator member;
+        /// the patch for the members of source diffed so far
+        basic_json patch;
+    };
+
+    /// @brief add the members of object @a target that @a source does not
+    ///        have to the merge patch @a result
+    static void merge_diff_added_members(basic_json& result, const basic_json& source, const basic_json& target)
+    {
+        for (auto it = target.begin(); it != target.end(); ++it)
+        {
+            auto itf = source.find(it.key());
+            if (itf == source.end())
+            {
+                result[it.key()] = it.value();
+            }
+        }
+    }
+
+    /*!
+    @brief @ref merge_diff of two objects at nesting level @a depth
+
+    Diffing two nested objects calls this function again, once per nesting
+    level, so values nested deeply enough would exhaust the call stack and
+    terminate the process. The descent is bounded here: once @ref
+    detail::recursion_depth_limit levels have been entered, @ref
+    merge_diff_iteratively diffs what is left without the call stack.
+    */
+    static basic_json merge_diff_recursively(const basic_json& source, const basic_json& target, const std::size_t depth)
+    {
+        JSON_ASSERT(source.is_object() && target.is_object());
+
+        if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit()))
+        {
+            return merge_diff_iteratively(source, target);
+        }
+
+        basic_json result(value_t::object);
+
+        for (auto it = source.begin(); it != source.end(); ++it)
+        {
+            auto itf = target.find(it.key());
+            if (itf != target.end())
+            {
+                if (it.value().is_object() && itf.value().is_object())
+                {
+                    auto diff = merge_diff_recursively(it.value(), itf.value(), depth + 1);
+                    if (!diff.empty())
+                    {
+                        result[it.key()] = std::move(diff);
+                    }
+                }
+                else if (it.value() != itf.value())
+                {
+                    result[it.key()] = itf.value();
+                }
+            }
+            else
+            {
+                result[it.key()] = value_t::null;
+            }
+        }
+
+        merge_diff_added_members(result, source, target);
+        return result;
+    }
+
+    /*!
+    @brief @ref merge_diff of two objects without the call stack
+
+    Produces the same patch as @ref merge_diff_recursively, keeping the objects
+    being diffed on an explicit stack instead of descending into them. Two
+    nested objects are still diffed completely before the next member, so the
+    patch has its members in the same order as in the recursive version. Only
+    reached for objects nested more deeply than @ref
+    detail::recursion_depth_limit.
+    */
+    static basic_json merge_diff_iteratively(const basic_json& source, const basic_json& target)
+    {
+        // the patch of source and target, set once their frame is popped
+        basic_json result;
+        std::vector<merge_diff_frame> stack;
+        stack.emplace_back(&source, &target);
+        while (!stack.empty())
+        {
+            // the frame's members are copied out and it is changed through
+            // stack.back(): pushing a frame for nested objects or popping
+            // this one would invalidate a reference to it
+            const basic_json* const s = stack.back().source;
+            const basic_json* const t = stack.back().target;
+            const const_iterator it = stack.back().member;
+
+            if (it != s->cend())
+            {
+                const const_iterator itf = t->find(it.key());
+                if (itf != t->cend())
+                {
+                    if (it.value().is_object() && itf.value().is_object())
+                    {
+                        // member is advanced once their patch is under its key
+                        stack.emplace_back(&it.value(), &itf.value());
+                        continue;
+                    }
+                    if (it.value() != itf.value())
+                    {
+                        stack.back().patch[it.key()] = itf.value();
+                    }
+                }
+                else
+                {
+                    stack.back().patch[it.key()] = value_t::null;
+                }
+                ++stack.back().member;
+                continue;
+            }
+
+            merge_diff_added_members(stack.back().patch, *s, *t);
+
+            // these objects are done: continue with the ones they are in
+            basic_json patch = std::move(stack.back().patch);
+            stack.pop_back();
+            if (stack.empty())
+            {
+                result = std::move(patch);
+            }
+            else
+            {
+                if (!patch.empty())
+                {
+                    stack.back().patch[stack.back().member.key()] = std::move(patch);
+                }
+                ++stack.back().member;
+            }
+        }
+        return result;
+    }
+
     /// @}
 };
 
@@ -7625,7 +7800,7 @@ struct formatter<nlohmann::NLOHMANN_BASIC_JSON_TPL, char> // NOLINT(cert-dcl58-c
 // where needed.
 #ifndef JSON_NO_AUTOMATIC_UDLS
     // NOLINTNEXTLINE(misc-header-include-cycle): json_literals.hpp includes this header
-    #include <nlohmann/json_literals.hpp>
+    #include <nlohmann/json_literals.hpp> // IWYU pragma: export
 #endif
 
 #endif  // INCLUDE_NLOHMANN_JSON_HPP_

tests/iwyu_consumer/conversions.cpp

@@ -0,0 +1,62 @@
+//     __ _____ _____ _____
+//  __|  |   __|     |   | |  JSON for Modern C++ (supporting code)
+// |  |  |__   |  |  | | | |  version 3.12.0
+// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+// See values.cpp: the conversion macros and nlohmann::adl_serializer must be available through
+// <nlohmann/json.hpp> alone. NLOHMANN_JSON_SERIALIZE_ENUM (from the same header as
+// NLOHMANN_DEFINE_TYPE_INTRUSIVE) is left out: IWYU attributes the basic_json comparison in its
+// expansion to an unrelated standard header, the ambiguity described for json.hpp in cmake/ci.cmake.
+
+#include <string>
+
+#include <nlohmann/json.hpp>
+
+namespace ns
+{
+struct person
+{
+    std::string name;
+    int age;
+    NLOHMANN_DEFINE_TYPE_INTRUSIVE(person, name, age)
+};
+
+struct legacy
+{
+    int value;
+};
+} // namespace ns
+
+namespace nlohmann
+{
+template<>
+struct adl_serializer<ns::legacy>
+{
+    static void to_json(json& j, const ns::legacy& l)
+    {
+        j = l.value;
+    }
+
+    static void from_json(const json& j, ns::legacy& l)
+    {
+        l.value = j.get<int>();
+    }
+};
+} // namespace nlohmann
+
+int main()
+{
+    ns::person ada;
+    ada.name = "Ada";
+    ada.age = 36;
+    ns::legacy one;
+    one.value = 1;
+
+    const nlohmann::json j = ada;
+    const auto p = j.get<ns::person>();
+    const auto l = nlohmann::json(one).get<ns::legacy>();
+    return p.age - 36 + l.value - 1;
+}

tests/iwyu_consumer/literals.cpp

@@ -0,0 +1,19 @@
+//     __ _____ _____ _____
+//  __|  |   __|     |   | |  JSON for Modern C++ (supporting code)
+// |  |  |__   |  |  | | | |  version 3.12.0
+// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+// See values.cpp: the user-defined literals must be available through <nlohmann/json.hpp> alone.
+
+#include <nlohmann/json.hpp>
+
+using namespace nlohmann::literals; // NOLINT(google-build-using-namespace)
+
+int main()
+{
+    const auto j = R"({"a": [1, 2]})"_json;
+    return j.at("/a/1"_json_pointer).get<int>() - 2;
+}

tests/iwyu_consumer/values.cpp

@@ -0,0 +1,30 @@
+//     __ _____ _____ _____
+//  __|  |   __|     |   | |  JSON for Modern C++ (supporting code)
+// |  |  |__   |  |  | | | |  version 3.12.0
+// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+// Uses the library the way a client does, including only <nlohmann/json.hpp>. ci_single_binaries
+// runs include-what-you-use and clang-tidy's misc-include-cleaner on this file and fails if either
+// asks for another nlohmann header, which would mean json.hpp does not export it (issue #5812).
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+#include <nlohmann/json.hpp>
+
+int main()
+{
+    // brace initialization (nlohmann::json, detail::json_ref)
+    nlohmann::json j = {{"pi", 3.141}, {"list", {1, 0, 2}}};
+    const nlohmann::ordered_json o = {{"b", 1}, {"a", 2}};
+
+    j["parsed"] = nlohmann::json::parse(R"({"happy": true})");
+    j["binary"] = nlohmann::json::binary(std::vector<std::uint8_t> {0xCA, 0xFE}, 42);
+
+    const std::string s = j.dump() + o.dump();
+    return s.empty() ? 1 : 0;
+}

tests/src/unit-class_lexer.cpp

@@ -938,6 +938,10 @@ TEST_CASE("Eisel-Lemire float conversion")
             const int k = i % 64;
             const std::uint64_t x = (std::uint64_t{1} << k) | (a & ((std::uint64_t{1} << k) - 1));
             CHECK(nlohmann::detail::count_leading_zeros(x) == 63 - k);
+
+            // the mirror image: bit k set, arbitrary bits above it
+            const std::uint64_t y = (std::uint64_t{1} << k) | (k == 63 ? 0 : (a << (k + 1)));
+            CHECK(nlohmann::detail::count_trailing_zeros(y) == k);
         }
     }
 

tests/src/unit-custom-binary-type.cpp

@@ -8,6 +8,13 @@
 
 #include "doctest_compatibility.h"
 
+// capture whether JSON_DISABLE_ENUM_SERIALIZATION was enabled on the command
+// line before including json.hpp, which #undefs it; std::byte is an enum, so
+// converting std::vector<std::byte> needs enum serialization
+#if defined(JSON_DISABLE_ENUM_SERIALIZATION) && (JSON_DISABLE_ENUM_SERIALIZATION == 1)
+    #define SKIP_TESTS_FOR_ENUM_SERIALIZATION
+#endif
+
 #include <nlohmann/json.hpp>
 
 #include <cstdint>
@@ -85,3 +92,53 @@ TEST_CASE("binary type whose value type is not std::uint8_t")
     }
 #endif
 }
+
+#ifdef JSON_HAS_CPP_17
+// moved from unit-regression3.cpp, so that file does not instantiate a second
+// basic_json specialization (MinGW section limit, see windows.yml)
+TEST_CASE("std::vector<std::byte> as binary type")
+{
+    SECTION("issue #4804: from_cbor incompatible with std::vector<std::byte> as binary_t")
+    {
+        const std::vector<std::uint8_t> data = {0x80};
+        const auto decoded = byte_binary_json::from_cbor(data);
+        CHECK((decoded == byte_binary_json::array()));
+    }
+
+#ifndef SKIP_TESTS_FOR_ENUM_SERIALIZATION
+    SECTION("discussion #4209 - custom BinaryType direct assignment and round-tripping")
+    {
+        // Test that assigning a custom BinaryType directly creates a binary value, not an array
+        const std::vector<std::byte> original{std::byte{1}, std::byte{2}, std::byte{3}};
+        const byte_binary_json j = original;
+        CHECK(j.is_binary());
+        CHECK(!j.is_array());
+
+        // Test round-tripping: extracting the binary value back as the custom container type
+        const auto extracted = j.get<std::vector<std::byte>>();
+        CHECK(extracted == original);
+
+        // Test that the default json alias behavior is unchanged: std::vector<uint8_t> -> array
+        const nlohmann::json default_json = std::vector<std::uint8_t> {1, 2, 3};
+        CHECK(default_json.is_array());
+        CHECK(!default_json.is_binary());
+    }
+
+    SECTION("discussion #4209 - custom BinaryType extraction from parsed array")
+    {
+        // Test that extracting a custom BinaryType from a parsed JSON array still works
+        // (not just from a binary-typed node)
+        const auto j = byte_binary_json::parse("[1,2,3]");
+        CHECK(j.is_array());
+        CHECK(!j.is_binary());
+
+        // Extracting as custom BinaryType should work from arrays
+        const auto extracted = j.get<std::vector<std::byte>>();
+        CHECK(extracted.size() == 3);
+        CHECK(extracted[0] == std::byte{1});
+        CHECK(extracted[1] == std::byte{2});
+        CHECK(extracted[2] == std::byte{3});
+    }
+#endif
+}
+#endif

tests/src/unit-deserialization.cpp

@@ -35,6 +35,7 @@ using nlohmann::json;
 #include <sstream>
 #include <string>
 #include <valarray>
+#include <vector>
 
 #include "sax_event_loggers.hpp"
 
@@ -1250,3 +1251,139 @@ TEST_CASE_TEMPLATE("deserialization of different character types (UTF-32)", T, c
     CHECK(json::sax_parse(v, &l));
     CHECK(l.events.size() == 1);
 }
+
+namespace
+{
+// records the position and message of a parse error; reporting the error via
+// SAX does not throw, so the test also runs with JSON_NOEXCEPTION
+struct SaxErrorRecorder : public SaxEventLogger
+{
+    bool parse_error(std::size_t position, const std::string& /*last_token*/, const json::exception& ex) override
+    {
+        error = std::to_string(position) + " " + ex.what();
+        return false;
+    }
+
+    std::string error {}; // NOLINT(readability-redundant-member-init)
+};
+} // namespace
+
+TEST_CASE("parse error positions are the same for contiguous and stream input")
+{
+    // contiguous input derives the position from the input on error, stream
+    // input counts it while reading (see lexer::lazy_position); the cases
+    // cover newlines before the error, ungetting a newline after a number,
+    // newlines inside a string, a comment, and reads at the end of input
+    struct test_case
+    {
+        std::string input;
+        std::size_t byte;
+        std::string message;
+    };
+    const std::vector<test_case> cases =
+    {
+        {"[\n  1,\n  2\n", 12, "[json.exception.parse_error.101] parse error at line 4, column 1: syntax error while parsing array - unexpected end of input; expected ']'"},
+        {"[1\n,2,\n]", 8, "[json.exception.parse_error.101] parse error at line 3, column 1: syntax error while parsing value - unexpected ']'; expected '[', '{', or a literal"},
+        {"[1\n x]", 5, "[json.exception.parse_error.101] parse error at line 2, column 2: syntax error while parsing array - invalid literal; last read: '1<U+000A> x'; expected ']'"},
+        {"1\n2", 3, "[json.exception.parse_error.101] parse error at line 2, column 1: syntax error while parsing value - unexpected number literal; expected end of input"},
+        {
+            "{\"a\":\n\"b\nc\"}", 9, R"([json.exception.parse_error.101] parse error at line 3, column 0: syntax error while parsing value - invalid string: control character U+000A (LF) must be escaped to \u000A or \n; last read: '"b<U+000A>')"
+        },
+        {"/* a\n b */\n x", 13, "[json.exception.parse_error.101] parse error at line 3, column 2: syntax error while parsing value - invalid literal; last read: '/* a<U+000A> b */<U+000A> x'"},
+        {"\n\n", 3, "[json.exception.parse_error.101] parse error at line 3, column 1: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal"},
+        {"[\"a\",\n 1e\n]", 10, "[json.exception.parse_error.101] parse error at line 3, column 0: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '1e<U+000A>'"},
+        {"[tru\n]", 5, "[json.exception.parse_error.101] parse error at line 2, column 0: syntax error while parsing value - invalid literal; last read: '[tru<U+000A>'"},
+    };
+
+    for (const auto& c : cases)
+    {
+        CAPTURE(c.input)
+        const std::string expected = std::to_string(c.byte) + " " + c.message;
+
+        SaxErrorRecorder string_sax;
+        CHECK_FALSE(json::sax_parse(c.input, &string_sax, json::input_format_t::json, true, true));
+        CHECK(string_sax.error == expected);
+
+        SaxErrorRecorder pointer_sax;
+        CHECK_FALSE(json::sax_parse(c.input.data(), c.input.data() + c.input.size(), &pointer_sax, json::input_format_t::json, true, true));
+        CHECK(pointer_sax.error == expected);
+
+        std::istringstream is(c.input);
+        SaxErrorRecorder stream_sax;
+        CHECK_FALSE(json::sax_parse(is, &stream_sax, json::input_format_t::json, true, true));
+        CHECK(stream_sax.error == expected);
+    }
+}
+
+#if !defined(JSON_NOEXCEPTION)
+TEST_CASE("indentation runs of every length around the 8-byte skip")
+{
+    // contiguous input skips the spaces after a newline 8 bytes at a time (see
+    // lexer::skip_indentation), stream input reads them one by one; results,
+    // byte offsets, and messages must be the same for runs that end anywhere
+    // relative to a word boundary, including at the end of the input
+    const auto parse_result = [](const std::string & input, bool contiguous)
+    {
+        try
+        {
+            if (contiguous)
+            {
+                return json::parse(input.data(), input.data() + input.size(), nullptr, true, true).dump();
+            }
+            std::istringstream is(input);
+            return json::parse(is, nullptr, true, true).dump();
+        }
+        catch (const json::parse_error& e)
+        {
+            return std::to_string(e.byte).append(" ").append(e.what());
+        }
+    };
+
+    // replace every '~' in a pattern by an indentation run
+    const auto indented = [](const std::string & pattern, const std::string & indent)
+    {
+        std::string result;
+        for (const char c : pattern)
+        {
+            if (c == '~')
+            {
+                result += indent;
+            }
+            else
+            {
+                result += c;
+            }
+        }
+        return result;
+    };
+
+    const std::vector<std::string> patterns =
+    {
+        "[\n~1,\n~2\n]",
+        "[\n~1,\n~x]",
+        "[\n~1,\n~",
+        "1\n~",
+        "[\n~\t1,\n~\r\n~2]",
+        "[\n~/* c */ 1]",
+        "[\n~1\n~2]",
+    };
+
+    for (std::size_t n = 0; n <= 33; ++n)
+    {
+        const std::string indent(n, ' ');
+        for (const auto& pattern : patterns)
+        {
+            const std::string input = indented(pattern, indent);
+            CAPTURE(n)
+            CAPTURE(input)
+            const std::string expected = parse_result(input, false);
+            CHECK(parse_result(input, true) == expected);
+            CHECK(json::accept(input.data(), input.data() + input.size(), true) == json::accept(input, true));
+        }
+    }
+
+    // and against fixed expectations, in case both paths broke the same way
+    CHECK(parse_result(indented("[\n~x]", std::string(16, ' ')), true) == "19 [json.exception.parse_error.101] parse error at line 2, column 17: syntax error while parsing value - invalid literal; last read: '[<U+000A>                x'");
+    CHECK(parse_result(indented("[\n~1]", std::string(17, ' ')), true) == "[1]");
+}
+#endif

tests/src/unit-diagnostics.cpp

@@ -567,9 +567,9 @@ TEST_CASE("Regression tests for extended diagnostics")
     }
 }
 
-TEST_CASE("Better diagnostics past the descent bound of update() and merge_patch()")
+TEST_CASE("Better diagnostics past the descent bound of update(), merge_patch() and merge_diff()")
 {
-    // Both merge objects nested more than detail::recursion_depth_limit()
+    // All three handle objects nested more than detail::recursion_depth_limit()
     // (128) levels deep without recursing; the values they add or replace
     // there must still know their parents.
     // The values are built rather than parsed, so that the expected messages
@@ -615,4 +615,19 @@ TEST_CASE("Better diagnostics past the descent bound of update() and merge_patch
         CHECK_THROWS_WITH_AS(p->at("x").at(0), expected_x.c_str(), json::type_error);
         CHECK_THROWS_WITH_AS(p->at("y").at(0), expected_y.c_str(), json::type_error);
     }
+
+    SECTION("merge_diff()")
+    {
+        // target and patch differ only in their innermost members, so the
+        // diff of the two is nested as deeply: {"a": ... {"x": null, "y": 2}}
+        const json diff = json::merge_diff(target, patch); // NOLINT(readability-suspicious-call-argument)
+
+        const json* p = &diff;
+        for (std::size_t i = 0; i < depth; ++i)
+        {
+            p = &p->at("a");
+        }
+        CHECK(p->at("x").is_null());
+        CHECK_THROWS_WITH_AS(p->at("y").at(0), expected_y.c_str(), json::type_error);
+    }
 }

tests/src/unit-regression3.cpp

@@ -58,11 +58,6 @@ using ordered_json = nlohmann::ordered_json;
         #include <optional>
     #elif __has_include(<experimental/optional>)
     #endif
-
-    /////////////////////////////////////////////////////////////////////
-    // for #4804
-    /////////////////////////////////////////////////////////////////////
-    using json_4804 = nlohmann::json::with_binary_t<std::vector<std::byte>>;
 #endif
 
 #ifdef JSON_HAS_CPP_20
@@ -458,49 +453,6 @@ TEST_CASE("regression tests 3")
 #endif
 
 #ifdef JSON_HAS_CPP_17
-    SECTION("issue #4804: from_cbor incompatible with std::vector<std::byte> as binary_t")
-    {
-        const std::vector<std::uint8_t> data = {0x80};
-        const auto decoded = json_4804::from_cbor(data);
-        CHECK((decoded == json_4804::array()));
-    }
-
-#ifndef SKIP_TESTS_FOR_ENUM_SERIALIZATION
-    SECTION("discussion #4209 - custom BinaryType direct assignment and round-tripping")
-    {
-        // Test that assigning a custom BinaryType directly creates a binary value, not an array
-        const std::vector<std::byte> original{std::byte{1}, std::byte{2}, std::byte{3}};
-        const json_4804 j = original;
-        CHECK(j.is_binary());
-        CHECK(!j.is_array());
-
-        // Test round-tripping: extracting the binary value back as the custom container type
-        const auto extracted = j.get<std::vector<std::byte>>();
-        CHECK(extracted == original);
-
-        // Test that the default json alias behavior is unchanged: std::vector<uint8_t> -> array
-        const json default_json = std::vector<std::uint8_t> {1, 2, 3};
-        CHECK(default_json.is_array());
-        CHECK(!default_json.is_binary());
-    }
-
-    SECTION("discussion #4209 - custom BinaryType extraction from parsed array")
-    {
-        // Test that extracting a custom BinaryType from a parsed JSON array still works
-        // (not just from a binary-typed node)
-        const auto j = json_4804::parse("[1,2,3]");
-        CHECK(j.is_array());
-        CHECK(!j.is_binary());
-
-        // Extracting as custom BinaryType should work from arrays
-        const auto extracted = j.get<std::vector<std::byte>>();
-        CHECK(extracted.size() == 3);
-        CHECK(extracted[0] == std::byte{1});
-        CHECK(extracted[1] == std::byte{2});
-        CHECK(extracted[2] == std::byte{3});
-    }
-#endif
-
     SECTION("issue #5046 - implicit conversion of return json to std::optional no longer implicit")
     {
         const json jval{};

tests/src/unit-serialization.cpp

@@ -923,66 +923,68 @@ TEST_CASE("serialization boundary values for the string buffer")
     // mid-string once fewer than 13 bytes remain (`string_buffer.size() - bytes
     // < 13`), 13 being one more than the most a single code point can ever
     // write at once (a surrogate pair: two back-to-back "\uXXXX" escapes, 12
-    // bytes). Every write into string_buffer that this check protects happens
-    // in steps of 2 (a simple "\\x" escape) or 6 (one "\uXXXX" unit), so
-    // `bytes` only ever takes even values at the point the check runs - the
-    // tightest values actually reachable are therefore 498 (512 - 498 == 14,
-    // one simple escape away from the threshold) and 500 (512 - 500 == 12,
-    // where the flush fires immediately and resets bytes to 0).
+    // bytes). Characters below 0x60 (control characters, '"' and '\\') are
+    // escaped straight into the write buffer and never reach string_buffer, so
+    // under ensure_ascii it fills in steps of 6 (one "\uXXXX" unit for a
+    // character below U+10000) or 12 (a surrogate pair). With 2-byte characters
+    // such as U+00E4, `bytes` takes multiples of 6 at the point the check runs;
+    // the tightest values reachable are therefore 498 (512 - 498 == 14, after
+    // 83 characters) and 504 (512 - 504 == 8, after 84 characters, where the
+    // flush fires immediately and resets bytes to 0).
 
-    SECTION("a run of 2-byte escapes lands bytes on, and one step past, the flush threshold")
+    SECTION("a run of 6-byte escapes lands bytes on, and one step past, the flush threshold")
     {
         for (const int count :
                 {
-                    249, 250, 251
+                    83, 84, 85
                 })
         {
             CAPTURE(count)
-            const json j = std::string(static_cast<std::size_t>(count), '\n');
+            std::string umlauts;
             std::string expected = "\"";
             for (int i = 0; i < count; ++i)
             {
-                expected += "\\n";
+                umlauts += "\xC3\xA4";
+                expected += "\\u00e4";
             }
             expected += '"';
-            CHECK(j.dump() == expected);
+            CHECK(json(umlauts).dump(-1, ' ', true) == expected);
         }
     }
 
-    SECTION("an ASCII prefix leaves the tightest reachable margin before a 12-byte surrogate pair")
+    SECTION("a prefix leaves the tightest reachable margin before a 12-byte surrogate pair")
     {
         // U+1F600 (the "\xF0\x9F\x98\x80" UTF-8 bytes) is dumped under
         // ensure_ascii as the 12-byte surrogate pair "\ud83d\ude00"; that
         // write happens in a single step with no intermediate flush check, so
         // it is the write most exposed by an off-by-one in the "< 13" guard.
-        // A prefix of 249 newlines leaves exactly 14 bytes of headroom
+        // A prefix of 83 U+00E4 characters leaves exactly 14 bytes of headroom
         // (512 - 498), the smallest margin the guard ever actually allows
-        // into a new code point; 250 newlines instead trigger the guard's own
+        // into a new code point; 84 characters instead trigger the guard's own
         // flush first, so the emoji starts from a freshly emptied (512-byte)
-        // buffer, and 251 repeats that with one more escape already past the
+        // buffer, and 85 repeats that with one more escape already past the
         // reset. Together they cover the margin the guard allows landing on,
         // one step before, and one step after - all must still produce the
         // identical, correct escapes.
         for (const int prefix_count :
                 {
-                    249, 250, 251
+                    83, 84, 85
                 })
         {
             CAPTURE(prefix_count)
-            const std::string prefix(static_cast<std::size_t>(prefix_count), '\n');
-            const std::string emoji = "\xF0\x9F\x98\x80";
-            const json j = prefix + emoji;
-
+            std::string prefix;
             std::string expected_prefix;
             for (int i = 0; i < prefix_count; ++i)
             {
-                expected_prefix += "\\n";
+                prefix += "\xC3\xA4";
+                expected_prefix += "\\u00e4";
             }
+            const std::string emoji = "\xF0\x9F\x98\x80";
+            const json j = prefix + emoji;
 
-            // newline escaping does not depend on ensure_ascii: only the
-            // emoji differs (raw UTF-8 bytes vs. a \u-escaped surrogate pair)
+            // without ensure_ascii, the whole string is copied unchanged
             std::string expected_raw = "\"";
-            expected_raw += expected_prefix;
+            expected_raw += prefix;
             expected_raw += emoji;
             expected_raw += '"';
             std::string expected_ascii = "\"";
@@ -1027,3 +1029,132 @@ TEST_CASE("serialization boundary values for the string buffer")
         }
     }
 }
+
+namespace
+{
+// reference escaper for ASCII strings, written independently of the table in
+// serializer::ascii_escapes()
+std::string reference_escape_ascii(const std::string& s, const bool ensure_ascii)
+{
+    std::string result = "\"";
+    for (const char c : s)
+    {
+        const auto byte = static_cast<unsigned char>(c);
+        switch (byte)
+        {
+            case '\b':
+                result += "\\b";
+                break;
+            case '\t':
+                result += "\\t";
+                break;
+            case '\n':
+                result += "\\n";
+                break;
+            case '\f':
+                result += "\\f";
+                break;
+            case '\r':
+                result += "\\r";
+                break;
+            case '"':
+                result += "\\\"";
+                break;
+            case '\\':
+                result += "\\\\";
+                break;
+            default:
+                if (byte < 0x20 || (ensure_ascii && byte == 0x7F))
+                {
+                    // byte < 0x80, so the first two hex digits are 0
+                    const char* const hex = "0123456789abcdef";
+                    result += "\\u00";
+                    result += hex[byte >> 4U];
+                    result += hex[byte & 0x0FU];
+                }
+                else
+                {
+                    result += c;
+                }
+                break;
+        }
+    }
+    result += '"';
+    return result;
+}
+} // namespace
+
+TEST_CASE("characters below 0x60 are escaped from a table straight into the write buffer")
+{
+    SECTION("every ASCII character at every position")
+    {
+        // lengths up to 17 cover the 8-byte strides of the bulk scanner on
+        // both sides of the character, so the scanner stops at it from every
+        // offset
+        for (std::size_t length = 1; length <= 17; ++length)
+        {
+            for (std::size_t pos = 0; pos < length; ++pos)
+            {
+                for (unsigned int c = 0; c < 0x80; ++c)
+                {
+                    std::string s(length, 'a');
+                    s[pos] = static_cast<char>(c);
+                    CAPTURE(length)
+                    CAPTURE(pos)
+                    CAPTURE(c)
+                    const json j = s;
+                    CHECK(j.dump(-1, ' ', false) == reference_escape_ascii(s, false));
+                    CHECK(j.dump(-1, ' ', true) == reference_escape_ascii(s, true));
+                }
+            }
+        }
+    }
+
+    SECTION("runs of escaped characters")
+    {
+        std::string s;
+        for (unsigned int c = 0; c < 0x20; ++c)
+        {
+            s += static_cast<char>(c);
+        }
+        s += R"("\"\)";
+        CHECK(json(s).dump() == reference_escape_ascii(s, false));
+        CHECK(json(s).dump(-1, ' ', true) == reference_escape_ascii(s, true));
+    }
+
+    SECTION("escapes straddle the end of the write buffer")
+    {
+        // the 8-byte store of a table entry needs room for all 8 bytes, even
+        // when the escape itself is shorter; a prefix of k plain characters
+        // shifts the 6-byte escapes across every offset of the 1024-byte
+        // write buffer
+        for (std::size_t k = 0; k < 8; ++k)
+        {
+            CAPTURE(k)
+            const std::string s = std::string(k, 'a') + std::string(400, '\x01');
+            CHECK(json(s).dump() == reference_escape_ascii(s, false));
+            CHECK(json::array({s, s}).dump() == "[" + reference_escape_ascii(s, false) + "," + reference_escape_ascii(s, false) + "]");
+        }
+    }
+
+    SECTION("pending \\u escapes of non-ASCII characters are written first")
+    {
+        // under ensure_ascii, U+00E4 is \u-escaped into the string buffer;
+        // a following escape from the table must not overtake it
+        const json j = "\xC3\xA4\n\xC3\xA4\"\xF0\x9F\x98\x80\\";
+        CHECK(j.dump(-1, ' ', true) == R"("\u00e4\n\u00e4\"\ud83d\ude00\\")");
+        CHECK(j.dump(-1, ' ', false) == "\"\xC3\xA4\\n\xC3\xA4\\\"\xF0\x9F\x98\x80\\\\\"");
+    }
+
+    SECTION("escapes after invalid UTF-8 for every error handler")
+    {
+        const json j = "a\xFF\n\xC3\"";
+        CHECK_THROWS_WITH_AS(utils::ignore_return_value(j.dump()), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0xFF", json::type_error&);
+        CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == R"("a\n\"")");
+        CHECK(j.dump(-1, ' ', true, json::error_handler_t::ignore) == R"("a\n\"")");
+        CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"a\xEF\xBF\xBD\\n\xEF\xBF\xBD\\\"\"");
+        CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == R"("a\ufffd\n\ufffd\"")");
+        CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"a\xFF\\n\xC3\\\"\"");
+        CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"a\xFF\\n\xC3\\\"\"");
+    }
+}