Keep only the beginning and the end of a long token for error messages

For "last read" in parse errors, the lexer keeps the text of the token it
reads. The token start moves only at strings and numbers, so everything
after a string or number belongs to it until the next one starts. Input
that cannot be read again copies that text while parsing (the istream and
FILE* adapters in their carry buffer, the lexer in token_string for
iterators), so valid input of 1 followed by 100,000,000 newlines peaked
at about 225 MB, and a long string was kept a second time.

Replace the carry buffer and token_string by token_text_buffer, which
keeps the first 4096 and at least the last 4112 characters of a token and
drops the characters in between where it would otherwise grow, so
appending stays as cheap as before. get_token_string() shows a token of
more than 8192 characters as its first and last 4096 characters around
"...", the same for every kind of input; contiguous input copies only
these parts.

The example input now peaks at 1.8 MB. Instruction counts of parsing
from std::string, std::istream and FILE* are unchanged (clang 21 and
GCC 15); parsing from std::list<char> needs 1 to 17 % fewer.

Fixes #6120.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

.github/workflows/ubuntu.yml

@@ -25,7 +25,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -38,7 +38,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Check that Meson and pkg-config offer the CMake options
         run: make check_build_options
       - name: Install Meson
@@ -92,7 +92,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -123,7 +123,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -142,7 +142,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -168,7 +168,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -244,7 +244,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: CXX=g++-${{ matrix.compiler }} cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -262,7 +262,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -279,7 +279,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Disable filesystem support (std::filesystem is unusable) for clang 7 / 8 / 9 / 10
         run: echo "JSON_FORCED_GLOBAL_COMPILE_OPTIONS=-DJSON_HAS_FILESYSTEM=0;-DJSON_HAS_EXPERIMENTAL_FILESYSTEM=0" >> "$GITHUB_ENV"
         if: ${{ matrix.compiler == '7' || matrix.compiler == '8' || matrix.compiler == '9' || matrix.compiler == '10' }}
@@ -299,7 +299,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -319,7 +319,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build with libc++
@@ -346,7 +346,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -362,11 +362,13 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
-      # The module test uses `import std;`, which needs CMake 4.5 or newer.
-      - name: Get CMake 4.5+ and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+      # The module test uses `import std;`, which needs CMake's experimental
+      # import-std support. Its opt-in token is CMake-version-specific, so pin
+      # CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt.
+      - name: Get pinned CMake and ninja
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
         with:
-          cmakeVersion: '^4.5.0'
+          cmakeVersion: 4.3.4
       # Clang: the std library module is provided by libc++ (the image's libstdc++
       # ships none), and the image's libc++ module manifest has a broken relative
       # path — repoint it at the real module sources.
@@ -390,7 +392,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -407,7 +409,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -421,7 +423,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
@@ -441,7 +443,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Run CMake
         run: cmake -S . -B build -DCMAKE_TOOLCHAIN_FILE=$EMSDK/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -GNinja
       - name: Build
@@ -482,7 +484,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Check that the job runs natively on arm64
         run: uname -m && test "$(uname -m)" = aarch64
       - name: Run CMake

.github/workflows/windows.yml

@@ -90,7 +90,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Get latest CMake and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
       - name: Set extra CXX_FLAGS for latest std_version
         # /wd5285 silences C5285 emitted by the bundled third-party doctest.h, which
         # specializes std::tuple (newly diagnosed by the VS2026 v145 toolset)
@@ -212,11 +212,13 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
-      # The module test uses `import std;`, which needs CMake 4.5 or newer.
-      - name: Get CMake 4.5+ and ninja
-        uses: lukka/get-cmake@16e8153a8b4918849d92aad131befe67a3705daa # v4.5.0
+      # The module test uses `import std;`, which needs CMake's experimental
+      # import-std support. Its opt-in token is CMake-version-specific, so pin
+      # CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt.
+      - name: Get pinned CMake and ninja
+        uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5
         with:
-          cmakeVersion: '^4.5.0'
+          cmakeVersion: 4.3.4
       # CMAKE_CXX_FLAGS replaces CMake's MSVC defaults (which include /EHsc), and
       # ci_module_cpp20 forwards it to the module build, so list /EHsc explicitly
       - name: Run CMake (Debug)

README.md

@@ -165,8 +165,7 @@ std::ifstream f("example.json");
 json data = json::parse(f);
 ```
 
-If using modules (enabled with `NLOHMANN_JSON_BUILD_MODULES`; `import std;` needs CMake 4.5 or newer, see the
-[modules documentation](https://json.nlohmann.me/features/modules/)), this example becomes:
+If using modules (enabled with `NLOHMANN_JSON_BUILD_MODULES`), this example becomes:
 ```cpp
 import std;
 import nlohmann.json;

docs/mkdocs/docs/features/modules.md

@@ -9,19 +9,7 @@ Importing the module instead of including the header reduces the compile time of
 ## Requirements
 The `nlohmann.json` module requires that the build system is configured to build and resolve modules when imported. Obviously, as modules were introduced in C++20, this feature can only be used in C++20 and subsequent versions.
 
-To enable building the `nlohmann.json` module (which is not done by default), set the CMake option
-[`NLOHMANN_JSON_BUILD_MODULES`](../integration/cmake.md#nlohmann_json_build_modules) and link the
-`nlohmann_json_modules` target.
-
-With CMake, building the module needs:
-
-- CMake 3.28 or newer, and
-- a generator that scans for modules: Ninja (1.11 or newer), Ninja Multi-Config, or Visual Studio 2022 or newer.
-
-The example below also uses `import std;`. CMake supports it without experimental opt-in since version 4.5, and then
-only with the Ninja generators and these toolchains: Clang 18.1.2 or newer (with libc++ or libstdc++), MSVC toolset 14.36
-(Visual Studio 17.6) or newer, and GCC 15 or newer (GCC 16 on macOS). With older CMake versions, `import std;` needs
-CMake's version-specific experimental opt-in.
+To enable building the `nlohmann.json` module (which is not done by default), the macro `NLOHMANN_JSON_BUILD_MODULES` must be passed to the build system.
 
 ## Example
 When using modules rather than headers, the previous example for creating a `json` object through a JSON file, would instead be:
@@ -79,7 +67,7 @@ and should not be used directly:
 
 ## Known issues
 
-C++20 modules support is exercised in CI with CMake 4.5 and `import std;` against current GCC and Clang on Ubuntu, and the default MSVC toolset on Windows Server 2022 — there is no documented minimum compiler version, unlike feature-test-macro-gated features such as [`JSON_HAS_RANGES`](../api/macros/json_has_ranges.md).
+C++20 modules support is exercised in CI against current GCC and Clang on Ubuntu, and the default MSVC toolset on Windows Server 2022 — there is no documented minimum compiler version, unlike feature-test-macro-gated features such as [`JSON_HAS_RANGES`](../api/macros/json_has_ranges.md).
 
 !!! info "Known compiler issues"
 

docs/mkdocs/docs/home/exceptions.md

@@ -140,6 +140,8 @@ Exceptions have ids 1xx.
 
 This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member `byte` indicates the error position.
 
+The message ends with the last read token (`last read: '...'`), with control characters written as `<U+XXXX>`. A token of more than 8192 characters is shown by its first and its last 4096 characters, separated by `...`.
+
 !!! failure "Example message"
 
     Input ended prematurely:

docs/mkdocs/docs/integration/cmake.md

@@ -269,9 +269,6 @@ target_link_libraries(myproject PRIVATE nlohmann_json_modules)
 target_compile_definitions(myproject PRIVATE NLOHMANN_JSON_BUILD_MODULES)
 ```
 
-If `main.cpp` also uses `import std;`, the project needs CMake 4.5 or newer and a Ninja generator; see
-[Modules](../features/modules.md#requirements) for the supported toolchains.
-
 ## See also
 
 - [Macros](../api/macros/index.md) - reference of all macros

include/nlohmann/detail/input/input_adapters.hpp

@@ -110,87 +110,164 @@ inline const char* empty_input_message() noexcept
     return "attempting to parse an empty input; check that your input string or stream contains the expected JSON";
 }
 
-////////////////////
-// input adapters //
-////////////////////
-
-#ifndef JSON_NO_IO
 /*!
+@brief the characters of the last read token, kept for error messages
+
 A growable array of characters for the part of a token that an input adapter
-copied out of its window: unlike std::vector<char>::push_back(), appending a
-character takes no reference to a local, and is a few instructions when there
-is room.
+copied out of its window, or that the lexer copies while it scans input it
+cannot read again: unlike std::vector::push_back(), appending a character
+takes no reference to a local, and is a few instructions when there is room.
+
+The token is only needed for error messages, which show its first head()
+characters and its last head() characters (see lexer::get_token_string). The
+buffer therefore keeps the first head() and at least the last tail()
+characters: when it is full and holds more than limit() characters, it drops
+the characters in between instead of growing. That happens where it would
+otherwise reallocate, so appending stays as cheap as before, and a long string
+or a long run of whitespace read from a stream is not kept a second time.
 */
-class token_carry_buffer
+template<typename CharType>
+class token_text_buffer
 {
   public:
-    token_carry_buffer() = default;
-    token_carry_buffer(const token_carry_buffer&) = delete;
-    token_carry_buffer& operator=(const token_carry_buffer&) = delete;
-    token_carry_buffer(token_carry_buffer&& rhs) noexcept
+    static constexpr std::size_t head() noexcept
+    {
+        return 4096;
+    }
+    /// a little more than head(), as the last character may be handed back
+    static constexpr std::size_t tail() noexcept
+    {
+        return head() + 16;
+    }
+    static constexpr std::size_t limit() noexcept
+    {
+        return head() + (2 * tail());
+    }
+
+    token_text_buffer() = default;
+    token_text_buffer(const token_text_buffer&) = delete;
+    token_text_buffer& operator=(const token_text_buffer&) = delete;
+    token_text_buffer(token_text_buffer&& rhs) noexcept
         : m_data(std::move(rhs.m_data)), m_end(rhs.m_end), m_capacity_end(rhs.m_capacity_end)
     {
         rhs.m_end = rhs.m_capacity_end = nullptr;
     }
-    token_carry_buffer& operator=(token_carry_buffer&&) = delete;
-    ~token_carry_buffer() = default;
+    token_text_buffer& operator=(token_text_buffer&& rhs) noexcept
+    {
+        m_data = std::move(rhs.m_data);
+        m_end = rhs.m_end;
+        m_capacity_end = rhs.m_capacity_end;
+        rhs.m_end = rhs.m_capacity_end = nullptr;
+        return *this;
+    }
+    ~token_text_buffer() = default;
 
     void clear() noexcept
     {
         m_end = m_data.get();
     }
 
-    void push_back(char c)
+    bool empty() const noexcept
+    {
+        return m_end == m_data.get();
+    }
+
+    void push_back(CharType c)
     {
         if (JSON_HEDLEY_UNLIKELY(m_end == m_capacity_end))
         {
-            grow(1);
+            make_room(1);
         }
         *m_end++ = c;
     }
 
-    void append(const char* first, const char* last)
+    /// drop the last character
+    void pop_back() noexcept
+    {
+        JSON_ASSERT(!empty());
+        --m_end;
+    }
+
+    void append(const CharType* first, const CharType* last)
     {
         const auto n = static_cast<std::size_t>(last - first);
         const auto room = static_cast<std::size_t>(m_capacity_end - m_end);
-        if (n > room)
+        if (JSON_HEDLEY_UNLIKELY(n > room))
         {
-            grow(n);
+            append_long(first, n);
+            return;
         }
         if (n != 0)
         {
-            std::memcpy(m_end, first, n);
+            std::memcpy(m_end, first, n * sizeof(CharType));
         }
         m_end += n;
     }
 
     /// replace the contents of @a out
-    void copy_to(std::vector<char>& out) const
+    void copy_to(std::vector<CharType>& out) const
     {
         out.assign(m_data.get(), m_end);
     }
 
   private:
-    /// make room for @a extra more characters
-    JSON_HEDLEY_NEVER_INLINE void grow(std::size_t extra)
+    /// append @a n characters that do not fit, in pieces of at most tail()
+    /// characters, so that dropping the middle keeps the last tail() of them
+    JSON_HEDLEY_NEVER_INLINE void append_long(const CharType* first, std::size_t n)
     {
-        const auto size = static_cast<std::size_t>(m_end - m_data.get());
-        const auto capacity = static_cast<std::size_t>(m_capacity_end - m_data.get());
-        const std::size_t new_capacity = (std::max)(capacity < 32 ? static_cast<std::size_t>(32) : 2 * capacity, size + extra);
-        std::unique_ptr<char[]> data(new char[new_capacity]); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
-        if (size != 0)
+        while (n != 0)
         {
-            std::memcpy(data.get(), m_data.get(), size);
+            const std::size_t piece = n < tail() ? n : tail();
+            make_room(piece);
+            std::memcpy(m_end, first, piece * sizeof(CharType));
+            m_end += piece;
+            first += piece;
+            n -= piece;
         }
-        m_data = std::move(data);
-        m_end = m_data.get() + size;
-        m_capacity_end = m_data.get() + new_capacity;
     }
 
-    std::unique_ptr<char[]> m_data{}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,readability-redundant-member-init)
-    char* m_end = nullptr;
-    char* m_capacity_end = nullptr;
+    /// make room for @a extra (at most tail()) more characters: drop the
+    /// middle of a long token instead of growing beyond limit()
+    JSON_HEDLEY_NEVER_INLINE void make_room(std::size_t extra)
+    {
+        // nothing is allocated before the first character
+        std::size_t size = m_data ? static_cast<std::size_t>(m_end - m_data.get()) : 0;
+        if (size + extra > limit() && size > head() + tail())
+        {
+            std::memmove(m_data.get() + head(), m_end - tail(), tail() * sizeof(CharType));
+            m_end = m_data.get() + head() + tail();
+            size = head() + tail();
+        }
+        const std::size_t capacity = m_data ? static_cast<std::size_t>(m_capacity_end - m_data.get()) : 0;
+        if (size + extra > capacity)
+        {
+            // never more than limit(): from there on, the middle is dropped
+            const std::size_t doubled = capacity < 32 ? static_cast<std::size_t>(32) : 2 * capacity;
+            const std::size_t new_capacity = (std::max)(doubled < limit() ? doubled : limit(), size + extra);
+            std::unique_ptr<CharType[]> data(new CharType[new_capacity]); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
+            if (size != 0)
+            {
+                std::memcpy(data.get(), m_data.get(), size * sizeof(CharType));
+            }
+            m_data = std::move(data);
+            m_end = m_data.get() + size;
+            m_capacity_end = m_data.get() + new_capacity;
+        }
+    }
+
+    std::unique_ptr<CharType[]> m_data{}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,readability-redundant-member-init)
+    CharType* m_end = nullptr;
+    CharType* m_capacity_end = nullptr;
 };
+
+/// the part of a token that an input adapter copied out of its window
+using token_carry_buffer = token_text_buffer<char>;
+
+////////////////////
+// input adapters //
+////////////////////
+
+#ifndef JSON_NO_IO
 /*!
 Input adapter for stdio file access. The file is locked for the lifetime of the
 adapter and read with the unlocked stdio functions (see

include/nlohmann/detail/input/lexer.hpp

@@ -494,7 +494,10 @@ class lexer : public lexer_base<BasicJsonType>
     /// streaming adapter: capture the characters
     void bulk_track_token(const char* data, std::size_t n, std::false_type /*lazy*/)
     {
-        token_string.insert(token_string.end(), data, data + n);
+        for (std::size_t i = 0; i < n; ++i)
+        {
+            token_string.push_back(static_cast<char_type>(data[i]));
+        }
     }
 
     /// streaming input: no bulk fast path
@@ -2237,15 +2240,28 @@ scan_number_done:
         const std::size_t stop = ia.get_consumed_count() - (pending_real_unget ? 1u : 0u);
         if (JSON_HEDLEY_LIKELY(stop >= token_string_start))
         {
-            ia.copy_consumed_range(token_string_start, stop, out);
+            // only the beginning and the end of a long token are shown (see
+            // get_token_string), so copy only these
+            constexpr std::size_t head = token_text_buffer<char_type>::head();
+            constexpr std::size_t tail = token_text_buffer<char_type>::tail();
+            if (stop - token_string_start > head + tail)
+            {
+                ia.copy_consumed_range(token_string_start, token_string_start + head, out);
+                ia.copy_consumed_range(stop - tail, stop, out);
+            }
+            else
+            {
+                ia.copy_consumed_range(token_string_start, stop, out);
+            }
         }
         return out;
     }
 
     /// streaming adapter: the token was copied eagerly while scanning
-    const std::vector<char_type>& collect_token_chars(std::vector<char_type>& /*out*/, std::false_type /*lazy*/) const
+    const std::vector<char_type>& collect_token_chars(std::vector<char_type>& out, std::false_type /*lazy*/) const
     {
-        return token_string;
+        token_string.copy_to(out);
+        return out;
     }
 
     /// return the last read token (for errors only).  Will never contain EOF
@@ -2256,10 +2272,32 @@ scan_number_done:
         std::vector<char_type> reconstructed;
         const std::vector<char_type>& chars = collect_token_chars(reconstructed, std::integral_constant<bool, lazy_token_string> {});
 
-        // escape control characters
+        // a long token is shown by its first and its last head() characters;
+        // the buffers that keep it while it is read keep no more than these
+        // and a few more (see token_text_buffer)
+        constexpr std::size_t shown = token_text_buffer<char_type>::head();
         std::string result;
-        for (const auto c : chars)
+        if (chars.size() > 2 * shown)
         {
+            append_escaped(result, chars.begin(), chars.begin() + static_cast<std::ptrdiff_t>(shown));
+            result += "...";
+            append_escaped(result, chars.end() - static_cast<std::ptrdiff_t>(shown), chars.end());
+        }
+        else
+        {
+            append_escaped(result, chars.begin(), chars.end());
+        }
+        return result;
+    }
+
+  private:
+    /// append the characters [first, last) to @a result, escaping control characters
+    template<typename Iterator>
+    static void append_escaped(std::string& result, Iterator first, Iterator last)
+    {
+        for (; first != last; ++first)
+        {
+            const auto c = *first;
             if (static_cast<unsigned char>(c) <= '\x1F')
             {
                 // escape control characters
@@ -2273,10 +2311,10 @@ scan_number_done:
                 result.push_back(static_cast<std::string::value_type>(c));
             }
         }
-
-        return result;
     }
 
+  public:
+
     /// return syntax error message
     JSON_HEDLEY_RETURNS_NON_NULL
     constexpr const char* get_error_message() const noexcept
@@ -2525,7 +2563,7 @@ scan_number_done:
 
     /// raw input token string for error messages; only populated for streaming
     /// adapters (seekable adapters reconstruct it lazily via token_string_start)
-    std::vector<char_type> token_string {};
+    token_text_buffer<char_type> token_string {};
 
     /// start offset of the current token within the input, used to reconstruct
     /// the last read token on error for seekable adapters (see collect_token_chars)

single_include/nlohmann/json.hpp

@@ -9323,87 +9323,164 @@ inline const char* empty_input_message() noexcept
     return "attempting to parse an empty input; check that your input string or stream contains the expected JSON";
 }
 
-////////////////////
-// input adapters //
-////////////////////
-
-#ifndef JSON_NO_IO
 /*!
+@brief the characters of the last read token, kept for error messages
+
 A growable array of characters for the part of a token that an input adapter
-copied out of its window: unlike std::vector<char>::push_back(), appending a
-character takes no reference to a local, and is a few instructions when there
-is room.
+copied out of its window, or that the lexer copies while it scans input it
+cannot read again: unlike std::vector::push_back(), appending a character
+takes no reference to a local, and is a few instructions when there is room.
+
+The token is only needed for error messages, which show its first head()
+characters and its last head() characters (see lexer::get_token_string). The
+buffer therefore keeps the first head() and at least the last tail()
+characters: when it is full and holds more than limit() characters, it drops
+the characters in between instead of growing. That happens where it would
+otherwise reallocate, so appending stays as cheap as before, and a long string
+or a long run of whitespace read from a stream is not kept a second time.
 */
-class token_carry_buffer
+template<typename CharType>
+class token_text_buffer
 {
   public:
-    token_carry_buffer() = default;
-    token_carry_buffer(const token_carry_buffer&) = delete;
-    token_carry_buffer& operator=(const token_carry_buffer&) = delete;
-    token_carry_buffer(token_carry_buffer&& rhs) noexcept
+    static constexpr std::size_t head() noexcept
+    {
+        return 4096;
+    }
+    /// a little more than head(), as the last character may be handed back
+    static constexpr std::size_t tail() noexcept
+    {
+        return head() + 16;
+    }
+    static constexpr std::size_t limit() noexcept
+    {
+        return head() + (2 * tail());
+    }
+
+    token_text_buffer() = default;
+    token_text_buffer(const token_text_buffer&) = delete;
+    token_text_buffer& operator=(const token_text_buffer&) = delete;
+    token_text_buffer(token_text_buffer&& rhs) noexcept
         : m_data(std::move(rhs.m_data)), m_end(rhs.m_end), m_capacity_end(rhs.m_capacity_end)
     {
         rhs.m_end = rhs.m_capacity_end = nullptr;
     }
-    token_carry_buffer& operator=(token_carry_buffer&&) = delete;
-    ~token_carry_buffer() = default;
+    token_text_buffer& operator=(token_text_buffer&& rhs) noexcept
+    {
+        m_data = std::move(rhs.m_data);
+        m_end = rhs.m_end;
+        m_capacity_end = rhs.m_capacity_end;
+        rhs.m_end = rhs.m_capacity_end = nullptr;
+        return *this;
+    }
+    ~token_text_buffer() = default;
 
     void clear() noexcept
     {
         m_end = m_data.get();
     }
 
-    void push_back(char c)
+    bool empty() const noexcept
+    {
+        return m_end == m_data.get();
+    }
+
+    void push_back(CharType c)
     {
         if (JSON_HEDLEY_UNLIKELY(m_end == m_capacity_end))
         {
-            grow(1);
+            make_room(1);
         }
         *m_end++ = c;
     }
 
-    void append(const char* first, const char* last)
+    /// drop the last character
+    void pop_back() noexcept
+    {
+        JSON_ASSERT(!empty());
+        --m_end;
+    }
+
+    void append(const CharType* first, const CharType* last)
     {
         const auto n = static_cast<std::size_t>(last - first);
         const auto room = static_cast<std::size_t>(m_capacity_end - m_end);
-        if (n > room)
+        if (JSON_HEDLEY_UNLIKELY(n > room))
         {
-            grow(n);
+            append_long(first, n);
+            return;
         }
         if (n != 0)
         {
-            std::memcpy(m_end, first, n);
+            std::memcpy(m_end, first, n * sizeof(CharType));
         }
         m_end += n;
     }
 
     /// replace the contents of @a out
-    void copy_to(std::vector<char>& out) const
+    void copy_to(std::vector<CharType>& out) const
     {
         out.assign(m_data.get(), m_end);
     }
 
   private:
-    /// make room for @a extra more characters
-    JSON_HEDLEY_NEVER_INLINE void grow(std::size_t extra)
+    /// append @a n characters that do not fit, in pieces of at most tail()
+    /// characters, so that dropping the middle keeps the last tail() of them
+    JSON_HEDLEY_NEVER_INLINE void append_long(const CharType* first, std::size_t n)
     {
-        const auto size = static_cast<std::size_t>(m_end - m_data.get());
-        const auto capacity = static_cast<std::size_t>(m_capacity_end - m_data.get());
-        const std::size_t new_capacity = (std::max)(capacity < 32 ? static_cast<std::size_t>(32) : 2 * capacity, size + extra);
-        std::unique_ptr<char[]> data(new char[new_capacity]); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
-        if (size != 0)
+        while (n != 0)
         {
-            std::memcpy(data.get(), m_data.get(), size);
+            const std::size_t piece = n < tail() ? n : tail();
+            make_room(piece);
+            std::memcpy(m_end, first, piece * sizeof(CharType));
+            m_end += piece;
+            first += piece;
+            n -= piece;
         }
-        m_data = std::move(data);
-        m_end = m_data.get() + size;
-        m_capacity_end = m_data.get() + new_capacity;
     }
 
-    std::unique_ptr<char[]> m_data{}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,readability-redundant-member-init)
-    char* m_end = nullptr;
-    char* m_capacity_end = nullptr;
+    /// make room for @a extra (at most tail()) more characters: drop the
+    /// middle of a long token instead of growing beyond limit()
+    JSON_HEDLEY_NEVER_INLINE void make_room(std::size_t extra)
+    {
+        // nothing is allocated before the first character
+        std::size_t size = m_data ? static_cast<std::size_t>(m_end - m_data.get()) : 0;
+        if (size + extra > limit() && size > head() + tail())
+        {
+            std::memmove(m_data.get() + head(), m_end - tail(), tail() * sizeof(CharType));
+            m_end = m_data.get() + head() + tail();
+            size = head() + tail();
+        }
+        const std::size_t capacity = m_data ? static_cast<std::size_t>(m_capacity_end - m_data.get()) : 0;
+        if (size + extra > capacity)
+        {
+            // never more than limit(): from there on, the middle is dropped
+            const std::size_t doubled = capacity < 32 ? static_cast<std::size_t>(32) : 2 * capacity;
+            const std::size_t new_capacity = (std::max)(doubled < limit() ? doubled : limit(), size + extra);
+            std::unique_ptr<CharType[]> data(new CharType[new_capacity]); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
+            if (size != 0)
+            {
+                std::memcpy(data.get(), m_data.get(), size * sizeof(CharType));
+            }
+            m_data = std::move(data);
+            m_end = m_data.get() + size;
+            m_capacity_end = m_data.get() + new_capacity;
+        }
+    }
+
+    std::unique_ptr<CharType[]> m_data{}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,readability-redundant-member-init)
+    CharType* m_end = nullptr;
+    CharType* m_capacity_end = nullptr;
 };
+
+/// the part of a token that an input adapter copied out of its window
+using token_carry_buffer = token_text_buffer<char>;
+
+////////////////////
+// input adapters //
+////////////////////
+
+#ifndef JSON_NO_IO
 /*!
 Input adapter for stdio file access. The file is locked for the lifetime of the
 adapter and read with the unlocked stdio functions (see
@@ -13898,7 +13975,10 @@ class lexer : public lexer_base<BasicJsonType>
     /// streaming adapter: capture the characters
     void bulk_track_token(const char* data, std::size_t n, std::false_type /*lazy*/)
     {
-        token_string.insert(token_string.end(), data, data + n);
+        for (std::size_t i = 0; i < n; ++i)
+        {
+            token_string.push_back(static_cast<char_type>(data[i]));
+        }
     }
 
     /// streaming input: no bulk fast path
@@ -15641,15 +15721,28 @@ scan_number_done:
         const std::size_t stop = ia.get_consumed_count() - (pending_real_unget ? 1u : 0u);
         if (JSON_HEDLEY_LIKELY(stop >= token_string_start))
         {
-            ia.copy_consumed_range(token_string_start, stop, out);
+            // only the beginning and the end of a long token are shown (see
+            // get_token_string), so copy only these
+            constexpr std::size_t head = token_text_buffer<char_type>::head();
+            constexpr std::size_t tail = token_text_buffer<char_type>::tail();
+            if (stop - token_string_start > head + tail)
+            {
+                ia.copy_consumed_range(token_string_start, token_string_start + head, out);
+                ia.copy_consumed_range(stop - tail, stop, out);
+            }
+            else
+            {
+                ia.copy_consumed_range(token_string_start, stop, out);
+            }
         }
         return out;
     }
 
     /// streaming adapter: the token was copied eagerly while scanning
-    const std::vector<char_type>& collect_token_chars(std::vector<char_type>& /*out*/, std::false_type /*lazy*/) const
+    const std::vector<char_type>& collect_token_chars(std::vector<char_type>& out, std::false_type /*lazy*/) const
     {
-        return token_string;
+        token_string.copy_to(out);
+        return out;
     }
 
     /// return the last read token (for errors only).  Will never contain EOF
@@ -15660,10 +15753,32 @@ scan_number_done:
         std::vector<char_type> reconstructed;
         const std::vector<char_type>& chars = collect_token_chars(reconstructed, std::integral_constant<bool, lazy_token_string> {});
 
-        // escape control characters
+        // a long token is shown by its first and its last head() characters;
+        // the buffers that keep it while it is read keep no more than these
+        // and a few more (see token_text_buffer)
+        constexpr std::size_t shown = token_text_buffer<char_type>::head();
         std::string result;
-        for (const auto c : chars)
+        if (chars.size() > 2 * shown)
         {
+            append_escaped(result, chars.begin(), chars.begin() + static_cast<std::ptrdiff_t>(shown));
+            result += "...";
+            append_escaped(result, chars.end() - static_cast<std::ptrdiff_t>(shown), chars.end());
+        }
+        else
+        {
+            append_escaped(result, chars.begin(), chars.end());
+        }
+        return result;
+    }
+
+  private:
+    /// append the characters [first, last) to @a result, escaping control characters
+    template<typename Iterator>
+    static void append_escaped(std::string& result, Iterator first, Iterator last)
+    {
+        for (; first != last; ++first)
+        {
+            const auto c = *first;
             if (static_cast<unsigned char>(c) <= '\x1F')
             {
                 // escape control characters
@@ -15677,10 +15792,10 @@ scan_number_done:
                 result.push_back(static_cast<std::string::value_type>(c));
             }
         }
-
-        return result;
     }
 
+  public:
+
     /// return syntax error message
     JSON_HEDLEY_RETURNS_NON_NULL
     constexpr const char* get_error_message() const noexcept
@@ -15929,7 +16044,7 @@ scan_number_done:
 
     /// raw input token string for error messages; only populated for streaming
     /// adapters (seekable adapters reconstruct it lazily via token_string_start)
-    std::vector<char_type> token_string {};
+    token_text_buffer<char_type> token_string {};
 
     /// start offset of the current token within the input, used to reconstruct
     /// the last read token on error for seekable adapters (see collect_token_chars)

tests/module_cpp20/CMakeLists.txt

@@ -1,6 +1,11 @@
-# `import std;` (used by main.cpp) is supported without experimental opt-in
-# since CMake 4.5: it is available to every target that scans for C++ modules.
-cmake_minimum_required(VERSION 4.5)
+cmake_minimum_required(VERSION 3.30)
+
+# `import std;` (used by main.cpp) needs CMake's experimental C++ import-std
+# support. The accepted token is CMake-version-specific; this value matches the
+# CMake version pinned in the ci_module_cpp20 CI jobs (4.3.x). If that pin is
+# bumped, this token must be updated to match (configuration fails loudly
+# otherwise).
+set(CMAKE_EXPERIMENTAL_CXX_IMPORT_STD "451f2fe2-a8a2-47c3-bc32-94786d8fc91b")
 
 project(json_test CXX)
 
@@ -8,6 +13,7 @@ project(json_test CXX)
 set(CMAKE_CXX_STANDARD 23)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_MODULE_STD ON)
 
 set(NLOHMANN_JSON_BUILD_MODULES ON CACHE BOOL "Enable nlohmann.json module support")
 

tests/module_cpp20/main.cpp

@@ -17,8 +17,8 @@ using namespace nlohmann::literals;
 // Standard-library facilities are pulled in via `import std;` rather than
 // textual `#include`s: mixing `import` with textual standard headers does not
 // compile under GCC's C++20 modules implementation, whereas `import std;` works
-// across GCC, Clang, and MSVC. This requires C++23 and CMake 4.5 or newer;
-// see CMakeLists.txt.
+// across GCC, Clang, and MSVC. This requires C++23 and CMake's (experimental)
+// import-std support; see CMakeLists.txt.
 int main()
 {
     // basic_json / json: parsing and value access

tests/src/unit-class_lexer.cpp

@@ -12,6 +12,7 @@
 #include <nlohmann/json.hpp>
 using nlohmann::json;
 
+#include <algorithm> // min
 #include <array> // array
 #include <cctype> // toupper
 #include <cstdint> // uint32_t, uint64_t
@@ -2233,3 +2234,123 @@ TEST_CASE("serialization of short strings")
     CHECK(json("caf\xC3\xA9").dump() == "\"caf\xC3\xA9\"");
     CHECK(json("caf\xC3\xA9").dump(-1, ' ', true) == "\"caf\\u00e9\"");
 }
+
+TEST_CASE("token_text_buffer keeps the beginning and the end of a long token")
+{
+    using buffer_t = nlohmann::detail::token_text_buffer<char>;
+    constexpr std::size_t head = buffer_t::head();
+    constexpr std::size_t tail = buffer_t::tail();
+
+    // the characters 0, 1, ..., 250, 0, 1, ...
+    const auto at = [](std::size_t i)
+    {
+        return static_cast<char>(i % 251);
+    };
+    // whether out holds the first head and the last tail characters of a
+    // token of n characters (and maybe more characters of its middle)
+    const auto keeps_ends = [&](const std::vector<char>& out, std::size_t n)
+    {
+        if (out.size() < head + tail || out.size() > buffer_t::limit())
+        {
+            return false;
+        }
+        for (std::size_t i = 0; i < head; ++i)
+        {
+            if (out[i] != at(i))
+            {
+                return false;
+            }
+        }
+        for (std::size_t i = 0; i < tail; ++i)
+        {
+            if (out[out.size() - tail + i] != at(n - tail + i))
+            {
+                return false;
+            }
+        }
+        return true;
+    };
+
+    SECTION("a short token is kept as it is")
+    {
+        buffer_t buffer;
+        CHECK(buffer.empty());
+        for (std::size_t i = 0; i < head + tail; ++i)
+        {
+            buffer.push_back(at(i));
+        }
+        std::vector<char> out;
+        buffer.copy_to(out);
+        REQUIRE(out.size() == head + tail);
+        bool same = true;
+        for (std::size_t i = 0; i < out.size(); ++i)
+        {
+            same = same && out[i] == at(i);
+        }
+        CHECK(same);
+    }
+
+    SECTION("characters added one by one")
+    {
+        for (const std::size_t n :
+                {
+                    head + tail + 1, buffer_t::limit(), buffer_t::limit() + 1, static_cast<std::size_t>(100000)
+                })
+        {
+            CAPTURE(n)
+            buffer_t buffer;
+            for (std::size_t i = 0; i < n; ++i)
+            {
+                buffer.push_back(at(i));
+            }
+            std::vector<char> out;
+            buffer.copy_to(out);
+            CHECK(keeps_ends(out, n));
+
+            // the last character can be handed back
+            buffer.pop_back();
+            buffer.copy_to(out);
+            CHECK(keeps_ends(out, n - 1));
+        }
+    }
+
+    SECTION("characters added in pieces of different sizes")
+    {
+        std::vector<char> input(200000);
+        for (std::size_t i = 0; i < input.size(); ++i)
+        {
+            input[i] = at(i);
+        }
+        for (const std::size_t piece :
+                {
+                    static_cast<std::size_t>(1), static_cast<std::size_t>(100), tail - 1, tail, tail + 1, static_cast<std::size_t>(65536), input.size()
+                })
+        {
+            CAPTURE(piece)
+            buffer_t buffer;
+            for (std::size_t pos = 0; pos < input.size(); pos += piece)
+            {
+                const std::size_t n = (std::min)(piece, input.size() - pos);
+                buffer.append(input.data() + pos, input.data() + pos + n);
+            }
+            std::vector<char> out;
+            buffer.copy_to(out);
+            CHECK(keeps_ends(out, input.size()));
+        }
+    }
+
+    SECTION("clear starts a new token")
+    {
+        buffer_t buffer;
+        for (std::size_t i = 0; i < 100000; ++i)
+        {
+            buffer.push_back('x');
+        }
+        buffer.clear();
+        CHECK(buffer.empty());
+        buffer.push_back('a');
+        std::vector<char> out;
+        buffer.copy_to(out);
+        CHECK(out == std::vector<char> {'a'});
+    }
+}

tests/src/unit-class_parser.cpp

@@ -3269,3 +3269,116 @@ TEST_CASE("a plain function is accepted as parser callback for containers")
     CHECK((nlohmann::detail::can_compare_ne<std::string::const_iterator, std::string::const_iterator>::value));
     CHECK((nlohmann::detail::can_compare_ne<const char*, const char*>::value));
 }
+
+#if !defined(JSON_NOEXCEPTION)
+TEST_CASE("parse error message of a long token")
+{
+    // the message of a parse error shows the last read token; a long token is
+    // shown by its first and its last 4096 characters, and the message is the
+    // same for every kind of input
+    const auto message = [](const std::string & text, int kind) -> std::string
+    {
+        json _;
+        try
+        {
+            if (kind == 0)
+            {
+                _ = json::parse(text);
+            }
+            else if (kind == 1)
+            {
+                std::istringstream stream(text);
+                _ = json::parse(stream);
+            }
+            else if (kind == 2)
+            {
+                FILE* const file = std::tmpfile(); // NOLINT(cppcoreguidelines-owning-memory)
+                REQUIRE(file != nullptr);
+                CHECK(std::fwrite(text.data(), 1, text.size(), file) == text.size());
+                CHECK(std::fseek(file, 0, SEEK_SET) == 0);
+                try
+                {
+                    _ = json::parse(file);
+                }
+                catch (...)
+                {
+                    std::fclose(file); // NOLINT(cert-err33-c,cppcoreguidelines-owning-memory)
+                    throw;
+                }
+                std::fclose(file); // NOLINT(cert-err33-c,cppcoreguidelines-owning-memory)
+            }
+            else
+            {
+                const std::list<char> chars(text.begin(), text.end());
+                _ = json::parse(chars.begin(), chars.end());
+            }
+        }
+        catch (const json::parse_error& e)
+        {
+            return e.what();
+        }
+        return "no error";
+    };
+
+    SECTION("a short token is shown completely")
+    {
+        const std::string text = "[\"" + std::string(8000, 'a') + "\x01\"]";
+        const std::string expected = "last read: '\"" + std::string(8000, 'a') + "<U+0001>'";
+        for (int kind = 0; kind < 4; ++kind)
+        {
+            CAPTURE(kind)
+            const std::string m = message(text, kind);
+            CHECK(m.size() > expected.size());
+            CHECK(m.compare(m.size() - expected.size(), expected.size(), expected) == 0);
+        }
+    }
+
+    SECTION("a long string")
+    {
+        const std::string text = "[\"" + std::string(100000, 'a') + "b\x01\"]";
+        // the token is '"', 100,000 times 'a', 'b', and the control character
+        const std::string expected = "last read: '\"" + std::string(4095, 'a') + "..." + std::string(4094, 'a') + "b<U+0001>'";
+        for (int kind = 0; kind < 4; ++kind)
+        {
+            CAPTURE(kind)
+            const std::string m = message(text, kind);
+            CHECK(m.size() > expected.size());
+            CHECK(m.compare(m.size() - expected.size(), expected.size(), expected) == 0);
+        }
+    }
+
+    SECTION("a long run of whitespace after a number")
+    {
+        // the token begins with the number and ends with the invalid character
+        const std::string text = "[1," + std::string(100000, '\n') + "?]";
+        std::string expected = "last read: '1,";
+        for (int i = 0; i < 4094; ++i)
+        {
+            expected += "<U+000A>";
+        }
+        expected += "...";
+        for (int i = 0; i < 4095; ++i)
+        {
+            expected += "<U+000A>";
+        }
+        expected += "?'";
+        for (int kind = 0; kind < 4; ++kind)
+        {
+            CAPTURE(kind)
+            const std::string m = message(text, kind);
+            CHECK(m.size() > expected.size());
+            CHECK(m.compare(m.size() - expected.size(), expected.size(), expected) == 0);
+        }
+    }
+
+    SECTION("valid input with a long run of whitespace")
+    {
+        const std::string text = "1" + std::string(100000, ' ');
+        for (int kind = 0; kind < 4; ++kind)
+        {
+            CAPTURE(kind)
+            CHECK(message(text, kind) == "no error");
+        }
+    }
+}
+#endif