Keep only the beginning and the end of a long token for error messages

For "last read" in parse errors, the lexer keeps the text of the token it
reads. The token start moves only at strings and numbers, so everything
after a string or number belongs to it until the next one starts. Input
that cannot be read again copies that text while parsing (the istream and
FILE* adapters in their carry buffer, the lexer in token_string for
iterators), so valid input of 1 followed by 100,000,000 newlines peaked
at about 225 MB, and a long string was kept a second time.

Replace the carry buffer and token_string by token_text_buffer, which
keeps the first 4096 and at least the last 4112 characters of a token and
drops the characters in between where it would otherwise grow, so
appending stays as cheap as before. get_token_string() shows a token of
more than 8192 characters as its first and last 4096 characters around
"...", the same for every kind of input; contiguous input copies only
these parts.

The example input now peaks at 1.8 MB. Instruction counts of parsing
from std::string, std::istream and FILE* are unchanged (clang 21 and
GCC 15); parsing from std::list<char> needs 1 to 17 % fewer.

Fixes #6120.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/input_adapters.hpp

@@ -110,87 +110,164 @@ inline const char* empty_input_message() noexcept
     return "attempting to parse an empty input; check that your input string or stream contains the expected JSON";
 }
 
-////////////////////
-// input adapters //
-////////////////////
-
-#ifndef JSON_NO_IO
 /*!
+@brief the characters of the last read token, kept for error messages
+
 A growable array of characters for the part of a token that an input adapter
-copied out of its window: unlike std::vector<char>::push_back(), appending a
-character takes no reference to a local, and is a few instructions when there
-is room.
+copied out of its window, or that the lexer copies while it scans input it
+cannot read again: unlike std::vector::push_back(), appending a character
+takes no reference to a local, and is a few instructions when there is room.
+
+The token is only needed for error messages, which show its first head()
+characters and its last head() characters (see lexer::get_token_string). The
+buffer therefore keeps the first head() and at least the last tail()
+characters: when it is full and holds more than limit() characters, it drops
+the characters in between instead of growing. That happens where it would
+otherwise reallocate, so appending stays as cheap as before, and a long string
+or a long run of whitespace read from a stream is not kept a second time.
 */
-class token_carry_buffer
+template<typename CharType>
+class token_text_buffer
 {
   public:
-    token_carry_buffer() = default;
-    token_carry_buffer(const token_carry_buffer&) = delete;
-    token_carry_buffer& operator=(const token_carry_buffer&) = delete;
-    token_carry_buffer(token_carry_buffer&& rhs) noexcept
+    static constexpr std::size_t head() noexcept
+    {
+        return 4096;
+    }
+    /// a little more than head(), as the last character may be handed back
+    static constexpr std::size_t tail() noexcept
+    {
+        return head() + 16;
+    }
+    static constexpr std::size_t limit() noexcept
+    {
+        return head() + (2 * tail());
+    }
+
+    token_text_buffer() = default;
+    token_text_buffer(const token_text_buffer&) = delete;
+    token_text_buffer& operator=(const token_text_buffer&) = delete;
+    token_text_buffer(token_text_buffer&& rhs) noexcept
         : m_data(std::move(rhs.m_data)), m_end(rhs.m_end), m_capacity_end(rhs.m_capacity_end)
     {
         rhs.m_end = rhs.m_capacity_end = nullptr;
     }
-    token_carry_buffer& operator=(token_carry_buffer&&) = delete;
-    ~token_carry_buffer() = default;
+    token_text_buffer& operator=(token_text_buffer&& rhs) noexcept
+    {
+        m_data = std::move(rhs.m_data);
+        m_end = rhs.m_end;
+        m_capacity_end = rhs.m_capacity_end;
+        rhs.m_end = rhs.m_capacity_end = nullptr;
+        return *this;
+    }
+    ~token_text_buffer() = default;
 
     void clear() noexcept
     {
         m_end = m_data.get();
     }
 
-    void push_back(char c)
+    bool empty() const noexcept
+    {
+        return m_end == m_data.get();
+    }
+
+    void push_back(CharType c)
     {
         if (JSON_HEDLEY_UNLIKELY(m_end == m_capacity_end))
         {
-            grow(1);
+            make_room(1);
         }
         *m_end++ = c;
     }
 
-    void append(const char* first, const char* last)
+    /// drop the last character
+    void pop_back() noexcept
+    {
+        JSON_ASSERT(!empty());
+        --m_end;
+    }
+
+    void append(const CharType* first, const CharType* last)
     {
         const auto n = static_cast<std::size_t>(last - first);
         const auto room = static_cast<std::size_t>(m_capacity_end - m_end);
-        if (n > room)
+        if (JSON_HEDLEY_UNLIKELY(n > room))
         {
-            grow(n);
+            append_long(first, n);
+            return;
         }
         if (n != 0)
         {
-            std::memcpy(m_end, first, n);
+            std::memcpy(m_end, first, n * sizeof(CharType));
         }
         m_end += n;
     }
 
     /// replace the contents of @a out
-    void copy_to(std::vector<char>& out) const
+    void copy_to(std::vector<CharType>& out) const
     {
         out.assign(m_data.get(), m_end);
     }
 
   private:
-    /// make room for @a extra more characters
-    JSON_HEDLEY_NEVER_INLINE void grow(std::size_t extra)
+    /// append @a n characters that do not fit, in pieces of at most tail()
+    /// characters, so that dropping the middle keeps the last tail() of them
+    JSON_HEDLEY_NEVER_INLINE void append_long(const CharType* first, std::size_t n)
     {
-        const auto size = static_cast<std::size_t>(m_end - m_data.get());
-        const auto capacity = static_cast<std::size_t>(m_capacity_end - m_data.get());
-        const std::size_t new_capacity = (std::max)(capacity < 32 ? static_cast<std::size_t>(32) : 2 * capacity, size + extra);
-        std::unique_ptr<char[]> data(new char[new_capacity]); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
-        if (size != 0)
+        while (n != 0)
         {
-            std::memcpy(data.get(), m_data.get(), size);
+            const std::size_t piece = n < tail() ? n : tail();
+            make_room(piece);
+            std::memcpy(m_end, first, piece * sizeof(CharType));
+            m_end += piece;
+            first += piece;
+            n -= piece;
         }
-        m_data = std::move(data);
-        m_end = m_data.get() + size;
-        m_capacity_end = m_data.get() + new_capacity;
     }
 
-    std::unique_ptr<char[]> m_data{}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,readability-redundant-member-init)
-    char* m_end = nullptr;
-    char* m_capacity_end = nullptr;
+    /// make room for @a extra (at most tail()) more characters: drop the
+    /// middle of a long token instead of growing beyond limit()
+    JSON_HEDLEY_NEVER_INLINE void make_room(std::size_t extra)
+    {
+        // nothing is allocated before the first character
+        std::size_t size = m_data ? static_cast<std::size_t>(m_end - m_data.get()) : 0;
+        if (size + extra > limit() && size > head() + tail())
+        {
+            std::memmove(m_data.get() + head(), m_end - tail(), tail() * sizeof(CharType));
+            m_end = m_data.get() + head() + tail();
+            size = head() + tail();
+        }
+        const std::size_t capacity = m_data ? static_cast<std::size_t>(m_capacity_end - m_data.get()) : 0;
+        if (size + extra > capacity)
+        {
+            // never more than limit(): from there on, the middle is dropped
+            const std::size_t doubled = capacity < 32 ? static_cast<std::size_t>(32) : 2 * capacity;
+            const std::size_t new_capacity = (std::max)(doubled < limit() ? doubled : limit(), size + extra);
+            std::unique_ptr<CharType[]> data(new CharType[new_capacity]); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
+            if (size != 0)
+            {
+                std::memcpy(data.get(), m_data.get(), size * sizeof(CharType));
+            }
+            m_data = std::move(data);
+            m_end = m_data.get() + size;
+            m_capacity_end = m_data.get() + new_capacity;
+        }
+    }
+
+    std::unique_ptr<CharType[]> m_data{}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,readability-redundant-member-init)
+    CharType* m_end = nullptr;
+    CharType* m_capacity_end = nullptr;
 };
+
+/// the part of a token that an input adapter copied out of its window
+using token_carry_buffer = token_text_buffer<char>;
+
+////////////////////
+// input adapters //
+////////////////////
+
+#ifndef JSON_NO_IO
 /*!
 Input adapter for stdio file access. The file is locked for the lifetime of the
 adapter and read with the unlocked stdio functions (see