Keep only the beginning and the end of a long token for error messages

For "last read" in parse errors, the lexer keeps the text of the token it
reads. The token start moves only at strings and numbers, so everything
after a string or number belongs to it until the next one starts. Input
that cannot be read again copies that text while parsing (the istream and
FILE* adapters in their carry buffer, the lexer in token_string for
iterators), so valid input of 1 followed by 100,000,000 newlines peaked
at about 225 MB, and a long string was kept a second time.

Replace the carry buffer and token_string by token_text_buffer, which
keeps the first 4096 and at least the last 4112 characters of a token and
drops the characters in between where it would otherwise grow, so
appending stays as cheap as before. get_token_string() shows a token of
more than 8192 characters as its first and last 4096 characters around
"...", the same for every kind of input; contiguous input copies only
these parts.

The example input now peaks at 1.8 MB. Instruction counts of parsing
from std::string, std::istream and FILE* are unchanged (clang 21 and
GCC 15); parsing from std::list<char> needs 1 to 17 % fewer.

Fixes #6120.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

tests/src/unit-class_lexer.cpp

@@ -12,6 +12,7 @@
 #include <nlohmann/json.hpp>
 using nlohmann::json;
 
+#include <algorithm> // min
 #include <array> // array
 #include <cctype> // toupper
 #include <cstdint> // uint32_t, uint64_t
@@ -2233,3 +2234,123 @@ TEST_CASE("serialization of short strings")
     CHECK(json("caf\xC3\xA9").dump() == "\"caf\xC3\xA9\"");
     CHECK(json("caf\xC3\xA9").dump(-1, ' ', true) == "\"caf\\u00e9\"");
 }
+
+TEST_CASE("token_text_buffer keeps the beginning and the end of a long token")
+{
+    using buffer_t = nlohmann::detail::token_text_buffer<char>;
+    constexpr std::size_t head = buffer_t::head();
+    constexpr std::size_t tail = buffer_t::tail();
+
+    // the characters 0, 1, ..., 250, 0, 1, ...
+    const auto at = [](std::size_t i)
+    {
+        return static_cast<char>(i % 251);
+    };
+    // whether out holds the first head and the last tail characters of a
+    // token of n characters (and maybe more characters of its middle)
+    const auto keeps_ends = [&](const std::vector<char>& out, std::size_t n)
+    {
+        if (out.size() < head + tail || out.size() > buffer_t::limit())
+        {
+            return false;
+        }
+        for (std::size_t i = 0; i < head; ++i)
+        {
+            if (out[i] != at(i))
+            {
+                return false;
+            }
+        }
+        for (std::size_t i = 0; i < tail; ++i)
+        {
+            if (out[out.size() - tail + i] != at(n - tail + i))
+            {
+                return false;
+            }
+        }
+        return true;
+    };
+
+    SECTION("a short token is kept as it is")
+    {
+        buffer_t buffer;
+        CHECK(buffer.empty());
+        for (std::size_t i = 0; i < head + tail; ++i)
+        {
+            buffer.push_back(at(i));
+        }
+        std::vector<char> out;
+        buffer.copy_to(out);
+        REQUIRE(out.size() == head + tail);
+        bool same = true;
+        for (std::size_t i = 0; i < out.size(); ++i)
+        {
+            same = same && out[i] == at(i);
+        }
+        CHECK(same);
+    }
+
+    SECTION("characters added one by one")
+    {
+        for (const std::size_t n :
+                {
+                    head + tail + 1, buffer_t::limit(), buffer_t::limit() + 1, static_cast<std::size_t>(100000)
+                })
+        {
+            CAPTURE(n)
+            buffer_t buffer;
+            for (std::size_t i = 0; i < n; ++i)
+            {
+                buffer.push_back(at(i));
+            }
+            std::vector<char> out;
+            buffer.copy_to(out);
+            CHECK(keeps_ends(out, n));
+
+            // the last character can be handed back
+            buffer.pop_back();
+            buffer.copy_to(out);
+            CHECK(keeps_ends(out, n - 1));
+        }
+    }
+
+    SECTION("characters added in pieces of different sizes")
+    {
+        std::vector<char> input(200000);
+        for (std::size_t i = 0; i < input.size(); ++i)
+        {
+            input[i] = at(i);
+        }
+        for (const std::size_t piece :
+                {
+                    static_cast<std::size_t>(1), static_cast<std::size_t>(100), tail - 1, tail, tail + 1, static_cast<std::size_t>(65536), input.size()
+                })
+        {
+            CAPTURE(piece)
+            buffer_t buffer;
+            for (std::size_t pos = 0; pos < input.size(); pos += piece)
+            {
+                const std::size_t n = (std::min)(piece, input.size() - pos);
+                buffer.append(input.data() + pos, input.data() + pos + n);
+            }
+            std::vector<char> out;
+            buffer.copy_to(out);
+            CHECK(keeps_ends(out, input.size()));
+        }
+    }
+
+    SECTION("clear starts a new token")
+    {
+        buffer_t buffer;
+        for (std::size_t i = 0; i < 100000; ++i)
+        {
+            buffer.push_back('x');
+        }
+        buffer.clear();
+        CHECK(buffer.empty());
+        buffer.push_back('a');
+        std::vector<char> out;
+        buffer.copy_to(out);
+        CHECK(out == std::vector<char> {'a'});
+    }
+}

tests/src/unit-class_parser.cpp

@@ -3269,3 +3269,116 @@ TEST_CASE("a plain function is accepted as parser callback for containers")
     CHECK((nlohmann::detail::can_compare_ne<std::string::const_iterator, std::string::const_iterator>::value));
     CHECK((nlohmann::detail::can_compare_ne<const char*, const char*>::value));
 }
+
+#if !defined(JSON_NOEXCEPTION)
+TEST_CASE("parse error message of a long token")
+{
+    // the message of a parse error shows the last read token; a long token is
+    // shown by its first and its last 4096 characters, and the message is the
+    // same for every kind of input
+    const auto message = [](const std::string & text, int kind) -> std::string
+    {
+        json _;
+        try
+        {
+            if (kind == 0)
+            {
+                _ = json::parse(text);
+            }
+            else if (kind == 1)
+            {
+                std::istringstream stream(text);
+                _ = json::parse(stream);
+            }
+            else if (kind == 2)
+            {
+                FILE* const file = std::tmpfile(); // NOLINT(cppcoreguidelines-owning-memory)
+                REQUIRE(file != nullptr);
+                CHECK(std::fwrite(text.data(), 1, text.size(), file) == text.size());
+                CHECK(std::fseek(file, 0, SEEK_SET) == 0);
+                try
+                {
+                    _ = json::parse(file);
+                }
+                catch (...)
+                {
+                    std::fclose(file); // NOLINT(cert-err33-c,cppcoreguidelines-owning-memory)
+                    throw;
+                }
+                std::fclose(file); // NOLINT(cert-err33-c,cppcoreguidelines-owning-memory)
+            }
+            else
+            {
+                const std::list<char> chars(text.begin(), text.end());
+                _ = json::parse(chars.begin(), chars.end());
+            }
+        }
+        catch (const json::parse_error& e)
+        {
+            return e.what();
+        }
+        return "no error";
+    };
+
+    SECTION("a short token is shown completely")
+    {
+        const std::string text = "[\"" + std::string(8000, 'a') + "\x01\"]";
+        const std::string expected = "last read: '\"" + std::string(8000, 'a') + "<U+0001>'";
+        for (int kind = 0; kind < 4; ++kind)
+        {
+            CAPTURE(kind)
+            const std::string m = message(text, kind);
+            CHECK(m.size() > expected.size());
+            CHECK(m.compare(m.size() - expected.size(), expected.size(), expected) == 0);
+        }
+    }
+
+    SECTION("a long string")
+    {
+        const std::string text = "[\"" + std::string(100000, 'a') + "b\x01\"]";
+        // the token is '"', 100,000 times 'a', 'b', and the control character
+        const std::string expected = "last read: '\"" + std::string(4095, 'a') + "..." + std::string(4094, 'a') + "b<U+0001>'";
+        for (int kind = 0; kind < 4; ++kind)
+        {
+            CAPTURE(kind)
+            const std::string m = message(text, kind);
+            CHECK(m.size() > expected.size());
+            CHECK(m.compare(m.size() - expected.size(), expected.size(), expected) == 0);
+        }
+    }
+
+    SECTION("a long run of whitespace after a number")
+    {
+        // the token begins with the number and ends with the invalid character
+        const std::string text = "[1," + std::string(100000, '\n') + "?]";
+        std::string expected = "last read: '1,";
+        for (int i = 0; i < 4094; ++i)
+        {
+            expected += "<U+000A>";
+        }
+        expected += "...";
+        for (int i = 0; i < 4095; ++i)
+        {
+            expected += "<U+000A>";
+        }
+        expected += "?'";
+        for (int kind = 0; kind < 4; ++kind)
+        {
+            CAPTURE(kind)
+            const std::string m = message(text, kind);
+            CHECK(m.size() > expected.size());
+            CHECK(m.compare(m.size() - expected.size(), expected.size(), expected) == 0);
+        }
+    }
+
+    SECTION("valid input with a long run of whitespace")
+    {
+        const std::string text = "1" + std::string(100000, ' ');
+        for (int kind = 0; kind < 4; ++kind)
+        {
+            CAPTURE(kind)
+            CHECK(message(text, kind) == "no error");
+        }
+    }
+}
+#endif