From 5423dd8de313916ba8553bb25869191dcc7654ae Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Thu, 15 Oct 2026 17:28:16 +0200 Subject: [PATCH] Copy the text of float tokens only when a SAX handler asks for it After the digits are converted while scanning, copying the token text into token_buffer (an out-of-line std::string::append) was about 10% of parsing canada.json. The only reader is the string argument of json_sax::number_float(), and the library's own handlers (json_sax_dom_parser, json_sax_dom_callback_parser, json_sax_acceptor) ignore it. For contiguous input, the fast path now keeps a pointer to the token in the input, and get_string() copies it on first use. The parser passes an empty string to the handlers marked by the new internal trait sax_ignores_number_text instead of calling get_string(), so json::parse() and json::accept() never copy float text. User handlers, and JSON_DIAGNOSTIC_POSITIONS (which reads get_string().size()), still see the exact text. Keys and strings, which never have pending text, use the unchecked get_string_token(). Input read through a window into a stream (std::istream, FILE*) copies the text right away, because the next read may refill the window. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/lexer.hpp | 47 +++++++++++-- include/nlohmann/detail/input/parser.hpp | 39 +++++++++-- single_include/nlohmann/json.hpp | 86 +++++++++++++++++++++--- tests/src/unit-class_parser.cpp | 75 +++++++++++++++++++++ 4 files changed, 231 insertions(+), 16 deletions(-) diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 0f10af759..64e0dafbd 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -1734,10 +1734,6 @@ scan_number_done: } #endif - // materialize the token exactly as scan_number() would. reset() already - // cleared token_buffer, so append() fills it (assign() is avoided - // because custom string_t types need not provide it) - token_buffer.append(reinterpret_cast(data), len); decimal_point_position = dot_index; bulk_consume(data + 1, len - 1); @@ -1753,10 +1749,27 @@ scan_number_done: s.negative = data[0] == '-'; if (convert_accumulated_float(s, std::integral_constant::value> {})) { + // the token text is only copied into token_buffer when + // get_string() asks for it; the library's own SAX handlers + // ignore it. A window into a stream may be refilled by the + // next read, so there the text is copied right away. + if (bulk_window) + { + token_buffer.append(reinterpret_cast(data), len); + } + else + { + pending_number_text = data; + pending_number_length = len; + } return token_type::value_float; } } + // materialize the token exactly as scan_number() would. reset() already + // cleared token_buffer, so append() fills it (assign() is avoided + // because custom string_t types need not provide it) + token_buffer.append(reinterpret_cast(data), len); return convert_number(number_type, mantissa_end); } @@ -1856,6 +1869,7 @@ scan_number_done: void reset() noexcept { token_buffer.clear(); + pending_number_text = nullptr; decimal_point_position = std::string::npos; #if JSON_DIAGNOSTIC_POSITIONS @@ -2115,10 +2129,30 @@ scan_number_done: /// return current string value string_t& get_string() { + if (pending_number_text != nullptr) + { + token_buffer.append(reinterpret_cast(pending_number_text), pending_number_length); + pending_number_text = nullptr; + } // a number token holds '.' regardless of the locale (#4084) return token_buffer; } + /// the value of a string token (key or string), which never has pending + /// number text, so get_string()'s check is not needed + string_t& get_string_token() noexcept + { + JSON_ASSERT(pending_number_text == nullptr); + return token_buffer; + } + + /// the text of a float token for a SAX handler that ignores it (the + /// library's own handlers): empty, without copying the token + const string_t& get_ignored_number_text() const noexcept + { + return token_buffer; + } + ///////////////////// // diagnostics ///////////////////// @@ -2514,6 +2548,11 @@ scan_number_done: /// whether the next get() call should just return current bool next_unget = false; + /// the text of the current float token in the input, not yet copied into + /// token_buffer (see get_string()), or nullptr + const char* pending_number_text = nullptr; + std::size_t pending_number_length = 0; + /// the start position of the current token position_t position {}; diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index 9f1618c84..493ff6f68 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -51,6 +51,20 @@ template using parser_callback_t = std::function; +/// whether a SAX handler ignores the text of float tokens (the library's own +/// handlers do), so that the parser need not materialize it +template +struct sax_ignores_number_text : std::false_type {}; + +template +struct sax_ignores_number_text> : std::true_type {}; + +template +struct sax_ignores_number_text> : std::true_type {}; + +template +struct sax_ignores_number_text> : std::true_type {}; + /*! @brief syntax analysis @@ -171,6 +185,23 @@ class parser } private: + /// the text of the current float token for @a sax (see sax_ignores_number_text) + template + const string_t& number_float_text(SAX* /*sax*/) + { + return number_float_text(std::integral_constant::value> {}); + } + + const string_t& number_float_text(std::true_type /*ignored*/) const noexcept + { + return m_lexer.get_ignored_number_text(); + } + + string_t& number_float_text(std::false_type /*ignored*/) + { + return m_lexer.get_string(); + } + /*! @brief run a DOM SAX parser to completion and position the lexer @@ -247,7 +278,7 @@ class parser { return syntax_error(*sax, exception_message(token_type::value_string, "object key")); } - if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) + if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string_token()))) { return false; } @@ -301,7 +332,7 @@ class parser out_of_range::create(exception_id::number_overflow, concat("number overflow parsing '", m_lexer.get_token_string(), '\''), nullptr)); } - if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, m_lexer.get_string()))) + if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, number_float_text(sax)))) { return false; } @@ -347,7 +378,7 @@ class parser case token_type::value_string: { - if (JSON_HEDLEY_UNLIKELY(!sax->string(m_lexer.get_string()))) + if (JSON_HEDLEY_UNLIKELY(!sax->string(m_lexer.get_string_token()))) { return false; } @@ -456,7 +487,7 @@ class parser return syntax_error(*sax, exception_message(token_type::value_string, "object key")); } - if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) + if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string_token()))) { return false; } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index a8498bd71..9a4e2b2d4 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12900,10 +12900,6 @@ scan_number_done: } #endif - // materialize the token exactly as scan_number() would. reset() already - // cleared token_buffer, so append() fills it (assign() is avoided - // because custom string_t types need not provide it) - token_buffer.append(reinterpret_cast(data), len); decimal_point_position = dot_index; bulk_consume(data + 1, len - 1); @@ -12919,10 +12915,27 @@ scan_number_done: s.negative = data[0] == '-'; if (convert_accumulated_float(s, std::integral_constant::value> {})) { + // the token text is only copied into token_buffer when + // get_string() asks for it; the library's own SAX handlers + // ignore it. A window into a stream may be refilled by the + // next read, so there the text is copied right away. + if (bulk_window) + { + token_buffer.append(reinterpret_cast(data), len); + } + else + { + pending_number_text = data; + pending_number_length = len; + } return token_type::value_float; } } + // materialize the token exactly as scan_number() would. reset() already + // cleared token_buffer, so append() fills it (assign() is avoided + // because custom string_t types need not provide it) + token_buffer.append(reinterpret_cast(data), len); return convert_number(number_type, mantissa_end); } @@ -13022,6 +13035,7 @@ scan_number_done: void reset() noexcept { token_buffer.clear(); + pending_number_text = nullptr; decimal_point_position = std::string::npos; #if JSON_DIAGNOSTIC_POSITIONS @@ -13281,10 +13295,30 @@ scan_number_done: /// return current string value string_t& get_string() { + if (pending_number_text != nullptr) + { + token_buffer.append(reinterpret_cast(pending_number_text), pending_number_length); + pending_number_text = nullptr; + } // a number token holds '.' regardless of the locale (#4084) return token_buffer; } + /// the value of a string token (key or string), which never has pending + /// number text, so get_string()'s check is not needed + string_t& get_string_token() noexcept + { + JSON_ASSERT(pending_number_text == nullptr); + return token_buffer; + } + + /// the text of a float token for a SAX handler that ignores it (the + /// library's own handlers): empty, without copying the token + const string_t& get_ignored_number_text() const noexcept + { + return token_buffer; + } + ///////////////////// // diagnostics ///////////////////// @@ -13680,6 +13714,11 @@ scan_number_done: /// whether the next get() call should just return current bool next_unget = false; + /// the text of the current float token in the input, not yet copied into + /// token_buffer (see get_string()), or nullptr + const char* pending_number_text = nullptr; + std::size_t pending_number_length = 0; + /// the start position of the current token position_t position {}; @@ -19482,6 +19521,20 @@ template using parser_callback_t = std::function; +/// whether a SAX handler ignores the text of float tokens (the library's own +/// handlers do), so that the parser need not materialize it +template +struct sax_ignores_number_text : std::false_type {}; + +template +struct sax_ignores_number_text> : std::true_type {}; + +template +struct sax_ignores_number_text> : std::true_type {}; + +template +struct sax_ignores_number_text> : std::true_type {}; + /*! @brief syntax analysis @@ -19602,6 +19655,23 @@ class parser } private: + /// the text of the current float token for @a sax (see sax_ignores_number_text) + template + const string_t& number_float_text(SAX* /*sax*/) + { + return number_float_text(std::integral_constant::value> {}); + } + + const string_t& number_float_text(std::true_type /*ignored*/) const noexcept + { + return m_lexer.get_ignored_number_text(); + } + + string_t& number_float_text(std::false_type /*ignored*/) + { + return m_lexer.get_string(); + } + /*! @brief run a DOM SAX parser to completion and position the lexer @@ -19678,7 +19748,7 @@ class parser { return syntax_error(*sax, exception_message(token_type::value_string, "object key")); } - if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) + if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string_token()))) { return false; } @@ -19732,7 +19802,7 @@ class parser out_of_range::create(exception_id::number_overflow, concat("number overflow parsing '", m_lexer.get_token_string(), '\''), nullptr)); } - if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, m_lexer.get_string()))) + if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, number_float_text(sax)))) { return false; } @@ -19778,7 +19848,7 @@ class parser case token_type::value_string: { - if (JSON_HEDLEY_UNLIKELY(!sax->string(m_lexer.get_string()))) + if (JSON_HEDLEY_UNLIKELY(!sax->string(m_lexer.get_string_token()))) { return false; } @@ -19887,7 +19957,7 @@ class parser return syntax_error(*sax, exception_message(token_type::value_string, "object key")); } - if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) + if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string_token()))) { return false; } diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index 3245d76b9..c3793f220 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -30,7 +30,9 @@ using nlohmann::json; #include #include #include +#include #include +#include #include #include #include @@ -2450,6 +2452,79 @@ std::string parse_error_message_range(IteratorType first, IteratorType last) } } // namespace +TEST_CASE("text of float tokens") +{ + // For contiguous input, the lexer copies the text of a float token into + // its buffer only when a SAX handler asks for it (the library's own + // handlers do not). A user handler must still see the exact text, for + // every input adapter and in every position of a document. + const std::string doc = R"([1.5, -0.0, 0.1, 1e5, -2.5E-4, 12345678901234567890.5, 0.12345678901234567890123, + {"a": 3.25, "b": [6.02e23, 1]}, "x", true, 7e-1])"; + const std::vector expected_floats = + { + "number_float(1.5)", "number_float(-0.0)", "number_float(0.1)", "number_float(1e5)", "number_float(-2.5E-4)", + "number_float(12345678901234567890.5)", "number_float(0.12345678901234567890123)", "number_float(3.25)", + "number_float(6.02e23)", "number_float(7e-1)" + }; + + const auto floats = [](const std::vector& events) + { + std::vector result; + std::copy_if(events.begin(), events.end(), std::back_inserter(result), [](const std::string & e) + { + return e.compare(0, 13, "number_float(") == 0; + }); + return result; + }; + + SECTION("contiguous input") + { + SaxEventLogger l; + CHECK(json::sax_parse(doc, &l)); + CHECK(floats(l.events) == expected_floats); + } + + SECTION("byte input") + { + const std::vector bytes(doc.begin(), doc.end()); + SaxEventLogger l; + CHECK(json::sax_parse(bytes, &l)); + CHECK(floats(l.events) == expected_floats); + } + + SECTION("stream input") + { + std::istringstream is(doc); + SaxEventLogger l; + CHECK(json::sax_parse(is, &l)); + CHECK(floats(l.events) == expected_floats); + } + + SECTION("non-contiguous iterators") + { + const std::list chars(doc.begin(), doc.end()); + SaxEventLogger l; + CHECK(json::sax_parse(chars.begin(), chars.end(), &l)); + CHECK(floats(l.events) == expected_floats); + } + + SECTION("the library's own handlers do not need the text") + { + using lexer_adapter = nlohmann::detail::iterator_input_adapter; + CHECK(nlohmann::detail::sax_ignores_number_text>::value); + CHECK(nlohmann::detail::sax_ignores_number_text>::value); + CHECK(nlohmann::detail::sax_ignores_number_text>::value); + CHECK_FALSE(nlohmann::detail::sax_ignores_number_text::value); + + // and the values are the same as when the text is read + const json j = json::parse(doc); + CHECK(j[0] == 1.5); + CHECK(j[5].get() == 12345678901234567890.5); + CHECK(j[7]["b"][0] == 6.02e23); + CHECK(json::accept(doc)); + } +} + TEST_CASE("last-read diagnostics are identical across input adapters") { // The lexer reconstructs the "last read" token lazily for seekable adapters