From 74f5ecbd354291fa74634275709b97ead1718ec5 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 31 Oct 2026 00:07:57 +0100 Subject: [PATCH] accept() checks strings without copying them json::accept() parses through json_sax_acceptor, which ignores the values of strings and object keys, but the lexer still copied every string token into its token buffer. That copy was 18 % of the instructions of json::accept() on twitter.json (x86, GCC 16). The lexer's flag for accept(), renamed from discard_number_values to discard_values, now also selects a second instantiation of scan_string() that performs the same checks (escapes, surrogates, UTF-8, control characters, with the same error messages) without writing to the token buffer. scan() chooses the instantiation once per string: a run-time check inside the string loop made number-heavy files up to 3 % slower on Apple M1. json::accept() over 19 files, time after/before (geomean): 0.876 on x86 with GCC 16.2, 0.884 with Clang 23, 0.932 on Apple M1; up to 0.74 on single files; files with few strings (canada, mesh, numbers) unchanged. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/lexer.hpp | 124 ++++++++++++++------- include/nlohmann/detail/input/parser.hpp | 4 +- include/nlohmann/json.hpp | 4 +- single_include/nlohmann/json.hpp | 132 +++++++++++++++-------- tests/src/unit-class_parser.cpp | 44 ++++++++ 5 files changed, 218 insertions(+), 90 deletions(-) diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 14d14f42a..d9e9a726d 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -268,10 +268,10 @@ class lexer : public lexer_base public: using token_type = typename lexer_base::token_type; - explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept + explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_values_ = false) noexcept : ia(std::move(adapter)) , ignore_comments(ignore_comments_) - , discard_number_values(discard_number_values_) + , discard_values(discard_values_) { // the token string of an error before the first token is everything // read so far @@ -388,6 +388,8 @@ class lexer : public lexer_base checks if it is inside the range. If a violation was detected, set up an error message and return false. Otherwise, return true. + @param[in] keep whether the bytes are added to token_buffer (false for + accept(), see scan()) @param[in] ranges list of integers; interpreted as list of pairs of inclusive lower and upper bound, respectively @@ -396,17 +398,18 @@ class lexer : public lexer_base @return true if and only if no range violation was detected */ - bool next_byte_in_range(std::initializer_list ranges) + template + bool next_byte_in_range(std::integral_constant keep, std::initializer_list ranges) { JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6); - add(current); + add_string_char(keep, current); for (auto range = ranges.begin(); range != ranges.end(); ++range) { get(); if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions) { - add(current); + add_string_char(keep, current); } else { @@ -419,14 +422,16 @@ class lexer : public lexer_base } /// contiguous input: bulk-append the run of ordinary characters and complete - /// well-formed UTF-8 sequences starting at the current read position, then + /// well-formed UTF-8 sequences starting at the current read position (only + /// skip it if @a keep is false), then /// read the first byte that needs individual handling (the closing quote, an /// escape, a control character, or an ill-formed UTF-8 byte) like get(). /// That byte is taken from the buffer that was just scanned. Through get(), /// Apple Clang merged this read with the other get() calls of the scan /// function, which load the read position again right after it was stored /// (3% of json::accept() on jeopardy.json on Apple M1). - char_int_type get_after_string_run(std::true_type /*bulk*/) + template + char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant keep) { // a pending unget must be consumed through the normal path first if (next_unget) @@ -444,7 +449,7 @@ class lexer : public lexer_base const std::size_t pos = lexer_string_bulk_run(data, remaining); if (pos != 0) { - token_buffer.append(reinterpret_cast(data), pos); + append_string_run(keep, data, pos); bulk_consume(reinterpret_cast(data), pos); } if (pos == remaining) @@ -488,7 +493,8 @@ class lexer : public lexer_base } /// streaming input: no bulk fast path - char_int_type get_after_string_run(std::false_type /*bulk*/) + template + char_int_type get_after_string_run(std::false_type /*bulk*/, std::integral_constant /*keep*/) { return get(); } @@ -502,13 +508,18 @@ class lexer : public lexer_base may contain \0 bytes), and token_buffer.size() is the number of bytes in the string. + @param[in] keep whether the string is copied into token_buffer; if false + (accept(), see scan()), it is only checked, with the same + errors, and token_buffer stays empty + @return token_type::value_string if string could be successfully scanned, token_type::parse_error otherwise @note In case of errors, variable error_message contains a textual description. */ - token_type scan_string() + template + token_type scan_string(std::integral_constant keep) { // reset token_buffer (ignore opening quote) reset(); @@ -520,7 +531,7 @@ class lexer : public lexer_base { // bulk-consume ordinary characters from contiguous input, then // get the next character, the one that ends the run - switch (get_after_string_run(std::integral_constant())) + switch (get_after_string_run(std::integral_constant(), keep)) { // end of file while parsing the string case char_traits::eof(): @@ -542,35 +553,35 @@ class lexer : public lexer_base { // quotation mark case '\"': - add('\"'); + add_string_char(keep, '\"'); break; // reverse solidus case '\\': - add('\\'); + add_string_char(keep, '\\'); break; // solidus case '/': - add('/'); + add_string_char(keep, '/'); break; // backspace case 'b': - add('\b'); + add_string_char(keep, '\b'); break; // form feed case 'f': - add('\f'); + add_string_char(keep, '\f'); break; // line feed case 'n': - add('\n'); + add_string_char(keep, '\n'); break; // carriage return case 'r': - add('\r'); + add_string_char(keep, '\r'); break; // tab case 't': - add('\t'); + add_string_char(keep, '\t'); break; // unicode escapes @@ -638,10 +649,7 @@ class lexer : public lexer_base JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF); // translate codepoint into bytes - encode_utf8(static_cast(codepoint), [this](std::uint32_t byte) - { - add(static_cast(byte)); - }); + add_codepoint(keep, codepoint); break; } @@ -824,7 +832,7 @@ class lexer : public lexer_base case 0x7E: case 0x7F: { - add(current); + add_string_char(keep, current); break; } @@ -860,7 +868,7 @@ class lexer : public lexer_base case 0xDE: case 0xDF: { - if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF}))) + if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range(keep, {0x80, 0xBF}))) { return token_type::parse_error; } @@ -870,7 +878,7 @@ class lexer : public lexer_base // U+0800..U+0FFF: bytes E0 A0..BF 80..BF case 0xE0: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0xA0, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -894,7 +902,7 @@ class lexer : public lexer_base case 0xEE: case 0xEF: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -904,7 +912,7 @@ class lexer : public lexer_base // U+D000..U+D7FF: bytes ED 80..9F 80..BF case 0xED: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x9F, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -914,7 +922,7 @@ class lexer : public lexer_base // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF case 0xF0: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -926,7 +934,7 @@ class lexer : public lexer_base case 0xF2: case 0xF3: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -936,7 +944,7 @@ class lexer : public lexer_base // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF case 0xF4: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -1427,7 +1435,7 @@ scan_number_done: token_type convert_number(token_type number_type, std::size_t mantissa_end) { // accept() only needs to know whether the input is valid, so it sets - // discard_number_values (see json.hpp), and an integer token whose + // discard_values (see json.hpp), and an integer token whose // digit count shows that it fits is reported without calling // convert_integer(). A number with up to 18 digits always fits into // both std::uint64_t and std::int64_t (18 nines is about 1e18, below @@ -1442,7 +1450,7 @@ scan_number_done: // json_sax_acceptor, whose number callbacks discard their argument and // return true, and the parser rejects neither integers nor finite // floats. value_unsigned/value_integer are left unset here, so a caller - // that reads the converted value must not set discard_number_values. + // that reads the converted value must not set discard_values. // // On contiguous input, scan_number_bulk_contiguous() converts integer // tokens itself and does not pass them to this function, unless @@ -1450,7 +1458,7 @@ scan_number_done: // reached for input without bulk access (e.g. streams), with // JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous() // falls back to scan_number(). - if (discard_number_values) + if (discard_values) { constexpr std::size_t safe_digit_count = 18; if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count) @@ -1701,7 +1709,7 @@ scan_number_done: // w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a // value that underflows is a finite number as well. Only a token that // may overflow has to be converted to find out. - if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits::max_exponent10)) + if (JSON_HEDLEY_UNLIKELY(discard_values && s.exponent + 19 <= std::numeric_limits::max_exponent10)) { value_float = 0; return true; @@ -2033,6 +2041,34 @@ scan_number_done: token_buffer.push_back(static_cast(c)); } + /// add a character of a string token; accept() checks the string but does + /// not keep its value (@a keep is false) + void add_string_char(std::true_type /*keep*/, char_int_type c) + { + token_buffer.push_back(static_cast(c)); + } + + static void add_string_char(std::false_type /*keep*/, char_int_type /*c*/) noexcept {} + + /// add the UTF-8 bytes of a code point from a \\u escape to a string token + void add_codepoint(std::true_type /*keep*/, int codepoint) + { + encode_utf8(static_cast(codepoint), [this](std::uint32_t byte) + { + add(static_cast(byte)); + }); + } + + static void add_codepoint(std::false_type /*keep*/, int /*codepoint*/) noexcept {} + + /// append a run of ordinary string characters (see get_after_string_run()) + void append_string_run(std::true_type /*keep*/, const unsigned char* data, std::size_t n) + { + token_buffer.append(reinterpret_cast(data), n); + } + + static void append_string_run(std::false_type /*keep*/, const unsigned char* /*data*/, std::size_t /*n*/) noexcept {} + public: ///////////////////// // value getters @@ -2427,7 +2463,11 @@ scan_number_done: // string case '\"': - return scan_string(); + // accept() (discard_values) checks strings without copying them + // into token_buffer. The choice is made once per string: a + // check inside the string loop made number-heavy documents up + // to 3 % slower (mesh.json, Apple M1) + return discard_values ? scan_string(std::false_type {}) : scan_string(std::true_type {}); // number case '-': @@ -2512,11 +2552,13 @@ scan_number_done: std::size_t decimal_point_position = std::string::npos; /// whether the caller only needs the token types and never looks at the - /// converted numeric values; set only by accept(), which parses through - /// json_sax_acceptor. When set, convert_number() skips converting integer - /// tokens whose digit count guarantees that they fit into 64 bits (see - /// there) - const bool discard_number_values = false; + /// values of number and string tokens; set only by accept(), which parses + /// through json_sax_acceptor. When set, convert_number() skips converting + /// integer tokens whose digit count guarantees that they fit into 64 bits + /// (see there), floats are converted only when they may overflow (see + /// convert_accumulated_float()), and strings are checked without being + /// copied into token_buffer (see scan()) + const bool discard_values = false; }; } // namespace detail diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index 5bf8f1b6b..e49c81a03 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -89,9 +89,9 @@ class parser const bool allow_exceptions_ = true, const bool ignore_comments = false, const bool ignore_trailing_commas_ = false, - const bool discard_number_values_ = false) + const bool discard_values_ = false) : callback(std::move(cb)) - , m_lexer(std::move(adapter), ignore_comments, discard_number_values_) + , m_lexer(std::move(adapter), ignore_comments, discard_values_) , allow_exceptions(allow_exceptions_) , ignore_trailing_commas(ignore_trailing_commas_) { diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 4d4f0a1cd..15510ac16 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -184,11 +184,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool allow_exceptions = true, const bool ignore_comments = false, const bool ignore_trailing_commas = false, - const bool discard_number_values = false + const bool discard_values = false ) { return ::nlohmann::detail::parser(std::move(adapter), - std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values); + std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_values); } private: diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 10b42dac1..3fb14819f 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -13124,10 +13124,10 @@ class lexer : public lexer_base public: using token_type = typename lexer_base::token_type; - explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept + explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_values_ = false) noexcept : ia(std::move(adapter)) , ignore_comments(ignore_comments_) - , discard_number_values(discard_number_values_) + , discard_values(discard_values_) { // the token string of an error before the first token is everything // read so far @@ -13244,6 +13244,8 @@ class lexer : public lexer_base checks if it is inside the range. If a violation was detected, set up an error message and return false. Otherwise, return true. + @param[in] keep whether the bytes are added to token_buffer (false for + accept(), see scan()) @param[in] ranges list of integers; interpreted as list of pairs of inclusive lower and upper bound, respectively @@ -13252,17 +13254,18 @@ class lexer : public lexer_base @return true if and only if no range violation was detected */ - bool next_byte_in_range(std::initializer_list ranges) + template + bool next_byte_in_range(std::integral_constant keep, std::initializer_list ranges) { JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6); - add(current); + add_string_char(keep, current); for (auto range = ranges.begin(); range != ranges.end(); ++range) { get(); if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions) { - add(current); + add_string_char(keep, current); } else { @@ -13275,14 +13278,16 @@ class lexer : public lexer_base } /// contiguous input: bulk-append the run of ordinary characters and complete - /// well-formed UTF-8 sequences starting at the current read position, then + /// well-formed UTF-8 sequences starting at the current read position (only + /// skip it if @a keep is false), then /// read the first byte that needs individual handling (the closing quote, an /// escape, a control character, or an ill-formed UTF-8 byte) like get(). /// That byte is taken from the buffer that was just scanned. Through get(), /// Apple Clang merged this read with the other get() calls of the scan /// function, which load the read position again right after it was stored /// (3% of json::accept() on jeopardy.json on Apple M1). - char_int_type get_after_string_run(std::true_type /*bulk*/) + template + char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant keep) { // a pending unget must be consumed through the normal path first if (next_unget) @@ -13300,7 +13305,7 @@ class lexer : public lexer_base const std::size_t pos = lexer_string_bulk_run(data, remaining); if (pos != 0) { - token_buffer.append(reinterpret_cast(data), pos); + append_string_run(keep, data, pos); bulk_consume(reinterpret_cast(data), pos); } if (pos == remaining) @@ -13344,7 +13349,8 @@ class lexer : public lexer_base } /// streaming input: no bulk fast path - char_int_type get_after_string_run(std::false_type /*bulk*/) + template + char_int_type get_after_string_run(std::false_type /*bulk*/, std::integral_constant /*keep*/) { return get(); } @@ -13358,13 +13364,18 @@ class lexer : public lexer_base may contain \0 bytes), and token_buffer.size() is the number of bytes in the string. + @param[in] keep whether the string is copied into token_buffer; if false + (accept(), see scan()), it is only checked, with the same + errors, and token_buffer stays empty + @return token_type::value_string if string could be successfully scanned, token_type::parse_error otherwise @note In case of errors, variable error_message contains a textual description. */ - token_type scan_string() + template + token_type scan_string(std::integral_constant keep) { // reset token_buffer (ignore opening quote) reset(); @@ -13376,7 +13387,7 @@ class lexer : public lexer_base { // bulk-consume ordinary characters from contiguous input, then // get the next character, the one that ends the run - switch (get_after_string_run(std::integral_constant())) + switch (get_after_string_run(std::integral_constant(), keep)) { // end of file while parsing the string case char_traits::eof(): @@ -13398,35 +13409,35 @@ class lexer : public lexer_base { // quotation mark case '\"': - add('\"'); + add_string_char(keep, '\"'); break; // reverse solidus case '\\': - add('\\'); + add_string_char(keep, '\\'); break; // solidus case '/': - add('/'); + add_string_char(keep, '/'); break; // backspace case 'b': - add('\b'); + add_string_char(keep, '\b'); break; // form feed case 'f': - add('\f'); + add_string_char(keep, '\f'); break; // line feed case 'n': - add('\n'); + add_string_char(keep, '\n'); break; // carriage return case 'r': - add('\r'); + add_string_char(keep, '\r'); break; // tab case 't': - add('\t'); + add_string_char(keep, '\t'); break; // unicode escapes @@ -13494,10 +13505,7 @@ class lexer : public lexer_base JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF); // translate codepoint into bytes - encode_utf8(static_cast(codepoint), [this](std::uint32_t byte) - { - add(static_cast(byte)); - }); + add_codepoint(keep, codepoint); break; } @@ -13680,7 +13688,7 @@ class lexer : public lexer_base case 0x7E: case 0x7F: { - add(current); + add_string_char(keep, current); break; } @@ -13716,7 +13724,7 @@ class lexer : public lexer_base case 0xDE: case 0xDF: { - if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF}))) + if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range(keep, {0x80, 0xBF}))) { return token_type::parse_error; } @@ -13726,7 +13734,7 @@ class lexer : public lexer_base // U+0800..U+0FFF: bytes E0 A0..BF 80..BF case 0xE0: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0xA0, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -13750,7 +13758,7 @@ class lexer : public lexer_base case 0xEE: case 0xEF: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -13760,7 +13768,7 @@ class lexer : public lexer_base // U+D000..U+D7FF: bytes ED 80..9F 80..BF case 0xED: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x9F, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -13770,7 +13778,7 @@ class lexer : public lexer_base // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF case 0xF0: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -13782,7 +13790,7 @@ class lexer : public lexer_base case 0xF2: case 0xF3: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -13792,7 +13800,7 @@ class lexer : public lexer_base // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF case 0xF4: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -14283,7 +14291,7 @@ scan_number_done: token_type convert_number(token_type number_type, std::size_t mantissa_end) { // accept() only needs to know whether the input is valid, so it sets - // discard_number_values (see json.hpp), and an integer token whose + // discard_values (see json.hpp), and an integer token whose // digit count shows that it fits is reported without calling // convert_integer(). A number with up to 18 digits always fits into // both std::uint64_t and std::int64_t (18 nines is about 1e18, below @@ -14298,7 +14306,7 @@ scan_number_done: // json_sax_acceptor, whose number callbacks discard their argument and // return true, and the parser rejects neither integers nor finite // floats. value_unsigned/value_integer are left unset here, so a caller - // that reads the converted value must not set discard_number_values. + // that reads the converted value must not set discard_values. // // On contiguous input, scan_number_bulk_contiguous() converts integer // tokens itself and does not pass them to this function, unless @@ -14306,7 +14314,7 @@ scan_number_done: // reached for input without bulk access (e.g. streams), with // JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous() // falls back to scan_number(). - if (discard_number_values) + if (discard_values) { constexpr std::size_t safe_digit_count = 18; if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count) @@ -14557,7 +14565,7 @@ scan_number_done: // w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a // value that underflows is a finite number as well. Only a token that // may overflow has to be converted to find out. - if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits::max_exponent10)) + if (JSON_HEDLEY_UNLIKELY(discard_values && s.exponent + 19 <= std::numeric_limits::max_exponent10)) { value_float = 0; return true; @@ -14889,6 +14897,34 @@ scan_number_done: token_buffer.push_back(static_cast(c)); } + /// add a character of a string token; accept() checks the string but does + /// not keep its value (@a keep is false) + void add_string_char(std::true_type /*keep*/, char_int_type c) + { + token_buffer.push_back(static_cast(c)); + } + + static void add_string_char(std::false_type /*keep*/, char_int_type /*c*/) noexcept {} + + /// add the UTF-8 bytes of a code point from a \\u escape to a string token + void add_codepoint(std::true_type /*keep*/, int codepoint) + { + encode_utf8(static_cast(codepoint), [this](std::uint32_t byte) + { + add(static_cast(byte)); + }); + } + + static void add_codepoint(std::false_type /*keep*/, int /*codepoint*/) noexcept {} + + /// append a run of ordinary string characters (see get_after_string_run()) + void append_string_run(std::true_type /*keep*/, const unsigned char* data, std::size_t n) + { + token_buffer.append(reinterpret_cast(data), n); + } + + static void append_string_run(std::false_type /*keep*/, const unsigned char* /*data*/, std::size_t /*n*/) noexcept {} + public: ///////////////////// // value getters @@ -15283,7 +15319,11 @@ scan_number_done: // string case '\"': - return scan_string(); + // accept() (discard_values) checks strings without copying them + // into token_buffer. The choice is made once per string: a + // check inside the string loop made number-heavy documents up + // to 3 % slower (mesh.json, Apple M1) + return discard_values ? scan_string(std::false_type {}) : scan_string(std::true_type {}); // number case '-': @@ -15368,11 +15408,13 @@ scan_number_done: std::size_t decimal_point_position = std::string::npos; /// whether the caller only needs the token types and never looks at the - /// converted numeric values; set only by accept(), which parses through - /// json_sax_acceptor. When set, convert_number() skips converting integer - /// tokens whose digit count guarantees that they fit into 64 bits (see - /// there) - const bool discard_number_values = false; + /// values of number and string tokens; set only by accept(), which parses + /// through json_sax_acceptor. When set, convert_number() skips converting + /// integer tokens whose digit count guarantees that they fit into 64 bits + /// (see there), floats are converted only when they may overflow (see + /// convert_accumulated_float()), and strings are checked without being + /// copied into token_buffer (see scan()) + const bool discard_values = false; }; } // namespace detail @@ -22581,9 +22623,9 @@ class parser const bool allow_exceptions_ = true, const bool ignore_comments = false, const bool ignore_trailing_commas_ = false, - const bool discard_number_values_ = false) + const bool discard_values_ = false) : callback(std::move(cb)) - , m_lexer(std::move(adapter), ignore_comments, discard_number_values_) + , m_lexer(std::move(adapter), ignore_comments, discard_values_) , allow_exceptions(allow_exceptions_) , ignore_trailing_commas(ignore_trailing_commas_) { @@ -36980,11 +37022,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool allow_exceptions = true, const bool ignore_comments = false, const bool ignore_trailing_commas = false, - const bool discard_number_values = false + const bool discard_values = false ) { return ::nlohmann::detail::parser(std::move(adapter), - std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values); + std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_values); } private: diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index 090ab4b41..0c9bdbcbb 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -2982,6 +2982,50 @@ TEST_CASE("accept() agrees with parse() on floats around the overflow limit") } } +TEST_CASE("accept() agrees with parse() on strings, which it checks without copying them") +{ + // accept() scans strings and object keys with a separate instantiation of + // the string scanner that does not copy them into the token buffer; it + // must accept and reject exactly what parse() does + const std::vector pieces = + { + "", "abc", "\\\"\\\\\\/\\b\\f\\n\\r\\t", "\\u0041\\u00e4\\u20AC", "\\ud83d\\ude00", "\\uD83D\\uDE00", + "\\ud800", "\\udc00", "\\ud800\\u0041", "\\ud800x", "\\udbff\\udfff", "\\u12", "\\u12g4", "\\x", "\\", + "\xc3\xa4", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xf4\x8f\xbf\xbf", "\xc0\xaf", "\xed\xa0\x80", "\xf4\x90\x80\x80", + "\xff", "\xe2\x82", "\xc3", "\x01", "\x1f", "\x7f", "\n", std::string(1, '\0') + }; + + for (std::size_t run = 0; run <= 70; ++run) + { + for (const std::string& piece : pieces) + { + // a run of ordinary characters puts the piece at every offset of the bulk scanners + const std::string str = std::string(run, 'a') + piece; + for (const std::string& text : + { + "\"" + str + "\"", "[\"" + str + "\"]", "{\"" + str + "\":1}", "{\"k\":\"" + str + "\",\"" + str + "\":[\"" + str + "\"]}", + "[\"" + str + }) + { + CAPTURE(text) + for (int options = 0; options < 4; ++options) + { + const bool ignore_comments = (options & 1) != 0; + const bool ignore_trailing_commas = (options & 2) != 0; + const bool parses = !json::parse(text, nullptr, false, ignore_comments, ignore_trailing_commas).is_discarded(); + CHECK(json::accept(text, ignore_comments, ignore_trailing_commas) == parses); + + std::istringstream stream(text); + CHECK(json::accept(stream, ignore_comments, ignore_trailing_commas) == parses); + + const std::list chars(text.begin(), text.end()); + CHECK(json::accept(chars.begin(), chars.end(), ignore_comments, ignore_trailing_commas) == parses); + } + } + } + } +} + namespace { // a DOM parser that reports parse errors without throwing, as in the sax_no_exception example