From 57424ab0b5994dc0a7bb883e960fa94917db9c0b Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Thu, 15 Oct 2026 17:53:52 +0200 Subject: [PATCH] Match literals without a stack array and keep parser states in bytes scan_after_whitespace() built the expected text of true, false, and null in local std::array objects. With -fstack-protector-strong (the default of Ubuntu's GCC, among others), a local array gives the function a stack canary check, which then ran on every token. scan_literal() now takes the text after the first character as a string literal, with its length as a template parameter; for contiguous input it compares the rest at once and falls back to the byte loop on a mismatch, so errors are reported at the same character. It is out of line, which keeps the token dispatch small. The parser kept the nesting of arrays and objects in a std::vector, whose push_back() GCC calls out of line; it is a std::vector now. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/lexer.hpp | 62 +++++++++++++------- include/nlohmann/detail/input/parser.hpp | 11 ++-- single_include/nlohmann/json.hpp | 73 +++++++++++++++--------- tests/src/unit-class_lexer.cpp | 58 +++++++++++++++++++ tests/src/unit-stream-bulk-scan.cpp | 3 +- 5 files changed, 154 insertions(+), 53 deletions(-) diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 64e0dafbd..fde164dd9 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -12,6 +12,7 @@ #include // size_t #include // uint32_t #include // snprintf +#include // memcmp #include // initializer_list #include // char_traits, string #include // move @@ -1841,18 +1842,23 @@ scan_number_done: } /*! - @param[in] literal_text the literal text to expect - @param[in] length the length of the passed literal text - @param[in] return_type the token type to return on success + @param[in] rest the literal text after its first character, which + is current (its length is a template parameter, so + that the comparison of the contiguous fast path is + a fixed-size load, not a call of memcmp) + @param[in] return_type the token type to return on success */ - JSON_HEDLEY_NON_NULL(2) - token_type scan_literal(const char_type* literal_text, const std::size_t length, - token_type return_type) + template + JSON_HEDLEY_NEVER_INLINE token_type scan_literal(const char (&rest)[N], token_type return_type) { - JSON_ASSERT(char_traits::to_char_type(current) == literal_text[0]); - for (std::size_t i = 1; i < length; ++i) + constexpr std::size_t length = N - 1; + if (scan_literal_bulk(rest, length, std::integral_constant {})) { - if (JSON_HEDLEY_UNLIKELY(char_traits::to_char_type(get()) != literal_text[i])) + return return_type; + } + for (std::size_t i = 0; i < length; ++i) + { + if (JSON_HEDLEY_UNLIKELY(char_traits::to_char_type(get()) != static_cast(rest[i]))) { error_message = "invalid literal"; return token_type::parse_error; @@ -1861,6 +1867,27 @@ scan_number_done: return return_type; } + /// contiguous input: compare the rest of a literal at once; false (and + /// nothing consumed) if it does not match, so that the byte path reports + /// the error at the right character + JSON_HEDLEY_ALWAYS_INLINE bool scan_literal_bulk(const char* rest, const std::size_t length, std::true_type /*bulk*/) + { + if (next_unget || ia.bulk_remaining() < length + || std::memcmp(ia.bulk_data(), rest, length) != 0) + { + return false; + } + bulk_consume(reinterpret_cast(ia.bulk_data()), length); + current = static_cast(static_cast(rest[length - 1])); + return true; + } + + /// streaming input: no bulk fast path + bool scan_literal_bulk(const char* /*rest*/, const std::size_t /*length*/, std::false_type /*bulk*/) const noexcept + { + return false; + } + ///////////////////// // input management ///////////////////// @@ -2486,21 +2513,14 @@ scan_number_done: return token_type::value_separator; // literals + // (the text after the first character; no local arrays, which + // would give this function a stack canary on every token) case 't': - { - std::array true_literal = {{static_cast('t'), static_cast('r'), static_cast('u'), static_cast('e')}}; - return scan_literal(true_literal.data(), true_literal.size(), token_type::literal_true); - } + return scan_literal("rue", token_type::literal_true); case 'f': - { - std::array false_literal = {{static_cast('f'), static_cast('a'), static_cast('l'), static_cast('s'), static_cast('e')}}; - return scan_literal(false_literal.data(), false_literal.size(), token_type::literal_false); - } + return scan_literal("alse", token_type::literal_false); case 'n': - { - std::array null_literal = {{static_cast('n'), static_cast('u'), static_cast('l'), static_cast('l')}}; - return scan_literal(null_literal.data(), null_literal.size(), token_type::literal_null); - } + return scan_literal("ull", token_type::literal_null); // string case '\"': diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index 493ff6f68..38bc9cdc9 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -244,8 +244,9 @@ class parser bool sax_parse_internal(SAX* sax) { // stack to remember the hierarchy of structured values we are parsing - // true = array; false = object - std::vector states; + // true = array; false = object (bytes, not std::vector, whose + // push_back() is a call with GCC) + std::vector states; // value to avoid a goto (see comment where set to true) bool skip_to_state_evaluation = false; @@ -290,7 +291,7 @@ class parser } // remember we are now inside an object - states.push_back(false); + states.push_back(0); // parse values get_token(); @@ -315,7 +316,7 @@ class parser } // remember we are now inside an array - states.push_back(true); + states.push_back(1); // parse values (no need to call get_token) continue; @@ -432,7 +433,7 @@ class parser return true; } - if (states.back()) // array + if (states.back() != 0) // array { // comma -> next value // or end of array (ignore_trailing_commas = true) diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 6c32488e5..ccd99cfb2 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -9215,6 +9215,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // size_t #include // uint32_t #include // snprintf +#include // memcmp #include // initializer_list #include // char_traits, string #include // move @@ -13426,18 +13427,23 @@ scan_number_done: } /*! - @param[in] literal_text the literal text to expect - @param[in] length the length of the passed literal text - @param[in] return_type the token type to return on success + @param[in] rest the literal text after its first character, which + is current (its length is a template parameter, so + that the comparison of the contiguous fast path is + a fixed-size load, not a call of memcmp) + @param[in] return_type the token type to return on success */ - JSON_HEDLEY_NON_NULL(2) - token_type scan_literal(const char_type* literal_text, const std::size_t length, - token_type return_type) + template + JSON_HEDLEY_NEVER_INLINE token_type scan_literal(const char (&rest)[N], token_type return_type) { - JSON_ASSERT(char_traits::to_char_type(current) == literal_text[0]); - for (std::size_t i = 1; i < length; ++i) + constexpr std::size_t length = N - 1; + if (scan_literal_bulk(rest, length, std::integral_constant {})) { - if (JSON_HEDLEY_UNLIKELY(char_traits::to_char_type(get()) != literal_text[i])) + return return_type; + } + for (std::size_t i = 0; i < length; ++i) + { + if (JSON_HEDLEY_UNLIKELY(char_traits::to_char_type(get()) != static_cast(rest[i]))) { error_message = "invalid literal"; return token_type::parse_error; @@ -13446,6 +13452,27 @@ scan_number_done: return return_type; } + /// contiguous input: compare the rest of a literal at once; false (and + /// nothing consumed) if it does not match, so that the byte path reports + /// the error at the right character + JSON_HEDLEY_ALWAYS_INLINE bool scan_literal_bulk(const char* rest, const std::size_t length, std::true_type /*bulk*/) + { + if (next_unget || ia.bulk_remaining() < length + || std::memcmp(ia.bulk_data(), rest, length) != 0) + { + return false; + } + bulk_consume(reinterpret_cast(ia.bulk_data()), length); + current = static_cast(static_cast(rest[length - 1])); + return true; + } + + /// streaming input: no bulk fast path + bool scan_literal_bulk(const char* /*rest*/, const std::size_t /*length*/, std::false_type /*bulk*/) const noexcept + { + return false; + } + ///////////////////// // input management ///////////////////// @@ -14071,21 +14098,14 @@ scan_number_done: return token_type::value_separator; // literals + // (the text after the first character; no local arrays, which + // would give this function a stack canary on every token) case 't': - { - std::array true_literal = {{static_cast('t'), static_cast('r'), static_cast('u'), static_cast('e')}}; - return scan_literal(true_literal.data(), true_literal.size(), token_type::literal_true); - } + return scan_literal("rue", token_type::literal_true); case 'f': - { - std::array false_literal = {{static_cast('f'), static_cast('a'), static_cast('l'), static_cast('s'), static_cast('e')}}; - return scan_literal(false_literal.data(), false_literal.size(), token_type::literal_false); - } + return scan_literal("alse", token_type::literal_false); case 'n': - { - std::array null_literal = {{static_cast('n'), static_cast('u'), static_cast('l'), static_cast('l')}}; - return scan_literal(null_literal.data(), null_literal.size(), token_type::literal_null); - } + return scan_literal("ull", token_type::literal_null); // string case '\"': @@ -20133,8 +20153,9 @@ class parser bool sax_parse_internal(SAX* sax) { // stack to remember the hierarchy of structured values we are parsing - // true = array; false = object - std::vector states; + // true = array; false = object (bytes, not std::vector, whose + // push_back() is a call with GCC) + std::vector states; // value to avoid a goto (see comment where set to true) bool skip_to_state_evaluation = false; @@ -20179,7 +20200,7 @@ class parser } // remember we are now inside an object - states.push_back(false); + states.push_back(0); // parse values get_token(); @@ -20204,7 +20225,7 @@ class parser } // remember we are now inside an array - states.push_back(true); + states.push_back(1); // parse values (no need to call get_token) continue; @@ -20321,7 +20342,7 @@ class parser return true; } - if (states.back()) // array + if (states.back() != 0) // array { // comma -> next value // or end of array (ignore_trailing_commas = true) diff --git a/tests/src/unit-class_lexer.cpp b/tests/src/unit-class_lexer.cpp index 1e9f19a41..c05727213 100644 --- a/tests/src/unit-class_lexer.cpp +++ b/tests/src/unit-class_lexer.cpp @@ -13,6 +13,7 @@ using nlohmann::json; #include // array +#include // toupper #include // uint32_t, uint64_t #include // snprintf #include // isfinite @@ -596,6 +597,63 @@ TEST_CASE("lexer number fast path") #endif } +TEST_CASE("lexer literal fast path") +{ + // For contiguous input, the rest of a literal is compared at once, and a + // mismatch is left to the byte path, which reports the error at the + // character that differs. Every truncation and every wrong character of + // each literal, at the end of the input or followed by more text, must + // give the byte path's result (a std::list is read one character at a + // time). + const auto outcome = [](const std::string & doc, bool byte_path) -> std::string + { + JSON_TRY + { + const std::list chars(doc.begin(), doc.end()); + return (byte_path ? json::parse(chars.begin(), chars.end()) : json::parse(doc)).dump(); + } + JSON_CATCH (const json::exception & e) + { + return e.what(); + } + }; + + std::vector mismatches; + for (const std::string literal : + {"true", "false", "null" + }) + { + std::vector variants; + for (std::size_t length = 1; length <= literal.size(); ++length) + { + variants.push_back(literal.substr(0, length)); + } + for (std::size_t i = 1; i < literal.size(); ++i) + { + std::string changed = literal; + changed[i] = 'X'; + variants.push_back(changed); + changed[i] = static_cast(std::toupper(static_cast(literal[i]))); + variants.push_back(changed); + } + for (const auto& variant : variants) + { + for (const std::string& doc : + { + variant, "[" + variant + "]", "[" + variant, variant + " ", variant + "1", "{\"k\":" + variant + "}" + }) + { + if (outcome(doc, false) != outcome(doc, true)) + { + mismatches.push_back(doc); + } + } + } + } + CAPTURE(mismatches) + CHECK(mismatches.empty()); +} + TEST_CASE("lexer string fast path") { // Build a byte string from explicit values: a hex escape in a string diff --git a/tests/src/unit-stream-bulk-scan.cpp b/tests/src/unit-stream-bulk-scan.cpp index b5b6ded15..b42b28e87 100644 --- a/tests/src/unit-stream-bulk-scan.cpp +++ b/tests/src/unit-stream-bulk-scan.cpp @@ -171,7 +171,8 @@ std::vector corpus() R"("unterminated)", "\"control\x01character\"", "\"ill-formed \xc3\x28 UTF-8\"", R"("bad \x escape")", R"("\ud800 lone surrogate")", // literals - "true", "false", "null", "tru", "nul", "falsey", + "true", "false", "null", "tru", "nul", "falsey", "[true,false,null]", "nulL", "truex", "[fals e]", "t", + "true1", "null null", "[false]", // structures "[]", "{}", "[1,2,3]", R"({"a":1,"b":[true,false,null],"c":{"d":"e"}})", "[1,]", R"({"a" 1})", "[[[[[]]]]]", // pretty-printed