diff --git a/docs/mkdocs/docs/api/basic_json_document/accept.md b/docs/mkdocs/docs/api/basic_json_document/accept.md index 1f1070d51..766fd20fa 100644 --- a/docs/mkdocs/docs/api/basic_json_document/accept.md +++ b/docs/mkdocs/docs/api/basic_json_document/accept.md @@ -13,9 +13,15 @@ function never throws an exception for invalid input, and the returned `#!cpp bo is returned. The result is the same as that of `BasicJsonType::accept()` by design; the difference is how the check is made. This -function runs the document's index builder (and discards the index), while `BasicJsonType::accept()` runs the SAX -parser with a handler that ignores every event. Use this one if you go on to parse the same input with -[`parse()`](parse.md) anyway, so that both use the same code; either is fine for a plain yes/no answer. +function runs the document's builder, which checks the text as [`parse()`](parse.md) does, while +`BasicJsonType::accept()` runs the SAX parser with a handler that ignores every event. Use this one if you go on to +parse the same input with [`parse()`](parse.md) anyway, so that both use the same code; either is fine for a plain +yes/no answer. + +When both `ignore_comments` and `ignore_trailing_commas` are `#!cpp false` (the default), the builder writes no index. +With either option set, it builds the index as [`parse()`](parse.md) does and discards it: the result is the same, only +the time differs. (Neither option needs the index; the variant that skips it exists only for the default options to +keep the code size down.) ## Template parameters diff --git a/include/nlohmann/detail/view/builder.hpp b/include/nlohmann/detail/view/builder.hpp index daf0d59bb..4b729ce39 100644 --- a/include/nlohmann/detail/view/builder.hpp +++ b/include/nlohmann/detail/view/builder.hpp @@ -69,7 +69,8 @@ struct parse_failure }; /// FloatType: the number_float_t of the document, whose overflow parse() rejects -template +/// Store: false for accept(): the grammar, number, string and UTF-8 checks run as for parse(), but no node is written +template class builder { public: @@ -245,11 +246,87 @@ class builder return p; } + /// the string from p, the first backslash, control character, ill-formed UTF-8 byte, or the end, checked and not + /// decoded (accept() stores nothing, so the arena is not written): the checks, errors and error positions of + /// slow_string(), for everything up to the closing quote + NLOHMANN_VIEW_NOINLINE decoded check_string(const unsigned char* p) + { + const unsigned char* const end = m_end; + for (;;) + { + if (p == end) + { + return failed(error_code::string_missing_quote, p); + } + const unsigned char c = *p; + if (c == '"') + { + ++p; + return decoded{p, 0, 0}; + } + if (c != '\\') + { + return failed(c < 0x20 ? error_code::string_control_character : error_code::string_utf8, p); + } + ++p; + if (p == end) + { + return failed(error_code::string_missing_quote, p); + } + const unsigned char d = *p++; + if (d == 'u') + { + const int cp = hex4(p); // (moves p past the four digits) + if (NLOHMANN_VIEW_UNLIKELY(cp < 0)) + { + return failed(error_code::string_unicode_hex, p); + } + if (NLOHMANN_VIEW_UNLIKELY((cp & 0xF800) == 0xD800)) // a surrogate + { + if (cp >= 0xDC00) + { + return failed(error_code::string_surrogate_low, p); + } + if (end - p < 2 || p[0] != '\\' || p[1] != 'u') + { + return failed(error_code::string_surrogate_high, p); + } + p += 2; + const int lo = hex4(p); // (moves p past the four digits) + if (lo < 0) + { + return failed(error_code::string_unicode_hex, p); + } + if (lo < 0xDC00 || lo > 0xDFFF) + { + return failed(error_code::string_surrogate_high, p); + } + } + } + else if (!(d == '"' || d == '\\' || d == '/' || d == 'b' || d == 'f' || d == 'n' || d == 'r' || d == 't')) + { + --p; + return failed(error_code::string_escape, p); + } + if (p != end && *p == '\\') + { + continue; // (escapes in a row: no run of text between them) + } + p = scan_string_run(p, end); // the text up to the next backslash, quote, control character, or ill-formed UTF-8 + } + } + /// s points to the start of the string, p to a backslash, a control /// character, an ill-formed UTF-8 byte, or the end: decode the string into /// the arena NLOHMANN_VIEW_NOINLINE decoded slow_string(const unsigned char* s, const unsigned char* p) { + if (!enabled(Store)) + { + // (accept(): the string is checked, not decoded) + static_cast(s); + return check_string(p); + } // single-character escapes; 0: invalid (and 'u', handled separately) static const std::array simple_escape = { @@ -625,10 +702,13 @@ run_found: NLOHMANN_VIEW_ALWAYS_INLINE bool run() { - cold.m_doc.reserve(estimate_nodes(reinterpret_cast(b), static_cast(e - b))); - base = cold.m_doc.nodes; - out = base; - cap = base + cold.m_doc.node_capacity; + if (enabled(Store)) + { + cold.m_doc.reserve(estimate_nodes(reinterpret_cast(b), static_cast(e - b))); + base = cold.m_doc.nodes; + out = base; + cap = base + cold.m_doc.node_capacity; + } if (e - p >= 3 && p[0] == 0xEF && p[1] == 0xBB && p[2] == 0xBF) { @@ -812,7 +892,10 @@ close_object: // an array pays nothing for this) if (NLOHMANN_VIEW_UNLIKELY(cur_count >= document_data::index_min_members)) { - cold.note_large_object(cur_idx); + if (enabled(Store)) + { + cold.note_large_object(cur_idx); + } } close_container: @@ -972,6 +1055,12 @@ indent_done: /// an integer's value); two stores on little-endian targets NLOHMANN_VIEW_ALWAYS_INLINE node* push_node(node_kind k, std::uint8_t flags, std::uint16_t extra, std::size_t off, std::uint64_t second) { + if (!enabled(Store)) + { + // accept(): nothing is stored, so what computes the arguments is not run either. base and out stay + // null: their difference, which open() takes, is 0. + return out; + } if (NLOHMANN_VIEW_UNLIKELY(out == cap)) { const auto n = static_cast(out - base); @@ -1030,9 +1119,12 @@ indent_done: NLOHMANN_VIEW_ALWAYS_INLINE void close() { - node& n = base[cur_idx]; - n.len = cur_count; - n.next = static_cast(out - base) - cur_idx; + if (enabled(Store)) + { + node& n = base[cur_idx]; + n.len = cur_count; + n.next = static_cast(out - base) - cur_idx; + } if (--depth != 0) { if (NLOHMANN_VIEW_LIKELY(depth <= 64)) @@ -1207,34 +1299,44 @@ indent_done: }; /// run the builder with compile-time options -template +template inline bool build_with(document_data& d, const char* src, std::size_t size, bool sentinel, parse_failure& failure) { if (sentinel) { - builder bld(d, src, size); + builder bld(d, src, size); const bool ok = bld.run(); failure = bld.failure(); return ok; } - builder bld(d, src, size); + builder bld(d, src, size); const bool ok = bld.run(); failure = bld.failure(); return ok; } /// sentinel: src[size] is readable and 0 (e.g. std::string); FloatType: the -/// number_float_t of the document +/// number_float_t of the document; validate_only: accept(), only the result is +/// used template -inline bool build(document_data& d, const char* src, std::size_t size, bool comments, bool trailing_commas, bool sentinel, parse_failure& failure) +inline bool build(document_data& d, const char* src, std::size_t size, bool comments, bool trailing_commas, bool sentinel, bool validate_only, + parse_failure& failure) { + if (validate_only && !comments && !trailing_commas) + { + // accept() of plain JSON: nothing is stored. The other option sets + // could skip the stores just as well; they keep the builder that + // stores only so that there are two more instantiations of the parse + // loop, not eight. + return build_with(d, src, size, sentinel, failure); + } if (comments) { - return trailing_commas ? build_with(d, src, size, sentinel, failure) - : build_with(d, src, size, sentinel, failure); + return trailing_commas ? build_with(d, src, size, sentinel, failure) + : build_with(d, src, size, sentinel, failure); } - return trailing_commas ? build_with(d, src, size, sentinel, failure) - : build_with(d, src, size, sentinel, failure); + return trailing_commas ? build_with(d, src, size, sentinel, failure) + : build_with(d, src, size, sentinel, failure); } } // namespace view diff --git a/include/nlohmann/detail/view/macro_scope.hpp b/include/nlohmann/detail/view/macro_scope.hpp index bbe1a240b..f19492648 100644 --- a/include/nlohmann/detail/view/macro_scope.hpp +++ b/include/nlohmann/detail/view/macro_scope.hpp @@ -40,6 +40,15 @@ #define NLOHMANN_VIEW_NOINLINE #endif +// GCC: a function compiled as written, without the clones in which interprocedural +// constant propagation drops an argument that every caller passes as the same constant +// (calling such a clone changes the code GCC generates for the parse loop that calls it) +#if defined(__GNUC__) && !defined(__clang__) + #define NLOHMANN_VIEW_NOCLONE __attribute__((noclone)) +#else + #define NLOHMANN_VIEW_NOCLONE +#endif + #if defined(__GNUC__) || defined(__clang__) #define NLOHMANN_VIEW_NODISCARD __attribute__((warn_unused_result)) #elif defined(_MSC_VER) diff --git a/include/nlohmann/detail/view/macro_unscope.hpp b/include/nlohmann/detail/view/macro_unscope.hpp index a9aec867b..0c4fab4d2 100644 --- a/include/nlohmann/detail/view/macro_unscope.hpp +++ b/include/nlohmann/detail/view/macro_unscope.hpp @@ -15,6 +15,7 @@ #undef NLOHMANN_VIEW_UNLIKELY #undef NLOHMANN_VIEW_ALWAYS_INLINE #undef NLOHMANN_VIEW_NOINLINE +#undef NLOHMANN_VIEW_NOCLONE #undef NLOHMANN_VIEW_NODISCARD #undef NLOHMANN_VIEW_THROW #undef NLOHMANN_VIEW_LITTLE_ENDIAN diff --git a/include/nlohmann/detail/view/simd.hpp b/include/nlohmann/detail/view/simd.hpp index 9dede9b09..f4745f6b8 100644 --- a/include/nlohmann/detail/view/simd.hpp +++ b/include/nlohmann/detail/view/simd.hpp @@ -289,9 +289,12 @@ compares, and the UTF-8 check covers the bytes up to it. Returns where the string scan stops, like scan_string_run: before ill-formed UTF-8 and for the last bytes of the input, the bytes are checked one sequence at a time. Out of line, so that no constants of the check occupy registers in the parse loop. -On x86-64, it is compiled for SSSE3 (see cpu_has_ssse3()). +On x86-64, it is compiled for SSSE3 (see cpu_has_ssse3()). Not cloned: with +the calls of accept()'s builder, GCC 16 would clone it without plain (always +string_plain()), and calling the clone made json_document::parse() run 0.7-1.2 % +more instructions. */ -NLOHMANN_VIEW_SSSE3_TARGET NLOHMANN_VIEW_NOINLINE inline const unsigned char* scan_string_vector(const unsigned char* p, const unsigned char* e, const std::uint8_t* plain) noexcept +NLOHMANN_VIEW_SSSE3_TARGET NLOHMANN_VIEW_NOINLINE NLOHMANN_VIEW_NOCLONE inline const unsigned char* scan_string_vector(const unsigned char* p, const unsigned char* e, const std::uint8_t* plain) noexcept { using lookup = utf8_lookup4<>; const unsigned char* block = p; diff --git a/include/nlohmann/json_view.hpp b/include/nlohmann/json_view.hpp index 9d26d12ea..097e489a4 100644 --- a/include/nlohmann/json_view.hpp +++ b/include/nlohmann/json_view.hpp @@ -1060,7 +1060,7 @@ class basic_json_document const bool ignore_trailing_commas = false) { basic_json_document d; - d.read_range(first, last, allow_exceptions, ignore_comments, ignore_trailing_commas); + d.read_range(first, last, error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas); return d; } @@ -1075,7 +1075,7 @@ class basic_json_document basic_json_document d; std::string buf; append_input(buf, std::forward(input)); - d.build_owned(std::move(buf), allow_exceptions, ignore_comments, ignore_trailing_commas); + d.build_owned(std::move(buf), error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas); return d; } @@ -1088,7 +1088,8 @@ class basic_json_document static bool accept(InputType&& input, const bool ignore_comments = false, const bool ignore_trailing_commas = false) { basic_json_document d; - d.read(std::forward(input), false, ignore_comments, ignore_trailing_commas); + d.read_kind(std::forward(input), on_error::validate, ignore_comments, ignore_trailing_commas, + std::integral_constant::value> {}); return !d.is_discarded(); } @@ -1104,7 +1105,7 @@ class basic_json_document const bool ignore_comments = false, const bool ignore_trailing_commas = false) { - read_kind(std::forward(input), allow_exceptions, ignore_comments, ignore_trailing_commas, + read_kind(std::forward(input), error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas, std::integral_constant::value> {}); } @@ -1463,6 +1464,22 @@ class basic_json_document return idx; } + /// what a failed parse does + enum class on_error : std::uint8_t + { + discard, ///< the document is discarded + raise, ///< parse_error is thrown + validate ///< accept(): the document is discarded, and no node is stored while parsing + }; + + static on_error error_mode(bool allow_exceptions) noexcept + { + return allow_exceptions ? on_error::raise : on_error::discard; + } + + /// NulIsEnd of the builder: a null byte ends the input unless strict_nul_handling + static constexpr bool nul_is_end = !detail::abi_config::strict_nul_handling; + /// create the storage (sized for the input) on first use void ensure_data(const char* src, std::size_t size) { @@ -1474,7 +1491,7 @@ class basic_json_document /// parse a buffer the document takes ownership of (it replaces the /// previous owned copy) - void build_owned(std::string&& buf, bool allow_exceptions, bool comments, bool trailing_commas) + void build_owned(std::string&& buf, on_error allow_exceptions, bool comments, bool trailing_commas) { ensure_data(buf.data(), buf.size()); m_data->owned = std::move(buf); @@ -1484,7 +1501,7 @@ class basic_json_document /// parse a copy of the input; the copy is appended to the cleared owned /// buffer, so that its capacity is reused template - void build_copy(T&& input, bool allow_exceptions, bool comments, bool trailing_commas) + void build_copy(T&& input, on_error allow_exceptions, bool comments, bool trailing_commas) { if (!m_data) { @@ -1505,7 +1522,7 @@ class basic_json_document } /// sentinel: src[size] is readable and 0 (std::string, C strings) - void build(const char* src, std::size_t size, bool allow_exceptions, bool comments, bool trailing_commas, bool owned, bool sentinel) + void build(const char* src, std::size_t size, on_error allow_exceptions, bool comments, bool trailing_commas, bool owned, bool sentinel) { ensure_data(src, size); document_data& d = *m_data; @@ -1532,7 +1549,8 @@ class basic_json_document } else { - ok = detail::view::build < typename BasicJsonType::number_float_t, !detail::abi_config::strict_nul_handling > (d, src, size, comments, trailing_commas, sentinel, failure); + ok = detail::view::build(d, src, size, comments, trailing_commas, sentinel, + allow_exceptions == on_error::validate, failure); } if (NLOHMANN_VIEW_LIKELY(ok)) { @@ -1545,7 +1563,7 @@ class basic_json_document return; } std::vector().swap(d.large_objects); - if (allow_exceptions) + if (allow_exceptions == on_error::raise) { detail::view::throw_parse_failure(failure, src, size, comments, trailing_commas); } @@ -1553,13 +1571,13 @@ class basic_json_document // --- input dispatch (see detail::view::input_kind) --- - void read_kind(std::string&& s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(std::string&& s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { build_owned(std::move(s), ae, c, tc); } template - void read_kind(CharT* s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(CharT* s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { if (s == nullptr) { @@ -1573,7 +1591,7 @@ class basic_json_document } template - void read_kind(Array& a, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(Array& a, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { using CharT = typename std::remove_cv::type>::type; const std::size_t n = std::extent::value; @@ -1584,7 +1602,7 @@ class basic_json_document } template - void read_kind(const T& s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(const T& s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { // std::basic_string guarantees data()[size()] == 0: use it as sentinel const auto size = static_cast(s.size()); @@ -1593,13 +1611,13 @@ class basic_json_document } template - void read_kind(const T& s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(const T& s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { build_copy(s, ae, c, tc); } template - void read_kind(CharT* s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(CharT* s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { if (s == nullptr) { @@ -1611,13 +1629,13 @@ class basic_json_document } template - void read_kind(T&& input, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(T&& input, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { build_copy(std::forward(input), ae, c, tc); } template - void read_range(IteratorType first, IteratorType last, bool ae, bool c, bool tc) + void read_range(IteratorType first, IteratorType last, on_error ae, bool c, bool tc) { using value_type = typename std::remove_cv::value_type>::type; // borrow the range where the library's input adapter scans it as one @@ -1628,14 +1646,14 @@ class basic_json_document } template - void read_range_impl(IteratorType first, IteratorType last, bool ae, bool c, bool tc, std::true_type /*contiguous bytes*/) + void read_range_impl(IteratorType first, IteratorType last, on_error ae, bool c, bool tc, std::true_type /*contiguous bytes*/) { const auto size = static_cast(std::distance(first, last)); build(size == 0 ? "" : reinterpret_cast(&*first), size, ae, c, tc, false, size == 0); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) } template - void read_range_impl(IteratorType first, IteratorType last, bool ae, bool c, bool tc, std::false_type /*other*/) + void read_range_impl(IteratorType first, IteratorType last, on_error ae, bool c, bool tc, std::false_type /*other*/) { build_copy(detail::view::make_adapter_input(detail::input_adapter(first, last)), ae, c, tc); } diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 7e20544ba..f446b273d 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -157,6 +157,15 @@ #define NLOHMANN_VIEW_NOINLINE #endif +// GCC: a function compiled as written, without the clones in which interprocedural +// constant propagation drops an argument that every caller passes as the same constant +// (calling such a clone changes the code GCC generates for the parse loop that calls it) +#if defined(__GNUC__) && !defined(__clang__) + #define NLOHMANN_VIEW_NOCLONE __attribute__((noclone)) +#else + #define NLOHMANN_VIEW_NOCLONE +#endif + #if defined(__GNUC__) || defined(__clang__) #define NLOHMANN_VIEW_NODISCARD __attribute__((warn_unused_result)) #elif defined(_MSC_VER) @@ -948,9 +957,12 @@ compares, and the UTF-8 check covers the bytes up to it. Returns where the string scan stops, like scan_string_run: before ill-formed UTF-8 and for the last bytes of the input, the bytes are checked one sequence at a time. Out of line, so that no constants of the check occupy registers in the parse loop. -On x86-64, it is compiled for SSSE3 (see cpu_has_ssse3()). +On x86-64, it is compiled for SSSE3 (see cpu_has_ssse3()). Not cloned: with +the calls of accept()'s builder, GCC 16 would clone it without plain (always +string_plain()), and calling the clone made json_document::parse() run 0.7-1.2 % +more instructions. */ -NLOHMANN_VIEW_SSSE3_TARGET NLOHMANN_VIEW_NOINLINE inline const unsigned char* scan_string_vector(const unsigned char* p, const unsigned char* e, const std::uint8_t* plain) noexcept +NLOHMANN_VIEW_SSSE3_TARGET NLOHMANN_VIEW_NOINLINE NLOHMANN_VIEW_NOCLONE inline const unsigned char* scan_string_vector(const unsigned char* p, const unsigned char* e, const std::uint8_t* plain) noexcept { using lookup = utf8_lookup4<>; const unsigned char* block = p; @@ -1311,7 +1323,8 @@ struct parse_failure }; /// FloatType: the number_float_t of the document, whose overflow parse() rejects -template +/// Store: false for accept(): the grammar, number, string and UTF-8 checks run as for parse(), but no node is written +template class builder { public: @@ -1487,11 +1500,87 @@ class builder return p; } + /// the string from p, the first backslash, control character, ill-formed UTF-8 byte, or the end, checked and not + /// decoded (accept() stores nothing, so the arena is not written): the checks, errors and error positions of + /// slow_string(), for everything up to the closing quote + NLOHMANN_VIEW_NOINLINE decoded check_string(const unsigned char* p) + { + const unsigned char* const end = m_end; + for (;;) + { + if (p == end) + { + return failed(error_code::string_missing_quote, p); + } + const unsigned char c = *p; + if (c == '"') + { + ++p; + return decoded{p, 0, 0}; + } + if (c != '\\') + { + return failed(c < 0x20 ? error_code::string_control_character : error_code::string_utf8, p); + } + ++p; + if (p == end) + { + return failed(error_code::string_missing_quote, p); + } + const unsigned char d = *p++; + if (d == 'u') + { + const int cp = hex4(p); // (moves p past the four digits) + if (NLOHMANN_VIEW_UNLIKELY(cp < 0)) + { + return failed(error_code::string_unicode_hex, p); + } + if (NLOHMANN_VIEW_UNLIKELY((cp & 0xF800) == 0xD800)) // a surrogate + { + if (cp >= 0xDC00) + { + return failed(error_code::string_surrogate_low, p); + } + if (end - p < 2 || p[0] != '\\' || p[1] != 'u') + { + return failed(error_code::string_surrogate_high, p); + } + p += 2; + const int lo = hex4(p); // (moves p past the four digits) + if (lo < 0) + { + return failed(error_code::string_unicode_hex, p); + } + if (lo < 0xDC00 || lo > 0xDFFF) + { + return failed(error_code::string_surrogate_high, p); + } + } + } + else if (!(d == '"' || d == '\\' || d == '/' || d == 'b' || d == 'f' || d == 'n' || d == 'r' || d == 't')) + { + --p; + return failed(error_code::string_escape, p); + } + if (p != end && *p == '\\') + { + continue; // (escapes in a row: no run of text between them) + } + p = scan_string_run(p, end); // the text up to the next backslash, quote, control character, or ill-formed UTF-8 + } + } + /// s points to the start of the string, p to a backslash, a control /// character, an ill-formed UTF-8 byte, or the end: decode the string into /// the arena NLOHMANN_VIEW_NOINLINE decoded slow_string(const unsigned char* s, const unsigned char* p) { + if (!enabled(Store)) + { + // (accept(): the string is checked, not decoded) + static_cast(s); + return check_string(p); + } // single-character escapes; 0: invalid (and 'u', handled separately) static const std::array simple_escape = { @@ -1867,10 +1956,13 @@ run_found: NLOHMANN_VIEW_ALWAYS_INLINE bool run() { - cold.m_doc.reserve(estimate_nodes(reinterpret_cast(b), static_cast(e - b))); - base = cold.m_doc.nodes; - out = base; - cap = base + cold.m_doc.node_capacity; + if (enabled(Store)) + { + cold.m_doc.reserve(estimate_nodes(reinterpret_cast(b), static_cast(e - b))); + base = cold.m_doc.nodes; + out = base; + cap = base + cold.m_doc.node_capacity; + } if (e - p >= 3 && p[0] == 0xEF && p[1] == 0xBB && p[2] == 0xBF) { @@ -2054,7 +2146,10 @@ close_object: // an array pays nothing for this) if (NLOHMANN_VIEW_UNLIKELY(cur_count >= document_data::index_min_members)) { - cold.note_large_object(cur_idx); + if (enabled(Store)) + { + cold.note_large_object(cur_idx); + } } close_container: @@ -2214,6 +2309,12 @@ indent_done: /// an integer's value); two stores on little-endian targets NLOHMANN_VIEW_ALWAYS_INLINE node* push_node(node_kind k, std::uint8_t flags, std::uint16_t extra, std::size_t off, std::uint64_t second) { + if (!enabled(Store)) + { + // accept(): nothing is stored, so what computes the arguments is not run either. base and out stay + // null: their difference, which open() takes, is 0. + return out; + } if (NLOHMANN_VIEW_UNLIKELY(out == cap)) { const auto n = static_cast(out - base); @@ -2272,9 +2373,12 @@ indent_done: NLOHMANN_VIEW_ALWAYS_INLINE void close() { - node& n = base[cur_idx]; - n.len = cur_count; - n.next = static_cast(out - base) - cur_idx; + if (enabled(Store)) + { + node& n = base[cur_idx]; + n.len = cur_count; + n.next = static_cast(out - base) - cur_idx; + } if (--depth != 0) { if (NLOHMANN_VIEW_LIKELY(depth <= 64)) @@ -2449,34 +2553,44 @@ indent_done: }; /// run the builder with compile-time options -template +template inline bool build_with(document_data& d, const char* src, std::size_t size, bool sentinel, parse_failure& failure) { if (sentinel) { - builder bld(d, src, size); + builder bld(d, src, size); const bool ok = bld.run(); failure = bld.failure(); return ok; } - builder bld(d, src, size); + builder bld(d, src, size); const bool ok = bld.run(); failure = bld.failure(); return ok; } /// sentinel: src[size] is readable and 0 (e.g. std::string); FloatType: the -/// number_float_t of the document +/// number_float_t of the document; validate_only: accept(), only the result is +/// used template -inline bool build(document_data& d, const char* src, std::size_t size, bool comments, bool trailing_commas, bool sentinel, parse_failure& failure) +inline bool build(document_data& d, const char* src, std::size_t size, bool comments, bool trailing_commas, bool sentinel, bool validate_only, + parse_failure& failure) { + if (validate_only && !comments && !trailing_commas) + { + // accept() of plain JSON: nothing is stored. The other option sets + // could skip the stores just as well; they keep the builder that + // stores only so that there are two more instantiations of the parse + // loop, not eight. + return build_with(d, src, size, sentinel, failure); + } if (comments) { - return trailing_commas ? build_with(d, src, size, sentinel, failure) - : build_with(d, src, size, sentinel, failure); + return trailing_commas ? build_with(d, src, size, sentinel, failure) + : build_with(d, src, size, sentinel, failure); } - return trailing_commas ? build_with(d, src, size, sentinel, failure) - : build_with(d, src, size, sentinel, failure); + return trailing_commas ? build_with(d, src, size, sentinel, failure) + : build_with(d, src, size, sentinel, failure); } } // namespace view @@ -10674,7 +10788,7 @@ class basic_json_document const bool ignore_trailing_commas = false) { basic_json_document d; - d.read_range(first, last, allow_exceptions, ignore_comments, ignore_trailing_commas); + d.read_range(first, last, error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas); return d; } @@ -10689,7 +10803,7 @@ class basic_json_document basic_json_document d; std::string buf; append_input(buf, std::forward(input)); - d.build_owned(std::move(buf), allow_exceptions, ignore_comments, ignore_trailing_commas); + d.build_owned(std::move(buf), error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas); return d; } @@ -10702,7 +10816,8 @@ class basic_json_document static bool accept(InputType&& input, const bool ignore_comments = false, const bool ignore_trailing_commas = false) { basic_json_document d; - d.read(std::forward(input), false, ignore_comments, ignore_trailing_commas); + d.read_kind(std::forward(input), on_error::validate, ignore_comments, ignore_trailing_commas, + std::integral_constant::value> {}); return !d.is_discarded(); } @@ -10718,7 +10833,7 @@ class basic_json_document const bool ignore_comments = false, const bool ignore_trailing_commas = false) { - read_kind(std::forward(input), allow_exceptions, ignore_comments, ignore_trailing_commas, + read_kind(std::forward(input), error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas, std::integral_constant::value> {}); } @@ -11077,6 +11192,22 @@ class basic_json_document return idx; } + /// what a failed parse does + enum class on_error : std::uint8_t + { + discard, ///< the document is discarded + raise, ///< parse_error is thrown + validate ///< accept(): the document is discarded, and no node is stored while parsing + }; + + static on_error error_mode(bool allow_exceptions) noexcept + { + return allow_exceptions ? on_error::raise : on_error::discard; + } + + /// NulIsEnd of the builder: a null byte ends the input unless strict_nul_handling + static constexpr bool nul_is_end = !detail::abi_config::strict_nul_handling; + /// create the storage (sized for the input) on first use void ensure_data(const char* src, std::size_t size) { @@ -11088,7 +11219,7 @@ class basic_json_document /// parse a buffer the document takes ownership of (it replaces the /// previous owned copy) - void build_owned(std::string&& buf, bool allow_exceptions, bool comments, bool trailing_commas) + void build_owned(std::string&& buf, on_error allow_exceptions, bool comments, bool trailing_commas) { ensure_data(buf.data(), buf.size()); m_data->owned = std::move(buf); @@ -11098,7 +11229,7 @@ class basic_json_document /// parse a copy of the input; the copy is appended to the cleared owned /// buffer, so that its capacity is reused template - void build_copy(T&& input, bool allow_exceptions, bool comments, bool trailing_commas) + void build_copy(T&& input, on_error allow_exceptions, bool comments, bool trailing_commas) { if (!m_data) { @@ -11119,7 +11250,7 @@ class basic_json_document } /// sentinel: src[size] is readable and 0 (std::string, C strings) - void build(const char* src, std::size_t size, bool allow_exceptions, bool comments, bool trailing_commas, bool owned, bool sentinel) + void build(const char* src, std::size_t size, on_error allow_exceptions, bool comments, bool trailing_commas, bool owned, bool sentinel) { ensure_data(src, size); document_data& d = *m_data; @@ -11146,7 +11277,8 @@ class basic_json_document } else { - ok = detail::view::build < typename BasicJsonType::number_float_t, !detail::abi_config::strict_nul_handling > (d, src, size, comments, trailing_commas, sentinel, failure); + ok = detail::view::build(d, src, size, comments, trailing_commas, sentinel, + allow_exceptions == on_error::validate, failure); } if (NLOHMANN_VIEW_LIKELY(ok)) { @@ -11159,7 +11291,7 @@ class basic_json_document return; } std::vector().swap(d.large_objects); - if (allow_exceptions) + if (allow_exceptions == on_error::raise) { detail::view::throw_parse_failure(failure, src, size, comments, trailing_commas); } @@ -11167,13 +11299,13 @@ class basic_json_document // --- input dispatch (see detail::view::input_kind) --- - void read_kind(std::string&& s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(std::string&& s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { build_owned(std::move(s), ae, c, tc); } template - void read_kind(CharT* s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(CharT* s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { if (s == nullptr) { @@ -11187,7 +11319,7 @@ class basic_json_document } template - void read_kind(Array& a, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(Array& a, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { using CharT = typename std::remove_cv::type>::type; const std::size_t n = std::extent::value; @@ -11198,7 +11330,7 @@ class basic_json_document } template - void read_kind(const T& s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(const T& s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { // std::basic_string guarantees data()[size()] == 0: use it as sentinel const auto size = static_cast(s.size()); @@ -11207,13 +11339,13 @@ class basic_json_document } template - void read_kind(const T& s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(const T& s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { build_copy(s, ae, c, tc); } template - void read_kind(CharT* s, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(CharT* s, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { if (s == nullptr) { @@ -11225,13 +11357,13 @@ class basic_json_document } template - void read_kind(T&& input, bool ae, bool c, bool tc, std::integral_constant /*unused*/) + void read_kind(T&& input, on_error ae, bool c, bool tc, std::integral_constant /*unused*/) { build_copy(std::forward(input), ae, c, tc); } template - void read_range(IteratorType first, IteratorType last, bool ae, bool c, bool tc) + void read_range(IteratorType first, IteratorType last, on_error ae, bool c, bool tc) { using value_type = typename std::remove_cv::value_type>::type; // borrow the range where the library's input adapter scans it as one @@ -11242,14 +11374,14 @@ class basic_json_document } template - void read_range_impl(IteratorType first, IteratorType last, bool ae, bool c, bool tc, std::true_type /*contiguous bytes*/) + void read_range_impl(IteratorType first, IteratorType last, on_error ae, bool c, bool tc, std::true_type /*contiguous bytes*/) { const auto size = static_cast(std::distance(first, last)); build(size == 0 ? "" : reinterpret_cast(&*first), size, ae, c, tc, false, size == 0); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) } template - void read_range_impl(IteratorType first, IteratorType last, bool ae, bool c, bool tc, std::false_type /*other*/) + void read_range_impl(IteratorType first, IteratorType last, on_error ae, bool c, bool tc, std::false_type /*other*/) { build_copy(detail::view::make_adapter_input(detail::input_adapter(first, last)), ae, c, tc); } @@ -11380,6 +11512,7 @@ class tuple_element> // NOLINT(cert #undef NLOHMANN_VIEW_UNLIKELY #undef NLOHMANN_VIEW_ALWAYS_INLINE #undef NLOHMANN_VIEW_NOINLINE +#undef NLOHMANN_VIEW_NOCLONE #undef NLOHMANN_VIEW_NODISCARD #undef NLOHMANN_VIEW_THROW #undef NLOHMANN_VIEW_LITTLE_ENDIAN diff --git a/tests/src/unit-json_view_builder.cpp b/tests/src/unit-json_view_builder.cpp index 7ac7a7d40..418f219da 100644 --- a/tests/src/unit-json_view_builder.cpp +++ b/tests/src/unit-json_view_builder.cpp @@ -13,6 +13,7 @@ #include #include #include + #include // json_document::accept #else #include // the single header contains the internal headers #endif @@ -59,7 +60,8 @@ built build(const std::string& text, bool comments, bool trailing_commas, bool s r.copy.assign(text.begin(), text.end()); src = r.copy.data(); } - r.ok = nlohmann::detail::view::build < FloatType, !nlohmann::detail::abi_config::strict_nul_handling > (*r.data, src, text.size(), comments, trailing_commas, sentinel, r.failure); + constexpr bool nul_is_end = !nlohmann::detail::abi_config::strict_nul_handling; + r.ok = nlohmann::detail::view::build(*r.data, src, text.size(), comments, trailing_commas, sentinel, false, r.failure); r.data->src = src; r.data->base[0] = src; r.data->base[1] = r.data->arena.data(); @@ -172,6 +174,22 @@ void check_string(const std::string& content) } } +// accept() stores no nodes: it must accept and reject what json::accept does, for all options and for inputs with +// and without a terminating NUL +void check_accept(const std::string& text) +{ + CAPTURE(text) + const std::vector bytes(text.begin(), text.end()); + for (int options = 0; options < 4; ++options) + { + const bool comments = (options & 1) != 0; + const bool trailing_commas = (options & 2) != 0; + const bool accepted = json::accept(text, comments, trailing_commas); + CHECK(nlohmann::json_document::accept(text, comments, trailing_commas) == accepted); + CHECK(nlohmann::json_document::accept(bytes, comments, trailing_commas) == accepted); + } +} + // a small deterministic generator of documents struct generator { @@ -423,6 +441,133 @@ TEST_CASE("json_view builder") } } + SECTION("accept() without an index") + { + for (const char* text : + { + "", " ", "\n", "null", "nul", "true", "[", "]", "[]", "{}", "[1,2,3]", "[1,2,3", "[1,2,3,]", "{\"a\":1,}", "{\"a\"}", "{\"a\":}", + "[1] x", "[1][2]", "\xEF\xBB\xBF[1]", "[\"a\\", R"(["\u12"])", R"(["\ud800"])", "[\"\x01\"]", "[\"\xff\"]", "[01]", "[1.]", + "[.5]", "[-]", "[1e]", "// c\n[1]", "/* c */ {\"a\" /* x */ : 1}", "/* c", "[1] // c", "1e400", "-1e400", "123456789012345678901234567890", + R"({"a":1,"a":2})", "[[[]]]", R"("\u00e4\n\ud83d\ude00")" + }) + { + check_accept(text); + } + // a null byte inside the text, and after it + check_accept(std::string("[1,\0]", 5)); + check_accept(std::string("[1]\0", 4)); + check_accept(std::string("\0", 1)); + + // generated documents and damaged copies, as for the builder above + generator g; + for (int i = 0; i < 2000; ++i) + { + std::string text; + g.value(text, 0); + check_accept(text); + std::string damaged = text; + const auto at = static_cast(g.r(static_cast(damaged.size()))); + static const char replacements[] = {'x', '"', '\\', ',', ':', ']', '}', '[', '{', '1', '-', '.', 'e', '\0', '\n', '/'}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + damaged[at] = replacements[g.r(16)]; + check_accept(damaged); + check_accept(text.substr(0, at)); + } + + // nesting beyond the 64 levels that are kept on the builder's own stack, closed and not closed + for (const std::size_t depth : + { + 63u, 64u, 65u, 66u, 200u, 1000u + }) + { + check_accept(std::string(depth, '[') + std::string(depth, ']')); + check_accept(std::string(depth, '[') + std::string(depth - 1, ']')); + std::string objects; + for (std::size_t i = 0; i < depth; ++i) + { + objects += "{\"k\":"; + } + objects += "1" + std::string(depth, '}'); + check_accept(objects); + check_accept(objects.substr(0, objects.size() - 1)); + } + + // objects with enough members to get a hash index when they are indexed (and duplicate keys) + std::string large = "{"; + for (int i = 0; i < 300; ++i) + { + large += (i == 0 ? "\"k" : ",\"k") + std::to_string(i % 200) + "\":" + std::to_string(i); + } + check_accept(large + "}"); + check_accept(large); + check_accept("[" + large + "},{" + large.substr(1) + "}]"); + + // the overflow check uses the floating-point type of the document + using float_json = nlohmann::basic_json; + for (const char* text : + { + "1e39", "-1e39", "3.4028235e38", "3.4028236e38", "[1.5e38, 3.5e38]", R"({"a": 1e-50, "b": 1e39})", "340282356779733661637539395458142568448" + }) + { + CAPTURE(text) + CHECK(nlohmann::basic_json_document::accept(text) == float_json::accept(text)); + } + + // strings are checked and not decoded: runs of text of every length from 0 to 40 between valid and invalid + // escapes, UTF-8 sequences and control characters, in values and in keys, cut near the end of the text + const std::array valid = {{"\\n", "\\\"", "\\\\", "\\/", "\\t", "\\u00e4", "\\u20ac", "\\ud83d\\ude00", "\\u0000"}}; + const std::array invalid = {{"\\x", "\\ud800", "\\udc00", "\\u12G4", "\\u12", "\\ud83dx", "\\ud83d\\u0041", "\\ud83d\\ud83d", "\\", "\x01", "\x1f", "\x7f"}}; + const std::array utf8 = {{"\xc3\xa4", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xc3", "\xff", "\x80"}}; + std::mt19937 rng(2029); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible + for (std::size_t run = 0; run <= 40; ++run) + { + for (int i = 0; i < 30; ++i) + { + std::string content = std::string(rng() % 20, 'a'); + const auto pieces = 1 + (rng() % 6); + for (unsigned k = 0; k < pieces; ++k) + { + content += valid[rng() % valid.size()]; + content += std::string(k == 0 ? run : rng() % 41, static_cast('b' + (rng() % 20))); + switch (rng() % 8) + { + case 0: + content += utf8[rng() % 3]; + break; + case 1: + content += utf8[rng() % utf8.size()]; + break; + case 2: + content += invalid[rng() % invalid.size()]; + break; + default: + break; + } + } + for (const std::string& text : + { + "[\"" + content + "\"]", "{\"" + content + "\":1}", "[\"" + content, "[\"" + content.substr(0, content.size() - (rng() % 4)) + "\"]" + }) + { + check_accept(text); + } + } + } + } + + SECTION("escapes in a row") + { + for (const char* content : + { + "\\n", R"(\n\t\\)", R"(\u00e9\u00e9\u4e2d)", "\\u00e9\\u00e", "\\u00e9\\", "\\n\\q", R"(\n\ud83d\ude00\n)", + R"(\ud83d\ude00\ud83d)", R"(\n\n\n abc \n\n)", "\\n\x01", "\\n\xC3\x28", "\\n\\" + }) + { + check_accept(std::string("[\"") + content + "\"]"); + check_accept(std::string("[\"") + content); + check_accept(std::string("{\"") + content + "\":1}"); + } + } + SECTION("test files") { for (const char* name : diff --git a/tests/src/unit-json_view_macros.cpp b/tests/src/unit-json_view_macros.cpp index 2cc8b7744..d2c9e197f 100644 --- a/tests/src/unit-json_view_macros.cpp +++ b/tests/src/unit-json_view_macros.cpp @@ -25,7 +25,7 @@ #endif // the view's own macros do not leak -#if defined(NLOHMANN_VIEW_LIKELY) || defined(NLOHMANN_VIEW_UNLIKELY) || defined(NLOHMANN_VIEW_ALWAYS_INLINE) || defined(NLOHMANN_VIEW_NOINLINE) \ +#if defined(NLOHMANN_VIEW_LIKELY) || defined(NLOHMANN_VIEW_UNLIKELY) || defined(NLOHMANN_VIEW_ALWAYS_INLINE) || defined(NLOHMANN_VIEW_NOINLINE) || defined(NLOHMANN_VIEW_NOCLONE) \ || defined(NLOHMANN_VIEW_NODISCARD) || defined(NLOHMANN_VIEW_THROW) || defined(NLOHMANN_VIEW_HAS_CPP_17) || defined(NLOHMANN_VIEW_LITTLE_ENDIAN) \ || defined(NLOHMANN_VIEW_REPEAT16) #error "json_view.hpp leaks a macro"