From 933952e09555735908b9323e21f8fe5e288399a5 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 6 Nov 2026 06:56:55 +0100 Subject: [PATCH] Parse JSON in one block of memory with a fused pointer-based parser The parser and the lexer exchange one token at a time, and the lexer keeps its state (the current character, a pending unget, where the token starts) in members, which every character pays for: json::accept() of twitter.json took 2.8x as long as json_document::accept(). detail::contiguous_parser reads text in one block of memory through a local pointer, with the grammar, the whitespace and the tokens in one loop (after yyjson's reader), and drives a SAX handler: parse()'s DOM handler, accept()'s handler, or the user's handler of sax_parse(). It is used with the default options (no callback, no comments, no trailing commas; sax_parse() with strict = true). Errors are reported by the library parser, so the exception, its message and its position stay the same. contiguous_parser stops at the token that the library parser would reject, before reporting it, so the events up to the error are the same; then: - parse() runs the library parser with a handler that only reports the error (json_sax_error_reporter), with the lexer of parse(), which is instantiated anyway; - sax_parse() runs it with a handler that forwards only parse_error() to the user's handler (json_sax_error_forwarder), so no event arrives twice; - accept() returns the verdict. A handler that returns false stops the parse as before. An error at the end of a large input costs parse() about 1.2x of what it cost before. On an Apple M1 Max: accept() 1.6-2.0x as fast, parse() 10-16% faster, ordered_json 11-16%, sax_parse() with a user's handler 1.36-1.46x, small messages 12-22% (parse()) and 2.1-2.4x (accept()). On x86-64 (GCC 15, Clang 21): accept() 1.6-1.9x, parse() 5-17%, ordered_json 12-19%, sax_parse() 1.32-1.57x. A translation unit that parses a string compiles 3% (-O0) and 9% (-O2) slower. unit-contiguous_parser.cpp compares the contiguous path with the byte path (std::list iterators) on corner cases, generated documents, their truncations and mutations, deep nesting, and narrow number types: the same values with the same number types and float bits, exception messages, accept() verdicts and sax_parse() events. Signed-off-by: Niels Lohmann --- .../macros/json_disable_stream_bulk_scan.md | 4 +- docs/mkdocs/docs/features/files.md | 10 +- docs/mkdocs/docs/features/parsing/index.md | 9 +- docs/mkdocs/docs/features/performance.md | 6 +- .../detail/input/contiguous_parser.hpp | 1013 ++++++++++++++++ include/nlohmann/detail/input/string_scan.hpp | 5 +- include/nlohmann/json.hpp | 29 +- single_include/nlohmann/json.hpp | 1058 ++++++++++++++++- tests/src/unit-contiguous_parser.cpp | 536 +++++++++ 9 files changed, 2654 insertions(+), 16 deletions(-) create mode 100644 include/nlohmann/detail/input/contiguous_parser.hpp create mode 100644 tests/src/unit-contiguous_parser.cpp diff --git a/docs/mkdocs/docs/api/macros/json_disable_stream_bulk_scan.md b/docs/mkdocs/docs/api/macros/json_disable_stream_bulk_scan.md index 7befb6cde..9af9c3aea 100644 --- a/docs/mkdocs/docs/api/macros/json_disable_stream_bulk_scan.md +++ b/docs/mkdocs/docs/api/macros/json_disable_stream_bulk_scan.md @@ -48,8 +48,8 @@ The default value is `0`: streams are scanned in bulk. !!! note "Performance" Parsing from a `std::ifstream` or `std::istringstream` is typically 1.5 to 2 times as fast as before, and reading - many small values from one stream with `operator>>` 1.4 to 4 times as fast. Parsing from a stream is then within - about 10 to 20 % of first reading the whole file into a `std::string` and parsing that. + many small values from one stream with `operator>>` 1.4 to 4 times as fast. Parsing from a stream then takes about + 1.15 to 1.45 times as long as first reading the whole file into a `std::string` and parsing that. A stream buffer without a get area (one that only overrides `underflow()` and `uflow()` and hands out every character from there) offers nothing to scan in bulk, so it is read one character at a time as before. diff --git a/docs/mkdocs/docs/features/files.md b/docs/mkdocs/docs/features/files.md index 3833f13db..aec2c7e74 100644 --- a/docs/mkdocs/docs/features/files.md +++ b/docs/mkdocs/docs/features/files.md @@ -32,11 +32,11 @@ json read_json_file(const std::string& path) column 1 instead of saying that the file is missing. A relative path is resolved against the working directory of the running program, which is often not the directory of the source file (see the [FAQ](../home/faq.md#parsing-a-file-fails-with-unexpected-end-of-input)). -- **Don't read the file into a string first.** Parsing from a `#!cpp std::ifstream` is only about 10 to 20 % slower - than reading the whole file into a string with one `read()` and parsing that, and it needs no memory for the copy (see - [input sources and performance](parsing/index.md#input-sources-and-performance)). The common way to read a file with - `#!cpp std::istreambuf_iterator` is so slow that it takes about twice as long as parsing from the stream. Read - the file into a string only if you need the text itself, for example for an error message. This advice is for +- **Read the file into a string only if the memory is no concern.** Parsing from a `#!cpp std::ifstream` takes about + 1.15 to 1.45 times as long as reading the whole file into a string with one `read()` and parsing that, but it needs no + memory for the copy (see [input sources and performance](parsing/index.md#input-sources-and-performance)). The common + way to read a file with `#!cpp std::istreambuf_iterator` is so slow that it takes about twice as long as parsing + from the stream. This advice is for `#!cpp json::parse`. [`json_document`](json_view.md) reads a stream into a buffer that it owns anyway, so there is no copy to save: reading the file into a `#!cpp std::string` yourself and passing that as an lvalue, which the document borrows, was about 6 to 12 % faster than parsing from the stream, with the same single buffer. diff --git a/docs/mkdocs/docs/features/parsing/index.md b/docs/mkdocs/docs/features/parsing/index.md index 93a02b501..f4ef0aadc 100644 --- a/docs/mkdocs/docs/features/parsing/index.md +++ b/docs/mkdocs/docs/features/parsing/index.md @@ -80,8 +80,13 @@ them faster than others: | `#!cpp FILE*` | one character at a time, with the file locked once for the whole parse | | other iterator ranges (e.g., `#!cpp std::list`, `#!cpp std::istreambuf_iterator`), wide strings | one character at a time | -Parsing a file from a `#!cpp std::ifstream` is therefore only about 10 to 20 % slower than reading the whole file into a -`#!cpp std::string` first and parsing that, and it needs no memory for a copy of the file. The stream is never read +Input in one block of memory is read even faster with the default options (no +[parser callback](../../api/basic_json/parser_callback_t.md), no comments, no trailing commas, and for `sax_parse()` +`strict = true`): `parse()`, `accept()`, and `sax_parse()` then read the grammar and the tokens in one loop over the +memory. Only after an error do `parse()` and `sax_parse()` read the input again, to report the error exactly as before. + +Parsing a file from a `#!cpp std::ifstream` therefore takes about 1.15 to 1.45 times as long as reading the whole file +into a `#!cpp std::string` first and parsing that, but it needs no memory for a copy of the file. The stream is never read ahead: after parsing, it is positioned exactly as if every character had been read one at a time, so [`operator>>`](../../api/operator_gtgt.md) can still read several values from one stream (see also [JSON Lines](json_lines.md)). diff --git a/docs/mkdocs/docs/features/performance.md b/docs/mkdocs/docs/features/performance.md index 8f4508057..f3c19a450 100644 --- a/docs/mkdocs/docs/features/performance.md +++ b/docs/mkdocs/docs/features/performance.md @@ -29,9 +29,9 @@ up. Nothing is read ahead, so the stream is left exactly where it would be after [`operator>>`](../api/operator_gtgt.md) can still read several values from one stream. A stream buffer without a get area is read one character at a time. A `#!cpp FILE*` is still read one character at a time, but `file_input_adapter` locks the file once per parse instead of once per character. See -[input sources and performance](parsing/index.md#input-sources-and-performance) for the full table. According to the -description of [#5828](https://github.com/nlohmann/json/pull/5828), parsing from a stream is within about 10 to 20 % of -parsing an equivalent `#!cpp std::string`. +[input sources and performance](parsing/index.md#input-sources-and-performance) for the full table. Parsing from a +stream takes about 1.15 to 1.45 times as long as parsing an equivalent `#!cpp std::string`, which `parse()` reads with +a parser of its own (see there). Inputs that are still read one character at a time: iterator ranges that are not contiguous (e.g. `#!cpp std::list` or `#!cpp std::istreambuf_iterator`), wide strings, and user-defined adapters. An diff --git a/include/nlohmann/detail/input/contiguous_parser.hpp b/include/nlohmann/detail/input/contiguous_parser.hpp new file mode 100644 index 000000000..bcaa9b543 --- /dev/null +++ b/include/nlohmann/detail/input/contiguous_parser.hpp @@ -0,0 +1,1013 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#pragma once + +#include // isfinite +#include // size_t, ptrdiff_t +#include // uint64_t, int64_t +#include // memcmp +#include // numeric_limits +#include // string +#include // integral_constant, true_type, false_type +#include // move +#include // vector + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief a parser for JSON text in one contiguous block, for parse(), accept(), + and sax_parse() + +The parser and the lexer exchange one token at a time, and the lexer keeps its +state (the current character, a pending unget, where the token starts) in +members, which every character pays for. This parser reads the block through a +local pointer instead, and the grammar, the whitespace and the tokens are one +loop, after yyjson's reader: the state stays in registers, and the expected +next token is tested where it is expected instead of being dispatched on. + +It reports what it reads to a SAX handler: the one parse() uses to build the +value, the one accept() uses, or the user's handler of sax_parse(). On a byte +the library parser would reject, it stops before reporting the token, and +returns false: the events up to there are those the library parser reports. +The caller then runs the library parser on the same input to report the error +with its exact message and position (report_contiguous_error(), +json_sax_error_forwarder), and parse() builds no value from what was read. + +What is accepted, and the values reported, are those of the library parser +with default options: the same number grammar and conversions, the same UTF-8 +and escape rules, a UTF-8 byte order mark at the start, and, without +JSON_STRICT_NUL_HANDLING, a NUL byte as the end of the input. + +@tparam KeepValues whether the handler needs the values of strings and + numbers (false for accept(), whose handler ignores them) +*/ +template +class contiguous_parser +{ + using string_t = typename BasicJsonType::string_t; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using char_t = typename string_t::value_type; + using uchar = unsigned char; + + public: + contiguous_parser(const uchar* first, const uchar* last, SAX* sax_) noexcept + : begin(first), end(last), sax(sax_) + {} + + contiguous_parser(const contiguous_parser&) = delete; + contiguous_parser(contiguous_parser&&) = delete; + contiguous_parser& operator=(const contiguous_parser&) = delete; + contiguous_parser& operator=(contiguous_parser&&) = delete; + ~contiguous_parser() = default; + + /// whether the whole block is one valid JSON text; the handler has seen all of it + JSON_HEDLEY_NEVER_INLINE bool parse() + { + const uchar* p = begin; + const uchar* const e = end; + + if (p != e && *p == 0xEF) + { + // a byte order mark, or the lexer reports a bad one + if (e - p < 3 || p[1] != 0xBB || p[2] != 0xBF) + { + return false; + } + p += 3; + } + + // The open arrays and objects: bit 0 of `kinds` is the innermost (1: + // object), so that the kind of the enclosing container is a shift away + // and stays in a register; every 64 levels, the word is pushed to + // `outer` and a new one is begun. + std::uint64_t kinds = 0; + std::size_t depth = 0; + std::vector outer; + + // the current byte, or 0 at the end (no JSON token starts with 0) +#define JSON_CONTIGUOUS_CUR() (p != e ? *p : 0) + // skip whitespace; most tokens have none before them +#define JSON_CONTIGUOUS_WS() \ + do \ + { \ + if (JSON_HEDLEY_UNLIKELY(JSON_CONTIGUOUS_CUR() <= ' ')) \ + { \ + p = skip_whitespace(p, e); \ + } \ + } \ + while (false) +#define JSON_CONTIGUOUS_OPEN(is_object) \ + do \ + { \ + if (JSON_HEDLEY_UNLIKELY(depth != 0 && (depth % 64) == 0)) \ + { \ + outer.push_back(kinds); \ + kinds = 0; \ + } \ + kinds = (kinds << 1u) | static_cast(is_object); \ + ++depth; \ + } \ + while (false) + + JSON_CONTIGUOUS_WS(); + switch (JSON_CONTIGUOUS_CUR()) + { + case '{': + ++p; + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_object(detail::unknown_size())))) + { + return false; + } + JSON_CONTIGUOUS_OPEN(1u); + goto object_first; + case '[': + ++p; + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_array(detail::unknown_size())))) + { + return false; + } + JSON_CONTIGUOUS_OPEN(0u); + goto array_first; + default: + if (JSON_HEDLEY_UNLIKELY(!scalar(p))) + { + return false; + } + goto done; + } + + // a value in an array or an object: expanded once for each, so that + // each dispatch has its own branch history (arrays tend to hold values + // of one kind) and the code after a scalar knows where it is +#define JSON_CONTIGUOUS_VALUE \ + switch (JSON_CONTIGUOUS_CUR()) \ + { \ + case '"': \ + if (JSON_HEDLEY_UNLIKELY(!string(p, false))) \ + { \ + return false; \ + } \ + break; \ + case '{': \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_object(detail::unknown_size())))) \ + { \ + return false; \ + } \ + JSON_CONTIGUOUS_OPEN(1u); \ + goto object_first; \ + case '[': \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_array(detail::unknown_size())))) \ + { \ + return false; \ + } \ + JSON_CONTIGUOUS_OPEN(0u); \ + goto array_first; \ + case '-': \ + case '0': \ + case '1': \ + case '2': \ + case '3': \ + case '4': \ + case '5': \ + case '6': \ + case '7': \ + case '8': \ + case '9': \ + if (JSON_HEDLEY_UNLIKELY(!number(p))) \ + { \ + return false; \ + } \ + break; \ + case 't': \ + if (JSON_HEDLEY_UNLIKELY(!literal(p, "true") || !report(sax->boolean(true)))) \ + { \ + return false; \ + } \ + break; \ + case 'f': \ + if (JSON_HEDLEY_UNLIKELY(!literal(p, "false") || !report(sax->boolean(false)))) \ + { \ + return false; \ + } \ + break; \ + case 'n': \ + if (JSON_HEDLEY_UNLIKELY(!literal(p, "null") || !report(sax->null()))) \ + { \ + return false; \ + } \ + break; \ + default: \ + return false; \ + } + + // what may follow a value in an array: a comma and the next value, or the end +#define JSON_CONTIGUOUS_AFTER_ELEMENT \ + JSON_CONTIGUOUS_WS(); \ + if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ',')) \ + { \ + ++p; \ + JSON_CONTIGUOUS_WS(); \ + goto array_value; \ + } \ + if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ']')) \ + { \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_array()))) \ + { \ + return false; \ + } \ + goto close; \ + } \ + return false; + + // what may follow a value in an object: a comma and the next member, or the end +#define JSON_CONTIGUOUS_AFTER_MEMBER \ + JSON_CONTIGUOUS_WS(); \ + if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ',')) \ + { \ + ++p; \ + JSON_CONTIGUOUS_WS(); \ + goto object_key; \ + } \ + if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == '}')) \ + { \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_object()))) \ + { \ + return false; \ + } \ + goto close; \ + } \ + return false; + +array_first: + JSON_CONTIGUOUS_WS(); + if (JSON_CONTIGUOUS_CUR() == ']') + { + ++p; + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_array()))) + { + return false; + } + goto close; + } +array_value: + JSON_CONTIGUOUS_VALUE + JSON_CONTIGUOUS_AFTER_ELEMENT + +object_first: + JSON_CONTIGUOUS_WS(); + if (JSON_CONTIGUOUS_CUR() == '}') + { + ++p; + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_object()))) + { + return false; + } + goto close; + } +object_key: + if (JSON_HEDLEY_UNLIKELY(JSON_CONTIGUOUS_CUR() != '"' || !string(p, true))) + { + return false; + } + if (JSON_HEDLEY_LIKELY(e - p >= 2 && p[0] == ':' && p[1] == ' ')) + { + p += 2; // ": " (pretty-printed input; a fast path of yyjson) + } + else + { + JSON_CONTIGUOUS_WS(); + if (JSON_HEDLEY_UNLIKELY(JSON_CONTIGUOUS_CUR() != ':')) + { + return false; + } + ++p; + } + JSON_CONTIGUOUS_WS(); + JSON_CONTIGUOUS_VALUE + JSON_CONTIGUOUS_AFTER_MEMBER + +close: + // an array or object was closed: continue in the enclosing one + --depth; + kinds >>= 1u; + if (JSON_HEDLEY_UNLIKELY(depth == 0)) + { + goto done; + } + if (JSON_HEDLEY_UNLIKELY((depth % 64) == 0)) + { + kinds = outer.back(); + outer.pop_back(); + } + if ((kinds & 1u) != 0) + { + JSON_CONTIGUOUS_AFTER_MEMBER + } + JSON_CONTIGUOUS_AFTER_ELEMENT + +done: + JSON_CONTIGUOUS_WS(); +#if JSON_STRICT_NUL_HANDLING + return p == e; +#else + // a NUL byte ends the input, as in the lexer + return p == e || *p == 0; +#endif + +#undef JSON_CONTIGUOUS_AFTER_MEMBER +#undef JSON_CONTIGUOUS_AFTER_ELEMENT +#undef JSON_CONTIGUOUS_VALUE +#undef JSON_CONTIGUOUS_OPEN +#undef JSON_CONTIGUOUS_WS +#undef JSON_CONTIGUOUS_CUR + } + + /// whether the handler stopped the parse (by returning false), as opposed + /// to a syntax error + bool aborted() const noexcept + { + return handler_stopped; + } + + private: + /// the result @a r of an event of the handler: false stops the parse + bool report(const bool r) noexcept + { + handler_stopped = handler_stopped || !r; + return r; + } + + /// the text of the float token [@a first, @a last), for a handler that reads it + const string_t& number_text(const char* first, const char* last) + { + return number_text(first, last, std::integral_constant::value> {}); + } + + const string_t& number_text(const char* /*first*/, const char* /*last*/, std::true_type /*ignored*/) const noexcept + { + return no_text; + } + + const string_t& number_text(const char* first, const char* last, std::false_type /*ignored*/) + { + float_text.clear(); + float_text.append(reinterpret_cast(first), static_cast(last - first)); + return float_text; + } + + /// KeepValues as a runtime condition: testing the template argument + /// directly makes `if` conditions constant, which MSVC reports as C4127 + static bool keep_values() noexcept + { + return KeepValues; + } + + /// the first byte at or after p that is not whitespace, or e + static JSON_HEDLEY_NEVER_INLINE const uchar* skip_whitespace(const uchar* p, const uchar* e) noexcept + { + while (p != e) + { + const uchar c = *p; + if (c == ' ' || c == '\t' || c == '\r') + { + ++p; + } + else if (c == '\n') + { + // the indentation of the next line, 8 spaces at a time + ++p; + while (e - p >= 8) + { + const std::uint64_t non_space = read_eight_bytes(p) ^ 0x2020202020202020ull; + if (non_space != 0) + { + p += static_cast(count_trailing_zeros(non_space)) / 8; + break; + } + p += 8; + } + } + else + { + break; + } + } + return p; + } + + /// the literal @a text at p, whose first byte the dispatch has seen + template + JSON_HEDLEY_ALWAYS_INLINE bool literal(const uchar*& p, const char (&text)[N]) const noexcept // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays): a string literal + { + constexpr std::size_t length = N - 1; + if (JSON_HEDLEY_UNLIKELY(static_cast(end - p) < length || std::memcmp(p, text, length) != 0)) + { + return false; + } + p += length; + return true; + } + + /// a JSON text that is a single scalar + bool scalar(const uchar*& p) + { + switch (p != end ? *p : 0) + { + case '"': + return string(p, false); + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return number(p); + case 't': + return literal(p, "true") && report(sax->boolean(true)); + case 'f': + return literal(p, "false") && report(sax->boolean(false)); + case 'n': + return literal(p, "null") && report(sax->null()); + default: + return false; + } + } + + /// the string whose opening quote is at p, reported as a key or a value; + /// p is moved past the closing quote + JSON_HEDLEY_ALWAYS_INLINE bool string(const uchar*& p, const bool is_key) + { + const uchar* const s = ++p; + // the text up to the first quote, backslash, control character or + // ill-formed UTF-8 (well-formed UTF-8 is part of the run) + p = s + lexer_string_bulk_run(s, static_cast(end - s)); + if (JSON_HEDLEY_LIKELY(p != end && *p == '"')) + { + if (keep_values()) + { + buffer.clear(); + buffer.append(reinterpret_cast(s), static_cast(p - s)); + } + ++p; + } + else if (JSON_HEDLEY_UNLIKELY(!escaped_string(s, p))) + { + return false; + } + return is_key ? report(sax->key(buffer)) : report(sax->string(buffer)); + } + + /// the rest of a string from p, a backslash, a control character, + /// ill-formed UTF-8 or the end; s is where the string starts. With + /// KeepValues, the string is decoded into buffer. + JSON_HEDLEY_NEVER_INLINE bool escaped_string(const uchar* s, const uchar*& pos) + { + const uchar* p = pos; + if (keep_values()) + { + buffer.clear(); + buffer.append(reinterpret_cast(s), static_cast(p - s)); + } + for (;;) + { + if (JSON_HEDLEY_UNLIKELY(p == end)) + { + return false; + } + if (*p == '"') + { + pos = p + 1; + return true; + } + if (JSON_HEDLEY_UNLIKELY(*p != '\\' || end - p < 2)) + { + return false; // a control character or ill-formed UTF-8 + } + const uchar escape = p[1]; + p += 2; + switch (escape) + { + case '"': + case '\\': + case '/': + add(static_cast(escape)); + break; + case 'b': + add('\b'); + break; + case 'f': + add('\f'); + break; + case 'n': + add('\n'); + break; + case 'r': + add('\r'); + break; + case 't': + add('\t'); + break; + case 'u': + { + if (JSON_HEDLEY_UNLIKELY(end - p < 4)) + { + return false; + } + int codepoint = hex_codepoint(p); + if (JSON_HEDLEY_UNLIKELY(codepoint < 0)) + { + return false; + } + p += 4; + if (JSON_HEDLEY_UNLIKELY((codepoint & 0xF800) == 0xD800)) + { + // a high surrogate must be followed by \u and a low surrogate + if (codepoint >= 0xDC00 || end - p < 6 || p[0] != '\\' || p[1] != 'u') + { + return false; + } + const int low = hex_codepoint(p + 2); + if (low < 0xDC00 || low > 0xDFFF) + { + return false; + } + p += 6; + codepoint = 0x10000 + ((codepoint - 0xD800) << 10) + (low - 0xDC00); + } + add_utf8(codepoint); + break; + } + default: + return false; + } + const uchar* const run = p; + p = run + lexer_string_bulk_run(run, static_cast(end - run)); + if (keep_values() && p != run) + { + buffer.append(reinterpret_cast(run), static_cast(p - run)); + } + } + } + + void add(const char_t c) + { + if (keep_values()) + { + buffer.push_back(c); + } + } + + /// add the UTF-8 encoding of @a codepoint (at most U+10FFFF) to buffer + void add_utf8(const int codepoint) + { + if (!keep_values()) + { + return; + } + const auto cp = static_cast(codepoint); + if (cp < 0x80) + { + buffer.push_back(static_cast(cp)); + } + else if (cp < 0x800) + { + buffer.push_back(static_cast(0xC0u | (cp >> 6u))); + buffer.push_back(static_cast(0x80u | (cp & 0x3Fu))); + } + else if (cp < 0x10000) + { + buffer.push_back(static_cast(0xE0u | (cp >> 12u))); + buffer.push_back(static_cast(0x80u | ((cp >> 6u) & 0x3Fu))); + buffer.push_back(static_cast(0x80u | (cp & 0x3Fu))); + } + else + { + buffer.push_back(static_cast(0xF0u | (cp >> 18u))); + buffer.push_back(static_cast(0x80u | ((cp >> 12u) & 0x3Fu))); + buffer.push_back(static_cast(0x80u | ((cp >> 6u) & 0x3Fu))); + buffer.push_back(static_cast(0x80u | (cp & 0x3Fu))); + } + } + + /*! + @brief the number at p, with the grammar and the conversions of the lexer's + contiguous path (lexer::scan_number_bulk_contiguous()) + + Integers that do not fit their type become floats; floats that overflow + are an error, which the library parser reports. + */ + JSON_HEDLEY_ALWAYS_INLINE bool number(const uchar*& pos) + { + const auto* const data = reinterpret_cast(pos); + const auto avail = static_cast(end - pos); + std::size_t i = 0; + std::size_t dot_index = std::string::npos; + bool is_float = false; + std::uint64_t w = 0; + const bool negative = data[0] == '-'; + if (negative) + { + i = 1; + } + const std::size_t integer_begin = i; + if (i < avail && data[i] == '0') + { + ++i; + } + else if (i < avail && data[i] >= '1' && data[i] <= '9') + { + i = accumulate_digits(data, i, avail, w); + } + else + { + return false; + } + std::size_t digit_count = i - integer_begin; + std::size_t fraction_digits = 0; + if (i < avail && data[i] == '.') + { + is_float = true; + dot_index = i; + ++i; + if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) + { + return false; + } + const std::size_t fraction_begin = i; + i = accumulate_digits(data, i, avail, w); + fraction_digits = i - fraction_begin; + digit_count += fraction_digits; + } + const std::size_t mantissa_end = i; + std::int64_t exponent = 0; + if (i < avail && (data[i] == 'e' || data[i] == 'E')) + { + is_float = true; + ++i; + bool negative_exponent = false; + if (i < avail && (data[i] == '+' || data[i] == '-')) + { + negative_exponent = data[i] == '-'; + ++i; + } + if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) + { + return false; + } + // saturated far beyond every range, see parse_float_exponent() + constexpr std::int64_t saturation = 100000000000000000; // 10^17 + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + if (exponent < saturation) + { + exponent = (exponent * 10) + (data[i] - '0'); + } + ++i; + } + exponent = negative_exponent ? -exponent : exponent; + } + pos += i; + + if (!is_float) + { + if (digit_count <= 18) + { + // up to 18 digits always fit into std::uint64_t and std::int64_t; + // a narrower type takes the value only if it round-trips + if (negative) + { + const std::int64_t x = -static_cast(w); + const auto value = static_cast(x); + if (static_cast(value) == x) + { + return report(sax->number_integer(value)); + } + } + else + { + const auto value = static_cast(w); + if (static_cast(value) == w) + { + return report(sax->number_unsigned(value)); + } + } + } + else if (negative) + { + number_integer_t value{}; + if (parse_integer_signed(data, data + i, value)) + { + return report(sax->number_integer(value)); + } + } + else + { + number_unsigned_t value{}; + if (parse_integer_unsigned(data, data + i, value)) + { + return report(sax->number_unsigned(value)); + } + } + } + + number_float_t value{}; + if (digit_count <= 19) + { + float_significand s; + s.w = w; + s.exponent = exponent - static_cast(fraction_digits); + s.negative = negative; + // accept() only needs to know that the value is finite: w < 10^19, + // so only a token that may reach the largest float is converted + if (!keep_values() && s.exponent + 19 <= std::numeric_limits::max_exponent10) + { + return report(sax->number_float(value, number_text(data, data + i))); + } + value = decimal_to_float(s); + } + else + { + static_cast(convert_float_fast(data, data + i, dot_index, mantissa_end, value)); + } + if (JSON_HEDLEY_UNLIKELY(!std::isfinite(value))) + { + return false; // the library parser reports the overflow + } + return report(sax->number_float(value, number_text(data, data + i))); + } + + const uchar* const begin; + const uchar* const end; + SAX* const sax; + /// the current string (with KeepValues); the handler may move from it + string_t buffer {}; + /// the text of float tokens, for the library's handlers, which ignore it + const string_t no_text {}; + /// the text of the current float token, for other handlers + string_t float_text {}; + /// whether the handler returned false + bool handler_stopped = false; +}; + +/// whether parse() and accept() can read the input behind @a InputAdapterType +/// with contiguous_parser: one contiguous block of bytes (not a window into a +/// stream), and floats in a format the library converts itself +template +struct can_parse_contiguous +{ + static constexpr bool value = +#if JSON_DIAGNOSTIC_POSITIONS + // the values record the positions the lexer reports + false && +#endif + input_adapter_supports_bulk_scan(is_detected {}) + && !input_adapter_bulk_window(is_detected {}) + && has_native_float_format::value; +}; + +/*! +@brief report the input behind @a adapter, one block of memory, to @a sax + +@return whether the input is valid and was reported completely; if not, + nothing was consumed from @a adapter, and the caller has the library + parser report the error +*/ +template +bool parse_contiguous(InputAdapterType& adapter, SAX& sax, std::true_type /*contiguous*/) +{ + const std::size_t size = adapter.bulk_remaining(); + if (size == 0) + { + return false; // the library parser reports the empty input + } + const auto* const first = reinterpret_cast(adapter.bulk_data()); + contiguous_parser p(first, first + size, &sax); + return p.parse(); +} + +/*! +@brief a SAX handler that only reports a parse error, as the handler of parse() + does: it throws the exception, unless exceptions are not allowed + +After contiguous_parser has rejected an input, the library parser is run again +with this handler to report the error: the error, its message and its position +are those of parse(), without building the values read up to it. +*/ +template +class json_sax_error_reporter : public json_sax_acceptor +{ + public: + explicit json_sax_error_reporter(const bool allow_exceptions_) noexcept + : allow_exceptions(allow_exceptions_) + {} + + /// (a template, as in json_sax_dom_parser: the parser passes the concrete + /// exception type, which is thrown as is) + template + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const Exception& ex) + { + if (allow_exceptions) + { + JSON_THROW(ex); + } + return false; + } + + private: + const bool allow_exceptions; +}; + +template +struct sax_ignores_number_text> : std::true_type {}; + +/*! +@brief report the error in the input behind @a adapter, which contiguous_parser + rejected, as parse() reports it: throw its exception, or, without + exceptions, return true + +@return false if the library parser accepts the input after all (which would + be a disagreement of the two parsers): the caller then parses it +*/ +template +bool report_contiguous_error(const InputAdapterType& adapter, const bool allow_exceptions) +{ + json_sax_error_reporter reporter(allow_exceptions); + // (a copy of the adapter, which the caller may need again) + // (the lexer of parse(), which the fallback in basic_json::parse_impl() + // instantiates anyway: only the grammar loop is instantiated again) + return !parser(InputAdapterType(adapter), nullptr, allow_exceptions, false, false).sax_parse(&reporter, true); +} + +/*! +@brief parse(): parse the input behind @a adapter into @a result, if it is one + block of memory + +@return whether @a result is the result of parse(): the value, or, after an + error and without exceptions, a discarded value (with exceptions, the + error is thrown); false if the input is not one block of memory, in + which case the caller parses it +*/ +template +bool parse_contiguous(InputAdapterType& adapter, BasicJsonType& result, const bool allow_exceptions, std::true_type /*contiguous*/) +{ + { + // the handler of parser::parse(), so that the values are built by the same code + json_sax_dom_parser sax(result, true); + if (parse_contiguous(adapter, sax, std::true_type {})) + { + return true; + } + } + // (what was read up to the error) + result = BasicJsonType(); + if (report_contiguous_error(adapter, allow_exceptions)) + { + result = BasicJsonType(value_t::discarded); + return true; + } + return false; // LCOV_EXCL_LINE (only if the two parsers disagree: a bug) +} + +template +bool parse_contiguous(InputAdapterType& /*adapter*/, BasicJsonType& /*result*/, const bool /*allow_exceptions*/, std::false_type /*contiguous*/) noexcept +{ + return false; +} + +/*! +@brief accept(): check the input behind @a adapter, if it is one block of memory + +@return whether @a accepted is the result of accept(); false if the input is + not one block of memory, in which case the caller checks it. + contiguous_parser accepts exactly what the library parser accepts. +*/ +template +bool accept_contiguous(InputAdapterType& adapter, bool& accepted, std::true_type /*contiguous*/) +{ + json_sax_acceptor sax; + accepted = parse_contiguous(adapter, sax, std::true_type {}); + return true; +} + +template +bool accept_contiguous(InputAdapterType& /*adapter*/, bool& /*accepted*/, std::false_type /*contiguous*/) noexcept +{ + return false; +} + +/*! +@brief a SAX handler that ignores the values and passes a parse error on to the + handler of the user + +After contiguous_parser has delivered the events of the input to the user's +handler and rejected it, the library parser is run again with this handler: +the user's handler gets the error, its message and its position from the +library parser, as before, but no event twice. +*/ +template +class json_sax_error_forwarder : public json_sax_acceptor +{ + public: + explicit json_sax_error_forwarder(SAX* user_) noexcept + : user(user_) + {} + + template + bool parse_error(std::size_t position, const std::string& last_token, const Exception& ex) + { + forwarded = true; + return user->parse_error(position, last_token, ex); + } + + /// whether the library parser reported an error + bool reported() const noexcept + { + return forwarded; + } + + private: + SAX* const user; + bool forwarded = false; +}; + +template +struct sax_ignores_number_text> : std::true_type {}; + +/*! +@brief sax_parse() of JSON in one block of memory with the strict option and + a user's handler @a sax + +contiguous_parser reports to @a sax what it reads, and stops at the token that +the library parser would reject, before reporting it: the events up to an error +are the same. The error itself is then reported by the library parser (see +json_sax_error_forwarder). + +@return whether @a result is the result of sax_parse(); false if the input is + not one block of memory, in which case the caller parses it +*/ +template +bool sax_parse_contiguous(InputAdapterType& adapter, SAX* sax, bool& result, std::true_type /*contiguous*/) +{ + const std::size_t size = adapter.bulk_remaining(); + if (size != 0) + { + const auto* const first = reinterpret_cast(adapter.bulk_data()); + contiguous_parser p(first, first + size, sax); + if (p.parse()) + { + result = true; + return true; + } + if (p.aborted()) + { + // the handler returned false, as with the library parser + result = false; + return true; + } + } + json_sax_error_forwarder forwarder(sax); + result = parser(InputAdapterType(adapter), nullptr, true, false, false).sax_parse(&forwarder, true); + // the library parser must reject the input as well (the events after the + // byte contiguous_parser stopped at cannot be delivered any more) + JSON_ASSERT(forwarder.reported()); + return true; +} + +template +bool sax_parse_contiguous(InputAdapterType& /*adapter*/, SAX* /*sax*/, bool& /*result*/, std::false_type /*contiguous*/) noexcept +{ + return false; +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/input/string_scan.hpp b/include/nlohmann/detail/input/string_scan.hpp index 10e4cc900..8564746ce 100644 --- a/include/nlohmann/detail/input/string_scan.hpp +++ b/include/nlohmann/detail/input/string_scan.hpp @@ -520,7 +520,10 @@ JSON_HEDLEY_NEVER_INLINE inline std::size_t string_bulk_run_from(const unsigned } #endif -// string_bulk_run() for the lexer. With NEON, only the first four words are +// string_bulk_run() for the lexer. The run must be the longest one: the lexer +// would only lose speed with a shorter run, but contiguous_parser rejects a +// string whose run ends at a byte other than a quote or a backslash, so a +// shorter run would reject valid input there. With NEON, only the first four words are // checked inline (most keys and short values end there), and longer runs and // non-ASCII text continue out of line. The lexer's scan function runs once per // token, and the vector code inlined into it needs more registers, which every diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 9be8a8c6c..0f808e1ec 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -58,6 +58,7 @@ #include // IWYU pragma: export #include #include +#include #include #include #include @@ -3109,6 +3110,15 @@ public: const bool ignore_trailing_commas) { basic_json result; + // JSON in one block of memory, with the default options, is read by + // the faster contiguous_parser; on an error, the library parser reads + // it again to report it (see contiguous_parser) + using contiguous = std::integral_constant::type>::value>; + if (cb == nullptr && !ignore_comments && !ignore_trailing_commas + && detail::parse_contiguous(adapter, result, allow_exceptions, contiguous {})) + { + return result; + } parser(std::forward(adapter), std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas).parse(true, result); return result; } @@ -3119,8 +3129,17 @@ public: const bool ignore_comments, const bool ignore_trailing_commas) { + // JSON in one block of memory is checked by the faster + // contiguous_parser (see parse_impl()) + using adapter_t = typename std::decay::type; + bool accepted = false; + if (!ignore_comments && !ignore_trailing_commas + && detail::accept_contiguous(adapter, accepted, std::integral_constant::value> {})) + { + return accepted; + } // the parser and lexer that discard the values of tokens (see lexer) - return ::nlohmann::detail::parser::type, true>(std::forward(adapter), + return ::nlohmann::detail::parser(std::forward(adapter), nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); } @@ -3135,6 +3154,14 @@ public: { if (format == input_format_t::json) { + // JSON in one block of memory is read by the faster + // contiguous_parser, with the default options (see parse_impl()) + bool result = false; + if (strict && !ignore_comments && !ignore_trailing_commas + && detail::sax_parse_contiguous(ia, sax, result, std::integral_constant::value> {})) + { + return result; + } return parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict); } return detail::binary_reader(std::move(ia), format).sax_parse(sax, strict, tag_handler); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index eedb9c561..30417876b 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12975,7 +12975,10 @@ JSON_HEDLEY_NEVER_INLINE inline std::size_t string_bulk_run_from(const unsigned } #endif -// string_bulk_run() for the lexer. With NEON, only the first four words are +// string_bulk_run() for the lexer. The run must be the longest one: the lexer +// would only lose speed with a shorter run, but contiguous_parser rejects a +// string whose run ends at a byte other than a quote or a backslash, so a +// shorter run would reject valid input there. With NEON, only the first four words are // checked inline (most keys and short values end there), and longer runs and // non-ASCII text continue out of line. The lexer's scan function runs once per // token, and the vector code inlined into it needs more registers, which every @@ -23208,12 +23211,37 @@ class binary_reader } // namespace detail NLOHMANN_JSON_NAMESPACE_END +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // isfinite +#include // size_t, ptrdiff_t +#include // uint64_t, int64_t +#include // memcmp +#include // numeric_limits +#include // string +#include // integral_constant, true_type, false_type +#include // move +#include // vector + +// #include + // #include // #include // #include +// #include + // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -23834,6 +23862,1006 @@ class parser } // namespace detail NLOHMANN_JSON_NAMESPACE_END +// #include + +// #include + +// #include + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief a parser for JSON text in one contiguous block, for parse(), accept(), + and sax_parse() + +The parser and the lexer exchange one token at a time, and the lexer keeps its +state (the current character, a pending unget, where the token starts) in +members, which every character pays for. This parser reads the block through a +local pointer instead, and the grammar, the whitespace and the tokens are one +loop, after yyjson's reader: the state stays in registers, and the expected +next token is tested where it is expected instead of being dispatched on. + +It reports what it reads to a SAX handler: the one parse() uses to build the +value, the one accept() uses, or the user's handler of sax_parse(). On a byte +the library parser would reject, it stops before reporting the token, and +returns false: the events up to there are those the library parser reports. +The caller then runs the library parser on the same input to report the error +with its exact message and position (report_contiguous_error(), +json_sax_error_forwarder), and parse() builds no value from what was read. + +What is accepted, and the values reported, are those of the library parser +with default options: the same number grammar and conversions, the same UTF-8 +and escape rules, a UTF-8 byte order mark at the start, and, without +JSON_STRICT_NUL_HANDLING, a NUL byte as the end of the input. + +@tparam KeepValues whether the handler needs the values of strings and + numbers (false for accept(), whose handler ignores them) +*/ +template +class contiguous_parser +{ + using string_t = typename BasicJsonType::string_t; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using char_t = typename string_t::value_type; + using uchar = unsigned char; + + public: + contiguous_parser(const uchar* first, const uchar* last, SAX* sax_) noexcept + : begin(first), end(last), sax(sax_) + {} + + contiguous_parser(const contiguous_parser&) = delete; + contiguous_parser(contiguous_parser&&) = delete; + contiguous_parser& operator=(const contiguous_parser&) = delete; + contiguous_parser& operator=(contiguous_parser&&) = delete; + ~contiguous_parser() = default; + + /// whether the whole block is one valid JSON text; the handler has seen all of it + JSON_HEDLEY_NEVER_INLINE bool parse() + { + const uchar* p = begin; + const uchar* const e = end; + + if (p != e && *p == 0xEF) + { + // a byte order mark, or the lexer reports a bad one + if (e - p < 3 || p[1] != 0xBB || p[2] != 0xBF) + { + return false; + } + p += 3; + } + + // The open arrays and objects: bit 0 of `kinds` is the innermost (1: + // object), so that the kind of the enclosing container is a shift away + // and stays in a register; every 64 levels, the word is pushed to + // `outer` and a new one is begun. + std::uint64_t kinds = 0; + std::size_t depth = 0; + std::vector outer; + + // the current byte, or 0 at the end (no JSON token starts with 0) +#define JSON_CONTIGUOUS_CUR() (p != e ? *p : 0) + // skip whitespace; most tokens have none before them +#define JSON_CONTIGUOUS_WS() \ + do \ + { \ + if (JSON_HEDLEY_UNLIKELY(JSON_CONTIGUOUS_CUR() <= ' ')) \ + { \ + p = skip_whitespace(p, e); \ + } \ + } \ + while (false) +#define JSON_CONTIGUOUS_OPEN(is_object) \ + do \ + { \ + if (JSON_HEDLEY_UNLIKELY(depth != 0 && (depth % 64) == 0)) \ + { \ + outer.push_back(kinds); \ + kinds = 0; \ + } \ + kinds = (kinds << 1u) | static_cast(is_object); \ + ++depth; \ + } \ + while (false) + + JSON_CONTIGUOUS_WS(); + switch (JSON_CONTIGUOUS_CUR()) + { + case '{': + ++p; + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_object(detail::unknown_size())))) + { + return false; + } + JSON_CONTIGUOUS_OPEN(1u); + goto object_first; + case '[': + ++p; + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_array(detail::unknown_size())))) + { + return false; + } + JSON_CONTIGUOUS_OPEN(0u); + goto array_first; + default: + if (JSON_HEDLEY_UNLIKELY(!scalar(p))) + { + return false; + } + goto done; + } + + // a value in an array or an object: expanded once for each, so that + // each dispatch has its own branch history (arrays tend to hold values + // of one kind) and the code after a scalar knows where it is +#define JSON_CONTIGUOUS_VALUE \ + switch (JSON_CONTIGUOUS_CUR()) \ + { \ + case '"': \ + if (JSON_HEDLEY_UNLIKELY(!string(p, false))) \ + { \ + return false; \ + } \ + break; \ + case '{': \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_object(detail::unknown_size())))) \ + { \ + return false; \ + } \ + JSON_CONTIGUOUS_OPEN(1u); \ + goto object_first; \ + case '[': \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_array(detail::unknown_size())))) \ + { \ + return false; \ + } \ + JSON_CONTIGUOUS_OPEN(0u); \ + goto array_first; \ + case '-': \ + case '0': \ + case '1': \ + case '2': \ + case '3': \ + case '4': \ + case '5': \ + case '6': \ + case '7': \ + case '8': \ + case '9': \ + if (JSON_HEDLEY_UNLIKELY(!number(p))) \ + { \ + return false; \ + } \ + break; \ + case 't': \ + if (JSON_HEDLEY_UNLIKELY(!literal(p, "true") || !report(sax->boolean(true)))) \ + { \ + return false; \ + } \ + break; \ + case 'f': \ + if (JSON_HEDLEY_UNLIKELY(!literal(p, "false") || !report(sax->boolean(false)))) \ + { \ + return false; \ + } \ + break; \ + case 'n': \ + if (JSON_HEDLEY_UNLIKELY(!literal(p, "null") || !report(sax->null()))) \ + { \ + return false; \ + } \ + break; \ + default: \ + return false; \ + } + + // what may follow a value in an array: a comma and the next value, or the end +#define JSON_CONTIGUOUS_AFTER_ELEMENT \ + JSON_CONTIGUOUS_WS(); \ + if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ',')) \ + { \ + ++p; \ + JSON_CONTIGUOUS_WS(); \ + goto array_value; \ + } \ + if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ']')) \ + { \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_array()))) \ + { \ + return false; \ + } \ + goto close; \ + } \ + return false; + + // what may follow a value in an object: a comma and the next member, or the end +#define JSON_CONTIGUOUS_AFTER_MEMBER \ + JSON_CONTIGUOUS_WS(); \ + if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ',')) \ + { \ + ++p; \ + JSON_CONTIGUOUS_WS(); \ + goto object_key; \ + } \ + if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == '}')) \ + { \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_object()))) \ + { \ + return false; \ + } \ + goto close; \ + } \ + return false; + +array_first: + JSON_CONTIGUOUS_WS(); + if (JSON_CONTIGUOUS_CUR() == ']') + { + ++p; + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_array()))) + { + return false; + } + goto close; + } +array_value: + JSON_CONTIGUOUS_VALUE + JSON_CONTIGUOUS_AFTER_ELEMENT + +object_first: + JSON_CONTIGUOUS_WS(); + if (JSON_CONTIGUOUS_CUR() == '}') + { + ++p; + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_object()))) + { + return false; + } + goto close; + } +object_key: + if (JSON_HEDLEY_UNLIKELY(JSON_CONTIGUOUS_CUR() != '"' || !string(p, true))) + { + return false; + } + if (JSON_HEDLEY_LIKELY(e - p >= 2 && p[0] == ':' && p[1] == ' ')) + { + p += 2; // ": " (pretty-printed input; a fast path of yyjson) + } + else + { + JSON_CONTIGUOUS_WS(); + if (JSON_HEDLEY_UNLIKELY(JSON_CONTIGUOUS_CUR() != ':')) + { + return false; + } + ++p; + } + JSON_CONTIGUOUS_WS(); + JSON_CONTIGUOUS_VALUE + JSON_CONTIGUOUS_AFTER_MEMBER + +close: + // an array or object was closed: continue in the enclosing one + --depth; + kinds >>= 1u; + if (JSON_HEDLEY_UNLIKELY(depth == 0)) + { + goto done; + } + if (JSON_HEDLEY_UNLIKELY((depth % 64) == 0)) + { + kinds = outer.back(); + outer.pop_back(); + } + if ((kinds & 1u) != 0) + { + JSON_CONTIGUOUS_AFTER_MEMBER + } + JSON_CONTIGUOUS_AFTER_ELEMENT + +done: + JSON_CONTIGUOUS_WS(); +#if JSON_STRICT_NUL_HANDLING + return p == e; +#else + // a NUL byte ends the input, as in the lexer + return p == e || *p == 0; +#endif + +#undef JSON_CONTIGUOUS_AFTER_MEMBER +#undef JSON_CONTIGUOUS_AFTER_ELEMENT +#undef JSON_CONTIGUOUS_VALUE +#undef JSON_CONTIGUOUS_OPEN +#undef JSON_CONTIGUOUS_WS +#undef JSON_CONTIGUOUS_CUR + } + + /// whether the handler stopped the parse (by returning false), as opposed + /// to a syntax error + bool aborted() const noexcept + { + return handler_stopped; + } + + private: + /// the result @a r of an event of the handler: false stops the parse + bool report(const bool r) noexcept + { + handler_stopped = handler_stopped || !r; + return r; + } + + /// the text of the float token [@a first, @a last), for a handler that reads it + const string_t& number_text(const char* first, const char* last) + { + return number_text(first, last, std::integral_constant::value> {}); + } + + const string_t& number_text(const char* /*first*/, const char* /*last*/, std::true_type /*ignored*/) const noexcept + { + return no_text; + } + + const string_t& number_text(const char* first, const char* last, std::false_type /*ignored*/) + { + float_text.clear(); + float_text.append(reinterpret_cast(first), static_cast(last - first)); + return float_text; + } + + /// KeepValues as a runtime condition: testing the template argument + /// directly makes `if` conditions constant, which MSVC reports as C4127 + static bool keep_values() noexcept + { + return KeepValues; + } + + /// the first byte at or after p that is not whitespace, or e + static JSON_HEDLEY_NEVER_INLINE const uchar* skip_whitespace(const uchar* p, const uchar* e) noexcept + { + while (p != e) + { + const uchar c = *p; + if (c == ' ' || c == '\t' || c == '\r') + { + ++p; + } + else if (c == '\n') + { + // the indentation of the next line, 8 spaces at a time + ++p; + while (e - p >= 8) + { + const std::uint64_t non_space = read_eight_bytes(p) ^ 0x2020202020202020ull; + if (non_space != 0) + { + p += static_cast(count_trailing_zeros(non_space)) / 8; + break; + } + p += 8; + } + } + else + { + break; + } + } + return p; + } + + /// the literal @a text at p, whose first byte the dispatch has seen + template + JSON_HEDLEY_ALWAYS_INLINE bool literal(const uchar*& p, const char (&text)[N]) const noexcept // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays): a string literal + { + constexpr std::size_t length = N - 1; + if (JSON_HEDLEY_UNLIKELY(static_cast(end - p) < length || std::memcmp(p, text, length) != 0)) + { + return false; + } + p += length; + return true; + } + + /// a JSON text that is a single scalar + bool scalar(const uchar*& p) + { + switch (p != end ? *p : 0) + { + case '"': + return string(p, false); + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return number(p); + case 't': + return literal(p, "true") && report(sax->boolean(true)); + case 'f': + return literal(p, "false") && report(sax->boolean(false)); + case 'n': + return literal(p, "null") && report(sax->null()); + default: + return false; + } + } + + /// the string whose opening quote is at p, reported as a key or a value; + /// p is moved past the closing quote + JSON_HEDLEY_ALWAYS_INLINE bool string(const uchar*& p, const bool is_key) + { + const uchar* const s = ++p; + // the text up to the first quote, backslash, control character or + // ill-formed UTF-8 (well-formed UTF-8 is part of the run) + p = s + lexer_string_bulk_run(s, static_cast(end - s)); + if (JSON_HEDLEY_LIKELY(p != end && *p == '"')) + { + if (keep_values()) + { + buffer.clear(); + buffer.append(reinterpret_cast(s), static_cast(p - s)); + } + ++p; + } + else if (JSON_HEDLEY_UNLIKELY(!escaped_string(s, p))) + { + return false; + } + return is_key ? report(sax->key(buffer)) : report(sax->string(buffer)); + } + + /// the rest of a string from p, a backslash, a control character, + /// ill-formed UTF-8 or the end; s is where the string starts. With + /// KeepValues, the string is decoded into buffer. + JSON_HEDLEY_NEVER_INLINE bool escaped_string(const uchar* s, const uchar*& pos) + { + const uchar* p = pos; + if (keep_values()) + { + buffer.clear(); + buffer.append(reinterpret_cast(s), static_cast(p - s)); + } + for (;;) + { + if (JSON_HEDLEY_UNLIKELY(p == end)) + { + return false; + } + if (*p == '"') + { + pos = p + 1; + return true; + } + if (JSON_HEDLEY_UNLIKELY(*p != '\\' || end - p < 2)) + { + return false; // a control character or ill-formed UTF-8 + } + const uchar escape = p[1]; + p += 2; + switch (escape) + { + case '"': + case '\\': + case '/': + add(static_cast(escape)); + break; + case 'b': + add('\b'); + break; + case 'f': + add('\f'); + break; + case 'n': + add('\n'); + break; + case 'r': + add('\r'); + break; + case 't': + add('\t'); + break; + case 'u': + { + if (JSON_HEDLEY_UNLIKELY(end - p < 4)) + { + return false; + } + int codepoint = hex_codepoint(p); + if (JSON_HEDLEY_UNLIKELY(codepoint < 0)) + { + return false; + } + p += 4; + if (JSON_HEDLEY_UNLIKELY((codepoint & 0xF800) == 0xD800)) + { + // a high surrogate must be followed by \u and a low surrogate + if (codepoint >= 0xDC00 || end - p < 6 || p[0] != '\\' || p[1] != 'u') + { + return false; + } + const int low = hex_codepoint(p + 2); + if (low < 0xDC00 || low > 0xDFFF) + { + return false; + } + p += 6; + codepoint = 0x10000 + ((codepoint - 0xD800) << 10) + (low - 0xDC00); + } + add_utf8(codepoint); + break; + } + default: + return false; + } + const uchar* const run = p; + p = run + lexer_string_bulk_run(run, static_cast(end - run)); + if (keep_values() && p != run) + { + buffer.append(reinterpret_cast(run), static_cast(p - run)); + } + } + } + + void add(const char_t c) + { + if (keep_values()) + { + buffer.push_back(c); + } + } + + /// add the UTF-8 encoding of @a codepoint (at most U+10FFFF) to buffer + void add_utf8(const int codepoint) + { + if (!keep_values()) + { + return; + } + const auto cp = static_cast(codepoint); + if (cp < 0x80) + { + buffer.push_back(static_cast(cp)); + } + else if (cp < 0x800) + { + buffer.push_back(static_cast(0xC0u | (cp >> 6u))); + buffer.push_back(static_cast(0x80u | (cp & 0x3Fu))); + } + else if (cp < 0x10000) + { + buffer.push_back(static_cast(0xE0u | (cp >> 12u))); + buffer.push_back(static_cast(0x80u | ((cp >> 6u) & 0x3Fu))); + buffer.push_back(static_cast(0x80u | (cp & 0x3Fu))); + } + else + { + buffer.push_back(static_cast(0xF0u | (cp >> 18u))); + buffer.push_back(static_cast(0x80u | ((cp >> 12u) & 0x3Fu))); + buffer.push_back(static_cast(0x80u | ((cp >> 6u) & 0x3Fu))); + buffer.push_back(static_cast(0x80u | (cp & 0x3Fu))); + } + } + + /*! + @brief the number at p, with the grammar and the conversions of the lexer's + contiguous path (lexer::scan_number_bulk_contiguous()) + + Integers that do not fit their type become floats; floats that overflow + are an error, which the library parser reports. + */ + JSON_HEDLEY_ALWAYS_INLINE bool number(const uchar*& pos) + { + const auto* const data = reinterpret_cast(pos); + const auto avail = static_cast(end - pos); + std::size_t i = 0; + std::size_t dot_index = std::string::npos; + bool is_float = false; + std::uint64_t w = 0; + const bool negative = data[0] == '-'; + if (negative) + { + i = 1; + } + const std::size_t integer_begin = i; + if (i < avail && data[i] == '0') + { + ++i; + } + else if (i < avail && data[i] >= '1' && data[i] <= '9') + { + i = accumulate_digits(data, i, avail, w); + } + else + { + return false; + } + std::size_t digit_count = i - integer_begin; + std::size_t fraction_digits = 0; + if (i < avail && data[i] == '.') + { + is_float = true; + dot_index = i; + ++i; + if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) + { + return false; + } + const std::size_t fraction_begin = i; + i = accumulate_digits(data, i, avail, w); + fraction_digits = i - fraction_begin; + digit_count += fraction_digits; + } + const std::size_t mantissa_end = i; + std::int64_t exponent = 0; + if (i < avail && (data[i] == 'e' || data[i] == 'E')) + { + is_float = true; + ++i; + bool negative_exponent = false; + if (i < avail && (data[i] == '+' || data[i] == '-')) + { + negative_exponent = data[i] == '-'; + ++i; + } + if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) + { + return false; + } + // saturated far beyond every range, see parse_float_exponent() + constexpr std::int64_t saturation = 100000000000000000; // 10^17 + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + if (exponent < saturation) + { + exponent = (exponent * 10) + (data[i] - '0'); + } + ++i; + } + exponent = negative_exponent ? -exponent : exponent; + } + pos += i; + + if (!is_float) + { + if (digit_count <= 18) + { + // up to 18 digits always fit into std::uint64_t and std::int64_t; + // a narrower type takes the value only if it round-trips + if (negative) + { + const std::int64_t x = -static_cast(w); + const auto value = static_cast(x); + if (static_cast(value) == x) + { + return report(sax->number_integer(value)); + } + } + else + { + const auto value = static_cast(w); + if (static_cast(value) == w) + { + return report(sax->number_unsigned(value)); + } + } + } + else if (negative) + { + number_integer_t value{}; + if (parse_integer_signed(data, data + i, value)) + { + return report(sax->number_integer(value)); + } + } + else + { + number_unsigned_t value{}; + if (parse_integer_unsigned(data, data + i, value)) + { + return report(sax->number_unsigned(value)); + } + } + } + + number_float_t value{}; + if (digit_count <= 19) + { + float_significand s; + s.w = w; + s.exponent = exponent - static_cast(fraction_digits); + s.negative = negative; + // accept() only needs to know that the value is finite: w < 10^19, + // so only a token that may reach the largest float is converted + if (!keep_values() && s.exponent + 19 <= std::numeric_limits::max_exponent10) + { + return report(sax->number_float(value, number_text(data, data + i))); + } + value = decimal_to_float(s); + } + else + { + static_cast(convert_float_fast(data, data + i, dot_index, mantissa_end, value)); + } + if (JSON_HEDLEY_UNLIKELY(!std::isfinite(value))) + { + return false; // the library parser reports the overflow + } + return report(sax->number_float(value, number_text(data, data + i))); + } + + const uchar* const begin; + const uchar* const end; + SAX* const sax; + /// the current string (with KeepValues); the handler may move from it + string_t buffer {}; + /// the text of float tokens, for the library's handlers, which ignore it + const string_t no_text {}; + /// the text of the current float token, for other handlers + string_t float_text {}; + /// whether the handler returned false + bool handler_stopped = false; +}; + +/// whether parse() and accept() can read the input behind @a InputAdapterType +/// with contiguous_parser: one contiguous block of bytes (not a window into a +/// stream), and floats in a format the library converts itself +template +struct can_parse_contiguous +{ + static constexpr bool value = +#if JSON_DIAGNOSTIC_POSITIONS + // the values record the positions the lexer reports + false && +#endif + input_adapter_supports_bulk_scan(is_detected {}) + && !input_adapter_bulk_window(is_detected {}) + && has_native_float_format::value; +}; + +/*! +@brief report the input behind @a adapter, one block of memory, to @a sax + +@return whether the input is valid and was reported completely; if not, + nothing was consumed from @a adapter, and the caller has the library + parser report the error +*/ +template +bool parse_contiguous(InputAdapterType& adapter, SAX& sax, std::true_type /*contiguous*/) +{ + const std::size_t size = adapter.bulk_remaining(); + if (size == 0) + { + return false; // the library parser reports the empty input + } + const auto* const first = reinterpret_cast(adapter.bulk_data()); + contiguous_parser p(first, first + size, &sax); + return p.parse(); +} + +/*! +@brief a SAX handler that only reports a parse error, as the handler of parse() + does: it throws the exception, unless exceptions are not allowed + +After contiguous_parser has rejected an input, the library parser is run again +with this handler to report the error: the error, its message and its position +are those of parse(), without building the values read up to it. +*/ +template +class json_sax_error_reporter : public json_sax_acceptor +{ + public: + explicit json_sax_error_reporter(const bool allow_exceptions_) noexcept + : allow_exceptions(allow_exceptions_) + {} + + /// (a template, as in json_sax_dom_parser: the parser passes the concrete + /// exception type, which is thrown as is) + template + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const Exception& ex) + { + if (allow_exceptions) + { + JSON_THROW(ex); + } + return false; + } + + private: + const bool allow_exceptions; +}; + +template +struct sax_ignores_number_text> : std::true_type {}; + +/*! +@brief report the error in the input behind @a adapter, which contiguous_parser + rejected, as parse() reports it: throw its exception, or, without + exceptions, return true + +@return false if the library parser accepts the input after all (which would + be a disagreement of the two parsers): the caller then parses it +*/ +template +bool report_contiguous_error(const InputAdapterType& adapter, const bool allow_exceptions) +{ + json_sax_error_reporter reporter(allow_exceptions); + // (a copy of the adapter, which the caller may need again) + // (the lexer of parse(), which the fallback in basic_json::parse_impl() + // instantiates anyway: only the grammar loop is instantiated again) + return !parser(InputAdapterType(adapter), nullptr, allow_exceptions, false, false).sax_parse(&reporter, true); +} + +/*! +@brief parse(): parse the input behind @a adapter into @a result, if it is one + block of memory + +@return whether @a result is the result of parse(): the value, or, after an + error and without exceptions, a discarded value (with exceptions, the + error is thrown); false if the input is not one block of memory, in + which case the caller parses it +*/ +template +bool parse_contiguous(InputAdapterType& adapter, BasicJsonType& result, const bool allow_exceptions, std::true_type /*contiguous*/) +{ + { + // the handler of parser::parse(), so that the values are built by the same code + json_sax_dom_parser sax(result, true); + if (parse_contiguous(adapter, sax, std::true_type {})) + { + return true; + } + } + // (what was read up to the error) + result = BasicJsonType(); + if (report_contiguous_error(adapter, allow_exceptions)) + { + result = BasicJsonType(value_t::discarded); + return true; + } + return false; // LCOV_EXCL_LINE (only if the two parsers disagree: a bug) +} + +template +bool parse_contiguous(InputAdapterType& /*adapter*/, BasicJsonType& /*result*/, const bool /*allow_exceptions*/, std::false_type /*contiguous*/) noexcept +{ + return false; +} + +/*! +@brief accept(): check the input behind @a adapter, if it is one block of memory + +@return whether @a accepted is the result of accept(); false if the input is + not one block of memory, in which case the caller checks it. + contiguous_parser accepts exactly what the library parser accepts. +*/ +template +bool accept_contiguous(InputAdapterType& adapter, bool& accepted, std::true_type /*contiguous*/) +{ + json_sax_acceptor sax; + accepted = parse_contiguous(adapter, sax, std::true_type {}); + return true; +} + +template +bool accept_contiguous(InputAdapterType& /*adapter*/, bool& /*accepted*/, std::false_type /*contiguous*/) noexcept +{ + return false; +} + +/*! +@brief a SAX handler that ignores the values and passes a parse error on to the + handler of the user + +After contiguous_parser has delivered the events of the input to the user's +handler and rejected it, the library parser is run again with this handler: +the user's handler gets the error, its message and its position from the +library parser, as before, but no event twice. +*/ +template +class json_sax_error_forwarder : public json_sax_acceptor +{ + public: + explicit json_sax_error_forwarder(SAX* user_) noexcept + : user(user_) + {} + + template + bool parse_error(std::size_t position, const std::string& last_token, const Exception& ex) + { + forwarded = true; + return user->parse_error(position, last_token, ex); + } + + /// whether the library parser reported an error + bool reported() const noexcept + { + return forwarded; + } + + private: + SAX* const user; + bool forwarded = false; +}; + +template +struct sax_ignores_number_text> : std::true_type {}; + +/*! +@brief sax_parse() of JSON in one block of memory with the strict option and + a user's handler @a sax + +contiguous_parser reports to @a sax what it reads, and stops at the token that +the library parser would reject, before reporting it: the events up to an error +are the same. The error itself is then reported by the library parser (see +json_sax_error_forwarder). + +@return whether @a result is the result of sax_parse(); false if the input is + not one block of memory, in which case the caller parses it +*/ +template +bool sax_parse_contiguous(InputAdapterType& adapter, SAX* sax, bool& result, std::true_type /*contiguous*/) +{ + const std::size_t size = adapter.bulk_remaining(); + if (size != 0) + { + const auto* const first = reinterpret_cast(adapter.bulk_data()); + contiguous_parser p(first, first + size, sax); + if (p.parse()) + { + result = true; + return true; + } + if (p.aborted()) + { + // the handler returned false, as with the library parser + result = false; + return true; + } + } + json_sax_error_forwarder forwarder(sax); + result = parser(InputAdapterType(adapter), nullptr, true, false, false).sax_parse(&forwarder, true); + // the library parser must reject the input as well (the events after the + // byte contiguous_parser stopped at cannot be delivered any more) + JSON_ASSERT(forwarder.reported()); + return true; +} + +template +bool sax_parse_contiguous(InputAdapterType& /*adapter*/, SAX* /*sax*/, bool& /*result*/, std::false_type /*contiguous*/) noexcept +{ + return false; +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include + +// #include + +// #include + +// #include + // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -40907,6 +41935,15 @@ public: const bool ignore_trailing_commas) { basic_json result; + // JSON in one block of memory, with the default options, is read by + // the faster contiguous_parser; on an error, the library parser reads + // it again to report it (see contiguous_parser) + using contiguous = std::integral_constant::type>::value>; + if (cb == nullptr && !ignore_comments && !ignore_trailing_commas + && detail::parse_contiguous(adapter, result, allow_exceptions, contiguous {})) + { + return result; + } parser(std::forward(adapter), std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas).parse(true, result); return result; } @@ -40917,8 +41954,17 @@ public: const bool ignore_comments, const bool ignore_trailing_commas) { + // JSON in one block of memory is checked by the faster + // contiguous_parser (see parse_impl()) + using adapter_t = typename std::decay::type; + bool accepted = false; + if (!ignore_comments && !ignore_trailing_commas + && detail::accept_contiguous(adapter, accepted, std::integral_constant::value> {})) + { + return accepted; + } // the parser and lexer that discard the values of tokens (see lexer) - return ::nlohmann::detail::parser::type, true>(std::forward(adapter), + return ::nlohmann::detail::parser(std::forward(adapter), nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); } @@ -40933,6 +41979,14 @@ public: { if (format == input_format_t::json) { + // JSON in one block of memory is read by the faster + // contiguous_parser, with the default options (see parse_impl()) + bool result = false; + if (strict && !ignore_comments && !ignore_trailing_commas + && detail::sax_parse_contiguous(ia, sax, result, std::integral_constant::value> {})) + { + return result; + } return parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict); } return detail::binary_reader(std::move(ia), format).sax_parse(sax, strict, tag_handler); diff --git a/tests/src/unit-contiguous_parser.cpp b/tests/src/unit-contiguous_parser.cpp new file mode 100644 index 000000000..043fa95c3 --- /dev/null +++ b/tests/src/unit-contiguous_parser.cpp @@ -0,0 +1,536 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// parse() and accept() read text in one block of memory with +// detail::contiguous_parser, and everything else (here: std::list iterators) +// with the lexer's byte path. These tests check that the two agree on every +// input: the same value, with the same number types and float bits, or the +// same exception with the same message. + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; +using nlohmann::ordered_json; + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + +/// the value with its number types and float bits; below 100 levels, the +/// library's dump() (whose recursion is bounded) describes the rest +template +std::string describe_value(const Json& j, int depth = 0) +{ + std::string out; + if (depth >= 100) + { + out = j.dump(); + return out; + } + switch (j.type()) + { + case Json::value_t::number_float: + { + const double d = j.template get(); + std::uint64_t bits = 0; + std::memcpy(&bits, &d, sizeof(bits)); + out += "f" + std::to_string(bits); + break; + } + case Json::value_t::number_integer: + out += "i" + j.dump(); + break; + case Json::value_t::number_unsigned: + out += "u" + j.dump(); + break; + case Json::value_t::array: + out += "["; + for (const auto& e : j) + { + out += describe_value(e, depth + 1) + ","; + } + out += "]"; + break; + case Json::value_t::object: + out += "{"; + for (auto it = j.begin(); it != j.end(); ++it) + { + out += Json(it.key()).dump() + ":" + describe_value(it.value(), depth + 1) + ","; + } + out += "}"; + break; + case Json::value_t::null: + case Json::value_t::boolean: + case Json::value_t::string: + case Json::value_t::binary: + case Json::value_t::discarded: + default: + out += j.dump(); + break; + } + return out; +} + +template +std::string outcome(Parse parse) +{ +#if !defined(JSON_NOEXCEPTION) + try + { + return describe_value(parse()); + } + catch (const typename Json::exception& e) + { + return std::string("exception: ") + e.what(); + } +#else + return describe_value(parse()); +#endif +} + +/// every SAX event with its values, the text of float tokens, and the parse +/// error with its position, token and message; returns false after +/// @a stop_after events (to stop the parse) +class event_log : public nlohmann::json_sax +{ + public: + explicit event_log(std::size_t stop_after_ = static_cast(-1), bool continue_after_error_ = false) + : stop_after(stop_after_), continue_after_error(continue_after_error_) + {} + + bool null() override + { + return add("null"); + } + bool boolean(bool val) override + { + return add(val ? "true" : "false"); + } + bool number_integer(number_integer_t val) override + { + return add("i" + std::to_string(val)); + } + bool number_unsigned(number_unsigned_t val) override + { + return add("u" + std::to_string(val)); + } + bool number_float(number_float_t val, const string_t& s) override + { + std::uint64_t bits = 0; + std::memcpy(&bits, &val, sizeof(bits)); + return add("f" + std::to_string(bits) + " " + s); + } + bool string(string_t& val) override + { + return add("s" + val); + } + bool binary(binary_t& /*val*/) override + { + return add("binary"); + } + bool start_object(std::size_t elements) override + { + return add("{" + std::to_string(elements)); + } + bool key(string_t& val) override + { + return add("k" + val); + } + bool end_object() override + { + return add("}"); + } + bool start_array(std::size_t elements) override + { + return add("[" + std::to_string(elements)); + } + bool end_array() override + { + return add("]"); + } + bool parse_error(std::size_t position, const std::string& last_token, const nlohmann::detail::exception& ex) override + { + events.push_back("error " + std::to_string(position) + " " + last_token + " " + ex.what()); + return continue_after_error; + } + + const std::vector& log() const noexcept + { + return events; + } + + private: + bool add(const std::string& event) + { + events.push_back(event); + return events.size() != stop_after; + } + + std::vector events {}; // NOLINT(readability-redundant-member-init) + std::size_t stop_after; + bool continue_after_error; +}; + +/// sax_parse() of the contiguous path against the byte path, also with a +/// handler that stops the parse +void check_same_events(const std::string& text, const std::list& bytes) +{ + for (const std::size_t stop_after : + { + static_cast(-1), std::size_t{1}, std::size_t{3} + }) + { + CAPTURE(stop_after) + event_log contiguous(stop_after); + event_log byte_path(stop_after); + const bool r1 = json::sax_parse(text, &contiguous); + const bool r2 = json::sax_parse(bytes.begin(), bytes.end(), &byte_path); + CHECK(r1 == r2); + CHECK(contiguous.log() == byte_path.log()); + } + // a handler that asks to go on after an error (the library parser then + // reports what follows, e.g., the missing end of the input) + event_log contiguous(static_cast(-1), true); + event_log byte_path(static_cast(-1), true); + const bool r1 = json::sax_parse(text, &contiguous); + const bool r2 = json::sax_parse(bytes.begin(), bytes.end(), &byte_path); + CHECK(r1 == r2); + CHECK(contiguous.log() == byte_path.log()); +} + +/// the contiguous path (a std::string) against the byte path (std::list iterators) +template +void check_same(const std::string& text) +{ + CAPTURE(text) + const std::list bytes(text.begin(), text.end()); + + const std::string contiguous = outcome([&] + { + return Json::parse(text); + }); + const std::string byte_path = outcome([&] + { + return Json::parse(bytes.begin(), bytes.end()); + }); + CHECK(contiguous == byte_path); + + // without exceptions: the value, or discarded for both + const Json c = Json::parse(text, nullptr, false); + const Json b = Json::parse(bytes.begin(), bytes.end(), nullptr, false); + CHECK(c.is_discarded() == b.is_discarded()); + + CHECK(Json::accept(text) == Json::accept(bytes.begin(), bytes.end())); + if (std::is_same::value) + { + check_same_events(text, bytes); + } + // a pointer range without a terminating NUL + std::vector exact(text.begin(), text.end()); + if (!exact.empty()) + { + CHECK(Json::accept(exact.data(), exact.data() + exact.size()) == Json::accept(bytes.begin(), bytes.end())); + } +} + +void check_both(const std::string& text) +{ + check_same(text); + check_same(text); +} + +/// a random JSON text, mostly valid, with the corner cases of the parser +class generator +{ + public: + explicit generator(std::uint32_t seed) : rng(seed) {} + + std::string document(int depth) + { + std::string out; + value(out, depth); + return out; + } + + private: + std::uint32_t pick(std::uint32_t n) + { + return static_cast(rng() % n); + } + + void whitespace(std::string& out) + { + static const char* const choices[] = {"", "", "", " ", "\n", "\n ", "\t", "\r\n ", "\n ", " "}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + out += choices[pick(10)]; + } + + void number(std::string& out) + { + static const char* const choices[] = // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + "0", "-0", "1", "-1", "42", "123456789012345678", "-123456789012345678", "1234567890123456789", + "-1234567890123456789", "9223372036854775807", "-9223372036854775808", "-9223372036854775809", + "18446744073709551615", "18446744073709551616", "123456789012345678901234567890", "0.0", "-0.0", + "1.5", "-65.613616999999977", "1e10", "1E+10", "1e-10", "2.2250738585072014e-308", "4.9e-324", + "1e-400", "1.7976931348623157e308", "1e308", "1e309", "-1e309", "123.456e-789", "0.1e1", + "12345678901234567890.5", "1.00000000000000011102230246251565404236316680908203125", + "9007199254740993", "0.30000000000000004", "1e22", "1e23", "100000000000000000000000", + "00", "01", "-", "1.", ".5", "1e", "1e+", "--1", "+1", "1.5e3.2", "0x10", "1_000" + }; + out += choices[pick(sizeof(choices) / sizeof(choices[0]))]; + } + + void string(std::string& out) + { + static const char* const pieces[] = // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + "a", "key", "a longer piece of plain ASCII text for the vector paths", "\\n", "\\t", "\\\"", + "\\\\", "\\/", "\\b", "\\f", "\\r", "\\u0000", "\\u00e4", "\\u20AC", "\\uD83D\\uDE00", + "\\uD83D", "\\uDE00", "\\uD83Dx", "\\u12", "\\x", "\xC3\xA4", "\xE2\x82\xAC", "\xF0\x9F\x98\x80", + "\xC0\xAF", "\xED\xA0\x80", "\xF4\x90\x80\x80", "\x80", "\xFF", "\x01", "\x1F", "\x7F", "\t" + }; + out += '"'; + const auto n = pick(6); + for (std::uint32_t i = 0; i < n; ++i) + { + // mostly well-formed pieces + out += pieces[pick(5) == 0 ? pick(sizeof(pieces) / sizeof(pieces[0])) : pick(20)]; + } + out += '"'; + } + + void value(std::string& out, int depth) + { + const auto kind = depth > 0 ? pick(9) : 3 + pick(6); + switch (kind) + { + case 0: + case 1: + { + out += '['; + whitespace(out); + const auto n = pick(6); + for (std::uint32_t i = 0; i < n; ++i) + { + if (i != 0) + { + out += ','; + whitespace(out); + } + value(out, depth - 1); + whitespace(out); + } + out += ']'; + break; + } + case 2: + { + out += '{'; + whitespace(out); + const auto n = pick(6); + for (std::uint32_t i = 0; i < n; ++i) + { + if (i != 0) + { + out += ','; + whitespace(out); + } + // a few duplicate keys + if (pick(4) == 0) + { + out += "\"dup\""; + } + else + { + string(out); + } + whitespace(out); + out += pick(2) == 0 ? ":" : ": "; + whitespace(out); + value(out, depth - 1); + whitespace(out); + } + out += '}'; + break; + } + case 3: + case 4: + number(out); + break; + case 5: + case 6: + string(out); + break; + case 7: + out += pick(2) == 0 ? "true" : "false"; + break; + default: + out += "null"; + break; + } + } + + std::mt19937 rng; +}; + +} // namespace + +TEST_CASE("contiguous parser") +{ + SECTION("fixed inputs") + { + const char* const inputs[] = // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + "", " ", "\n", "null", "true", "false", "nul", "tru", "fals", "nullx", "[]", "{}", "[", "{", "]", "}", + "[1,]", "[,1]", R"({"a":1,})", R"({"a" 1})", R"({"a":})", "{1:2}", "[1 2]", R"("")", R"("abc)", R"("\")", + "\xEF\xBB\xBF[1]", "\xEF\xBB[1]", "\xEF[1]", "\xEF\xBB\xBF", " [1] ", "[1]x", "[1] [2]", "1 2", + R"({"a":1,"a":2})", R"({"b":1,"a":2,"b":3})", R"(["\u0000"])", "[1e400]", "[-1e400]", + R"({"a": [1, 2, {"b": null}], "c": "d"})", "[true,false,null]", "[[[[[]]]]]", + }; + for (const char* input : inputs) + { + check_both(input); + } + + // NUL bytes: by default the end of the input (between values), else an error + check_both(std::string("[1]\0garbage", 11)); + check_both(std::string("[1,\0]", 5)); + check_both(std::string("\"a\0b\"", 5)); + check_both(std::string("\0", 1)); + check_both(std::string("1\0", 2)); + } + + SECTION("deep nesting") + { + // across the 64-level words of the nesting stack + const std::vector depths = {63, 64, 65, 127, 128, 129, 200, 1000, 10000}; + for (const std::size_t depth : depths) + { + check_both(std::string(depth, '[') + std::string(depth, ']')); + std::string objects; + for (std::size_t i = 0; i < depth; ++i) + { + objects += (i % 2 == 0) ? "{\"k\":" : "["; + } + objects += "1"; + for (std::size_t i = depth; i-- > 0;) + { + objects += (i % 2 == 0) ? "}" : "]"; + } + check_both(objects); + // unbalanced + check_both(std::string(depth, '[') + std::string(depth - 1, ']')); + check_both(std::string(depth, '[') + std::string(depth + 1, ']')); + } + // a value after a container that closes a 64-level word + std::string mixed = std::string(64, '[') + "1" + std::string(63, ']') + ",{\"a\":2}]"; + check_both(mixed); + mixed = "{\"x\":" + std::string(64, '[') + std::string(64, ']') + ",\"y\":[1]}"; + check_both(mixed); + } + + SECTION("generated documents and their mutations") + { + generator gen(20261105u); + std::mt19937 rng(42u); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible + const char alphabet[] = "{}[]:,\"\\ \t\n\r0123456789-+.eEtrufalsn/u\x80\xbf\xc3\xe2\xed\xf0\xff"; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + for (int round = 0; round < 1500; ++round) + { + const std::string text = gen.document(1 + (round % 5)); + check_both(text); + // every truncation of short documents, and a few mutations + if (text.size() < 60) + { + for (std::size_t n = 0; n < text.size(); ++n) + { + check_same(text.substr(0, n)); + } + } + for (int m = 0; m < 3 && !text.empty(); ++m) + { + std::string mutated = text; + const std::size_t pos = rng() % mutated.size(); + switch (rng() % 3) + { + case 0: + mutated[pos] = alphabet[rng() % (sizeof(alphabet) - 1)]; + break; + case 1: + mutated.insert(pos, 1, alphabet[rng() % (sizeof(alphabet) - 1)]); + break; + default: + mutated.erase(pos, 1); + break; + } + check_same(mutated); + } + } + } + + SECTION("narrow number types") + { + // integers that do not fit become floats, floats are converted to float + using narrow_json = nlohmann::basic_json; + const std::vector inputs = + { + "[2147483647, 2147483648, -2147483648, -2147483649, 4294967295, 4294967296, 18446744073709551615]", + "[1.5, 3.4028234663852886e38, 3.4028235677973366e38, 1e39, -1e39, 1e-46, 0.1, 16777217]", + "[-0, 0.0, 123456789012345678901234567890, -9223372036854775809]", "1e39", "4294967296", "-2147483649" + }; + for (const auto& input : inputs) + { + check_same(input); + } + generator gen(17u); + for (int round = 0; round < 200; ++round) + { + check_same(gen.document(1 + (round % 4))); + } + } + + SECTION("number corner cases at the end of the input") + { + // the number ends at the end of the block (no closing bracket after it) + for (const char* number : + {"0", "-0", "1", "12345678", "123456789", "1234567890123456789", "18446744073709551616", + "1.5", "1e5", "1e", "1.", "-", "1.5e-3", "123456789012345678901234567890e-10" + }) + { + check_both(number); + check_both(std::string("[") + number + "]"); + check_both(std::string("{\"n\":") + number + "}"); + } + } + + SECTION("callbacks, comments and trailing commas take the library parser") + { + // (the results are those of the options; the contiguous parser does not support them) + CHECK(json::parse("[1,]", nullptr, true, false, true) == json({1})); + CHECK(json::parse("[1/*c*/]", nullptr, true, true) == json({1})); + CHECK(json::accept("[1,]", false, true)); + CHECK(json::accept("// c\n[1]", true)); + int calls = 0; + const json j = json::parse("[1,2]", [&calls](int /*depth*/, json::parse_event_t /*event*/, json& /*parsed*/) + { + ++calls; + return true; + }); + CHECK(j == json({1, 2})); + CHECK(calls == 4); + } +}