diff --git a/BUILD.bazel b/BUILD.bazel index 2b1c9f146..b7908bcf7 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -30,6 +30,7 @@ cc_library( "include/nlohmann/detail/exceptions.hpp", "include/nlohmann/detail/hash.hpp", "include/nlohmann/detail/input/binary_reader.hpp", + "include/nlohmann/detail/input/contiguous_parser.hpp", "include/nlohmann/detail/input/input_adapters.hpp", "include/nlohmann/detail/input/json_sax.hpp", "include/nlohmann/detail/input/lexer.hpp", diff --git a/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md b/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md index 4dac9f9ff..340498acb 100644 --- a/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md +++ b/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md @@ -34,6 +34,9 @@ or quotes. Note that enabling this macro increases the size of every JSON value by two `std::size_t` fields and adds slight runtime overhead to parsing, copying JSON value objects, and the generation of error messages for exceptions. It also causes these values to be reported in those error messages. +In addition, `parse()`, `accept()`, and `sax_parse()` no longer use the +[faster path for input in one block of memory](../../features/parsing/index.md#input-sources-and-performance), so +parsing a string or a buffer is slower than without the macro. If [`JSON_DIAGNOSTICS`](json_diagnostics.md) is also set, the message contains the path and the byte range, such as `(/1) (bytes 4-8)`. An exception about the root value has no path and only the range, such as `(bytes 0-4)`. diff --git a/docs/mkdocs/docs/api/macros/json_disable_stream_bulk_scan.md b/docs/mkdocs/docs/api/macros/json_disable_stream_bulk_scan.md index 7befb6cde..9af9c3aea 100644 --- a/docs/mkdocs/docs/api/macros/json_disable_stream_bulk_scan.md +++ b/docs/mkdocs/docs/api/macros/json_disable_stream_bulk_scan.md @@ -48,8 +48,8 @@ The default value is `0`: streams are scanned in bulk. !!! note "Performance" Parsing from a `std::ifstream` or `std::istringstream` is typically 1.5 to 2 times as fast as before, and reading - many small values from one stream with `operator>>` 1.4 to 4 times as fast. Parsing from a stream is then within - about 10 to 20 % of first reading the whole file into a `std::string` and parsing that. + many small values from one stream with `operator>>` 1.4 to 4 times as fast. Parsing from a stream then takes about + 1.15 to 1.45 times as long as first reading the whole file into a `std::string` and parsing that. A stream buffer without a get area (one that only overrides `underflow()` and `uflow()` and hands out every character from there) offers nothing to scan in bulk, so it is read one character at a time as before. diff --git a/docs/mkdocs/docs/features/files.md b/docs/mkdocs/docs/features/files.md index bf3632e89..c69879a45 100644 --- a/docs/mkdocs/docs/features/files.md +++ b/docs/mkdocs/docs/features/files.md @@ -32,11 +32,11 @@ json read_json_file(const std::string& path) column 1 instead of saying that the file is missing. A relative path is resolved against the working directory of the running program, which is often not the directory of the source file (see the [FAQ](../home/faq.md#parsing-a-file-fails-with-unexpected-end-of-input)). -- **Don't read the file into a string first.** Parsing from a `#!cpp std::ifstream` is only about 10 to 20 % slower - than reading the whole file into a string with one `read()` and parsing that, and it needs no memory for the copy (see - [input sources and performance](parsing/index.md#input-sources-and-performance)). The common way to read a file with - `#!cpp std::istreambuf_iterator` is so slow that it takes about twice as long as parsing from the stream. Read - the file into a string only if you need the text itself, for example for an error message. This advice is for +- **Read the file into a string only if the memory is no concern.** Parsing from a `#!cpp std::ifstream` takes about + 1.15 to 1.45 times as long as reading the whole file into a string with one `read()` and parsing that, but it needs no + memory for the copy (see [input sources and performance](parsing/index.md#input-sources-and-performance)). The common + way to read a file with `#!cpp std::istreambuf_iterator` is so slow that it takes about twice as long as parsing + from the stream. This advice is for `#!cpp json::parse`. [`json_document`](json_view/index.md) reads a stream into a buffer that it owns anyway, so there is no copy to save: reading the file into a `#!cpp std::string` yourself and passing that as an lvalue, which the document borrows, was about 6 to 12 % faster than parsing from the stream, with the same single buffer. diff --git a/docs/mkdocs/docs/features/parsing/index.md b/docs/mkdocs/docs/features/parsing/index.md index 984de9f7e..38e523f88 100644 --- a/docs/mkdocs/docs/features/parsing/index.md +++ b/docs/mkdocs/docs/features/parsing/index.md @@ -80,8 +80,16 @@ them faster than others: | `#!cpp FILE*` | with glibc and the Apple C library, the characters the `FILE` holds in its buffer are scanned in bulk like a stream's; elsewhere one character at a time; the file is locked once for the whole parse | | other iterator ranges (e.g., `#!cpp std::list`, `#!cpp std::istreambuf_iterator`), wide strings | one character at a time | -Parsing a file from a `#!cpp std::ifstream` is therefore only about 10 to 20 % slower than reading the whole file into a -`#!cpp std::string` first and parsing that, and it needs no memory for a copy of the file. The stream is never read +Input in one block of memory is read even faster with the default options (no +[parser callback](../../api/basic_json/parser_callback_t.md), no comments, no trailing commas, and for `sax_parse()` +`strict = true`): `parse()`, `accept()`, and `sax_parse()` then read the grammar and the tokens in one loop over the +memory. Only after an error do `parse()` and `sax_parse()` read the input again, to report the error exactly as before. +This faster path is not used when [`JSON_DIAGNOSTIC_POSITIONS`](../../api/macros/json_diagnostic_positions.md) is +enabled, because the positions are recorded by the default parser: the input is then parsed as before, at the same speed +as before. + +Parsing a file from a `#!cpp std::ifstream` therefore takes about 1.15 to 1.45 times as long as reading the whole file +into a `#!cpp std::string` first and parsing that, but it needs no memory for a copy of the file. The stream is never read ahead: after parsing, it is positioned exactly as if every character had been read one at a time, so [`operator>>`](../../api/operator_gtgt.md) can still read several values from one stream (see also [JSON Lines](json_lines.md)). diff --git a/docs/mkdocs/docs/features/performance.md b/docs/mkdocs/docs/features/performance.md index 570130055..b901e7ff3 100644 --- a/docs/mkdocs/docs/features/performance.md +++ b/docs/mkdocs/docs/features/performance.md @@ -29,9 +29,9 @@ up. Nothing is read ahead, so the stream is left exactly where it would be after [`operator>>`](../api/operator_gtgt.md) can still read several values from one stream. A stream buffer without a get area is read one character at a time. A `#!cpp FILE*` is locked once per parse instead of once per character, and with glibc and the Apple C library, the characters in its buffer are scanned in bulk as a stream's get area is. See -[input sources and performance](parsing/index.md#input-sources-and-performance) for the full table. According to the -description of [#5828](https://github.com/nlohmann/json/pull/5828), parsing from a stream is within about 10 to 20 % of -parsing an equivalent `#!cpp std::string`. +[input sources and performance](parsing/index.md#input-sources-and-performance) for the full table. Parsing from a +stream takes about 1.15 to 1.45 times as long as parsing an equivalent `#!cpp std::string`, which `parse()` reads with +a parser of its own (see there). Inputs that are still read one character at a time: iterator ranges that are not contiguous (e.g. `#!cpp std::list` or `#!cpp std::istreambuf_iterator`), wide strings, and user-defined adapters. An diff --git a/include/nlohmann/detail/input/contiguous_parser.hpp b/include/nlohmann/detail/input/contiguous_parser.hpp new file mode 100644 index 000000000..29a090a3b --- /dev/null +++ b/include/nlohmann/detail/input/contiguous_parser.hpp @@ -0,0 +1,1032 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#pragma once + +#include // isfinite +#include // size_t, ptrdiff_t +#include // uint64_t, int64_t +#include // memcmp +#include // numeric_limits +#include // string +#include // integral_constant, true_type, false_type +#include // move +#include // vector + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief a parser for JSON text in one contiguous block, for parse(), accept(), + and sax_parse() + +The default parser and lexer exchange one token at a time, and the lexer keeps +its state (the current character, a pending unget, where the token starts) in +members, which every character pays for. This parser reads the block through a +local pointer instead, and the grammar, the whitespace and the tokens are one +loop, after yyjson's reader: the state stays in registers, and the expected +next token is tested where it is expected instead of being dispatched on. + +It reports what it reads to a SAX handler: the one parse() uses to build the +value, the one accept() uses, or the user's handler of sax_parse(). On a byte +the default parser would reject, it stops before reporting the token, and +returns false: the events up to there are those the default parser reports. +The caller then runs the default parser on the same input to report the error +with its exact message and position (report_contiguous_error(), +json_sax_error_forwarder), and parse() builds no value from what was read. + +What is accepted, and the values reported, are those of the default parser +with default options: the same number grammar and conversions, the same UTF-8 +and escape rules, a UTF-8 byte order mark at the start, and, without +JSON_STRICT_NUL_HANDLING, a NUL byte as the end of the input. + +@tparam KeepValues whether the handler needs the values of strings and + numbers (false for accept(), whose handler ignores them) +*/ +template +class contiguous_parser +{ + using string_t = typename BasicJsonType::string_t; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using char_t = typename string_t::value_type; + using uchar = unsigned char; + + public: + contiguous_parser(const uchar* first, const uchar* last, SAX* sax_) noexcept + : begin(first), end(last), sax(sax_) + {} + + contiguous_parser(const contiguous_parser&) = delete; + contiguous_parser(contiguous_parser&&) = delete; + contiguous_parser& operator=(const contiguous_parser&) = delete; + contiguous_parser& operator=(contiguous_parser&&) = delete; + ~contiguous_parser() = default; + + /// whether the whole block is one valid JSON text; the handler has seen all of it + JSON_HEDLEY_NEVER_INLINE bool parse() + { + const uchar* p = begin; + const uchar* const e = end; + + if (p != e && *p == 0xEF) + { + // a byte order mark, or the lexer reports a bad one + if (e - p < 3 || p[1] != 0xBB || p[2] != 0xBF) + { + return false; + } + p += 3; + } + + // The open arrays and objects: bit 0 of `kinds` is the innermost (1: + // object), so that the kind of the enclosing container is a shift away + // and stays in a register; every 64 levels, the word is pushed to + // `outer` and a new one is begun. + std::uint64_t kinds = 0; + std::size_t depth = 0; + std::vector outer; + + // the current byte, or 0 at the end (no JSON token starts with 0) +#define JSON_CONTIGUOUS_CUR() current(p, e) + // skip whitespace; most tokens have none before them +#define JSON_CONTIGUOUS_WS() \ + do \ + { \ + if (JSON_HEDLEY_UNLIKELY(JSON_CONTIGUOUS_CUR() <= ' ')) \ + { \ + p = skip_whitespace(p, e); \ + } \ + } \ + while (false) +#define JSON_CONTIGUOUS_OPEN(is_object) \ + do \ + { \ + if (JSON_HEDLEY_UNLIKELY(depth != 0 && (depth % 64) == 0)) /* NOLINT(readability/braces) */ \ + { \ + outer.push_back(kinds); \ + kinds = 0; \ + } \ + kinds = (kinds << 1u) | static_cast(is_object); \ + ++depth; \ + } \ + while (false) + + JSON_CONTIGUOUS_WS(); + switch (JSON_CONTIGUOUS_CUR()) + { + case '{': + ++p; + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_object(detail::unknown_size())))) + { + return false; + } + JSON_CONTIGUOUS_OPEN(1u); + goto object_first; + case '[': + ++p; + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_array(detail::unknown_size())))) + { + return false; + } + JSON_CONTIGUOUS_OPEN(0u); + goto array_first; + default: + if (JSON_HEDLEY_UNLIKELY(!scalar(p))) + { + return false; + } + goto done; + } + + // a value in an array or an object: expanded once for each, so that + // each dispatch has its own branch history (arrays tend to hold values + // of one kind) and the code after a scalar knows where it is +#define JSON_CONTIGUOUS_VALUE \ + switch (JSON_CONTIGUOUS_CUR()) \ + { \ + case '"': \ + if (JSON_HEDLEY_UNLIKELY(!string(p, false))) \ + { \ + return false; \ + } \ + break; \ + case '{': \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_object(detail::unknown_size())))) \ + { \ + return false; \ + } \ + JSON_CONTIGUOUS_OPEN(1u); \ + goto object_first; \ + case '[': \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_array(detail::unknown_size())))) \ + { \ + return false; \ + } \ + JSON_CONTIGUOUS_OPEN(0u); \ + goto array_first; \ + case '-': \ + case '0': \ + case '1': \ + case '2': \ + case '3': \ + case '4': \ + case '5': \ + case '6': \ + case '7': \ + case '8': \ + case '9': \ + if (JSON_HEDLEY_UNLIKELY(!number(p))) \ + { \ + return false; \ + } \ + break; \ + case 't': \ + if (JSON_HEDLEY_UNLIKELY(!literal(p, "true") || !report(sax->boolean(true)))) \ + { \ + return false; \ + } \ + break; \ + case 'f': \ + if (JSON_HEDLEY_UNLIKELY(!literal(p, "false") || !report(sax->boolean(false)))) \ + { \ + return false; \ + } \ + break; \ + case 'n': \ + if (JSON_HEDLEY_UNLIKELY(!literal(p, "null") || !report(sax->null()))) \ + { \ + return false; \ + } \ + break; \ + default: \ + return false; \ + } + + // what may follow a value in an array: a comma and the next value, or the end +#define JSON_CONTIGUOUS_AFTER_ELEMENT \ + JSON_CONTIGUOUS_WS(); \ + if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ',')) /* NOLINT(readability/braces) */ \ + { \ + ++p; \ + JSON_CONTIGUOUS_WS(); \ + goto array_value; \ + } \ + if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ']')) /* NOLINT(readability/braces) */ \ + { \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_array()))) \ + { \ + return false; \ + } \ + goto close; \ + } \ + return false; + + // what may follow a value in an object: a comma and the next member, or the end +#define JSON_CONTIGUOUS_AFTER_MEMBER \ + JSON_CONTIGUOUS_WS(); \ + if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ',')) /* NOLINT(readability/braces) */ \ + { \ + ++p; \ + JSON_CONTIGUOUS_WS(); \ + goto object_key; \ + } \ + if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == '}')) /* NOLINT(readability/braces) */ \ + { \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_object()))) \ + { \ + return false; \ + } \ + goto close; \ + } \ + return false; + +array_first: + JSON_CONTIGUOUS_WS(); + if (JSON_CONTIGUOUS_CUR() == ']') + { + ++p; + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_array()))) + { + return false; + } + goto close; + } +array_value: + JSON_CONTIGUOUS_VALUE + JSON_CONTIGUOUS_AFTER_ELEMENT + +object_first: + JSON_CONTIGUOUS_WS(); + if (JSON_CONTIGUOUS_CUR() == '}') + { + ++p; + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_object()))) + { + return false; + } + goto close; + } +object_key: + if (JSON_HEDLEY_UNLIKELY(JSON_CONTIGUOUS_CUR() != '"' || !string(p, true))) + { + return false; + } + if (JSON_HEDLEY_LIKELY(e - p >= 2 && p[0] == ':' && p[1] == ' ')) + { + p += 2; // ": " (pretty-printed input; a fast path of yyjson) + } + else + { + JSON_CONTIGUOUS_WS(); + if (JSON_HEDLEY_UNLIKELY(JSON_CONTIGUOUS_CUR() != ':')) + { + return false; + } + ++p; + } + JSON_CONTIGUOUS_WS(); + JSON_CONTIGUOUS_VALUE + JSON_CONTIGUOUS_AFTER_MEMBER + +close: + // an array or object was closed: continue in the enclosing one + --depth; + kinds >>= 1u; + if (JSON_HEDLEY_UNLIKELY(depth == 0)) + { + goto done; + } + if (JSON_HEDLEY_UNLIKELY((depth % 64) == 0)) + { + kinds = outer.back(); + outer.pop_back(); + } + if ((kinds & 1u) != 0) + { + JSON_CONTIGUOUS_AFTER_MEMBER + } + JSON_CONTIGUOUS_AFTER_ELEMENT + +done: + JSON_CONTIGUOUS_WS(); +#if JSON_STRICT_NUL_HANDLING + return p == e; +#else + // a NUL byte ends the input, as in the lexer + return p == e || *p == 0; +#endif + +#undef JSON_CONTIGUOUS_AFTER_MEMBER +#undef JSON_CONTIGUOUS_AFTER_ELEMENT +#undef JSON_CONTIGUOUS_VALUE +#undef JSON_CONTIGUOUS_OPEN +#undef JSON_CONTIGUOUS_WS +#undef JSON_CONTIGUOUS_CUR + } + + /// whether the handler stopped the parse (by returning false), as opposed + /// to a syntax error + bool aborted() const noexcept + { + return handler_stopped; + } + + private: + /// the result @a r of an event of the handler: false stops the parse + bool report(const bool r) noexcept + { + handler_stopped = handler_stopped || !r; + return r; + } + + /// the text of the float token [@a first, @a last), for a handler that reads it + const string_t& number_text(const char* first, const char* last) + { + return number_text(first, last, std::integral_constant::value> {}); + } + + const string_t& number_text(const char* /*first*/, const char* /*last*/, std::true_type /*ignored*/) const noexcept + { + return no_text; + } + + const string_t& number_text(const char* first, const char* last, std::false_type /*ignored*/) + { + float_text.clear(); + float_text.append(reinterpret_cast(first), static_cast(last - first)); + return float_text; + } + + /// KeepValues as a runtime condition: testing the template argument + /// directly makes `if` conditions constant, which MSVC reports as C4127 + static bool keep_values() noexcept + { + return KeepValues; + } + + /// the byte at @a p, or 0 at the end @a e (no JSON token starts with 0) + static JSON_HEDLEY_ALWAYS_INLINE uchar current(const uchar* p, const uchar* e) noexcept + { + return p != e ? *p : 0; + } + + /// the first byte at or after p that is not whitespace, or e + static JSON_HEDLEY_NEVER_INLINE const uchar* skip_whitespace(const uchar* p, const uchar* e) noexcept + { + while (p != e) + { + const uchar c = *p; + if (c == ' ' || c == '\t' || c == '\r') + { + ++p; + } + else if (c == '\n') + { + // the indentation of the next line, 8 spaces at a time + ++p; + while (e - p >= 8) + { + const std::uint64_t non_space = read_eight_bytes(p) ^ 0x2020202020202020ull; + if (non_space != 0) + { + p += static_cast(count_trailing_zeros(non_space)) / 8; + break; + } + p += 8; + } + } + else + { + break; + } + } + return p; + } + + /// the literal @a text at p, whose first byte the dispatch has seen + template + JSON_HEDLEY_ALWAYS_INLINE bool literal(const uchar*& p, const char (&text)[N]) const noexcept // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays): a string literal + { + constexpr std::size_t length = N - 1; + if (JSON_HEDLEY_UNLIKELY(static_cast(end - p) < length || std::memcmp(p, text, length) != 0)) + { + return false; + } + p += length; + return true; + } + + /// a JSON text that is a single scalar + bool scalar(const uchar*& p) + { + switch (current(p, end)) + { + case '"': + return string(p, false); + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return number(p); + case 't': + return literal(p, "true") && report(sax->boolean(true)); + case 'f': + return literal(p, "false") && report(sax->boolean(false)); + case 'n': + return literal(p, "null") && report(sax->null()); + default: + return false; + } + } + + /// the string whose opening quote is at p, reported as a key or a value; + /// p is moved past the closing quote + JSON_HEDLEY_ALWAYS_INLINE bool string(const uchar*& p, const bool is_key) + { + const uchar* const s = ++p; + // the text up to the first quote, backslash, control character or + // ill-formed UTF-8 (well-formed UTF-8 is part of the run) + p = s + lexer_string_bulk_run(s, static_cast(end - s)); + if (JSON_HEDLEY_LIKELY(p != end && *p == '"')) + { + if (keep_values()) + { + buffer.clear(); + buffer.append(reinterpret_cast(s), static_cast(p - s)); + } + ++p; + } + else if (JSON_HEDLEY_UNLIKELY(!escaped_string(s, p))) + { + return false; + } + return is_key ? report(sax->key(buffer)) : report(sax->string(buffer)); + } + + /// the rest of a string from p, a backslash, a control character, + /// ill-formed UTF-8 or the end; s is where the string starts. With + /// KeepValues, the string is decoded into buffer. + JSON_HEDLEY_NEVER_INLINE bool escaped_string(const uchar* s, const uchar*& pos) + { + const uchar* p = pos; + if (keep_values()) + { + buffer.clear(); + buffer.append(reinterpret_cast(s), static_cast(p - s)); + } + for (;;) + { + if (JSON_HEDLEY_UNLIKELY(p == end)) + { + return false; + } + if (*p == '"') + { + pos = p + 1; + return true; + } + if (JSON_HEDLEY_UNLIKELY(*p != '\\' || end - p < 2)) + { + return false; // a control character or ill-formed UTF-8 + } + const uchar escape = p[1]; + p += 2; + switch (escape) + { + case '"': + case '\\': + case '/': + add(static_cast(escape)); + break; + case 'b': + add('\b'); + break; + case 'f': + add('\f'); + break; + case 'n': + add('\n'); + break; + case 'r': + add('\r'); + break; + case 't': + add('\t'); + break; + case 'u': + { + if (JSON_HEDLEY_UNLIKELY(end - p < 4)) + { + return false; + } + int codepoint = hex_codepoint(p); + if (JSON_HEDLEY_UNLIKELY(codepoint < 0)) + { + return false; + } + p += 4; + if (JSON_HEDLEY_UNLIKELY((codepoint & 0xF800) == 0xD800)) + { + // a high surrogate must be followed by \u and a low surrogate + if (codepoint >= 0xDC00 || end - p < 6 || p[0] != '\\' || p[1] != 'u') + { + return false; + } + const int low = hex_codepoint(p + 2); + if (low < 0xDC00 || low > 0xDFFF) + { + return false; + } + p += 6; + codepoint = 0x10000 + ((codepoint - 0xD800) << 10) + (low - 0xDC00); + } + add_utf8(codepoint); + break; + } + default: + return false; + } + const uchar* const run = p; + p = run + lexer_string_bulk_run(run, static_cast(end - run)); + if (keep_values() && p != run) + { + buffer.append(reinterpret_cast(run), static_cast(p - run)); + } + } + } + + void add(const char_t c) + { + if (keep_values()) + { + buffer.push_back(c); + } + } + + /// add the UTF-8 encoding of @a codepoint (at most U+10FFFF) to buffer + void add_utf8(const int codepoint) + { + if (!keep_values()) + { + return; + } + const auto cp = static_cast(codepoint); + if (cp < 0x80) + { + buffer.push_back(static_cast(cp)); + } + else if (cp < 0x800) + { + buffer.push_back(static_cast(0xC0u | (cp >> 6u))); + buffer.push_back(static_cast(0x80u | (cp & 0x3Fu))); + } + else if (cp < 0x10000) + { + buffer.push_back(static_cast(0xE0u | (cp >> 12u))); + buffer.push_back(static_cast(0x80u | ((cp >> 6u) & 0x3Fu))); + buffer.push_back(static_cast(0x80u | (cp & 0x3Fu))); + } + else + { + buffer.push_back(static_cast(0xF0u | (cp >> 18u))); + buffer.push_back(static_cast(0x80u | ((cp >> 12u) & 0x3Fu))); + buffer.push_back(static_cast(0x80u | ((cp >> 6u) & 0x3Fu))); + buffer.push_back(static_cast(0x80u | (cp & 0x3Fu))); + } + } + + /*! + @brief the number at p, with the grammar and the conversions of the lexer's + contiguous path (lexer::scan_number_bulk_contiguous()) + + Integers that do not fit their type become floats; floats that overflow + are an error, which the default parser reports. + */ + JSON_HEDLEY_ALWAYS_INLINE bool number(const uchar*& pos) + { + const auto* const data = reinterpret_cast(pos); + const auto avail = static_cast(end - pos); + std::size_t i = 0; + std::size_t dot_index = std::string::npos; + bool is_float = false; + std::uint64_t w = 0; + const bool negative = data[0] == '-'; + if (negative) + { + i = 1; + } + const std::size_t integer_begin = i; + if (i < avail && data[i] == '0') + { + ++i; + } + else if (i < avail && data[i] >= '1' && data[i] <= '9') + { + // accept() needs the value only for a float with a large exponent, + // which computes it below; its handler ignores the values + i = keep_values() ? accumulate_digits(data, i, avail, w) : skip_digits(data, i, avail); + } + else + { + return false; + } + std::size_t digit_count = i - integer_begin; + std::size_t fraction_digits = 0; + if (i < avail && data[i] == '.') + { + is_float = true; + dot_index = i; + ++i; + if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) + { + return false; + } + const std::size_t fraction_begin = i; + i = keep_values() ? accumulate_digits(data, i, avail, w) : skip_digits(data, i, avail); + fraction_digits = i - fraction_begin; + digit_count += fraction_digits; + } + const std::size_t mantissa_end = i; + std::int64_t exponent = 0; + if (i < avail && (data[i] == 'e' || data[i] == 'E')) + { + is_float = true; + ++i; + bool negative_exponent = false; + if (i < avail && (data[i] == '+' || data[i] == '-')) + { + negative_exponent = data[i] == '-'; + ++i; + } + if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) + { + return false; + } + // saturated far beyond every range, see parse_float_exponent() + constexpr std::int64_t saturation = 100000000000000000; // 10^17 + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + if (exponent < saturation) + { + exponent = (exponent * 10) + (data[i] - '0'); + } + ++i; + } + exponent = negative_exponent ? -exponent : exponent; + } + pos += i; + + if (!is_float) + { + if (digit_count <= 18) + { + // up to 18 digits always fit into std::uint64_t and std::int64_t; + // a narrower type takes the value only if it round-trips + if (negative) + { + const std::int64_t x = -static_cast(w); + const auto value = static_cast(x); + if (static_cast(value) == x) + { + return report(sax->number_integer(value)); + } + } + else + { + const auto value = static_cast(w); + if (static_cast(value) == w) + { + return report(sax->number_unsigned(value)); + } + } + } + else if (negative) + { + number_integer_t value{}; + if (parse_integer_signed(data, data + i, value)) + { + return report(sax->number_integer(value)); + } + } + else + { + number_unsigned_t value{}; + if (parse_integer_unsigned(data, data + i, value)) + { + return report(sax->number_unsigned(value)); + } + } + } + + number_float_t value{}; + if (digit_count <= 19) + { + float_significand s; + s.w = w; + s.exponent = exponent - static_cast(fraction_digits); + s.negative = negative; + // accept() only needs to know that the value is finite: w < 10^19, + // so only a token that may reach the largest float is converted + if (!keep_values() && s.exponent + 19 <= std::numeric_limits::max_exponent10) + { + return report(sax->number_float(value, number_text(data, data + i))); + } + if (!keep_values()) + { + // the digits were skipped: their value now + s.w = 0; + const std::size_t integer_end = accumulate_digits(data, integer_begin, avail, s.w); + if (dot_index != std::string::npos) + { + static_cast(accumulate_digits(data, integer_end + 1, avail, s.w)); + } + } + value = decimal_to_float(s); + } + else + { + static_cast(convert_float_fast(data, data + i, dot_index, mantissa_end, value)); + } + if (JSON_HEDLEY_UNLIKELY(!std::isfinite(value))) + { + return false; // the default parser reports the overflow + } + return report(sax->number_float(value, number_text(data, data + i))); + } + + const uchar* const begin; + const uchar* const end; + SAX* const sax; + /// the current string (with KeepValues); the handler may move from it + string_t buffer {}; + /// the text of float tokens, for the library's handlers, which ignore it + const string_t no_text {}; + /// the text of the current float token, for other handlers + string_t float_text {}; + /// whether the handler returned false + bool handler_stopped = false; +}; + +/// whether parse() and accept() can read the input behind @a InputAdapterType +/// with contiguous_parser: one contiguous block of bytes (not a window into a +/// stream), and floats in a format the library converts itself +template +struct can_parse_contiguous +{ + static constexpr bool value = +#if JSON_DIAGNOSTIC_POSITIONS + // the values record the positions the lexer reports + false && +#endif + input_adapter_supports_bulk_scan(is_detected {}) + && !input_adapter_bulk_window(is_detected {}) + && has_native_float_format::value; +}; + +/*! +@brief report the input behind @a adapter, one block of memory, to @a sax + +@return whether the input is valid and was reported completely; if not, + nothing was consumed from @a adapter, and the caller has the default + parser report the error +*/ +template +bool parse_contiguous(InputAdapterType& adapter, SAX& sax, std::true_type /*contiguous*/) +{ + const std::size_t size = adapter.bulk_remaining(); + if (size == 0) + { + return false; // the default parser reports the empty input + } + const auto* const first = reinterpret_cast(adapter.bulk_data()); + contiguous_parser p(first, first + size, &sax); + return p.parse(); +} + +/*! +@brief a SAX handler that only reports a parse error, as the handler of parse() + does: it throws the exception, unless exceptions are not allowed + +After contiguous_parser has rejected an input, the default parser is run again +with this handler to report the error: the error, its message and its position +are those of parse(), without building the values read up to it. +*/ +template +class json_sax_error_reporter : public json_sax_acceptor +{ + public: + explicit json_sax_error_reporter(const bool allow_exceptions_) noexcept + : allow_exceptions(allow_exceptions_) + {} + + /// (a template, as in json_sax_dom_parser: the parser passes the concrete + /// exception type, which is thrown as is) + template + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const Exception& ex) + { + static_cast(ex); // (unused without exceptions) + if (allow_exceptions) + { + JSON_THROW(ex); + } + return false; + } + + private: + const bool allow_exceptions; +}; + +template +struct sax_ignores_number_text> : std::true_type {}; + +/*! +@brief report the error in the input behind @a adapter, which contiguous_parser + rejected, as parse() reports it: throw its exception, or, without + exceptions, return true + +@return false if the default parser accepts the input after all (which would + be a disagreement of the two parsers): the caller then parses it +*/ +template +bool report_contiguous_error(const InputAdapterType& adapter, const bool allow_exceptions) +{ + json_sax_error_reporter reporter(allow_exceptions); + // (a copy of the adapter, which the caller may need again) + // (the lexer of parse(), which the fallback in basic_json::parse_impl() + // instantiates anyway: only the grammar loop is instantiated again) + return !parser(InputAdapterType(adapter), nullptr, allow_exceptions, false, false).sax_parse(&reporter, true); +} + +/*! +@brief parse(): parse the input behind @a adapter into @a result, if it is one + block of memory + +@return whether @a result is the result of parse(): the value, or, after an + error and without exceptions, a discarded value (with exceptions, the + error is thrown); false if the input is not one block of memory, in + which case the caller parses it +*/ +template +bool parse_contiguous(InputAdapterType& adapter, BasicJsonType& result, const bool allow_exceptions, std::true_type /*contiguous*/) +{ + { + // the handler of parser::parse(), so that the values are built by the same code + json_sax_dom_parser sax(result, true); + if (parse_contiguous(adapter, sax, std::true_type {})) + { + return true; + } + } + // (what was read up to the error) + result = BasicJsonType(); + if (report_contiguous_error(adapter, allow_exceptions)) + { + result = BasicJsonType(value_t::discarded); + return true; + } + return false; // LCOV_EXCL_LINE (only if the two parsers disagree: a bug) +} + +template +bool parse_contiguous(InputAdapterType& /*adapter*/, BasicJsonType& /*result*/, const bool /*allow_exceptions*/, std::false_type /*contiguous*/) noexcept +{ + return false; +} + +/*! +@brief accept(): check the input behind @a adapter, if it is one block of memory + +@return whether @a accepted is the result of accept(); false if the input is + not one block of memory, in which case the caller checks it. + contiguous_parser accepts exactly what the default parser accepts. +*/ +template +bool accept_contiguous(InputAdapterType& adapter, bool& accepted, std::true_type /*contiguous*/) +{ + json_sax_acceptor sax; + accepted = parse_contiguous(adapter, sax, std::true_type {}); + return true; +} + +template +bool accept_contiguous(InputAdapterType& /*adapter*/, bool& /*accepted*/, std::false_type /*contiguous*/) noexcept +{ + return false; +} + +/*! +@brief a SAX handler that ignores the values and passes a parse error on to the + handler of the user + +After contiguous_parser has delivered the events of the input to the user's +handler and rejected it, the default parser is run again with this handler: +the user's handler gets the error, its message and its position from the +default parser, as before, but no event twice. +*/ +template +class json_sax_error_forwarder : public json_sax_acceptor +{ + public: + explicit json_sax_error_forwarder(SAX* user_) noexcept + : user(user_) + {} + + template + bool parse_error(std::size_t position, const std::string& last_token, const Exception& ex) + { + forwarded = true; + return user->parse_error(position, last_token, ex); + } + + /// whether the default parser reported an error + bool reported() const noexcept + { + return forwarded; + } + + private: + SAX* const user; + bool forwarded = false; +}; + +template +struct sax_ignores_number_text> : std::true_type {}; + +/*! +@brief sax_parse() of JSON in one block of memory with the strict option and + a user's handler @a sax + +contiguous_parser reports to @a sax what it reads, and stops at the token that +the default parser would reject, before reporting it: the events up to an error +are the same. The error itself is then reported by the default parser (see +json_sax_error_forwarder). + +@return whether @a result is the result of sax_parse(); false if the input is + not one block of memory, in which case the caller parses it +*/ +template +bool sax_parse_contiguous(InputAdapterType& adapter, SAX* sax, bool& result, std::true_type /*contiguous*/) +{ + const std::size_t size = adapter.bulk_remaining(); + if (size != 0) + { + const auto* const first = reinterpret_cast(adapter.bulk_data()); + contiguous_parser p(first, first + size, sax); + if (p.parse()) + { + result = true; + return true; + } + if (p.aborted()) + { + // the handler returned false, as with the default parser + result = false; + return true; + } + } + json_sax_error_forwarder forwarder(sax); + result = parser(InputAdapterType(adapter), nullptr, true, false, false).sax_parse(&forwarder, true); + // the default parser must reject the input as well (the events after the + // byte contiguous_parser stopped at cannot be delivered any more) + JSON_ASSERT(forwarder.reported()); + return true; +} + +template +bool sax_parse_contiguous(InputAdapterType& /*adapter*/, SAX* /*sax*/, bool& /*result*/, std::false_type /*contiguous*/) noexcept +{ + return false; +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/input/number_parse.hpp b/include/nlohmann/detail/input/number_parse.hpp index 852e1a6c8..71215d3df 100644 --- a/include/nlohmann/detail/input/number_parse.hpp +++ b/include/nlohmann/detail/input/number_parse.hpp @@ -316,6 +316,30 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t accumulate_digits(const char* data, std::s return i; } +/// the index of the first byte at or after @a i that is not a digit: the +/// digits of accumulate_digits() without their value +JSON_HEDLEY_ALWAYS_INLINE std::size_t skip_digits(const char* data, std::size_t i, std::size_t avail) noexcept +{ + while (avail - i >= 8) + { + if (!is_eight_digits(read_eight_bytes(data + i))) + { + // a byte of the word is not a digit: no bounds checks before it + while (static_cast(data[i] - '0') <= 9) + { + ++i; + } + return i; + } + i += 8; + } + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + ++i; + } + return i; +} + /// whether [first, last) contains a digit other than '0'; a plain loop, as /// GCC calls std::any_of out of line even for the usual empty range JSON_HEDLEY_ALWAYS_INLINE bool has_nonzero_digit(const char* first, const char* last) noexcept diff --git a/include/nlohmann/detail/input/string_scan.hpp b/include/nlohmann/detail/input/string_scan.hpp index 10e4cc900..8564746ce 100644 --- a/include/nlohmann/detail/input/string_scan.hpp +++ b/include/nlohmann/detail/input/string_scan.hpp @@ -520,7 +520,10 @@ JSON_HEDLEY_NEVER_INLINE inline std::size_t string_bulk_run_from(const unsigned } #endif -// string_bulk_run() for the lexer. With NEON, only the first four words are +// string_bulk_run() for the lexer. The run must be the longest one: the lexer +// would only lose speed with a shorter run, but contiguous_parser rejects a +// string whose run ends at a byte other than a quote or a backslash, so a +// shorter run would reject valid input there. With NEON, only the first four words are // checked inline (most keys and short values end there), and longer runs and // non-ASCII text continue out of line. The lexer's scan function runs once per // token, and the vector code inlined into it needs more registers, which every diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 9be8a8c6c..bd06811d9 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -58,6 +58,7 @@ #include // IWYU pragma: export #include #include +#include #include #include #include @@ -3109,6 +3110,15 @@ public: const bool ignore_trailing_commas) { basic_json result; + // JSON in one block of memory, with the default options, is read by + // the faster contiguous_parser; on an error, the default parser reads + // it again to report it (see contiguous_parser) + using contiguous = std::integral_constant::type>::value>; + if (cb == nullptr && !ignore_comments && !ignore_trailing_commas + && detail::parse_contiguous(adapter, result, allow_exceptions, contiguous {})) + { + return result; + } parser(std::forward(adapter), std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas).parse(true, result); return result; } @@ -3119,8 +3129,17 @@ public: const bool ignore_comments, const bool ignore_trailing_commas) { + // JSON in one block of memory, with the default options, is checked by + // the faster contiguous_parser (see parse_impl()) + using adapter_t = typename std::decay::type; + bool accepted = false; + if (!ignore_comments && !ignore_trailing_commas + && detail::accept_contiguous(adapter, accepted, std::integral_constant::value> {})) + { + return accepted; + } // the parser and lexer that discard the values of tokens (see lexer) - return ::nlohmann::detail::parser::type, true>(std::forward(adapter), + return ::nlohmann::detail::parser(std::forward(adapter), nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); } @@ -3135,6 +3154,15 @@ public: { if (format == input_format_t::json) { + // JSON in one block of memory, with the default options, is read by + // the faster contiguous_parser; on an error, the default parser + // reads it again to report it (see parse_impl()) + bool result = false; + if (strict && !ignore_comments && !ignore_trailing_commas + && detail::sax_parse_contiguous(ia, sax, result, std::integral_constant::value> {})) + { + return result; + } return parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict); } return detail::binary_reader(std::move(ia), format).sax_parse(sax, strict, tag_handler); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 1ed74f706..3ef7ac4dd 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -11432,6 +11432,30 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t accumulate_digits(const char* data, std::s return i; } +/// the index of the first byte at or after @a i that is not a digit: the +/// digits of accumulate_digits() without their value +JSON_HEDLEY_ALWAYS_INLINE std::size_t skip_digits(const char* data, std::size_t i, std::size_t avail) noexcept +{ + while (avail - i >= 8) + { + if (!is_eight_digits(read_eight_bytes(data + i))) + { + // a byte of the word is not a digit: no bounds checks before it + while (static_cast(data[i] - '0') <= 9) + { + ++i; + } + return i; + } + i += 8; + } + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + ++i; + } + return i; +} + /// whether [first, last) contains a digit other than '0'; a plain loop, as /// GCC calls std::any_of out of line even for the usual empty range JSON_HEDLEY_ALWAYS_INLINE bool has_nonzero_digit(const char* first, const char* last) noexcept @@ -13150,7 +13174,10 @@ JSON_HEDLEY_NEVER_INLINE inline std::size_t string_bulk_run_from(const unsigned } #endif -// string_bulk_run() for the lexer. With NEON, only the first four words are +// string_bulk_run() for the lexer. The run must be the longest one: the lexer +// would only lose speed with a shorter run, but contiguous_parser rejects a +// string whose run ends at a byte other than a quote or a backslash, so a +// shorter run would reject valid input there. With NEON, only the first four words are // checked inline (most keys and short values end there), and longer runs and // non-ASCII text continue out of line. The lexer's scan function runs once per // token, and the vector code inlined into it needs more registers, which every @@ -23421,12 +23448,37 @@ class binary_reader } // namespace detail NLOHMANN_JSON_NAMESPACE_END +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // isfinite +#include // size_t, ptrdiff_t +#include // uint64_t, int64_t +#include // memcmp +#include // numeric_limits +#include // string +#include // integral_constant, true_type, false_type +#include // move +#include // vector + +// #include + // #include // #include // #include +// #include + // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -24047,6 +24099,1025 @@ class parser } // namespace detail NLOHMANN_JSON_NAMESPACE_END +// #include + +// #include + +// #include + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief a parser for JSON text in one contiguous block, for parse(), accept(), + and sax_parse() + +The default parser and lexer exchange one token at a time, and the lexer keeps +its state (the current character, a pending unget, where the token starts) in +members, which every character pays for. This parser reads the block through a +local pointer instead, and the grammar, the whitespace and the tokens are one +loop, after yyjson's reader: the state stays in registers, and the expected +next token is tested where it is expected instead of being dispatched on. + +It reports what it reads to a SAX handler: the one parse() uses to build the +value, the one accept() uses, or the user's handler of sax_parse(). On a byte +the default parser would reject, it stops before reporting the token, and +returns false: the events up to there are those the default parser reports. +The caller then runs the default parser on the same input to report the error +with its exact message and position (report_contiguous_error(), +json_sax_error_forwarder), and parse() builds no value from what was read. + +What is accepted, and the values reported, are those of the default parser +with default options: the same number grammar and conversions, the same UTF-8 +and escape rules, a UTF-8 byte order mark at the start, and, without +JSON_STRICT_NUL_HANDLING, a NUL byte as the end of the input. + +@tparam KeepValues whether the handler needs the values of strings and + numbers (false for accept(), whose handler ignores them) +*/ +template +class contiguous_parser +{ + using string_t = typename BasicJsonType::string_t; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using char_t = typename string_t::value_type; + using uchar = unsigned char; + + public: + contiguous_parser(const uchar* first, const uchar* last, SAX* sax_) noexcept + : begin(first), end(last), sax(sax_) + {} + + contiguous_parser(const contiguous_parser&) = delete; + contiguous_parser(contiguous_parser&&) = delete; + contiguous_parser& operator=(const contiguous_parser&) = delete; + contiguous_parser& operator=(contiguous_parser&&) = delete; + ~contiguous_parser() = default; + + /// whether the whole block is one valid JSON text; the handler has seen all of it + JSON_HEDLEY_NEVER_INLINE bool parse() + { + const uchar* p = begin; + const uchar* const e = end; + + if (p != e && *p == 0xEF) + { + // a byte order mark, or the lexer reports a bad one + if (e - p < 3 || p[1] != 0xBB || p[2] != 0xBF) + { + return false; + } + p += 3; + } + + // The open arrays and objects: bit 0 of `kinds` is the innermost (1: + // object), so that the kind of the enclosing container is a shift away + // and stays in a register; every 64 levels, the word is pushed to + // `outer` and a new one is begun. + std::uint64_t kinds = 0; + std::size_t depth = 0; + std::vector outer; + + // the current byte, or 0 at the end (no JSON token starts with 0) +#define JSON_CONTIGUOUS_CUR() current(p, e) + // skip whitespace; most tokens have none before them +#define JSON_CONTIGUOUS_WS() \ + do \ + { \ + if (JSON_HEDLEY_UNLIKELY(JSON_CONTIGUOUS_CUR() <= ' ')) \ + { \ + p = skip_whitespace(p, e); \ + } \ + } \ + while (false) +#define JSON_CONTIGUOUS_OPEN(is_object) \ + do \ + { \ + if (JSON_HEDLEY_UNLIKELY(depth != 0 && (depth % 64) == 0)) /* NOLINT(readability/braces) */ \ + { \ + outer.push_back(kinds); \ + kinds = 0; \ + } \ + kinds = (kinds << 1u) | static_cast(is_object); \ + ++depth; \ + } \ + while (false) + + JSON_CONTIGUOUS_WS(); + switch (JSON_CONTIGUOUS_CUR()) + { + case '{': + ++p; + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_object(detail::unknown_size())))) + { + return false; + } + JSON_CONTIGUOUS_OPEN(1u); + goto object_first; + case '[': + ++p; + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_array(detail::unknown_size())))) + { + return false; + } + JSON_CONTIGUOUS_OPEN(0u); + goto array_first; + default: + if (JSON_HEDLEY_UNLIKELY(!scalar(p))) + { + return false; + } + goto done; + } + + // a value in an array or an object: expanded once for each, so that + // each dispatch has its own branch history (arrays tend to hold values + // of one kind) and the code after a scalar knows where it is +#define JSON_CONTIGUOUS_VALUE \ + switch (JSON_CONTIGUOUS_CUR()) \ + { \ + case '"': \ + if (JSON_HEDLEY_UNLIKELY(!string(p, false))) \ + { \ + return false; \ + } \ + break; \ + case '{': \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_object(detail::unknown_size())))) \ + { \ + return false; \ + } \ + JSON_CONTIGUOUS_OPEN(1u); \ + goto object_first; \ + case '[': \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_array(detail::unknown_size())))) \ + { \ + return false; \ + } \ + JSON_CONTIGUOUS_OPEN(0u); \ + goto array_first; \ + case '-': \ + case '0': \ + case '1': \ + case '2': \ + case '3': \ + case '4': \ + case '5': \ + case '6': \ + case '7': \ + case '8': \ + case '9': \ + if (JSON_HEDLEY_UNLIKELY(!number(p))) \ + { \ + return false; \ + } \ + break; \ + case 't': \ + if (JSON_HEDLEY_UNLIKELY(!literal(p, "true") || !report(sax->boolean(true)))) \ + { \ + return false; \ + } \ + break; \ + case 'f': \ + if (JSON_HEDLEY_UNLIKELY(!literal(p, "false") || !report(sax->boolean(false)))) \ + { \ + return false; \ + } \ + break; \ + case 'n': \ + if (JSON_HEDLEY_UNLIKELY(!literal(p, "null") || !report(sax->null()))) \ + { \ + return false; \ + } \ + break; \ + default: \ + return false; \ + } + + // what may follow a value in an array: a comma and the next value, or the end +#define JSON_CONTIGUOUS_AFTER_ELEMENT \ + JSON_CONTIGUOUS_WS(); \ + if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ',')) /* NOLINT(readability/braces) */ \ + { \ + ++p; \ + JSON_CONTIGUOUS_WS(); \ + goto array_value; \ + } \ + if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ']')) /* NOLINT(readability/braces) */ \ + { \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_array()))) \ + { \ + return false; \ + } \ + goto close; \ + } \ + return false; + + // what may follow a value in an object: a comma and the next member, or the end +#define JSON_CONTIGUOUS_AFTER_MEMBER \ + JSON_CONTIGUOUS_WS(); \ + if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ',')) /* NOLINT(readability/braces) */ \ + { \ + ++p; \ + JSON_CONTIGUOUS_WS(); \ + goto object_key; \ + } \ + if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == '}')) /* NOLINT(readability/braces) */ \ + { \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_object()))) \ + { \ + return false; \ + } \ + goto close; \ + } \ + return false; + +array_first: + JSON_CONTIGUOUS_WS(); + if (JSON_CONTIGUOUS_CUR() == ']') + { + ++p; + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_array()))) + { + return false; + } + goto close; + } +array_value: + JSON_CONTIGUOUS_VALUE + JSON_CONTIGUOUS_AFTER_ELEMENT + +object_first: + JSON_CONTIGUOUS_WS(); + if (JSON_CONTIGUOUS_CUR() == '}') + { + ++p; + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_object()))) + { + return false; + } + goto close; + } +object_key: + if (JSON_HEDLEY_UNLIKELY(JSON_CONTIGUOUS_CUR() != '"' || !string(p, true))) + { + return false; + } + if (JSON_HEDLEY_LIKELY(e - p >= 2 && p[0] == ':' && p[1] == ' ')) + { + p += 2; // ": " (pretty-printed input; a fast path of yyjson) + } + else + { + JSON_CONTIGUOUS_WS(); + if (JSON_HEDLEY_UNLIKELY(JSON_CONTIGUOUS_CUR() != ':')) + { + return false; + } + ++p; + } + JSON_CONTIGUOUS_WS(); + JSON_CONTIGUOUS_VALUE + JSON_CONTIGUOUS_AFTER_MEMBER + +close: + // an array or object was closed: continue in the enclosing one + --depth; + kinds >>= 1u; + if (JSON_HEDLEY_UNLIKELY(depth == 0)) + { + goto done; + } + if (JSON_HEDLEY_UNLIKELY((depth % 64) == 0)) + { + kinds = outer.back(); + outer.pop_back(); + } + if ((kinds & 1u) != 0) + { + JSON_CONTIGUOUS_AFTER_MEMBER + } + JSON_CONTIGUOUS_AFTER_ELEMENT + +done: + JSON_CONTIGUOUS_WS(); +#if JSON_STRICT_NUL_HANDLING + return p == e; +#else + // a NUL byte ends the input, as in the lexer + return p == e || *p == 0; +#endif + +#undef JSON_CONTIGUOUS_AFTER_MEMBER +#undef JSON_CONTIGUOUS_AFTER_ELEMENT +#undef JSON_CONTIGUOUS_VALUE +#undef JSON_CONTIGUOUS_OPEN +#undef JSON_CONTIGUOUS_WS +#undef JSON_CONTIGUOUS_CUR + } + + /// whether the handler stopped the parse (by returning false), as opposed + /// to a syntax error + bool aborted() const noexcept + { + return handler_stopped; + } + + private: + /// the result @a r of an event of the handler: false stops the parse + bool report(const bool r) noexcept + { + handler_stopped = handler_stopped || !r; + return r; + } + + /// the text of the float token [@a first, @a last), for a handler that reads it + const string_t& number_text(const char* first, const char* last) + { + return number_text(first, last, std::integral_constant::value> {}); + } + + const string_t& number_text(const char* /*first*/, const char* /*last*/, std::true_type /*ignored*/) const noexcept + { + return no_text; + } + + const string_t& number_text(const char* first, const char* last, std::false_type /*ignored*/) + { + float_text.clear(); + float_text.append(reinterpret_cast(first), static_cast(last - first)); + return float_text; + } + + /// KeepValues as a runtime condition: testing the template argument + /// directly makes `if` conditions constant, which MSVC reports as C4127 + static bool keep_values() noexcept + { + return KeepValues; + } + + /// the byte at @a p, or 0 at the end @a e (no JSON token starts with 0) + static JSON_HEDLEY_ALWAYS_INLINE uchar current(const uchar* p, const uchar* e) noexcept + { + return p != e ? *p : 0; + } + + /// the first byte at or after p that is not whitespace, or e + static JSON_HEDLEY_NEVER_INLINE const uchar* skip_whitespace(const uchar* p, const uchar* e) noexcept + { + while (p != e) + { + const uchar c = *p; + if (c == ' ' || c == '\t' || c == '\r') + { + ++p; + } + else if (c == '\n') + { + // the indentation of the next line, 8 spaces at a time + ++p; + while (e - p >= 8) + { + const std::uint64_t non_space = read_eight_bytes(p) ^ 0x2020202020202020ull; + if (non_space != 0) + { + p += static_cast(count_trailing_zeros(non_space)) / 8; + break; + } + p += 8; + } + } + else + { + break; + } + } + return p; + } + + /// the literal @a text at p, whose first byte the dispatch has seen + template + JSON_HEDLEY_ALWAYS_INLINE bool literal(const uchar*& p, const char (&text)[N]) const noexcept // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays): a string literal + { + constexpr std::size_t length = N - 1; + if (JSON_HEDLEY_UNLIKELY(static_cast(end - p) < length || std::memcmp(p, text, length) != 0)) + { + return false; + } + p += length; + return true; + } + + /// a JSON text that is a single scalar + bool scalar(const uchar*& p) + { + switch (current(p, end)) + { + case '"': + return string(p, false); + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return number(p); + case 't': + return literal(p, "true") && report(sax->boolean(true)); + case 'f': + return literal(p, "false") && report(sax->boolean(false)); + case 'n': + return literal(p, "null") && report(sax->null()); + default: + return false; + } + } + + /// the string whose opening quote is at p, reported as a key or a value; + /// p is moved past the closing quote + JSON_HEDLEY_ALWAYS_INLINE bool string(const uchar*& p, const bool is_key) + { + const uchar* const s = ++p; + // the text up to the first quote, backslash, control character or + // ill-formed UTF-8 (well-formed UTF-8 is part of the run) + p = s + lexer_string_bulk_run(s, static_cast(end - s)); + if (JSON_HEDLEY_LIKELY(p != end && *p == '"')) + { + if (keep_values()) + { + buffer.clear(); + buffer.append(reinterpret_cast(s), static_cast(p - s)); + } + ++p; + } + else if (JSON_HEDLEY_UNLIKELY(!escaped_string(s, p))) + { + return false; + } + return is_key ? report(sax->key(buffer)) : report(sax->string(buffer)); + } + + /// the rest of a string from p, a backslash, a control character, + /// ill-formed UTF-8 or the end; s is where the string starts. With + /// KeepValues, the string is decoded into buffer. + JSON_HEDLEY_NEVER_INLINE bool escaped_string(const uchar* s, const uchar*& pos) + { + const uchar* p = pos; + if (keep_values()) + { + buffer.clear(); + buffer.append(reinterpret_cast(s), static_cast(p - s)); + } + for (;;) + { + if (JSON_HEDLEY_UNLIKELY(p == end)) + { + return false; + } + if (*p == '"') + { + pos = p + 1; + return true; + } + if (JSON_HEDLEY_UNLIKELY(*p != '\\' || end - p < 2)) + { + return false; // a control character or ill-formed UTF-8 + } + const uchar escape = p[1]; + p += 2; + switch (escape) + { + case '"': + case '\\': + case '/': + add(static_cast(escape)); + break; + case 'b': + add('\b'); + break; + case 'f': + add('\f'); + break; + case 'n': + add('\n'); + break; + case 'r': + add('\r'); + break; + case 't': + add('\t'); + break; + case 'u': + { + if (JSON_HEDLEY_UNLIKELY(end - p < 4)) + { + return false; + } + int codepoint = hex_codepoint(p); + if (JSON_HEDLEY_UNLIKELY(codepoint < 0)) + { + return false; + } + p += 4; + if (JSON_HEDLEY_UNLIKELY((codepoint & 0xF800) == 0xD800)) + { + // a high surrogate must be followed by \u and a low surrogate + if (codepoint >= 0xDC00 || end - p < 6 || p[0] != '\\' || p[1] != 'u') + { + return false; + } + const int low = hex_codepoint(p + 2); + if (low < 0xDC00 || low > 0xDFFF) + { + return false; + } + p += 6; + codepoint = 0x10000 + ((codepoint - 0xD800) << 10) + (low - 0xDC00); + } + add_utf8(codepoint); + break; + } + default: + return false; + } + const uchar* const run = p; + p = run + lexer_string_bulk_run(run, static_cast(end - run)); + if (keep_values() && p != run) + { + buffer.append(reinterpret_cast(run), static_cast(p - run)); + } + } + } + + void add(const char_t c) + { + if (keep_values()) + { + buffer.push_back(c); + } + } + + /// add the UTF-8 encoding of @a codepoint (at most U+10FFFF) to buffer + void add_utf8(const int codepoint) + { + if (!keep_values()) + { + return; + } + const auto cp = static_cast(codepoint); + if (cp < 0x80) + { + buffer.push_back(static_cast(cp)); + } + else if (cp < 0x800) + { + buffer.push_back(static_cast(0xC0u | (cp >> 6u))); + buffer.push_back(static_cast(0x80u | (cp & 0x3Fu))); + } + else if (cp < 0x10000) + { + buffer.push_back(static_cast(0xE0u | (cp >> 12u))); + buffer.push_back(static_cast(0x80u | ((cp >> 6u) & 0x3Fu))); + buffer.push_back(static_cast(0x80u | (cp & 0x3Fu))); + } + else + { + buffer.push_back(static_cast(0xF0u | (cp >> 18u))); + buffer.push_back(static_cast(0x80u | ((cp >> 12u) & 0x3Fu))); + buffer.push_back(static_cast(0x80u | ((cp >> 6u) & 0x3Fu))); + buffer.push_back(static_cast(0x80u | (cp & 0x3Fu))); + } + } + + /*! + @brief the number at p, with the grammar and the conversions of the lexer's + contiguous path (lexer::scan_number_bulk_contiguous()) + + Integers that do not fit their type become floats; floats that overflow + are an error, which the default parser reports. + */ + JSON_HEDLEY_ALWAYS_INLINE bool number(const uchar*& pos) + { + const auto* const data = reinterpret_cast(pos); + const auto avail = static_cast(end - pos); + std::size_t i = 0; + std::size_t dot_index = std::string::npos; + bool is_float = false; + std::uint64_t w = 0; + const bool negative = data[0] == '-'; + if (negative) + { + i = 1; + } + const std::size_t integer_begin = i; + if (i < avail && data[i] == '0') + { + ++i; + } + else if (i < avail && data[i] >= '1' && data[i] <= '9') + { + // accept() needs the value only for a float with a large exponent, + // which computes it below; its handler ignores the values + i = keep_values() ? accumulate_digits(data, i, avail, w) : skip_digits(data, i, avail); + } + else + { + return false; + } + std::size_t digit_count = i - integer_begin; + std::size_t fraction_digits = 0; + if (i < avail && data[i] == '.') + { + is_float = true; + dot_index = i; + ++i; + if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) + { + return false; + } + const std::size_t fraction_begin = i; + i = keep_values() ? accumulate_digits(data, i, avail, w) : skip_digits(data, i, avail); + fraction_digits = i - fraction_begin; + digit_count += fraction_digits; + } + const std::size_t mantissa_end = i; + std::int64_t exponent = 0; + if (i < avail && (data[i] == 'e' || data[i] == 'E')) + { + is_float = true; + ++i; + bool negative_exponent = false; + if (i < avail && (data[i] == '+' || data[i] == '-')) + { + negative_exponent = data[i] == '-'; + ++i; + } + if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) + { + return false; + } + // saturated far beyond every range, see parse_float_exponent() + constexpr std::int64_t saturation = 100000000000000000; // 10^17 + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + if (exponent < saturation) + { + exponent = (exponent * 10) + (data[i] - '0'); + } + ++i; + } + exponent = negative_exponent ? -exponent : exponent; + } + pos += i; + + if (!is_float) + { + if (digit_count <= 18) + { + // up to 18 digits always fit into std::uint64_t and std::int64_t; + // a narrower type takes the value only if it round-trips + if (negative) + { + const std::int64_t x = -static_cast(w); + const auto value = static_cast(x); + if (static_cast(value) == x) + { + return report(sax->number_integer(value)); + } + } + else + { + const auto value = static_cast(w); + if (static_cast(value) == w) + { + return report(sax->number_unsigned(value)); + } + } + } + else if (negative) + { + number_integer_t value{}; + if (parse_integer_signed(data, data + i, value)) + { + return report(sax->number_integer(value)); + } + } + else + { + number_unsigned_t value{}; + if (parse_integer_unsigned(data, data + i, value)) + { + return report(sax->number_unsigned(value)); + } + } + } + + number_float_t value{}; + if (digit_count <= 19) + { + float_significand s; + s.w = w; + s.exponent = exponent - static_cast(fraction_digits); + s.negative = negative; + // accept() only needs to know that the value is finite: w < 10^19, + // so only a token that may reach the largest float is converted + if (!keep_values() && s.exponent + 19 <= std::numeric_limits::max_exponent10) + { + return report(sax->number_float(value, number_text(data, data + i))); + } + if (!keep_values()) + { + // the digits were skipped: their value now + s.w = 0; + const std::size_t integer_end = accumulate_digits(data, integer_begin, avail, s.w); + if (dot_index != std::string::npos) + { + static_cast(accumulate_digits(data, integer_end + 1, avail, s.w)); + } + } + value = decimal_to_float(s); + } + else + { + static_cast(convert_float_fast(data, data + i, dot_index, mantissa_end, value)); + } + if (JSON_HEDLEY_UNLIKELY(!std::isfinite(value))) + { + return false; // the default parser reports the overflow + } + return report(sax->number_float(value, number_text(data, data + i))); + } + + const uchar* const begin; + const uchar* const end; + SAX* const sax; + /// the current string (with KeepValues); the handler may move from it + string_t buffer {}; + /// the text of float tokens, for the library's handlers, which ignore it + const string_t no_text {}; + /// the text of the current float token, for other handlers + string_t float_text {}; + /// whether the handler returned false + bool handler_stopped = false; +}; + +/// whether parse() and accept() can read the input behind @a InputAdapterType +/// with contiguous_parser: one contiguous block of bytes (not a window into a +/// stream), and floats in a format the library converts itself +template +struct can_parse_contiguous +{ + static constexpr bool value = +#if JSON_DIAGNOSTIC_POSITIONS + // the values record the positions the lexer reports + false && +#endif + input_adapter_supports_bulk_scan(is_detected {}) + && !input_adapter_bulk_window(is_detected {}) + && has_native_float_format::value; +}; + +/*! +@brief report the input behind @a adapter, one block of memory, to @a sax + +@return whether the input is valid and was reported completely; if not, + nothing was consumed from @a adapter, and the caller has the default + parser report the error +*/ +template +bool parse_contiguous(InputAdapterType& adapter, SAX& sax, std::true_type /*contiguous*/) +{ + const std::size_t size = adapter.bulk_remaining(); + if (size == 0) + { + return false; // the default parser reports the empty input + } + const auto* const first = reinterpret_cast(adapter.bulk_data()); + contiguous_parser p(first, first + size, &sax); + return p.parse(); +} + +/*! +@brief a SAX handler that only reports a parse error, as the handler of parse() + does: it throws the exception, unless exceptions are not allowed + +After contiguous_parser has rejected an input, the default parser is run again +with this handler to report the error: the error, its message and its position +are those of parse(), without building the values read up to it. +*/ +template +class json_sax_error_reporter : public json_sax_acceptor +{ + public: + explicit json_sax_error_reporter(const bool allow_exceptions_) noexcept + : allow_exceptions(allow_exceptions_) + {} + + /// (a template, as in json_sax_dom_parser: the parser passes the concrete + /// exception type, which is thrown as is) + template + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const Exception& ex) + { + static_cast(ex); // (unused without exceptions) + if (allow_exceptions) + { + JSON_THROW(ex); + } + return false; + } + + private: + const bool allow_exceptions; +}; + +template +struct sax_ignores_number_text> : std::true_type {}; + +/*! +@brief report the error in the input behind @a adapter, which contiguous_parser + rejected, as parse() reports it: throw its exception, or, without + exceptions, return true + +@return false if the default parser accepts the input after all (which would + be a disagreement of the two parsers): the caller then parses it +*/ +template +bool report_contiguous_error(const InputAdapterType& adapter, const bool allow_exceptions) +{ + json_sax_error_reporter reporter(allow_exceptions); + // (a copy of the adapter, which the caller may need again) + // (the lexer of parse(), which the fallback in basic_json::parse_impl() + // instantiates anyway: only the grammar loop is instantiated again) + return !parser(InputAdapterType(adapter), nullptr, allow_exceptions, false, false).sax_parse(&reporter, true); +} + +/*! +@brief parse(): parse the input behind @a adapter into @a result, if it is one + block of memory + +@return whether @a result is the result of parse(): the value, or, after an + error and without exceptions, a discarded value (with exceptions, the + error is thrown); false if the input is not one block of memory, in + which case the caller parses it +*/ +template +bool parse_contiguous(InputAdapterType& adapter, BasicJsonType& result, const bool allow_exceptions, std::true_type /*contiguous*/) +{ + { + // the handler of parser::parse(), so that the values are built by the same code + json_sax_dom_parser sax(result, true); + if (parse_contiguous(adapter, sax, std::true_type {})) + { + return true; + } + } + // (what was read up to the error) + result = BasicJsonType(); + if (report_contiguous_error(adapter, allow_exceptions)) + { + result = BasicJsonType(value_t::discarded); + return true; + } + return false; // LCOV_EXCL_LINE (only if the two parsers disagree: a bug) +} + +template +bool parse_contiguous(InputAdapterType& /*adapter*/, BasicJsonType& /*result*/, const bool /*allow_exceptions*/, std::false_type /*contiguous*/) noexcept +{ + return false; +} + +/*! +@brief accept(): check the input behind @a adapter, if it is one block of memory + +@return whether @a accepted is the result of accept(); false if the input is + not one block of memory, in which case the caller checks it. + contiguous_parser accepts exactly what the default parser accepts. +*/ +template +bool accept_contiguous(InputAdapterType& adapter, bool& accepted, std::true_type /*contiguous*/) +{ + json_sax_acceptor sax; + accepted = parse_contiguous(adapter, sax, std::true_type {}); + return true; +} + +template +bool accept_contiguous(InputAdapterType& /*adapter*/, bool& /*accepted*/, std::false_type /*contiguous*/) noexcept +{ + return false; +} + +/*! +@brief a SAX handler that ignores the values and passes a parse error on to the + handler of the user + +After contiguous_parser has delivered the events of the input to the user's +handler and rejected it, the default parser is run again with this handler: +the user's handler gets the error, its message and its position from the +default parser, as before, but no event twice. +*/ +template +class json_sax_error_forwarder : public json_sax_acceptor +{ + public: + explicit json_sax_error_forwarder(SAX* user_) noexcept + : user(user_) + {} + + template + bool parse_error(std::size_t position, const std::string& last_token, const Exception& ex) + { + forwarded = true; + return user->parse_error(position, last_token, ex); + } + + /// whether the default parser reported an error + bool reported() const noexcept + { + return forwarded; + } + + private: + SAX* const user; + bool forwarded = false; +}; + +template +struct sax_ignores_number_text> : std::true_type {}; + +/*! +@brief sax_parse() of JSON in one block of memory with the strict option and + a user's handler @a sax + +contiguous_parser reports to @a sax what it reads, and stops at the token that +the default parser would reject, before reporting it: the events up to an error +are the same. The error itself is then reported by the default parser (see +json_sax_error_forwarder). + +@return whether @a result is the result of sax_parse(); false if the input is + not one block of memory, in which case the caller parses it +*/ +template +bool sax_parse_contiguous(InputAdapterType& adapter, SAX* sax, bool& result, std::true_type /*contiguous*/) +{ + const std::size_t size = adapter.bulk_remaining(); + if (size != 0) + { + const auto* const first = reinterpret_cast(adapter.bulk_data()); + contiguous_parser p(first, first + size, sax); + if (p.parse()) + { + result = true; + return true; + } + if (p.aborted()) + { + // the handler returned false, as with the default parser + result = false; + return true; + } + } + json_sax_error_forwarder forwarder(sax); + result = parser(InputAdapterType(adapter), nullptr, true, false, false).sax_parse(&forwarder, true); + // the default parser must reject the input as well (the events after the + // byte contiguous_parser stopped at cannot be delivered any more) + JSON_ASSERT(forwarder.reported()); + return true; +} + +template +bool sax_parse_contiguous(InputAdapterType& /*adapter*/, SAX* /*sax*/, bool& /*result*/, std::false_type /*contiguous*/) noexcept +{ + return false; +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include + +// #include + +// #include + +// #include + // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -41124,6 +42195,15 @@ public: const bool ignore_trailing_commas) { basic_json result; + // JSON in one block of memory, with the default options, is read by + // the faster contiguous_parser; on an error, the default parser reads + // it again to report it (see contiguous_parser) + using contiguous = std::integral_constant::type>::value>; + if (cb == nullptr && !ignore_comments && !ignore_trailing_commas + && detail::parse_contiguous(adapter, result, allow_exceptions, contiguous {})) + { + return result; + } parser(std::forward(adapter), std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas).parse(true, result); return result; } @@ -41134,8 +42214,17 @@ public: const bool ignore_comments, const bool ignore_trailing_commas) { + // JSON in one block of memory, with the default options, is checked by + // the faster contiguous_parser (see parse_impl()) + using adapter_t = typename std::decay::type; + bool accepted = false; + if (!ignore_comments && !ignore_trailing_commas + && detail::accept_contiguous(adapter, accepted, std::integral_constant::value> {})) + { + return accepted; + } // the parser and lexer that discard the values of tokens (see lexer) - return ::nlohmann::detail::parser::type, true>(std::forward(adapter), + return ::nlohmann::detail::parser(std::forward(adapter), nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); } @@ -41150,6 +42239,15 @@ public: { if (format == input_format_t::json) { + // JSON in one block of memory, with the default options, is read by + // the faster contiguous_parser; on an error, the default parser + // reads it again to report it (see parse_impl()) + bool result = false; + if (strict && !ignore_comments && !ignore_trailing_commas + && detail::sax_parse_contiguous(ia, sax, result, std::integral_constant::value> {})) + { + return result; + } return parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict); } return detail::binary_reader(std::move(ia), format).sax_parse(sax, strict, tag_handler); diff --git a/tests/src/unit-contiguous_parser.cpp b/tests/src/unit-contiguous_parser.cpp new file mode 100644 index 000000000..4c3ec39b4 --- /dev/null +++ b/tests/src/unit-contiguous_parser.cpp @@ -0,0 +1,564 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// parse() and accept() read text in one block of memory with +// detail::contiguous_parser, and everything else (here: std::list iterators) +// with the lexer's byte path. These tests check that the two agree on every +// input: the same value, with the same number types and float bits, or the +// same exception with the same message. + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; +using nlohmann::ordered_json; + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + +/// the value with its number types and float bits; below 100 levels, the +/// library's dump() (whose recursion is bounded) describes the rest +template +std::string describe_value(const Json& j, int depth = 0) +{ + std::string out; + if (depth >= 100) + { + out = j.dump(); + return out; + } + switch (j.type()) + { + case Json::value_t::number_float: + { + const double d = j.template get(); + std::uint64_t bits = 0; + std::memcpy(&bits, &d, sizeof(bits)); + out += "f" + std::to_string(bits); + break; + } + case Json::value_t::number_integer: + out += "i" + j.dump(); + break; + case Json::value_t::number_unsigned: + out += "u" + j.dump(); + break; + case Json::value_t::array: + out += "["; + for (const auto& e : j) + { + out += describe_value(e, depth + 1) + ","; + } + out += "]"; + break; + case Json::value_t::object: + out += "{"; + for (auto it = j.begin(); it != j.end(); ++it) + { + out += Json(it.key()).dump() + ":" + describe_value(it.value(), depth + 1) + ","; + } + out += "}"; + break; + case Json::value_t::null: + case Json::value_t::boolean: + case Json::value_t::string: + case Json::value_t::binary: + case Json::value_t::discarded: + default: + out += j.dump(); + break; + } + return out; +} + +#if !defined(JSON_NOEXCEPTION) +template +std::string outcome(Parse parse) +{ + try + { + return describe_value(parse()); + } + catch (const typename Json::exception& e) + { + return std::string("exception: ") + e.what(); + } +} +#endif + +/// every SAX event with its values, the text of float tokens, and the parse +/// error with its position, token and message; returns false after +/// @a stop_after events (to stop the parse) +class event_log : public nlohmann::json_sax +{ + public: + explicit event_log(std::size_t stop_after_ = static_cast(-1), bool continue_after_error_ = false) + : stop_after(stop_after_), continue_after_error(continue_after_error_) + {} + + bool null() override + { + return add("null"); + } + bool boolean(bool val) override + { + return add(val ? "true" : "false"); + } + bool number_integer(number_integer_t val) override + { + return add("i" + std::to_string(val)); + } + bool number_unsigned(number_unsigned_t val) override + { + return add("u" + std::to_string(val)); + } + bool number_float(number_float_t val, const string_t& s) override + { + std::uint64_t bits = 0; + std::memcpy(&bits, &val, sizeof(bits)); + return add("f" + std::to_string(bits) + " " + s); + } + bool string(string_t& val) override + { + return add("s" + val); + } + bool binary(binary_t& /*val*/) override + { + return add("binary"); + } + bool start_object(std::size_t elements) override + { + return add("{" + std::to_string(elements)); + } + bool key(string_t& val) override + { + return add("k" + val); + } + bool end_object() override + { + return add("}"); + } + bool start_array(std::size_t elements) override + { + return add("[" + std::to_string(elements)); + } + bool end_array() override + { + return add("]"); + } + bool parse_error(std::size_t position, const std::string& last_token, const nlohmann::detail::exception& ex) override + { + events.push_back("error " + std::to_string(position) + " " + last_token + " " + ex.what()); + return continue_after_error; + } + + const std::vector& log() const noexcept + { + return events; + } + + private: + bool add(const std::string& event) + { + events.push_back(event); + return events.size() != stop_after; + } + + std::vector events {}; // NOLINT(readability-redundant-member-init) + std::size_t stop_after; + bool continue_after_error; +}; + +/// sax_parse() of the contiguous path against the byte path, also with a +/// handler that stops the parse +void check_same_events(const std::string& text, const std::list& bytes) +{ + for (const std::size_t stop_after : + { + static_cast(-1), std::size_t{1}, std::size_t{3} + }) + { + CAPTURE(stop_after) + event_log contiguous(stop_after); + event_log byte_path(stop_after); + const bool r1 = json::sax_parse(text, &contiguous); + const bool r2 = json::sax_parse(bytes.begin(), bytes.end(), &byte_path); + CHECK(r1 == r2); + CHECK(contiguous.log() == byte_path.log()); + } + // a handler that asks to go on after an error (the default parser then + // reports what follows, e.g., the missing end of the input) + event_log contiguous(static_cast(-1), true); + event_log byte_path(static_cast(-1), true); + const bool r1 = json::sax_parse(text, &contiguous); + const bool r2 = json::sax_parse(bytes.begin(), bytes.end(), &byte_path); + CHECK(r1 == r2); + CHECK(contiguous.log() == byte_path.log()); +} + +/// the contiguous path (a std::string) against the byte path (std::list iterators) +template +void check_same(const std::string& text) +{ + CAPTURE(text) + const std::list bytes(text.begin(), text.end()); + +#if !defined(JSON_NOEXCEPTION) + // with exceptions: the value, or the same exception for both + const std::string contiguous = outcome([&] + { + return Json::parse(text); + }); + const std::string byte_path = outcome([&] + { + return Json::parse(bytes.begin(), bytes.end()); + }); + CHECK(contiguous == byte_path); +#endif + + // without exceptions: the value, or discarded for both + const Json c = Json::parse(text, nullptr, false); + const Json b = Json::parse(bytes.begin(), bytes.end(), nullptr, false); + CHECK(c.is_discarded() == b.is_discarded()); + + CHECK(Json::accept(text) == Json::accept(bytes.begin(), bytes.end())); + if (std::is_same::value) + { + check_same_events(text, bytes); + } + // a pointer range without a terminating NUL + std::vector exact(text.begin(), text.end()); + if (!exact.empty()) + { + CHECK(Json::accept(exact.data(), exact.data() + exact.size()) == Json::accept(bytes.begin(), bytes.end())); + } +} + +void check_both(const std::string& text) +{ + check_same(text); + check_same(text); +} + +/// a random JSON text, mostly valid, with the corner cases of the parser +class generator +{ + public: + explicit generator(std::uint32_t seed) : rng(seed) {} + + std::string document(int depth) + { + std::string out; + value(out, depth); + return out; + } + + private: + std::uint32_t pick(std::uint32_t n) + { + return static_cast(rng() % n); + } + + void whitespace(std::string& out) + { + static const char* const choices[] = {"", "", "", " ", "\n", "\n ", "\t", "\r\n ", "\n ", " "}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + out += choices[pick(10)]; + } + + void number(std::string& out) + { + static const char* const choices[] = // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + "0", "-0", "1", "-1", "42", "123456789012345678", "-123456789012345678", "1234567890123456789", + "-1234567890123456789", "9223372036854775807", "-9223372036854775808", "-9223372036854775809", + "18446744073709551615", "18446744073709551616", "123456789012345678901234567890", "0.0", "-0.0", + "1.5", "-65.613616999999977", "1e10", "1E+10", "1e-10", "2.2250738585072014e-308", "4.9e-324", + "1e-400", "1.7976931348623157e308", "1e308", "1e309", "-1e309", "123.456e-789", "0.1e1", + "12345678901234567890.5", "1.00000000000000011102230246251565404236316680908203125", + "9007199254740993", "0.30000000000000004", "1e22", "1e23", "100000000000000000000000", + "00", "01", "-", "1.", ".5", "1e", "1e+", "--1", "+1", "1.5e3.2", "0x10", "1_000" + }; + out += choices[pick(sizeof(choices) / sizeof(choices[0]))]; + } + + void string(std::string& out) + { + static const char* const pieces[] = // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + "a", "key", "a longer piece of plain ASCII text for the vector paths", "\\n", "\\t", "\\\"", + "\\\\", "\\/", "\\b", "\\f", "\\r", "\\u0000", "\\u00e4", "\\u20AC", "\\uD83D\\uDE00", + "\\uD83D", "\\uDE00", "\\uD83Dx", "\\u12", "\\x", "\xC3\xA4", "\xE2\x82\xAC", "\xF0\x9F\x98\x80", + "\xC0\xAF", "\xED\xA0\x80", "\xF4\x90\x80\x80", "\x80", "\xFF", "\x01", "\x1F", "\x7F", "\t" + }; + out += '"'; + const auto n = pick(6); + for (std::uint32_t i = 0; i < n; ++i) + { + // mostly well-formed pieces + out += pieces[pick(5) == 0 ? pick(sizeof(pieces) / sizeof(pieces[0])) : pick(20)]; + } + out += '"'; + } + + void value(std::string& out, int depth) + { + const auto kind = depth > 0 ? pick(9) : 3 + pick(6); + switch (kind) + { + case 0: + case 1: + { + out += '['; + whitespace(out); + const auto n = pick(6); + for (std::uint32_t i = 0; i < n; ++i) + { + if (i != 0) + { + out += ','; + whitespace(out); + } + value(out, depth - 1); + whitespace(out); + } + out += ']'; + break; + } + case 2: + { + out += '{'; + whitespace(out); + const auto n = pick(6); + for (std::uint32_t i = 0; i < n; ++i) + { + if (i != 0) + { + out += ','; + whitespace(out); + } + // a few duplicate keys + if (pick(4) == 0) + { + out += "\"dup\""; + } + else + { + string(out); + } + whitespace(out); + out += pick(2) == 0 ? ":" : ": "; + whitespace(out); + value(out, depth - 1); + whitespace(out); + } + out += '}'; + break; + } + case 3: + case 4: + number(out); + break; + case 5: + case 6: + string(out); + break; + case 7: + out += pick(2) == 0 ? "true" : "false"; + break; + default: + out += "null"; + break; + } + } + + std::mt19937 rng; +}; + +} // namespace + +TEST_CASE("contiguous parser") +{ + SECTION("fixed inputs") + { + const char* const inputs[] = // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + "", " ", "\n", "null", "true", "false", "nul", "tru", "fals", "nullx", "[]", "{}", "[", "{", "]", "}", + "[1,]", "[,1]", R"({"a":1,})", R"({"a" 1})", R"({"a":})", "{1:2}", "[1 2]", R"("")", R"("abc)", R"("\")", + "\xEF\xBB\xBF[1]", "\xEF\xBB[1]", "\xEF[1]", "\xEF\xBB\xBF", " [1] ", "[1]x", "[1] [2]", "1 2", + R"({"a":1,"a":2})", R"({"b":1,"a":2,"b":3})", R"(["\u0000"])", "[1e400]", "[-1e400]", + R"({"a": [1, 2, {"b": null}], "c": "d"})", "[true,false,null]", "[[[[[]]]]]", + }; + for (const char* input : inputs) + { + check_both(input); + } + + // NUL bytes: by default the end of the input (between values), else an error + check_both(std::string("[1]\0garbage", 11)); + check_both(std::string("[1,\0]", 5)); + check_both(std::string("\"a\0b\"", 5)); + check_both(std::string("\0", 1)); + check_both(std::string("1\0", 2)); + } + + SECTION("deep nesting") + { + // across the 64-level words of the nesting stack + const std::vector depths = {63, 64, 65, 127, 128, 129, 200, 1000, 10000}; + for (const std::size_t depth : depths) + { + check_both(std::string(depth, '[') + std::string(depth, ']')); + std::string objects; + for (std::size_t i = 0; i < depth; ++i) + { + objects += (i % 2 == 0) ? "{\"k\":" : "["; + } + objects += "1"; + for (std::size_t i = depth; i-- > 0;) + { + objects += (i % 2 == 0) ? "}" : "]"; + } + check_both(objects); + // unbalanced + check_both(std::string(depth, '[') + std::string(depth - 1, ']')); + check_both(std::string(depth, '[') + std::string(depth + 1, ']')); + } + // a value after a container that closes a 64-level word + std::string mixed = std::string(64, '[') + "1" + std::string(63, ']') + ",{\"a\":2}]"; + check_both(mixed); + mixed = "{\"x\":" + std::string(64, '[') + std::string(64, ']') + ",\"y\":[1]}"; + check_both(mixed); + } + + SECTION("generated documents and their mutations") + { + generator gen(20261105u); + std::mt19937 rng(42u); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible + const char alphabet[] = "{}[]:,\"\\ \t\n\r0123456789-+.eEtrufalsn/u\x80\xbf\xc3\xe2\xed\xf0\xff"; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + for (int round = 0; round < 1500; ++round) + { + const std::string text = gen.document(1 + (round % 5)); + check_both(text); + // every truncation of short documents, and a few mutations + if (text.size() < 60) + { + for (std::size_t n = 0; n < text.size(); ++n) + { + check_same(text.substr(0, n)); + } + } + for (int m = 0; m < 3 && !text.empty(); ++m) + { + std::string mutated = text; + const std::size_t pos = rng() % mutated.size(); + switch (rng() % 3) + { + case 0: + mutated[pos] = alphabet[rng() % (sizeof(alphabet) - 1)]; + break; + case 1: + mutated.insert(pos, 1, alphabet[rng() % (sizeof(alphabet) - 1)]); + break; + default: + mutated.erase(pos, 1); + break; + } + check_same(mutated); + } + } + } + + SECTION("narrow number types") + { + // integers that do not fit become floats, floats are converted to float + using narrow_json = nlohmann::basic_json; + const std::vector inputs = + { + "[2147483647, 2147483648, -2147483648, -2147483649, 4294967295, 4294967296, 18446744073709551615]", + "[1.5, 3.4028234663852886e38, 3.4028235677973366e38, 1e39, -1e39, 1e-46, 0.1, 16777217]", + "[-0, 0.0, 123456789012345678901234567890, -9223372036854775809]", "1e39", "4294967296", "-2147483649" + }; + for (const auto& input : inputs) + { + check_same(input); + } + generator gen(17u); + for (int round = 0; round < 200; ++round) + { + check_same(gen.document(1 + (round % 4))); + } + } + + SECTION("number corner cases at the end of the input") + { + // the number ends at the end of the block (no closing bracket after it) + for (const char* number : + {"0", "-0", "1", "12345678", "123456789", "1234567890123456789", "18446744073709551616", + "1.5", "1e5", "1e", "1.", "-", "1.5e-3", "123456789012345678901234567890e-10" + }) + { + check_both(number); + check_both(std::string("[") + number + "]"); + check_both(std::string("{\"n\":") + number + "}"); + } + } + + SECTION("accept() of floats near the largest value") + { + // accept() skips digits without their value and computes it only for + // a float whose exponent may reach the largest value + for (const char* number : + {"17976931348623158e292", "17976931348623159e292", "1797693134862315.8e293", "1797693134862315.9e293", + "0.17976931348623158e309", "0.17976931348623159e309", "-17976931348623159e292", "9999999999999999999e289" + }) + { + check_both(number); + check_both(std::string("[") + number + "]"); + } + CHECK(json::accept("17976931348623158e292")); + CHECK_FALSE(json::accept("17976931348623159e292")); + CHECK(json::accept("[1797693134862315.8e293]")); + CHECK_FALSE(json::accept("[1797693134862315.9e293]")); + CHECK(json::accept("0.17976931348623158e309")); + CHECK_FALSE(json::accept("0.17976931348623159e309")); + CHECK_FALSE(json::accept("-17976931348623159e292")); + + using narrow_json = nlohmann::basic_json; + check_same("34028235e31"); + check_same("34028236e31"); + CHECK(narrow_json::accept("34028235e31")); + CHECK_FALSE(narrow_json::accept("34028236e31")); + } + + SECTION("callbacks, comments and trailing commas take the default parser") + { + // (the results are those of the options; the contiguous parser does not support them) + CHECK(json::parse("[1,]", nullptr, true, false, true) == json({1})); + CHECK(json::parse("[1/*c*/]", nullptr, true, true) == json({1})); + CHECK(json::accept("[1,]", false, true)); + CHECK(json::accept("// c\n[1]", true)); + int calls = 0; + const json j = json::parse("[1,2]", [&calls](int /*depth*/, json::parse_event_t /*event*/, json& /*parsed*/) noexcept + { + ++calls; + return true; + }); + CHECK(j == json({1, 2})); + CHECK(calls == 4); + } +}