From 7dcd213e4c0b41372fa69764a62b6911f2659388 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 7 Nov 2026 08:33:18 +0000 Subject: [PATCH 1/8] Tests: mark parser callback in unit-contiguous_parser noexcept Fixes the -Werror=noexcept failure with GCC 26 (ci_test_standards_gcc (26)). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz Signed-off-by: Niels Lohmann --- tests/src/unit-contiguous_parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/unit-contiguous_parser.cpp b/tests/src/unit-contiguous_parser.cpp index 5579c0d1d..bcebe2047 100644 --- a/tests/src/unit-contiguous_parser.cpp +++ b/tests/src/unit-contiguous_parser.cpp @@ -553,7 +553,7 @@ TEST_CASE("contiguous parser") CHECK(json::accept("[1,]", false, true)); CHECK(json::accept("// c\n[1]", true)); int calls = 0; - const json j = json::parse("[1,2]", [&calls](int /*depth*/, json::parse_event_t /*event*/, json& /*parsed*/) + const json j = json::parse("[1,2]", [&calls](int /*depth*/, json::parse_event_t /*event*/, json& /*parsed*/) noexcept { ++calls; return true; From 4f012e9b14a70786ff7bb7ed718a3225e1529309 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 7 Nov 2026 08:33:18 +0000 Subject: [PATCH 2/8] Tests: sweep_allocator::construct() accepts pointers to const Recent libc++ (emsdk latest) probes construct(const T*, const T&) in an unevaluated context while checking for trivial copies; the static_cast to void* in the noexcept specification then fails to compile. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz (cherry picked from commit 794bf70823c12a2a1cb2a8a95f34dfb6357746a8) Signed-off-by: Niels Lohmann --- tests/src/unit-allocator.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/src/unit-allocator.cpp b/tests/src/unit-allocator.cpp index 8fa9034b5..ca215751e 100644 --- a/tests/src/unit-allocator.cpp +++ b/tests/src/unit-allocator.cpp @@ -1073,10 +1073,12 @@ struct sweep_allocator return (std::numeric_limits::max)() / sizeof(T); } + // p may point to a const object when libc++ probes whether construct() is + // usable for copying, which static_cast(p) would not accept template - void construct(U* p, Args&& ... args) noexcept(noexcept(::new (static_cast(p)) U(std::forward(args)...))) + void construct(U* p, Args&& ... args) noexcept(noexcept(::new (const_cast(static_cast(p))) U(std::forward(args)...))) { - ::new (static_cast(p)) U(std::forward(args)...); + ::new (const_cast(static_cast(p))) U(std::forward(args)...); } template From 8b70bf87679c647c8b5d0e706f3d3fd3ff077811 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 7 Nov 2026 08:33:18 +0000 Subject: [PATCH 3/8] Tests: guard sweep_allocator::construct() against const types instead of const_cast The const_cast added in the previous commit is rejected by cppcoreguidelines-pro-type-const-cast (clang-tidy). Make construct() not viable for const U instead, which is what libc++ probes for. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz (cherry picked from commit de099fa607001b710862fd1bbecef3609c54cbc4) Signed-off-by: Niels Lohmann --- tests/src/unit-allocator.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/src/unit-allocator.cpp b/tests/src/unit-allocator.cpp index ca215751e..3c719bf26 100644 --- a/tests/src/unit-allocator.cpp +++ b/tests/src/unit-allocator.cpp @@ -14,6 +14,7 @@ using nlohmann::json; #include #include +#include #include #if JSON_HAS_RANGES #include @@ -1073,12 +1074,12 @@ struct sweep_allocator return (std::numeric_limits::max)() / sizeof(T); } - // p may point to a const object when libc++ probes whether construct() is - // usable for copying, which static_cast(p) would not accept - template - void construct(U* p, Args&& ... args) noexcept(noexcept(::new (const_cast(static_cast(p))) U(std::forward(args)...))) + // libc++ probes whether construct() is usable for const objects (when + // checking for trivial copies); this must not be viable then + template::value>::type> + void construct(U* p, Args&& ... args) noexcept(noexcept(::new (static_cast(p)) U(std::forward(args)...))) { - ::new (const_cast(static_cast(p))) U(std::forward(args)...); + ::new (static_cast(p)) U(std::forward(args)...); } template From 7960f33af84d7bdf817d934c4509c6850f1df8f1 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 7 Nov 2026 09:08:24 +0000 Subject: [PATCH 4/8] Fix CI: astyle and clang-tidy for sweep_allocator::construct() astyle 3.4.13 reformats the enable_if default template argument (Check amalgamation). Delete the const overload instead, which makes the libc++ probe for const objects fail in the SFINAE context just the same and needs no . Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz Signed-off-by: Niels Lohmann --- tests/src/unit-allocator.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/src/unit-allocator.cpp b/tests/src/unit-allocator.cpp index 3c719bf26..9862e8c5e 100644 --- a/tests/src/unit-allocator.cpp +++ b/tests/src/unit-allocator.cpp @@ -14,7 +14,6 @@ using nlohmann::json; #include #include -#include #include #if JSON_HAS_RANGES #include @@ -1074,14 +1073,17 @@ struct sweep_allocator return (std::numeric_limits::max)() / sizeof(T); } - // libc++ probes whether construct() is usable for const objects (when - // checking for trivial copies); this must not be viable then - template::value>::type> + template void construct(U* p, Args&& ... args) noexcept(noexcept(::new (static_cast(p)) U(std::forward(args)...))) { ::new (static_cast(p)) U(std::forward(args)...); } + // libc++ probes whether construct() is usable for const objects (when + // checking for trivial copies); this must not be viable then + template + void construct(const U* /*unused*/, const Args& ... /*unused*/) = delete; + template void destroy(U* p) noexcept { From cbb9e14889f43cd10ffe54870977338381104c6b Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 7 Nov 2026 10:10:21 +0100 Subject: [PATCH 5/8] Fix ci_emscripten: make sweep_allocator::construct() not viable for const types (#6036) * Tests: make sweep_allocator::construct() not viable for const types The newest libc++ (emsdk latest, used by ci_emscripten) probes construct(const T*, const T&) in an unevaluated context while checking whether copies are trivial. For the test allocator, instantiating the noexcept specification then fails to compile, because static_cast cannot drop the const qualifier. Constrain construct() to non-const types, which makes the probe fail in the SFINAE context instead of with a hard error. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz Signed-off-by: Niels Lohmann * Fix CI: astyle and clang-tidy for sweep_allocator::construct() astyle 3.4.13 reformats the enable_if default template argument (Check amalgamation). Delete the const overload instead, which makes the libc++ probe for const objects fail in the SFINAE context just the same and needs no . Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann Co-authored-by: Claude Sonnet 5.5 --- tests/src/unit-allocator.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/src/unit-allocator.cpp b/tests/src/unit-allocator.cpp index 8fa9034b5..9862e8c5e 100644 --- a/tests/src/unit-allocator.cpp +++ b/tests/src/unit-allocator.cpp @@ -1079,6 +1079,11 @@ struct sweep_allocator ::new (static_cast(p)) U(std::forward(args)...); } + // libc++ probes whether construct() is usable for const objects (when + // checking for trivial copies); this must not be viable then + template + void construct(const U* /*unused*/, const Args& ... /*unused*/) = delete; + template void destroy(U* p) noexcept { From 951271127bacede4e1b3bfc272013a7440ac5b94 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 7 Nov 2026 10:10:50 +0100 Subject: [PATCH 6/8] Binary readers: append strings straight from contiguous input (#6028) * Binary readers: append strings straight from contiguous input get_bytes() read every string into a zero-filled resize() and copied the bytes over it, so each byte was written twice. A string that the input holds in one block (bulk_remaining() at least its length) is now appended from it directly. The claimed length is checked against the input first, so a huge claimed length still allocates nothing it cannot fill; other inputs and binary values keep the chunked loop. from_cbor() and from_msgpack() of twitter.json and citm_catalog.json get 5.5-9.1% faster on an Apple M1 Max and 2.7-8.6% on x86-64 (GCC 15, Clang 21); canada.json, which has few strings, is unchanged. Signed-off-by: Niels Lohmann * Binary readers: keep the tag argument of append_from_block() astyle-proof Signed-off-by: Niels Lohmann * Fix CI: bugprone-signed-char-misuse in append_from_block Convert the (positive) length to its unsigned type before widening it to std::size_t, as clang-tidy flags the int8_t to std::size_t cast. Signed-off-by: Niels Lohmann * Tests: sweep_allocator::construct() accepts pointers to const Recent libc++ (emsdk latest) probes construct(const T*, const T&) in an unevaluated context while checking for trivial copies; the static_cast to void* in the noexcept specification then fails to compile. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz (cherry picked from commit 794bf70823c12a2a1cb2a8a95f34dfb6357746a8) Signed-off-by: Niels Lohmann * Tests: guard sweep_allocator::construct() against const types instead of const_cast The const_cast added in the previous commit is rejected by cppcoreguidelines-pro-type-const-cast (clang-tidy). Make construct() not viable for const U instead, which is what libc++ probes for. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz (cherry picked from commit de099fa607001b710862fd1bbecef3609c54cbc4) Signed-off-by: Niels Lohmann * Fix CI: astyle and clang-tidy for sweep_allocator::construct() astyle 3.4.13 reformats the enable_if default template argument (Check amalgamation). Delete the const overload instead, which makes the libc++ probe for const objects fail in the SFINAE context just the same and needs no . Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann Co-authored-by: Claude Sonnet 5.5 --- .../nlohmann/detail/input/binary_reader.hpp | 38 +++++++++++++++++++ single_include/nlohmann/json.hpp | 38 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 93f707097..81a3ebd42 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -4887,6 +4887,14 @@ class binary_reader const char* context, ContainerType& result) { + // a string that is in the block of input as a whole is appended from + // it, without the zero-filling resize() below + constexpr bool from_block = bulk_scan && std::is_same::value; + if (append_from_block(len, result, std::integral_constant {})) + { + return true; + } + // upper bound on the number of bytes read (and allocated) per chunk constexpr std::size_t chunk_size = 4096; @@ -4923,6 +4931,36 @@ class binary_reader return true; } + /// append the @a len bytes at the current position to @a result, if the + /// input holds them in one block; false (with nothing read) otherwise + template + bool append_from_block(const NumberType len, string_t& result, std::true_type /*block*/) + { + if (len <= 0) + { + return false; + } + // (len > 0, so the unsigned type holds it; casting a signed char + // straight to std::size_t would trip bugprone-signed-char-misuse) + const auto ulen = static_cast::type>(len); + const std::size_t remaining = ia.bulk_remaining(); + if (static_cast(ulen) > static_cast(remaining)) + { + return false; // (the loop reports the premature end) + } + const auto n = static_cast(ulen); + result.append(reinterpret_cast(ia.bulk_data()), n); + ia.bulk_skip(n); + chars_read += n; + return true; + } + + template + static bool append_from_block(const NumberType /*len*/, ContainerType& /*result*/, std::false_type /*block*/) noexcept + { + return false; + } + /*! @param[in] context further context information (for diagnostics) @return whether the last read character is not EOF diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index eedb9c561..6e2ead1b4 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -22903,6 +22903,14 @@ class binary_reader const char* context, ContainerType& result) { + // a string that is in the block of input as a whole is appended from + // it, without the zero-filling resize() below + constexpr bool from_block = bulk_scan && std::is_same::value; + if (append_from_block(len, result, std::integral_constant {})) + { + return true; + } + // upper bound on the number of bytes read (and allocated) per chunk constexpr std::size_t chunk_size = 4096; @@ -22939,6 +22947,36 @@ class binary_reader return true; } + /// append the @a len bytes at the current position to @a result, if the + /// input holds them in one block; false (with nothing read) otherwise + template + bool append_from_block(const NumberType len, string_t& result, std::true_type /*block*/) + { + if (len <= 0) + { + return false; + } + // (len > 0, so the unsigned type holds it; casting a signed char + // straight to std::size_t would trip bugprone-signed-char-misuse) + const auto ulen = static_cast::type>(len); + const std::size_t remaining = ia.bulk_remaining(); + if (static_cast(ulen) > static_cast(remaining)) + { + return false; // (the loop reports the premature end) + } + const auto n = static_cast(ulen); + result.append(reinterpret_cast(ia.bulk_data()), n); + ia.bulk_skip(n); + chars_read += n; + return true; + } + + template + static bool append_from_block(const NumberType /*len*/, ContainerType& /*result*/, std::false_type /*block*/) noexcept + { + return false; + } + /*! @param[in] context further context information (for diagnostics) @return whether the last read character is not EOF From 18138227dc754f0b6293beadd05192de3221b794 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 7 Nov 2026 10:11:09 +0100 Subject: [PATCH 7/8] Scan the buffer of a FILE in bulk with glibc and the Apple C library (#6030) * Scan the buffer of a FILE in bulk with glibc and the Apple C library Parsing from a FILE* still read one character at a time through getc_unlocked(), while a std::istream has its get area scanned in bulk. file_input_adapter now offers the lexer the same kind of window into the buffer of the FILE: it reads the fields that the C library's own getc_unlocked() macro reads (glibc: _IO_read_ptr and _IO_read_end; Apple: _p and _r) and consumes exactly the characters getc_unlocked() would return, so the file is left where it was left before. The part of the current token in a buffer that is about to be refilled is kept for error messages, as for streams. The window needs the file locked for the whole parse, so JSON_DISABLE_UNLOCKED_FILE_IO and MemorySanitizer builds turn it off; uClibc, which defines __GLIBC__ but has other fields, does not get it. parse() from a FILE* gets 2.0-2.6x as fast on an Apple M1 Max and 1.7-2.2x on x86-64 (glibc, GCC 15 and Clang 21), accept() 2.9-4.4x; that is about as fast as parsing from a std::ifstream. The new tests read with FILE buffers of 1 to 70 bytes and unbuffered files, after ungetc(), with SAX positions and with binary strings across the buffer's edges. Signed-off-by: Niels Lohmann * Tests: no setvbuf() buffer below 2 bytes on Windows The Microsoft C runtime rejects a _IOFBF buffer size below 2 as an invalid parameter and ends the process (0xc0000409), which made test-file_input fail on all MSVC and clang-cl jobs. Signed-off-by: Niels Lohmann * Fix CI: astyle formatting in unit-file_input.cpp Signed-off-by: Niels Lohmann * Tests: limit the multi-character ungetc() test to glibc and Apple The Windows C runtime only supports one pushed-back character, which broke the Windows CI jobs. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz Signed-off-by: Niels Lohmann * Tests: sweep_allocator::construct() accepts pointers to const Recent libc++ (emsdk latest) probes construct(const T*, const T&) in an unevaluated context while checking for trivial copies; the static_cast to void* in the noexcept specification then fails to compile. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz (cherry picked from commit 794bf70823c12a2a1cb2a8a95f34dfb6357746a8) Signed-off-by: Niels Lohmann * Tests: guard sweep_allocator::construct() against const types instead of const_cast The const_cast added in the previous commit is rejected by cppcoreguidelines-pro-type-const-cast (clang-tidy). Make construct() not viable for const U instead, which is what libc++ probes for. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz (cherry picked from commit de099fa607001b710862fd1bbecef3609c54cbc4) Signed-off-by: Niels Lohmann * Fix CI: astyle and clang-tidy for sweep_allocator::construct() astyle 3.4.13 reformats the enable_if default template argument (Check amalgamation). Delete the const overload instead, which makes the libc++ probe for const objects fail in the SFINAE context just the same and needs no . Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01N5NasEsrzHhSNouxSoEwqz Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann Co-authored-by: Claude Sonnet 5.5 --- .../macros/json_disable_unlocked_file_io.md | 18 +- docs/mkdocs/docs/features/parsing/index.md | 2 +- docs/mkdocs/docs/features/performance.md | 4 +- .../nlohmann/detail/input/input_adapters.hpp | 179 ++++++++++++++++- include/nlohmann/detail/macro_unscope.hpp | 1 + single_include/nlohmann/json.hpp | 180 +++++++++++++++++- tests/src/unit-file_input.cpp | 133 +++++++++++++ 7 files changed, 506 insertions(+), 11 deletions(-) diff --git a/docs/mkdocs/docs/api/macros/json_disable_unlocked_file_io.md b/docs/mkdocs/docs/api/macros/json_disable_unlocked_file_io.md index 2c266ba2d..031566cfe 100644 --- a/docs/mkdocs/docs/api/macros/json_disable_unlocked_file_io.md +++ b/docs/mkdocs/docs/api/macros/json_disable_unlocked_file_io.md @@ -5,7 +5,9 @@ ``` When defined to `1`, parsing from a `#!cpp FILE*` reads every character with `std::fgetc`, as in version 3.12.0 and -earlier, instead of locking the file once and reading it with the unlocked stdio functions. +earlier, instead of locking the file once and reading it with the unlocked stdio functions. This also turns off the +scan of the `FILE`'s buffer in bulk (see below), which [`JSON_DISABLE_STREAM_BULK_SCAN`](json_disable_stream_bulk_scan.md) +does not affect. It does not change any result: values, exceptions, and the position of the file after parsing are the same either way. Only the speed and the locking of the file differ (see the notes on locking below). @@ -37,9 +39,17 @@ The default value is `0`: the file is locked once and read with the unlocked std | Microsoft (MSVC, clang-cl) | `_lock_file`, `_fgetc_nolock`, `_unlock_file` | | others (e.g., musl, MinGW, newlib) | `std::fgetc`, as before | - This makes parsing from a `FILE*` 3 to 4 times as fast with the Apple C library and 1.1 to 1.3 times as fast with - glibc, whose locking is cheaper. The binary formats ([`from_cbor`](../basic_json/from_cbor.md), - [`from_msgpack`](../basic_json/from_msgpack.md), ...) read from a `FILE*` the same way. + With glibc and the Apple C library, the parser also scans the characters that the `FILE` already holds in its + buffer in bulk, as it scans the get area of a stream (see + [`JSON_DISABLE_STREAM_BULK_SCAN`](json_disable_stream_bulk_scan.md)): it reads the same fields of the `FILE` that + the C library's own `getc_unlocked` reads, and it consumes exactly the characters that `getc_unlocked` would have + returned, so the file is left where it was left before. + + Locking the file once makes parsing from a `FILE*` 3 to 4 times as fast with the Apple C library and 1.1 to 1.3 + times as fast with glibc, whose locking is cheaper. Scanning the buffer in bulk makes `parse()` another 1.7 to 2.6 + times as fast (and `accept()` 2.9 to 4.4 times), about as fast as parsing from a `std::ifstream`. The binary + formats ([`from_cbor`](../basic_json/from_cbor.md), [`from_msgpack`](../basic_json/from_msgpack.md), ...) read + from a `FILE*` the same way. !!! note "Locking" diff --git a/docs/mkdocs/docs/features/parsing/index.md b/docs/mkdocs/docs/features/parsing/index.md index 93a02b501..fa2bec462 100644 --- a/docs/mkdocs/docs/features/parsing/index.md +++ b/docs/mkdocs/docs/features/parsing/index.md @@ -77,7 +77,7 @@ them faster than others: |--------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------| | strings, string views, string literals, contiguous containers, pointer or iterator ranges over contiguous memory | the whole input is scanned in bulk: runs of string characters, numbers, and indentation are recognized several bytes at a time | | `#!cpp std::istream` (`#!cpp std::ifstream`, `#!cpp std::istringstream`, ...) | the characters the stream buffer holds are scanned in bulk like a string; the stream buffer is asked for more when they are used up | -| `#!cpp FILE*` | one character at a time, with the file locked once for the whole parse | +| `#!cpp FILE*` | with glibc and the Apple C library, the characters the `FILE` holds in its buffer are scanned in bulk like a stream's; elsewhere one character at a time; the file is locked once for the whole parse | | other iterator ranges (e.g., `#!cpp std::list`, `#!cpp std::istreambuf_iterator`), wide strings | one character at a time | Parsing a file from a `#!cpp std::ifstream` is therefore only about 10 to 20 % slower than reading the whole file into a diff --git a/docs/mkdocs/docs/features/performance.md b/docs/mkdocs/docs/features/performance.md index 8f4508057..3d0714ca9 100644 --- a/docs/mkdocs/docs/features/performance.md +++ b/docs/mkdocs/docs/features/performance.md @@ -27,8 +27,8 @@ A `#!cpp std::istream` (including `#!cpp std::ifstream`) gets the same bulk scan characters its stream buffer already holds (the *get area*) and asks the stream buffer for more only when they are used up. Nothing is read ahead, so the stream is left exactly where it would be after reading one character at a time, and [`operator>>`](../api/operator_gtgt.md) can still read several values from one stream. A stream buffer without a get -area is read one character at a time. A `#!cpp FILE*` is still read one character at a time, but `file_input_adapter` -locks the file once per parse instead of once per character. See +area is read one character at a time. A `#!cpp FILE*` is locked once per parse instead of once per character, and with +glibc and the Apple C library, the characters in its buffer are scanned in bulk as a stream's get area is. See [input sources and performance](parsing/index.md#input-sources-and-performance) for the full table. According to the description of [#5828](https://github.com/nlohmann/json/pull/5828), parsing from a stream is within about 10 to 20 % of parsing an equivalent `#!cpp std::string`. diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index 1d925588f..7b7222f06 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -58,6 +58,19 @@ #define JSON_UNLOCKED_FILE_IO 0 #endif #undef JSON_UNLOCKED_FILE_IO_MSAN + + // Whether file_input_adapter scans the buffer of the FILE directly, as + // getc_unlocked() reads it: the fields it uses are those that the C + // library's own getc_unlocked() macro uses (glibc: _IO_read_ptr and + // _IO_read_end; Apple: _p and _r). It needs the file locked for the whole + // parse, as JSON_UNLOCKED_FILE_IO == 1 has it (so JSON_DISABLE_UNLOCKED_FILE_IO + // and MemorySanitizer builds turn it off). + // (uClibc defines __GLIBC__ as well, but its FILE has other fields) + #if JSON_UNLOCKED_FILE_IO == 1 && ((defined(__GLIBC__) && !defined(__UCLIBC__)) || defined(__APPLE__)) + #define JSON_FILE_WINDOW 1 + #else + #define JSON_FILE_WINDOW 0 + #endif #endif // JSON_NO_IO NLOHMANN_JSON_NAMESPACE_BEGIN @@ -179,8 +192,12 @@ class token_carry_buffer char* m_capacity_end = nullptr; }; /*! -Input adapter for stdio file access. This adapter read only 1 byte and do not use any - buffer. This adapter is a very low level adapter. +Input adapter for stdio file access. The file is locked for the lifetime of the +adapter and read with the unlocked stdio functions (see +JSON_DISABLE_UNLOCKED_FILE_IO). With glibc and the Apple C library, the +characters in the buffer of the FILE are also scanned in bulk, as the get area +of a stream is (see input_stream_adapter): exactly the characters getc_unlocked() +would return are consumed, so the file is left where it was left before. */ class file_input_adapter { @@ -199,6 +216,12 @@ class file_input_adapter file_input_adapter(const file_input_adapter&) = delete; file_input_adapter(file_input_adapter&& rhs) noexcept : m_file(rhs.m_file) +#if JSON_FILE_WINDOW + , token_carry(std::move(rhs.token_carry)) + , token_segment(rhs.token_segment) + , current_in_buffer(rhs.current_in_buffer) + , tracking_token(rhs.tracking_token) +#endif { // the lock is owned by the thread, not the adapter: hand it over rhs.m_file = nullptr; @@ -210,6 +233,20 @@ class file_input_adapter unlock(); } +#if JSON_FILE_WINDOW + std::char_traits::int_type get_character() + { + // what getc_unlocked() does, while the buffer has characters + const char* const next = buffer_ptr(); + if (JSON_HEDLEY_LIKELY(next != buffer_end())) + { + current_in_buffer = true; + buffer_bump(1); + return std::char_traits::to_int_type(*next); + } + return get_character_slow(); + } +#else std::char_traits::int_type get_character() noexcept { // the file is locked for the lifetime of the adapter, so the @@ -222,6 +259,7 @@ class file_input_adapter return std::fgetc(m_file); #endif } +#endif // returns the number of characters successfully read template @@ -230,7 +268,134 @@ class file_input_adapter return fread(dest, 1, sizeof(T) * count, m_file); } +#if JSON_FILE_WINDOW + // The lexer may scan the buffer of the FILE directly, as the window of + // input_stream_adapter: the buffer ends wherever the C library's buffer + // ends, and is refilled by the next get_character(). + static constexpr bool supports_bulk_scan = true; + static constexpr bool bulk_window = true; + + // Pointer to the next unread character; only valid when bulk_remaining() > 0. + const char* bulk_data() const + { + return buffer_ptr(); + } + + // Number of unread characters in the buffer. + std::size_t bulk_remaining() const + { + if (!current_in_buffer) + { + return 0; + } + return static_cast(buffer_end() - buffer_ptr()); + } + + // Consume @a n characters previously inspected via bulk_data(). + void bulk_skip(std::size_t n) + { + JSON_ASSERT(n <= bulk_remaining()); + buffer_bump(n); + } + + // The adapter keeps the current token for the lexer's error messages, as + // input_stream_adapter does. + static constexpr bool supports_token_tracking = true; + + void begin_token() noexcept + { + tracking_token = true; + token_carry.clear(); + token_segment = buffer_ptr(); + } + + void mark_token_start(std::char_traits::int_type c) + { + tracking_token = true; + token_carry.clear(); + if (current_in_buffer) + { + token_segment = buffer_ptr() - 1; + } + else + { + token_segment = nullptr; + token_carry.push_back(std::char_traits::to_char_type(c)); + } + } + + void copy_token(std::vector& out, bool drop_last) const + { + token_carry.copy_to(out); + if (token_segment != nullptr) + { + out.insert(out.end(), token_segment, buffer_ptr()); + } + if (drop_last && !out.empty()) + { + out.pop_back(); + } + } +#endif + private: +#if JSON_FILE_WINDOW + // the buffer of the FILE: [buffer_ptr(), buffer_end()) are the characters + // getc_unlocked() would return next +#if defined(__GLIBC__) + const char* buffer_ptr() const noexcept + { + return m_file->_IO_read_ptr; + } + const char* buffer_end() const noexcept + { + return m_file->_IO_read_end; + } + void buffer_bump(std::size_t n) noexcept + { + m_file->_IO_read_ptr += n; + } +#else + const char* buffer_ptr() const noexcept + { + return reinterpret_cast(m_file->_p); + } + const char* buffer_end() const noexcept + { + return m_file->_r > 0 ? reinterpret_cast(m_file->_p) + m_file->_r : reinterpret_cast(m_file->_p); + } + void buffer_bump(std::size_t n) noexcept + { + m_file->_p += n; + m_file->_r -= static_cast(n); + } +#endif + + // the buffer is exhausted: getc_unlocked() refills it + JSON_HEDLEY_NEVER_INLINE std::char_traits::int_type get_character_slow() + { + if (token_segment != nullptr) + { + token_carry.append(token_segment, buffer_end()); + token_segment = nullptr; + } + const auto res = getc_unlocked(m_file); + if (res == EOF) + { + current_in_buffer = false; + return std::char_traits::eof(); + } + // the character was taken from the refilled buffer, at buffer_ptr() - 1 + current_in_buffer = true; + if (tracking_token) + { + token_segment = buffer_ptr() - 1; + } + return res; + } + +#endif + // Lock the file once for the whole parse instead of once per character. // The lock is recursive, so fread() and the user's own stdio calls on the // same thread keep working. @@ -259,6 +424,16 @@ class file_input_adapter /// the file pointer to read from std::FILE* m_file; +#if JSON_FILE_WINDOW + /// the part of the current token that was in earlier buffers + token_carry_buffer token_carry{}; // NOLINT(readability-redundant-member-init) + /// start of the current token's part in the buffer, or nullptr + const char* token_segment = nullptr; + /// whether the character last returned by get_character() is in the buffer + bool current_in_buffer = false; + /// whether a token is tracked at all; the binary readers never start one + bool tracking_token = false; +#endif }; /*! diff --git a/include/nlohmann/detail/macro_unscope.hpp b/include/nlohmann/detail/macro_unscope.hpp index d3cdb0d3f..b61db5dc6 100644 --- a/include/nlohmann/detail/macro_unscope.hpp +++ b/include/nlohmann/detail/macro_unscope.hpp @@ -22,6 +22,7 @@ #undef JSON_DISABLE_ENUM_SERIALIZATION #undef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION #undef JSON_UNLOCKED_FILE_IO +#undef JSON_FILE_WINDOW #undef JSON_DTOA_SSE2 #undef JSON_DTOA_NEON #undef JSON_LITTLE_ENDIAN_TARGET diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 6e2ead1b4..f91671e82 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -9167,6 +9167,19 @@ NLOHMANN_JSON_NAMESPACE_END #define JSON_UNLOCKED_FILE_IO 0 #endif #undef JSON_UNLOCKED_FILE_IO_MSAN + + // Whether file_input_adapter scans the buffer of the FILE directly, as + // getc_unlocked() reads it: the fields it uses are those that the C + // library's own getc_unlocked() macro uses (glibc: _IO_read_ptr and + // _IO_read_end; Apple: _p and _r). It needs the file locked for the whole + // parse, as JSON_UNLOCKED_FILE_IO == 1 has it (so JSON_DISABLE_UNLOCKED_FILE_IO + // and MemorySanitizer builds turn it off). + // (uClibc defines __GLIBC__ as well, but its FILE has other fields) + #if JSON_UNLOCKED_FILE_IO == 1 && ((defined(__GLIBC__) && !defined(__UCLIBC__)) || defined(__APPLE__)) + #define JSON_FILE_WINDOW 1 + #else + #define JSON_FILE_WINDOW 0 + #endif #endif // JSON_NO_IO NLOHMANN_JSON_NAMESPACE_BEGIN @@ -9288,8 +9301,12 @@ class token_carry_buffer char* m_capacity_end = nullptr; }; /*! -Input adapter for stdio file access. This adapter read only 1 byte and do not use any - buffer. This adapter is a very low level adapter. +Input adapter for stdio file access. The file is locked for the lifetime of the +adapter and read with the unlocked stdio functions (see +JSON_DISABLE_UNLOCKED_FILE_IO). With glibc and the Apple C library, the +characters in the buffer of the FILE are also scanned in bulk, as the get area +of a stream is (see input_stream_adapter): exactly the characters getc_unlocked() +would return are consumed, so the file is left where it was left before. */ class file_input_adapter { @@ -9308,6 +9325,12 @@ class file_input_adapter file_input_adapter(const file_input_adapter&) = delete; file_input_adapter(file_input_adapter&& rhs) noexcept : m_file(rhs.m_file) +#if JSON_FILE_WINDOW + , token_carry(std::move(rhs.token_carry)) + , token_segment(rhs.token_segment) + , current_in_buffer(rhs.current_in_buffer) + , tracking_token(rhs.tracking_token) +#endif { // the lock is owned by the thread, not the adapter: hand it over rhs.m_file = nullptr; @@ -9319,6 +9342,20 @@ class file_input_adapter unlock(); } +#if JSON_FILE_WINDOW + std::char_traits::int_type get_character() + { + // what getc_unlocked() does, while the buffer has characters + const char* const next = buffer_ptr(); + if (JSON_HEDLEY_LIKELY(next != buffer_end())) + { + current_in_buffer = true; + buffer_bump(1); + return std::char_traits::to_int_type(*next); + } + return get_character_slow(); + } +#else std::char_traits::int_type get_character() noexcept { // the file is locked for the lifetime of the adapter, so the @@ -9331,6 +9368,7 @@ class file_input_adapter return std::fgetc(m_file); #endif } +#endif // returns the number of characters successfully read template @@ -9339,7 +9377,134 @@ class file_input_adapter return fread(dest, 1, sizeof(T) * count, m_file); } +#if JSON_FILE_WINDOW + // The lexer may scan the buffer of the FILE directly, as the window of + // input_stream_adapter: the buffer ends wherever the C library's buffer + // ends, and is refilled by the next get_character(). + static constexpr bool supports_bulk_scan = true; + static constexpr bool bulk_window = true; + + // Pointer to the next unread character; only valid when bulk_remaining() > 0. + const char* bulk_data() const + { + return buffer_ptr(); + } + + // Number of unread characters in the buffer. + std::size_t bulk_remaining() const + { + if (!current_in_buffer) + { + return 0; + } + return static_cast(buffer_end() - buffer_ptr()); + } + + // Consume @a n characters previously inspected via bulk_data(). + void bulk_skip(std::size_t n) + { + JSON_ASSERT(n <= bulk_remaining()); + buffer_bump(n); + } + + // The adapter keeps the current token for the lexer's error messages, as + // input_stream_adapter does. + static constexpr bool supports_token_tracking = true; + + void begin_token() noexcept + { + tracking_token = true; + token_carry.clear(); + token_segment = buffer_ptr(); + } + + void mark_token_start(std::char_traits::int_type c) + { + tracking_token = true; + token_carry.clear(); + if (current_in_buffer) + { + token_segment = buffer_ptr() - 1; + } + else + { + token_segment = nullptr; + token_carry.push_back(std::char_traits::to_char_type(c)); + } + } + + void copy_token(std::vector& out, bool drop_last) const + { + token_carry.copy_to(out); + if (token_segment != nullptr) + { + out.insert(out.end(), token_segment, buffer_ptr()); + } + if (drop_last && !out.empty()) + { + out.pop_back(); + } + } +#endif + private: +#if JSON_FILE_WINDOW + // the buffer of the FILE: [buffer_ptr(), buffer_end()) are the characters + // getc_unlocked() would return next +#if defined(__GLIBC__) + const char* buffer_ptr() const noexcept + { + return m_file->_IO_read_ptr; + } + const char* buffer_end() const noexcept + { + return m_file->_IO_read_end; + } + void buffer_bump(std::size_t n) noexcept + { + m_file->_IO_read_ptr += n; + } +#else + const char* buffer_ptr() const noexcept + { + return reinterpret_cast(m_file->_p); + } + const char* buffer_end() const noexcept + { + return m_file->_r > 0 ? reinterpret_cast(m_file->_p) + m_file->_r : reinterpret_cast(m_file->_p); + } + void buffer_bump(std::size_t n) noexcept + { + m_file->_p += n; + m_file->_r -= static_cast(n); + } +#endif + + // the buffer is exhausted: getc_unlocked() refills it + JSON_HEDLEY_NEVER_INLINE std::char_traits::int_type get_character_slow() + { + if (token_segment != nullptr) + { + token_carry.append(token_segment, buffer_end()); + token_segment = nullptr; + } + const auto res = getc_unlocked(m_file); + if (res == EOF) + { + current_in_buffer = false; + return std::char_traits::eof(); + } + // the character was taken from the refilled buffer, at buffer_ptr() - 1 + current_in_buffer = true; + if (tracking_token) + { + token_segment = buffer_ptr() - 1; + } + return res; + } + +#endif + // Lock the file once for the whole parse instead of once per character. // The lock is recursive, so fread() and the user's own stdio calls on the // same thread keep working. @@ -9368,6 +9533,16 @@ class file_input_adapter /// the file pointer to read from std::FILE* m_file; +#if JSON_FILE_WINDOW + /// the part of the current token that was in earlier buffers + token_carry_buffer token_carry{}; // NOLINT(readability-redundant-member-init) + /// start of the current token's part in the buffer, or nullptr + const char* token_segment = nullptr; + /// whether the character last returned by get_character() is in the buffer + bool current_in_buffer = false; + /// whether a token is tracked at all; the binary readers never start one + bool tracking_token = false; +#endif }; /*! @@ -46386,6 +46561,7 @@ struct formatter // NOLINT(cert-dcl58-c #undef JSON_DISABLE_ENUM_SERIALIZATION #undef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION #undef JSON_UNLOCKED_FILE_IO +#undef JSON_FILE_WINDOW #undef JSON_DTOA_SSE2 #undef JSON_DTOA_NEON #undef JSON_LITTLE_ENDIAN_TARGET diff --git a/tests/src/unit-file_input.cpp b/tests/src/unit-file_input.cpp index e88b6f4d0..f74f23267 100644 --- a/tests/src/unit-file_input.cpp +++ b/tests/src/unit-file_input.cpp @@ -49,6 +49,14 @@ file_ptr open_with(const char* name, const std::string& content) return {std::fopen(name, "rb"), &std::fclose}; } +// the smallest size setvbuf() accepts for _IOFBF: the Microsoft C runtime +// rejects sizes below 2 as an invalid parameter, which ends the process +#if defined(_WIN32) + constexpr int min_full_buffer = 2; +#else + constexpr int min_full_buffer = 1; +#endif + // The stdio lock is recursive, so whether parsing released it can only be // seen from another thread: ftrylockfile() fails there while this thread // holds the lock. ftrylockfile() is POSIX; elsewhere, the check is skipped. @@ -225,6 +233,131 @@ TEST_CASE("parsing from a FILE*") check_unlocked(f.get()); } +#if !defined(JSON_NOEXCEPTION) + SECTION("tokens across the edges of the file's buffer") + { + // the adapter reads the buffer of the FILE directly where it can (see + // JSON_FILE_WINDOW): with buffers of 1 to 70 bytes and none, every + // token crosses an edge somewhere; the results, the error messages + // and their positions must be those of the string + const std::vector documents = + { + input, + R"({"long": "a string that is longer than the buffer of the file, with \"escapes\" and \u00e4", "n": [123456789012345678901, -0.000123e-45, 1.5E+300, 0, -0]})", + "[\n \"indented\",\n \"caf\xc3\xa9 \xe2\x82\xac \xf0\x9f\x98\x80\",\n {\n \"key\": 1\n }\n]", + R"({"a": [1, 2, 3], "b": "unterminated)", + R"([1, 2, 3, 1e999])", + R"({"a": "invalid escape \x"})", + "[\"invalid UTF-8 \xc3\x28\"]", + "[1, 2, 3] trailing", + R"([truex])", + }; + for (const auto& document : documents) + { + std::string expected; + try + { + expected = json::parse(document).dump(); + } + catch (const json::exception& e) + { + expected = e.what(); + } + const bool expected_accept = json::accept(document); + for (int size = 0; size <= 70; ++size) + { + if (size != 0 && size < min_full_buffer) + { + continue; + } + CAPTURE(document) + CAPTURE(size) + std::vector buffer(static_cast(size) + 1); + for (int pass = 0; pass < 2; ++pass) + { + const file_ptr f = open_with(name, document); + REQUIRE(f != nullptr); + // size 0: unbuffered + REQUIRE(std::setvbuf(f.get(), size == 0 ? nullptr : buffer.data(), size == 0 ? _IONBF : _IOFBF, static_cast(size)) == 0); + if (pass == 0) + { + std::string actual; + try + { + actual = json::parse(f.get()).dump(); + } + catch (const json::exception& e) + { + actual = e.what(); + } + CHECK(actual == expected); + } + else + { + CHECK(json::accept(f.get()) == expected_accept); + } + check_unlocked(f.get()); + } + } + } + } +#endif + + // the C standard guarantees only one pushed-back character (and the + // Windows C runtime fails further calls to ungetc()), so this is limited + // to the C libraries that keep them in a separate buffer +#if !defined(JSON_NOEXCEPTION) && ((defined(__GLIBC__) && !defined(__UCLIBC__)) || defined(__APPLE__)) + SECTION("characters pushed back with ungetc() before parsing") + { + const file_ptr f = open_with(name, "2, 3]"); + REQUIRE(f != nullptr); + REQUIRE(std::ungetc(',', f.get()) == ','); + REQUIRE(std::ungetc('1', f.get()) == '1'); + REQUIRE(std::ungetc('[', f.get()) == '['); + CHECK(json::parse(f.get()) == json({1, 2, 3})); + } +#endif + + SECTION("SAX events across the edges of the file's buffer") + { + const std::string document = R"({"key": "a value that is longer than the buffer", "n": [1, -2.5e3, true, null]})"; + std::vector expected; + { + const file_ptr f = open_with(name, document); + REQUIRE(f != nullptr); + position_reading_sax sax(f.get()); + CHECK(json::sax_parse(f.get(), &sax)); + expected = sax.positions(); + } + for (int size = min_full_buffer; size <= 20; ++size) + { + CAPTURE(size) + std::vector buffer(static_cast(size)); + const file_ptr f = open_with(name, document); + REQUIRE(f != nullptr); + REQUIRE(std::setvbuf(f.get(), buffer.data(), _IOFBF, buffer.size()) == 0); + position_reading_sax sax(f.get()); + CHECK(json::sax_parse(f.get(), &sax)); + // the events happen at the same positions of the file + CHECK(sax.positions() == expected); + } + } + + SECTION("binary strings across the edges of the file's buffer") + { + const json value = {{"a string that is longer than the buffer of the file", "and a value that is longer, too"}, {"n", {1, 2}}}; + const std::vector cbor = json::to_cbor(value); + for (int size = min_full_buffer; size <= 20; ++size) + { + CAPTURE(size) + std::vector buffer(static_cast(size)); + const file_ptr f = open_with(name, std::string(cbor.begin(), cbor.end())); + REQUIRE(f != nullptr); + REQUIRE(std::setvbuf(f.get(), buffer.data(), _IOFBF, buffer.size()) == 0); + CHECK(json::from_cbor(f.get()) == value); + } + } + SECTION("binary formats read from a FILE*") { const json j = {{"string", std::string(300, 'x')}, {"array", {1, 2, 3}}, {"number", 1.5}}; From 181f41487f86545d1278145178d6b63f560c7a48 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 7 Nov 2026 10:16:20 +0100 Subject: [PATCH 8/8] Contiguous parser: review fixes (wording, macros, diagnostic positions) - "default parser" instead of "library parser" in comments and tests - the three comments in json.hpp say the same thing - current() replaces the repeated `p != end ? *p : 0` - the backslashes of each macro are aligned - the docs say that JSON_DIAGNOSTIC_POSITIONS turns the fast path off Signed-off-by: Niels Lohmann --- .../api/macros/json_diagnostic_positions.md | 3 + docs/mkdocs/docs/features/parsing/index.md | 3 + .../detail/input/contiguous_parser.hpp | 268 +++++++++-------- include/nlohmann/json.hpp | 11 +- single_include/nlohmann/json.hpp | 279 +++++++++--------- tests/src/unit-contiguous_parser.cpp | 4 +- 6 files changed, 294 insertions(+), 274 deletions(-) diff --git a/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md b/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md index 4dac9f9ff..340498acb 100644 --- a/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md +++ b/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md @@ -34,6 +34,9 @@ or quotes. Note that enabling this macro increases the size of every JSON value by two `std::size_t` fields and adds slight runtime overhead to parsing, copying JSON value objects, and the generation of error messages for exceptions. It also causes these values to be reported in those error messages. +In addition, `parse()`, `accept()`, and `sax_parse()` no longer use the +[faster path for input in one block of memory](../../features/parsing/index.md#input-sources-and-performance), so +parsing a string or a buffer is slower than without the macro. If [`JSON_DIAGNOSTICS`](json_diagnostics.md) is also set, the message contains the path and the byte range, such as `(/1) (bytes 4-8)`. An exception about the root value has no path and only the range, such as `(bytes 0-4)`. diff --git a/docs/mkdocs/docs/features/parsing/index.md b/docs/mkdocs/docs/features/parsing/index.md index f4ef0aadc..ed5c80434 100644 --- a/docs/mkdocs/docs/features/parsing/index.md +++ b/docs/mkdocs/docs/features/parsing/index.md @@ -84,6 +84,9 @@ Input in one block of memory is read even faster with the default options (no [parser callback](../../api/basic_json/parser_callback_t.md), no comments, no trailing commas, and for `sax_parse()` `strict = true`): `parse()`, `accept()`, and `sax_parse()` then read the grammar and the tokens in one loop over the memory. Only after an error do `parse()` and `sax_parse()` read the input again, to report the error exactly as before. +This faster path is not used when [`JSON_DIAGNOSTIC_POSITIONS`](../../api/macros/json_diagnostic_positions.md) is +enabled, because the positions are recorded by the default parser: the input is then parsed as before, at the same speed +as before. Parsing a file from a `#!cpp std::ifstream` therefore takes about 1.15 to 1.45 times as long as reading the whole file into a `#!cpp std::string` first and parsing that, but it needs no memory for a copy of the file. The stream is never read diff --git a/include/nlohmann/detail/input/contiguous_parser.hpp b/include/nlohmann/detail/input/contiguous_parser.hpp index 04c972bb8..29a090a3b 100644 --- a/include/nlohmann/detail/input/contiguous_parser.hpp +++ b/include/nlohmann/detail/input/contiguous_parser.hpp @@ -37,8 +37,8 @@ namespace detail @brief a parser for JSON text in one contiguous block, for parse(), accept(), and sax_parse() -The parser and the lexer exchange one token at a time, and the lexer keeps its -state (the current character, a pending unget, where the token starts) in +The default parser and lexer exchange one token at a time, and the lexer keeps +its state (the current character, a pending unget, where the token starts) in members, which every character pays for. This parser reads the block through a local pointer instead, and the grammar, the whitespace and the tokens are one loop, after yyjson's reader: the state stays in registers, and the expected @@ -46,13 +46,13 @@ next token is tested where it is expected instead of being dispatched on. It reports what it reads to a SAX handler: the one parse() uses to build the value, the one accept() uses, or the user's handler of sax_parse(). On a byte -the library parser would reject, it stops before reporting the token, and -returns false: the events up to there are those the library parser reports. -The caller then runs the library parser on the same input to report the error +the default parser would reject, it stops before reporting the token, and +returns false: the events up to there are those the default parser reports. +The caller then runs the default parser on the same input to report the error with its exact message and position (report_contiguous_error(), json_sax_error_forwarder), and parse() builds no value from what was read. -What is accepted, and the values reported, are those of the library parser +What is accepted, and the values reported, are those of the default parser with default options: the same number grammar and conversions, the same UTF-8 and escape rules, a UTF-8 byte order mark at the start, and, without JSON_STRICT_NUL_HANDLING, a NUL byte as the end of the input. @@ -106,28 +106,28 @@ class contiguous_parser std::vector outer; // the current byte, or 0 at the end (no JSON token starts with 0) -#define JSON_CONTIGUOUS_CUR() (p != e ? *p : 0) +#define JSON_CONTIGUOUS_CUR() current(p, e) // skip whitespace; most tokens have none before them -#define JSON_CONTIGUOUS_WS() \ - do \ - { \ - if (JSON_HEDLEY_UNLIKELY(JSON_CONTIGUOUS_CUR() <= ' ')) \ - { \ - p = skip_whitespace(p, e); \ - } \ - } \ +#define JSON_CONTIGUOUS_WS() \ + do \ + { \ + if (JSON_HEDLEY_UNLIKELY(JSON_CONTIGUOUS_CUR() <= ' ')) \ + { \ + p = skip_whitespace(p, e); \ + } \ + } \ while (false) -#define JSON_CONTIGUOUS_OPEN(is_object) \ - do \ - { \ +#define JSON_CONTIGUOUS_OPEN(is_object) \ + do \ + { \ if (JSON_HEDLEY_UNLIKELY(depth != 0 && (depth % 64) == 0)) /* NOLINT(readability/braces) */ \ - { \ - outer.push_back(kinds); \ - kinds = 0; \ - } \ - kinds = (kinds << 1u) | static_cast(is_object); \ - ++depth; \ - } \ + { \ + outer.push_back(kinds); \ + kinds = 0; \ + } \ + kinds = (kinds << 1u) | static_cast(is_object); \ + ++depth; \ + } \ while (false) JSON_CONTIGUOUS_WS(); @@ -160,107 +160,107 @@ class contiguous_parser // a value in an array or an object: expanded once for each, so that // each dispatch has its own branch history (arrays tend to hold values // of one kind) and the code after a scalar knows where it is -#define JSON_CONTIGUOUS_VALUE \ - switch (JSON_CONTIGUOUS_CUR()) \ - { \ - case '"': \ - if (JSON_HEDLEY_UNLIKELY(!string(p, false))) \ - { \ - return false; \ - } \ - break; \ - case '{': \ - ++p; \ - if (JSON_HEDLEY_UNLIKELY(!report(sax->start_object(detail::unknown_size())))) \ - { \ - return false; \ - } \ - JSON_CONTIGUOUS_OPEN(1u); \ - goto object_first; \ - case '[': \ - ++p; \ - if (JSON_HEDLEY_UNLIKELY(!report(sax->start_array(detail::unknown_size())))) \ - { \ - return false; \ - } \ - JSON_CONTIGUOUS_OPEN(0u); \ - goto array_first; \ - case '-': \ - case '0': \ - case '1': \ - case '2': \ - case '3': \ - case '4': \ - case '5': \ - case '6': \ - case '7': \ - case '8': \ - case '9': \ - if (JSON_HEDLEY_UNLIKELY(!number(p))) \ - { \ - return false; \ - } \ - break; \ - case 't': \ - if (JSON_HEDLEY_UNLIKELY(!literal(p, "true") || !report(sax->boolean(true)))) \ - { \ - return false; \ - } \ - break; \ - case 'f': \ - if (JSON_HEDLEY_UNLIKELY(!literal(p, "false") || !report(sax->boolean(false)))) \ - { \ - return false; \ - } \ - break; \ - case 'n': \ - if (JSON_HEDLEY_UNLIKELY(!literal(p, "null") || !report(sax->null()))) \ - { \ - return false; \ - } \ - break; \ - default: \ - return false; \ +#define JSON_CONTIGUOUS_VALUE \ + switch (JSON_CONTIGUOUS_CUR()) \ + { \ + case '"': \ + if (JSON_HEDLEY_UNLIKELY(!string(p, false))) \ + { \ + return false; \ + } \ + break; \ + case '{': \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_object(detail::unknown_size())))) \ + { \ + return false; \ + } \ + JSON_CONTIGUOUS_OPEN(1u); \ + goto object_first; \ + case '[': \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_array(detail::unknown_size())))) \ + { \ + return false; \ + } \ + JSON_CONTIGUOUS_OPEN(0u); \ + goto array_first; \ + case '-': \ + case '0': \ + case '1': \ + case '2': \ + case '3': \ + case '4': \ + case '5': \ + case '6': \ + case '7': \ + case '8': \ + case '9': \ + if (JSON_HEDLEY_UNLIKELY(!number(p))) \ + { \ + return false; \ + } \ + break; \ + case 't': \ + if (JSON_HEDLEY_UNLIKELY(!literal(p, "true") || !report(sax->boolean(true)))) \ + { \ + return false; \ + } \ + break; \ + case 'f': \ + if (JSON_HEDLEY_UNLIKELY(!literal(p, "false") || !report(sax->boolean(false)))) \ + { \ + return false; \ + } \ + break; \ + case 'n': \ + if (JSON_HEDLEY_UNLIKELY(!literal(p, "null") || !report(sax->null()))) \ + { \ + return false; \ + } \ + break; \ + default: \ + return false; \ } // what may follow a value in an array: a comma and the next value, or the end -#define JSON_CONTIGUOUS_AFTER_ELEMENT \ - JSON_CONTIGUOUS_WS(); \ +#define JSON_CONTIGUOUS_AFTER_ELEMENT \ + JSON_CONTIGUOUS_WS(); \ if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ',')) /* NOLINT(readability/braces) */ \ - { \ - ++p; \ - JSON_CONTIGUOUS_WS(); \ - goto array_value; \ - } \ + { \ + ++p; \ + JSON_CONTIGUOUS_WS(); \ + goto array_value; \ + } \ if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ']')) /* NOLINT(readability/braces) */ \ - { \ - ++p; \ - if (JSON_HEDLEY_UNLIKELY(!report(sax->end_array()))) \ - { \ - return false; \ - } \ - goto close; \ - } \ + { \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_array()))) \ + { \ + return false; \ + } \ + goto close; \ + } \ return false; // what may follow a value in an object: a comma and the next member, or the end -#define JSON_CONTIGUOUS_AFTER_MEMBER \ - JSON_CONTIGUOUS_WS(); \ +#define JSON_CONTIGUOUS_AFTER_MEMBER \ + JSON_CONTIGUOUS_WS(); \ if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ',')) /* NOLINT(readability/braces) */ \ - { \ - ++p; \ - JSON_CONTIGUOUS_WS(); \ - goto object_key; \ - } \ + { \ + ++p; \ + JSON_CONTIGUOUS_WS(); \ + goto object_key; \ + } \ if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == '}')) /* NOLINT(readability/braces) */ \ - { \ - ++p; \ - if (JSON_HEDLEY_UNLIKELY(!report(sax->end_object()))) \ - { \ - return false; \ - } \ - goto close; \ - } \ + { \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_object()))) \ + { \ + return false; \ + } \ + goto close; \ + } \ return false; array_first: @@ -387,6 +387,12 @@ done: return KeepValues; } + /// the byte at @a p, or 0 at the end @a e (no JSON token starts with 0) + static JSON_HEDLEY_ALWAYS_INLINE uchar current(const uchar* p, const uchar* e) noexcept + { + return p != e ? *p : 0; + } + /// the first byte at or after p that is not whitespace, or e static JSON_HEDLEY_NEVER_INLINE const uchar* skip_whitespace(const uchar* p, const uchar* e) noexcept { @@ -436,7 +442,7 @@ done: /// a JSON text that is a single scalar bool scalar(const uchar*& p) { - switch (p != end ? *p : 0) + switch (current(p, end)) { case '"': return string(p, false); @@ -624,7 +630,7 @@ done: contiguous path (lexer::scan_number_bulk_contiguous()) Integers that do not fit their type become floats; floats that overflow - are an error, which the library parser reports. + are an error, which the default parser reports. */ JSON_HEDLEY_ALWAYS_INLINE bool number(const uchar*& pos) { @@ -773,7 +779,7 @@ done: } if (JSON_HEDLEY_UNLIKELY(!std::isfinite(value))) { - return false; // the library parser reports the overflow + return false; // the default parser reports the overflow } return report(sax->number_float(value, number_text(data, data + i))); } @@ -811,7 +817,7 @@ struct can_parse_contiguous @brief report the input behind @a adapter, one block of memory, to @a sax @return whether the input is valid and was reported completely; if not, - nothing was consumed from @a adapter, and the caller has the library + nothing was consumed from @a adapter, and the caller has the default parser report the error */ template @@ -820,7 +826,7 @@ bool parse_contiguous(InputAdapterType& adapter, SAX& sax, std::true_type /*cont const std::size_t size = adapter.bulk_remaining(); if (size == 0) { - return false; // the library parser reports the empty input + return false; // the default parser reports the empty input } const auto* const first = reinterpret_cast(adapter.bulk_data()); contiguous_parser p(first, first + size, &sax); @@ -831,7 +837,7 @@ bool parse_contiguous(InputAdapterType& adapter, SAX& sax, std::true_type /*cont @brief a SAX handler that only reports a parse error, as the handler of parse() does: it throws the exception, unless exceptions are not allowed -After contiguous_parser has rejected an input, the library parser is run again +After contiguous_parser has rejected an input, the default parser is run again with this handler to report the error: the error, its message and its position are those of parse(), without building the values read up to it. */ @@ -868,7 +874,7 @@ struct sax_ignores_number_text> : std::tr rejected, as parse() reports it: throw its exception, or, without exceptions, return true -@return false if the library parser accepts the input after all (which would +@return false if the default parser accepts the input after all (which would be a disagreement of the two parsers): the caller then parses it */ template @@ -922,7 +928,7 @@ bool parse_contiguous(InputAdapterType& /*adapter*/, BasicJsonType& /*result*/, @return whether @a accepted is the result of accept(); false if the input is not one block of memory, in which case the caller checks it. - contiguous_parser accepts exactly what the library parser accepts. + contiguous_parser accepts exactly what the default parser accepts. */ template bool accept_contiguous(InputAdapterType& adapter, bool& accepted, std::true_type /*contiguous*/) @@ -943,9 +949,9 @@ bool accept_contiguous(InputAdapterType& /*adapter*/, bool& /*accepted*/, std::f handler of the user After contiguous_parser has delivered the events of the input to the user's -handler and rejected it, the library parser is run again with this handler: +handler and rejected it, the default parser is run again with this handler: the user's handler gets the error, its message and its position from the -library parser, as before, but no event twice. +default parser, as before, but no event twice. */ template class json_sax_error_forwarder : public json_sax_acceptor @@ -962,7 +968,7 @@ class json_sax_error_forwarder : public json_sax_acceptor return user->parse_error(position, last_token, ex); } - /// whether the library parser reported an error + /// whether the default parser reported an error bool reported() const noexcept { return forwarded; @@ -981,8 +987,8 @@ struct sax_ignores_number_text> : s a user's handler @a sax contiguous_parser reports to @a sax what it reads, and stops at the token that -the library parser would reject, before reporting it: the events up to an error -are the same. The error itself is then reported by the library parser (see +the default parser would reject, before reporting it: the events up to an error +are the same. The error itself is then reported by the default parser (see json_sax_error_forwarder). @return whether @a result is the result of sax_parse(); false if the input is @@ -1003,14 +1009,14 @@ bool sax_parse_contiguous(InputAdapterType& adapter, SAX* sax, bool& result, std } if (p.aborted()) { - // the handler returned false, as with the library parser + // the handler returned false, as with the default parser result = false; return true; } } json_sax_error_forwarder forwarder(sax); result = parser(InputAdapterType(adapter), nullptr, true, false, false).sax_parse(&forwarder, true); - // the library parser must reject the input as well (the events after the + // the default parser must reject the input as well (the events after the // byte contiguous_parser stopped at cannot be delivered any more) JSON_ASSERT(forwarder.reported()); return true; diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 0f808e1ec..bd06811d9 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -3111,7 +3111,7 @@ public: { basic_json result; // JSON in one block of memory, with the default options, is read by - // the faster contiguous_parser; on an error, the library parser reads + // the faster contiguous_parser; on an error, the default parser reads // it again to report it (see contiguous_parser) using contiguous = std::integral_constant::type>::value>; if (cb == nullptr && !ignore_comments && !ignore_trailing_commas @@ -3129,8 +3129,8 @@ public: const bool ignore_comments, const bool ignore_trailing_commas) { - // JSON in one block of memory is checked by the faster - // contiguous_parser (see parse_impl()) + // JSON in one block of memory, with the default options, is checked by + // the faster contiguous_parser (see parse_impl()) using adapter_t = typename std::decay::type; bool accepted = false; if (!ignore_comments && !ignore_trailing_commas @@ -3154,8 +3154,9 @@ public: { if (format == input_format_t::json) { - // JSON in one block of memory is read by the faster - // contiguous_parser, with the default options (see parse_impl()) + // JSON in one block of memory, with the default options, is read by + // the faster contiguous_parser; on an error, the default parser + // reads it again to report it (see parse_impl()) bool result = false; if (strict && !ignore_comments && !ignore_trailing_commas && detail::sax_parse_contiguous(ia, sax, result, std::integral_constant::value> {})) diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index b5801e12a..9811a1375 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -23903,8 +23903,8 @@ namespace detail @brief a parser for JSON text in one contiguous block, for parse(), accept(), and sax_parse() -The parser and the lexer exchange one token at a time, and the lexer keeps its -state (the current character, a pending unget, where the token starts) in +The default parser and lexer exchange one token at a time, and the lexer keeps +its state (the current character, a pending unget, where the token starts) in members, which every character pays for. This parser reads the block through a local pointer instead, and the grammar, the whitespace and the tokens are one loop, after yyjson's reader: the state stays in registers, and the expected @@ -23912,13 +23912,13 @@ next token is tested where it is expected instead of being dispatched on. It reports what it reads to a SAX handler: the one parse() uses to build the value, the one accept() uses, or the user's handler of sax_parse(). On a byte -the library parser would reject, it stops before reporting the token, and -returns false: the events up to there are those the library parser reports. -The caller then runs the library parser on the same input to report the error +the default parser would reject, it stops before reporting the token, and +returns false: the events up to there are those the default parser reports. +The caller then runs the default parser on the same input to report the error with its exact message and position (report_contiguous_error(), json_sax_error_forwarder), and parse() builds no value from what was read. -What is accepted, and the values reported, are those of the library parser +What is accepted, and the values reported, are those of the default parser with default options: the same number grammar and conversions, the same UTF-8 and escape rules, a UTF-8 byte order mark at the start, and, without JSON_STRICT_NUL_HANDLING, a NUL byte as the end of the input. @@ -23972,28 +23972,28 @@ class contiguous_parser std::vector outer; // the current byte, or 0 at the end (no JSON token starts with 0) -#define JSON_CONTIGUOUS_CUR() (p != e ? *p : 0) +#define JSON_CONTIGUOUS_CUR() current(p, e) // skip whitespace; most tokens have none before them -#define JSON_CONTIGUOUS_WS() \ - do \ - { \ - if (JSON_HEDLEY_UNLIKELY(JSON_CONTIGUOUS_CUR() <= ' ')) \ - { \ - p = skip_whitespace(p, e); \ - } \ - } \ +#define JSON_CONTIGUOUS_WS() \ + do \ + { \ + if (JSON_HEDLEY_UNLIKELY(JSON_CONTIGUOUS_CUR() <= ' ')) \ + { \ + p = skip_whitespace(p, e); \ + } \ + } \ while (false) -#define JSON_CONTIGUOUS_OPEN(is_object) \ - do \ - { \ +#define JSON_CONTIGUOUS_OPEN(is_object) \ + do \ + { \ if (JSON_HEDLEY_UNLIKELY(depth != 0 && (depth % 64) == 0)) /* NOLINT(readability/braces) */ \ - { \ - outer.push_back(kinds); \ - kinds = 0; \ - } \ - kinds = (kinds << 1u) | static_cast(is_object); \ - ++depth; \ - } \ + { \ + outer.push_back(kinds); \ + kinds = 0; \ + } \ + kinds = (kinds << 1u) | static_cast(is_object); \ + ++depth; \ + } \ while (false) JSON_CONTIGUOUS_WS(); @@ -24026,107 +24026,107 @@ class contiguous_parser // a value in an array or an object: expanded once for each, so that // each dispatch has its own branch history (arrays tend to hold values // of one kind) and the code after a scalar knows where it is -#define JSON_CONTIGUOUS_VALUE \ - switch (JSON_CONTIGUOUS_CUR()) \ - { \ - case '"': \ - if (JSON_HEDLEY_UNLIKELY(!string(p, false))) \ - { \ - return false; \ - } \ - break; \ - case '{': \ - ++p; \ - if (JSON_HEDLEY_UNLIKELY(!report(sax->start_object(detail::unknown_size())))) \ - { \ - return false; \ - } \ - JSON_CONTIGUOUS_OPEN(1u); \ - goto object_first; \ - case '[': \ - ++p; \ - if (JSON_HEDLEY_UNLIKELY(!report(sax->start_array(detail::unknown_size())))) \ - { \ - return false; \ - } \ - JSON_CONTIGUOUS_OPEN(0u); \ - goto array_first; \ - case '-': \ - case '0': \ - case '1': \ - case '2': \ - case '3': \ - case '4': \ - case '5': \ - case '6': \ - case '7': \ - case '8': \ - case '9': \ - if (JSON_HEDLEY_UNLIKELY(!number(p))) \ - { \ - return false; \ - } \ - break; \ - case 't': \ - if (JSON_HEDLEY_UNLIKELY(!literal(p, "true") || !report(sax->boolean(true)))) \ - { \ - return false; \ - } \ - break; \ - case 'f': \ - if (JSON_HEDLEY_UNLIKELY(!literal(p, "false") || !report(sax->boolean(false)))) \ - { \ - return false; \ - } \ - break; \ - case 'n': \ - if (JSON_HEDLEY_UNLIKELY(!literal(p, "null") || !report(sax->null()))) \ - { \ - return false; \ - } \ - break; \ - default: \ - return false; \ +#define JSON_CONTIGUOUS_VALUE \ + switch (JSON_CONTIGUOUS_CUR()) \ + { \ + case '"': \ + if (JSON_HEDLEY_UNLIKELY(!string(p, false))) \ + { \ + return false; \ + } \ + break; \ + case '{': \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_object(detail::unknown_size())))) \ + { \ + return false; \ + } \ + JSON_CONTIGUOUS_OPEN(1u); \ + goto object_first; \ + case '[': \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->start_array(detail::unknown_size())))) \ + { \ + return false; \ + } \ + JSON_CONTIGUOUS_OPEN(0u); \ + goto array_first; \ + case '-': \ + case '0': \ + case '1': \ + case '2': \ + case '3': \ + case '4': \ + case '5': \ + case '6': \ + case '7': \ + case '8': \ + case '9': \ + if (JSON_HEDLEY_UNLIKELY(!number(p))) \ + { \ + return false; \ + } \ + break; \ + case 't': \ + if (JSON_HEDLEY_UNLIKELY(!literal(p, "true") || !report(sax->boolean(true)))) \ + { \ + return false; \ + } \ + break; \ + case 'f': \ + if (JSON_HEDLEY_UNLIKELY(!literal(p, "false") || !report(sax->boolean(false)))) \ + { \ + return false; \ + } \ + break; \ + case 'n': \ + if (JSON_HEDLEY_UNLIKELY(!literal(p, "null") || !report(sax->null()))) \ + { \ + return false; \ + } \ + break; \ + default: \ + return false; \ } // what may follow a value in an array: a comma and the next value, or the end -#define JSON_CONTIGUOUS_AFTER_ELEMENT \ - JSON_CONTIGUOUS_WS(); \ +#define JSON_CONTIGUOUS_AFTER_ELEMENT \ + JSON_CONTIGUOUS_WS(); \ if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ',')) /* NOLINT(readability/braces) */ \ - { \ - ++p; \ - JSON_CONTIGUOUS_WS(); \ - goto array_value; \ - } \ + { \ + ++p; \ + JSON_CONTIGUOUS_WS(); \ + goto array_value; \ + } \ if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ']')) /* NOLINT(readability/braces) */ \ - { \ - ++p; \ - if (JSON_HEDLEY_UNLIKELY(!report(sax->end_array()))) \ - { \ - return false; \ - } \ - goto close; \ - } \ + { \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_array()))) \ + { \ + return false; \ + } \ + goto close; \ + } \ return false; // what may follow a value in an object: a comma and the next member, or the end -#define JSON_CONTIGUOUS_AFTER_MEMBER \ - JSON_CONTIGUOUS_WS(); \ +#define JSON_CONTIGUOUS_AFTER_MEMBER \ + JSON_CONTIGUOUS_WS(); \ if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == ',')) /* NOLINT(readability/braces) */ \ - { \ - ++p; \ - JSON_CONTIGUOUS_WS(); \ - goto object_key; \ - } \ + { \ + ++p; \ + JSON_CONTIGUOUS_WS(); \ + goto object_key; \ + } \ if (JSON_HEDLEY_LIKELY(JSON_CONTIGUOUS_CUR() == '}')) /* NOLINT(readability/braces) */ \ - { \ - ++p; \ - if (JSON_HEDLEY_UNLIKELY(!report(sax->end_object()))) \ - { \ - return false; \ - } \ - goto close; \ - } \ + { \ + ++p; \ + if (JSON_HEDLEY_UNLIKELY(!report(sax->end_object()))) \ + { \ + return false; \ + } \ + goto close; \ + } \ return false; array_first: @@ -24253,6 +24253,12 @@ done: return KeepValues; } + /// the byte at @a p, or 0 at the end @a e (no JSON token starts with 0) + static JSON_HEDLEY_ALWAYS_INLINE uchar current(const uchar* p, const uchar* e) noexcept + { + return p != e ? *p : 0; + } + /// the first byte at or after p that is not whitespace, or e static JSON_HEDLEY_NEVER_INLINE const uchar* skip_whitespace(const uchar* p, const uchar* e) noexcept { @@ -24302,7 +24308,7 @@ done: /// a JSON text that is a single scalar bool scalar(const uchar*& p) { - switch (p != end ? *p : 0) + switch (current(p, end)) { case '"': return string(p, false); @@ -24490,7 +24496,7 @@ done: contiguous path (lexer::scan_number_bulk_contiguous()) Integers that do not fit their type become floats; floats that overflow - are an error, which the library parser reports. + are an error, which the default parser reports. */ JSON_HEDLEY_ALWAYS_INLINE bool number(const uchar*& pos) { @@ -24639,7 +24645,7 @@ done: } if (JSON_HEDLEY_UNLIKELY(!std::isfinite(value))) { - return false; // the library parser reports the overflow + return false; // the default parser reports the overflow } return report(sax->number_float(value, number_text(data, data + i))); } @@ -24677,7 +24683,7 @@ struct can_parse_contiguous @brief report the input behind @a adapter, one block of memory, to @a sax @return whether the input is valid and was reported completely; if not, - nothing was consumed from @a adapter, and the caller has the library + nothing was consumed from @a adapter, and the caller has the default parser report the error */ template @@ -24686,7 +24692,7 @@ bool parse_contiguous(InputAdapterType& adapter, SAX& sax, std::true_type /*cont const std::size_t size = adapter.bulk_remaining(); if (size == 0) { - return false; // the library parser reports the empty input + return false; // the default parser reports the empty input } const auto* const first = reinterpret_cast(adapter.bulk_data()); contiguous_parser p(first, first + size, &sax); @@ -24697,7 +24703,7 @@ bool parse_contiguous(InputAdapterType& adapter, SAX& sax, std::true_type /*cont @brief a SAX handler that only reports a parse error, as the handler of parse() does: it throws the exception, unless exceptions are not allowed -After contiguous_parser has rejected an input, the library parser is run again +After contiguous_parser has rejected an input, the default parser is run again with this handler to report the error: the error, its message and its position are those of parse(), without building the values read up to it. */ @@ -24734,7 +24740,7 @@ struct sax_ignores_number_text> : std::tr rejected, as parse() reports it: throw its exception, or, without exceptions, return true -@return false if the library parser accepts the input after all (which would +@return false if the default parser accepts the input after all (which would be a disagreement of the two parsers): the caller then parses it */ template @@ -24788,7 +24794,7 @@ bool parse_contiguous(InputAdapterType& /*adapter*/, BasicJsonType& /*result*/, @return whether @a accepted is the result of accept(); false if the input is not one block of memory, in which case the caller checks it. - contiguous_parser accepts exactly what the library parser accepts. + contiguous_parser accepts exactly what the default parser accepts. */ template bool accept_contiguous(InputAdapterType& adapter, bool& accepted, std::true_type /*contiguous*/) @@ -24809,9 +24815,9 @@ bool accept_contiguous(InputAdapterType& /*adapter*/, bool& /*accepted*/, std::f handler of the user After contiguous_parser has delivered the events of the input to the user's -handler and rejected it, the library parser is run again with this handler: +handler and rejected it, the default parser is run again with this handler: the user's handler gets the error, its message and its position from the -library parser, as before, but no event twice. +default parser, as before, but no event twice. */ template class json_sax_error_forwarder : public json_sax_acceptor @@ -24828,7 +24834,7 @@ class json_sax_error_forwarder : public json_sax_acceptor return user->parse_error(position, last_token, ex); } - /// whether the library parser reported an error + /// whether the default parser reported an error bool reported() const noexcept { return forwarded; @@ -24847,8 +24853,8 @@ struct sax_ignores_number_text> : s a user's handler @a sax contiguous_parser reports to @a sax what it reads, and stops at the token that -the library parser would reject, before reporting it: the events up to an error -are the same. The error itself is then reported by the library parser (see +the default parser would reject, before reporting it: the events up to an error +are the same. The error itself is then reported by the default parser (see json_sax_error_forwarder). @return whether @a result is the result of sax_parse(); false if the input is @@ -24869,14 +24875,14 @@ bool sax_parse_contiguous(InputAdapterType& adapter, SAX* sax, bool& result, std } if (p.aborted()) { - // the handler returned false, as with the library parser + // the handler returned false, as with the default parser result = false; return true; } } json_sax_error_forwarder forwarder(sax); result = parser(InputAdapterType(adapter), nullptr, true, false, false).sax_parse(&forwarder, true); - // the library parser must reject the input as well (the events after the + // the default parser must reject the input as well (the events after the // byte contiguous_parser stopped at cannot be delivered any more) JSON_ASSERT(forwarder.reported()); return true; @@ -41973,7 +41979,7 @@ public: { basic_json result; // JSON in one block of memory, with the default options, is read by - // the faster contiguous_parser; on an error, the library parser reads + // the faster contiguous_parser; on an error, the default parser reads // it again to report it (see contiguous_parser) using contiguous = std::integral_constant::type>::value>; if (cb == nullptr && !ignore_comments && !ignore_trailing_commas @@ -41991,8 +41997,8 @@ public: const bool ignore_comments, const bool ignore_trailing_commas) { - // JSON in one block of memory is checked by the faster - // contiguous_parser (see parse_impl()) + // JSON in one block of memory, with the default options, is checked by + // the faster contiguous_parser (see parse_impl()) using adapter_t = typename std::decay::type; bool accepted = false; if (!ignore_comments && !ignore_trailing_commas @@ -42016,8 +42022,9 @@ public: { if (format == input_format_t::json) { - // JSON in one block of memory is read by the faster - // contiguous_parser, with the default options (see parse_impl()) + // JSON in one block of memory, with the default options, is read by + // the faster contiguous_parser; on an error, the default parser + // reads it again to report it (see parse_impl()) bool result = false; if (strict && !ignore_comments && !ignore_trailing_commas && detail::sax_parse_contiguous(ia, sax, result, std::integral_constant::value> {})) diff --git a/tests/src/unit-contiguous_parser.cpp b/tests/src/unit-contiguous_parser.cpp index bcebe2047..4c3ec39b4 100644 --- a/tests/src/unit-contiguous_parser.cpp +++ b/tests/src/unit-contiguous_parser.cpp @@ -201,7 +201,7 @@ void check_same_events(const std::string& text, const std::list& bytes) CHECK(r1 == r2); CHECK(contiguous.log() == byte_path.log()); } - // a handler that asks to go on after an error (the library parser then + // a handler that asks to go on after an error (the default parser then // reports what follows, e.g., the missing end of the input) event_log contiguous(static_cast(-1), true); event_log byte_path(static_cast(-1), true); @@ -545,7 +545,7 @@ TEST_CASE("contiguous parser") CHECK_FALSE(narrow_json::accept("34028236e31")); } - SECTION("callbacks, comments and trailing commas take the library parser") + SECTION("callbacks, comments and trailing commas take the default parser") { // (the results are those of the options; the contiguous parser does not support them) CHECK(json::parse("[1,]", nullptr, true, false, true) == json({1}));