From 0115bb47b362867add4615aeb8d4aa68f9f142b9 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 30 Oct 2026 15:17:24 +0100 Subject: [PATCH 1/7] Name the source and license of adapted code in the file comments (#5965) The files with code adapted from fast_float, Zmij and simdjson carried the authors' SPDX copyright lines, but their file comments did not say what was taken or under which license: number_parse.hpp, to_chars.hpp, simd_scan.hpp and view/simd.hpp now do, as zmij.hpp and the yyjson-based view files already did. README and the license page now also name fast_float's eight-digit parsing and the lexer's use of simdjson's UTF-8 check. Comments only. Signed-off-by: Niels Lohmann --- README.md | 4 ++-- docs/mkdocs/docs/home/license.md | 4 ++-- .../nlohmann/detail/conversions/to_chars.hpp | 5 +++++ .../nlohmann/detail/input/number_parse.hpp | 7 +++++++ include/nlohmann/detail/input/simd_scan.hpp | 7 +++++++ include/nlohmann/detail/view/simd.hpp | 7 +++++++ single_include/nlohmann/json.hpp | 19 +++++++++++++++++++ single_include/nlohmann/json_view.hpp | 7 +++++++ 8 files changed, 56 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 87c70fc96..2551df959 100644 --- a/README.md +++ b/README.md @@ -1408,9 +1408,9 @@ THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR I - The class contains a port of the shortest double-to-decimal conversion of [Żmij](https://github.com/vitaut/zmij) by Victor Zverovich, including the conversion of the digits to text by Xiang JunBo and the SIMD instruction sequence of Dougall Johnson, which is licensed under the [MIT License](https://opensource.org/licenses/MIT) (see above). Copyright © 2025 [Victor Zverovich](https://github.com/vitaut) - The class contains a copy of [Hedley](https://nemequ.github.io/hedley/) from Evan Nemerson which is licensed as [CC0-1.0](https://creativecommons.org/publicdomain/zero/1.0/). - The class contains parts of [Google Abseil](https://github.com/abseil/abseil-cpp) which is licensed under the [Apache 2.0 License](https://opensource.org/licenses/Apache-2.0). -- The class contains an adapted version of the Eisel-Lemire algorithm, its table of powers of five, and its digit comparison for long numbers from [fast_float](https://github.com/fastfloat/fast_float) by Daniel Lemire and contributors, which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here), the Apache 2.0 License, and the Boost Software License. Copyright © 2021 The fast_float authors +- The class contains an adapted version of the Eisel-Lemire algorithm, its table of powers of five, its digit comparison for long numbers, and its parsing of eight digits at a time from [fast_float](https://github.com/fastfloat/fast_float) by Daniel Lemire and contributors, which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here), the Apache 2.0 License, and the Boost Software License. Copyright © 2021 The fast_float authors - The view's parser (``) contains techniques and code adapted from [yyjson](https://github.com/ibireme/yyjson) by YaoYuan, which is licensed under the [MIT License](https://opensource.org/licenses/MIT) (see above): table-driven decoding of `\u` escapes and fixed-offset unrolled checks. -- The view's parser (``) validates non-ASCII strings with the vector UTF-8 check of [simdjson](https://github.com/simdjson/simdjson) by Daniel Lemire, Geoff Langdale, John Keiser, and contributors (its "lookup4" algorithm and tables, after J. Keiser and D. Lemire, "Validating UTF-8 In Less Than One Instruction Per Byte", 2021), which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here) and the Apache 2.0 License. Copyright © 2018-2025 The simdjson authors +- The class's lexer and the view's parser (``) validate non-ASCII strings with the vector UTF-8 check of [simdjson](https://github.com/simdjson/simdjson) by Daniel Lemire, Geoff Langdale, John Keiser, and contributors (its "lookup4" algorithm and tables, after J. Keiser and D. Lemire, "Validating UTF-8 In Less Than One Instruction Per Byte", 2021), which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here) and the Apache 2.0 License. Copyright © 2018-2025 The simdjson authors REUSE Software diff --git a/docs/mkdocs/docs/home/license.md b/docs/mkdocs/docs/home/license.md index 714e971ed..c0bb94633 100644 --- a/docs/mkdocs/docs/home/license.md +++ b/docs/mkdocs/docs/home/license.md @@ -22,8 +22,8 @@ The class contains a port of the shortest double-to-decimal conversion of [Żmij The class contains a copy of [Hedley](https://nemequ.github.io/hedley/) from Evan Nemerson which is licensed as [CC0-1.0](https://creativecommons.org/publicdomain/zero/1.0/). -The class contains an adapted version of the Eisel-Lemire algorithm, its table of powers of five, and its digit comparison for long numbers from [fast_float](https://github.com/fastfloat/fast_float) by Daniel Lemire and contributors, which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here), the Apache 2.0 License, and the Boost Software License. Copyright © 2021 The fast_float authors +The class contains an adapted version of the Eisel-Lemire algorithm, its table of powers of five, its digit comparison for long numbers, and its parsing of eight digits at a time from [fast_float](https://github.com/fastfloat/fast_float) by Daniel Lemire and contributors, which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here), the Apache 2.0 License, and the Boost Software License. Copyright © 2021 The fast_float authors The view's parser (``) contains techniques and code adapted from [yyjson](https://github.com/ibireme/yyjson) by YaoYuan, which is licensed under the [MIT License](https://opensource.org/licenses/MIT) (see above): table-driven decoding of `\u` escapes and fixed-offset unrolled checks. -The view's parser (``) validates non-ASCII strings with the vector UTF-8 check of [simdjson](https://github.com/simdjson/simdjson) by Daniel Lemire, Geoff Langdale, John Keiser, and contributors (its "lookup4" algorithm and tables, after J. Keiser and D. Lemire, "Validating UTF-8 In Less Than One Instruction Per Byte", 2021), which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here) and the Apache 2.0 License. Copyright © 2018-2025 The simdjson authors +The class's lexer and the view's parser (``) validate non-ASCII strings with the vector UTF-8 check of [simdjson](https://github.com/simdjson/simdjson) by Daniel Lemire, Geoff Langdale, John Keiser, and contributors (its "lookup4" algorithm and tables, after J. Keiser and D. Lemire, "Validating UTF-8 In Less Than One Instruction Per Byte", 2021), which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here) and the Apache 2.0 License. Copyright © 2018-2025 The simdjson authors diff --git a/include/nlohmann/detail/conversions/to_chars.hpp b/include/nlohmann/detail/conversions/to_chars.hpp index a977e797b..6cfb9b8e7 100644 --- a/include/nlohmann/detail/conversions/to_chars.hpp +++ b/include/nlohmann/detail/conversions/to_chars.hpp @@ -19,6 +19,11 @@ #include // conditional +// The shortest-digit conversion of doubles (see zmij.hpp) and the writing of +// its digits, including the conversion of the digits to text by Xiang JunBo +// and the SIMD instruction sequence by Dougall Johnson, are ported from Zmij +// by Victor Zverovich (https://github.com/vitaut/zmij, MIT license). +// // SSE2 (every x86-64 CPU) and NEON (every 64-bit Arm CPU) convert the 16 // digits of a double at once (unless JSON_NO_SIMD selects the portable code) #if !defined(JSON_NO_SIMD) && (defined(__x86_64__) || (defined(_M_X64) && !defined(_M_ARM64EC))) diff --git a/include/nlohmann/detail/input/number_parse.hpp b/include/nlohmann/detail/input/number_parse.hpp index dce0105de..852e1a6c8 100644 --- a/include/nlohmann/detail/input/number_parse.hpp +++ b/include/nlohmann/detail/input/number_parse.hpp @@ -44,6 +44,13 @@ // use std::from_chars or std::strtold. They are free functions so the lexer // stays focused on scanning (see lexer::convert_number()) and so that other // parsers of JSON text can convert tokens exactly like it does. +// +// The Eisel-Lemire conversion (eisel_lemire()), the digit comparison for the +// cases it cannot decide, and the eight-digits-at-a-time helpers +// (is_eight_digits(), parse_eight_digits()) are adapted from fast_float +// (https://github.com/fastfloat/fast_float, Daniel Lemire and contributors, +// available under the MIT, Apache 2.0, and Boost Software licenses; used here +// under the MIT license). The table of powers of five is in pow5_table.hpp. NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail diff --git a/include/nlohmann/detail/input/simd_scan.hpp b/include/nlohmann/detail/input/simd_scan.hpp index 8caed1814..3abb911ee 100644 --- a/include/nlohmann/detail/input/simd_scan.hpp +++ b/include/nlohmann/detail/input/simd_scan.hpp @@ -30,6 +30,13 @@ // NLOHMANN_JSON_NO_TARGET_ATTRIBUTE (for compilers that ignore the target // attribute and then reject the SSSE3 intrinsics) keeps the check portable // without disabling the SSE2 code. +// +// The vector UTF-8 check (utf8_lookup4) is adapted from simdjson +// (https://github.com/simdjson/simdjson, Daniel Lemire, Geoff Langdale, John +// Keiser, and contributors, available under the Apache 2.0 and MIT licenses; +// used here under the MIT license): the "lookup4" algorithm and tables of +// J. Keiser and D. Lemire, "Validating UTF-8 In Less Than One Instruction Per +// Byte" (2021). #if !defined(JSON_NO_SIMD) && defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) && defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ #include #define JSON_SCAN_NEON 1 diff --git a/include/nlohmann/detail/view/simd.hpp b/include/nlohmann/detail/view/simd.hpp index dbfbdc7a9..9dede9b09 100644 --- a/include/nlohmann/detail/view/simd.hpp +++ b/include/nlohmann/detail/view/simd.hpp @@ -29,6 +29,13 @@ // NLOHMANN_VIEW_NO_TARGET_ATTRIBUTE (set by the GCC module interface, where GCC // ignores the target attribute and then rejects the SSSE3 intrinsics) keeps the // check portable without disabling the SSE2 code. +// +// The vector UTF-8 check (utf8_lookup4) is adapted from simdjson +// (https://github.com/simdjson/simdjson, Daniel Lemire, Geoff Langdale, John +// Keiser, and contributors, available under the Apache 2.0 and MIT licenses; +// used here under the MIT license): the "lookup4" algorithm and tables of +// J. Keiser and D. Lemire, "Validating UTF-8 In Less Than One Instruction Per +// Byte" (2021). #if !defined(JSON_NO_SIMD) && defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) && NLOHMANN_VIEW_LITTLE_ENDIAN #include #define NLOHMANN_VIEW_NEON 1 diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 98d6045fa..10b42dac1 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -10970,6 +10970,13 @@ NLOHMANN_JSON_NAMESPACE_END // use std::from_chars or std::strtold. They are free functions so the lexer // stays focused on scanning (see lexer::convert_number()) and so that other // parsers of JSON text can convert tokens exactly like it does. +// +// The Eisel-Lemire conversion (eisel_lemire()), the digit comparison for the +// cases it cannot decide, and the eight-digits-at-a-time helpers +// (is_eight_digits(), parse_eight_digits()) are adapted from fast_float +// (https://github.com/fastfloat/fast_float, Daniel Lemire and contributors, +// available under the MIT, Apache 2.0, and Boost Software licenses; used here +// under the MIT license). The table of powers of five is in pow5_table.hpp. NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -12078,6 +12085,13 @@ NLOHMANN_JSON_NAMESPACE_END // NLOHMANN_JSON_NO_TARGET_ATTRIBUTE (for compilers that ignore the target // attribute and then reject the SSSE3 intrinsics) keeps the check portable // without disabling the SSE2 code. +// +// The vector UTF-8 check (utf8_lookup4) is adapted from simdjson +// (https://github.com/simdjson/simdjson, Daniel Lemire, Geoff Langdale, John +// Keiser, and contributors, available under the Apache 2.0 and MIT licenses; +// used here under the MIT license): the "lookup4" algorithm and tables of +// J. Keiser and D. Lemire, "Validating UTF-8 In Less Than One Instruction Per +// Byte" (2021). #if !defined(JSON_NO_SIMD) && defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) && defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ #include #define JSON_SCAN_NEON 1 @@ -32495,6 +32509,11 @@ NLOHMANN_JSON_NAMESPACE_END #include // conditional +// The shortest-digit conversion of doubles (see zmij.hpp) and the writing of +// its digits, including the conversion of the digits to text by Xiang JunBo +// and the SIMD instruction sequence by Dougall Johnson, are ported from Zmij +// by Victor Zverovich (https://github.com/vitaut/zmij, MIT license). +// // SSE2 (every x86-64 CPU) and NEON (every 64-bit Arm CPU) convert the 16 // digits of a double at once (unless JSON_NO_SIMD selects the portable code) #if !defined(JSON_NO_SIMD) && (defined(__x86_64__) || (defined(_M_X64) && !defined(_M_ARM64EC))) diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 90593b2d1..7992e2c78 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -688,6 +688,13 @@ NLOHMANN_JSON_NAMESPACE_END // NLOHMANN_VIEW_NO_TARGET_ATTRIBUTE (set by the GCC module interface, where GCC // ignores the target attribute and then rejects the SSSE3 intrinsics) keeps the // check portable without disabling the SSE2 code. +// +// The vector UTF-8 check (utf8_lookup4) is adapted from simdjson +// (https://github.com/simdjson/simdjson, Daniel Lemire, Geoff Langdale, John +// Keiser, and contributors, available under the Apache 2.0 and MIT licenses; +// used here under the MIT license): the "lookup4" algorithm and tables of +// J. Keiser and D. Lemire, "Validating UTF-8 In Less Than One Instruction Per +// Byte" (2021). #if !defined(JSON_NO_SIMD) && defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) && NLOHMANN_VIEW_LITTLE_ENDIAN #include #define NLOHMANN_VIEW_NEON 1 From 0b91f9ddd8314b3739f1a9ddf929eb5501e39136 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 30 Oct 2026 15:17:55 +0100 Subject: [PATCH 2/7] json_view: check_utf8 skips well-formed runs with the parser's string scan (#5926) * json_view: check_utf8 skips well-formed runs with the parser's string scan Strings of 32 bytes or more that are written into an editable document are validated with scan_string_run(), which skips well-formed UTF-8 16 bytes at a time; the byte-wise check runs only from where the scan found ill-formed UTF-8, to report the same type_error.316. Shorter strings keep the byte-wise check (inlined with a constant start, so that this path is the code it was), where the vector scan costs more than it saves. set()/push_back() of strings of 32 bytes or more take 0.29-0.76 of the time (x86-64 GCC, averaged over code layouts) and 0.25-0.80 (AArch64 clang); shorter strings take 1.5-4 % fewer instructions; the json_view rows of bench_edit execute at most 0.04 % more instructions. Signed-off-by: Niels Lohmann * tests: avoid sign conversion in json_view_edit UTF-8 check Signed-off-by: Niels Lohmann * json_view: use while in check_utf8_bytewise Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- include/nlohmann/detail/view/edit.hpp | 43 ++++++++- single_include/nlohmann/json_view.hpp | 44 +++++++++- tests/src/unit-json_view_edit.cpp | 120 ++++++++++++++++++++++++++ 3 files changed, 201 insertions(+), 6 deletions(-) diff --git a/include/nlohmann/detail/view/edit.hpp b/include/nlohmann/detail/view/edit.hpp index f47bf5985..c542c1141 100644 --- a/include/nlohmann/detail/view/edit.hpp +++ b/include/nlohmann/detail/view/edit.hpp @@ -26,6 +26,7 @@ #include #include #include +#include #include NLOHMANN_JSON_NAMESPACE_BEGIN @@ -46,10 +47,9 @@ struct first_true : std::integral_constant < int, 1 + firs /// (type_error.316 with the same message), so that an editable document /// only holds valid UTF-8: the error is at the first byte that no /// well-formed sequence can continue with (Unicode, Table 3-7). -inline void check_utf8(const char* s, std::size_t n) +NLOHMANN_VIEW_ALWAYS_INLINE void check_utf8_bytewise(const unsigned char* p, std::size_t n, std::size_t i) { - const auto* const p = reinterpret_cast(s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - for (std::size_t i = 0; i < n;) + while (i < n) { const unsigned char c = p[i]; if (c < 0x80) @@ -79,6 +79,43 @@ inline void check_utf8(const char* s, std::size_t n) } } +/// The long-string path of check_utf8(): skips well-formed runs with the +/// parser's string scan (which stops at a quote, a backslash, or a control +/// character, all legal here, and at ill-formed UTF-8); the byte-wise check +/// runs only from where the scan found ill-formed UTF-8, to report it. +NLOHMANN_VIEW_NOINLINE inline void check_utf8_scan(const unsigned char* p, std::size_t n) +{ + const unsigned char* const e = p + n; + const unsigned char* q = p; + for (;;) + { + q = scan_string_run(q, e); + if (q == e) + { + return; + } + if (*q >= 0x80) + { + check_utf8_bytewise(p, n, static_cast(q - p)); + return; // not reached: the scan stopped at ill-formed UTF-8 + } + ++q; // a quote, backslash, or control character + } +} + +inline void check_utf8(const char* s, std::size_t n) +{ + const auto* const p = reinterpret_cast(s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + if (NLOHMANN_VIEW_UNLIKELY(n >= 32)) + { + check_utf8_scan(p, n); + return; + } + // short strings: the vector scan costs more than it saves; (the loop is + // inlined with a constant start, so this path is the plain byte-wise check) + check_utf8_bytewise(p, n, 0); +} + /*! @brief the edits of an editable basic_json_document diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 7992e2c78..20de9f188 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -3765,6 +3765,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -3838,10 +3840,9 @@ struct first_true : std::integral_constant < int, 1 + firs /// (type_error.316 with the same message), so that an editable document /// only holds valid UTF-8: the error is at the first byte that no /// well-formed sequence can continue with (Unicode, Table 3-7). -inline void check_utf8(const char* s, std::size_t n) +NLOHMANN_VIEW_ALWAYS_INLINE void check_utf8_bytewise(const unsigned char* p, std::size_t n, std::size_t i) { - const auto* const p = reinterpret_cast(s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - for (std::size_t i = 0; i < n;) + while (i < n) { const unsigned char c = p[i]; if (c < 0x80) @@ -3871,6 +3872,43 @@ inline void check_utf8(const char* s, std::size_t n) } } +/// The long-string path of check_utf8(): skips well-formed runs with the +/// parser's string scan (which stops at a quote, a backslash, or a control +/// character, all legal here, and at ill-formed UTF-8); the byte-wise check +/// runs only from where the scan found ill-formed UTF-8, to report it. +NLOHMANN_VIEW_NOINLINE inline void check_utf8_scan(const unsigned char* p, std::size_t n) +{ + const unsigned char* const e = p + n; + const unsigned char* q = p; + for (;;) + { + q = scan_string_run(q, e); + if (q == e) + { + return; + } + if (*q >= 0x80) + { + check_utf8_bytewise(p, n, static_cast(q - p)); + return; // not reached: the scan stopped at ill-formed UTF-8 + } + ++q; // a quote, backslash, or control character + } +} + +inline void check_utf8(const char* s, std::size_t n) +{ + const auto* const p = reinterpret_cast(s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + if (NLOHMANN_VIEW_UNLIKELY(n >= 32)) + { + check_utf8_scan(p, n); + return; + } + // short strings: the vector scan costs more than it saves; (the loop is + // inlined with a constant start, so this path is the plain byte-wise check) + check_utf8_bytewise(p, n, 0); +} + /*! @brief the edits of an editable basic_json_document diff --git a/tests/src/unit-json_view_edit.cpp b/tests/src/unit-json_view_edit.cpp index 4a6fbe69d..1b1a24ec8 100644 --- a/tests/src/unit-json_view_edit.cpp +++ b/tests/src/unit-json_view_edit.cpp @@ -941,3 +941,123 @@ TEST_CASE("json_view edits: duplicate keys") CHECK(d.root()["o"]["d"].get() == 0); } } + +#if !defined(JSON_NOEXCEPTION) +TEST_CASE("json_view edits: check_utf8 agrees with the byte-wise check") +{ + namespace dv = nlohmann::detail::view; + + // "ok" or the exception message + const auto run = [](bool fast, const std::string & s) -> std::string + { + try + { + if (fast) + { + dv::check_utf8(s.data(), s.size()); + } + else + { + dv::check_utf8_bytewise(reinterpret_cast(s.data()), s.size(), 0); + } + } + catch (const json::type_error& e) + { + return {e.what()}; + } + return {"ok"}; + }; + std::size_t errors = 0; + const auto same = [&](const std::string & s) + { + const std::string a = run(true, s); + const std::string b = run(false, s); + if (a != b) + { + INFO("fast: " << a << ", byte-wise: " << b << ", length " << s.size()); + CHECK(a == b); + } + errors += b != "ok" ? 1u : 0u; + }; + + const std::vector pieces = + { + "a", "Z", " ", "\"", "\\", "\n", "\x01", "\x7f", "\xc3\xa4", "\xe4\xb8\xad", "\xf0\x9f\x98\x80", + "\x80", "\xbf", "\xc0\x80", "\xc1\xbf", "\xc2", "\xe0\x80\x80", "\xe0\x9f\xbf", "\xed\xa0\x80", "\xed\xbf\xbf", + "\xf0\x80\x80\x80", "\xf0\x8f\xbf\xbf", "\xf4\x90\x80\x80", "\xf4\x8f\xbf\xbf", "\xf5", "\xf8", "\xff", "\xfe", + "\xe4\xb8", "\xf0\x9f\x98", "\xf0\x9f", "\xf0" + }; + + // every piece behind prefixes of every length, and followed by more bytes + for (std::size_t prefix = 0; prefix <= 64; ++prefix) + { + for (const auto& piece : pieces) + { + same(std::string(prefix, 'x') + piece); + same(std::string(prefix, 'x') + piece + "yz"); + same(std::string(prefix, '"') + piece); + same(std::string(prefix, '\xc3') + piece); + same(std::string(prefix / 2, '\xc3').append(prefix / 2, '\xa4') + piece); + same(std::string(prefix, 'x') + "\xe4\xb8\xad" + piece); + } + } + // an invalid byte at every position of valid strings of every length + for (std::size_t len = 0; len <= 64; ++len) + { + const std::string ascii(len, 'a'); + std::string mixed; + while (mixed.size() < len) + { + if (mixed.size() % 7 == 0) + { + mixed += "\xc3\xa4"; + } + else if (mixed.size() % 11 == 0) + { + mixed += "\xf0\x9f\x98\x80"; + } + else + { + mixed += "a"; + } + } + same(ascii); + same(mixed); + for (std::size_t pos = 0; pos < len; ++pos) + { + for (const char bad : + {'\x80', '\xc0', '\xc2', '\xe0', '\xf4', '\xff', '"', '\\', '\x1f' + }) + { + std::string s = ascii; + s[pos] = bad; + same(s); + s = mixed; + s[pos] = bad; + same(s); + } + } + } + // random bytes, biased to the interesting ones + std::mt19937 gen(5869); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible + static const std::array interesting = {{0x00, 0x1f, 0x22, 0x5c, 0x7f, 0x80, 0x8f, 0x90, 0x9f, 0xa0, 0xbf, 0xc0, 0xc2, 0xdf, 0xe0, 0xe1, 0xed, 0xef, 0xf0, 0xf1, 0xf4, 0xf5, 0xff}}; + for (int i = 0; i < 100000; ++i) + { + std::string s(gen() % 80, 'a'); + for (auto& c : s) + { + const unsigned r = gen() % 8; + if (r == 0) + { + c = static_cast(gen() & 0xff); + } + else if (r == 1) + { + c = static_cast(interesting[gen() % interesting.size()]); + } + } + same(s); + } + CHECK(errors > 1000); +} +#endif From 28f71914b5d7352dbd25074a2e4636d64244cef0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:18:44 +0100 Subject: [PATCH 3/7] Bump egor-tensin/setup-mingw (#5963) Bumps [egor-tensin/setup-mingw](https://github.com/egor-tensin/setup-mingw) from 41b837e47d7f85214629d255b9c4bc3fcbe9fd63 to d54f5ef0ce585bd3fbb1f8a71b5b254f59dbf931. - [Release notes](https://github.com/egor-tensin/setup-mingw/releases) - [Changelog](https://github.com/egor-tensin/setup-mingw/blob/master/CHANGELOG.md) - [Commits](https://github.com/egor-tensin/setup-mingw/compare/41b837e47d7f85214629d255b9c4bc3fcbe9fd63...d54f5ef0ce585bd3fbb1f8a71b5b254f59dbf931) --- updated-dependencies: - dependency-name: egor-tensin/setup-mingw dependency-version: d54f5ef0ce585bd3fbb1f8a71b5b254f59dbf931 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/windows.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 6f1cf8cae..59647ed94 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -28,7 +28,7 @@ jobs: with: persist-credentials: false - name: Set up MinGW - uses: egor-tensin/setup-mingw@41b837e47d7f85214629d255b9c4bc3fcbe9fd63 # v3.0 + uses: egor-tensin/setup-mingw@d54f5ef0ce585bd3fbb1f8a71b5b254f59dbf931 # v3.0 with: platform: ${{ matrix.architecture }} version: 12.2.0 # https://github.com/egor-tensin/setup-mingw/issues/14 @@ -153,7 +153,7 @@ jobs: - name: Install Clang run: curl -fsSL -o LLVM${{ matrix.version }}.exe https://github.com/llvm/llvm-project/releases/download/llvmorg-${{ matrix.version }}/LLVM-${{ matrix.version }}-win64.exe ; 7z x LLVM${{ matrix.version }}.exe -y -o"C:/Program Files/LLVM" - name: Set up MinGW - uses: egor-tensin/setup-mingw@41b837e47d7f85214629d255b9c4bc3fcbe9fd63 # v3.0 + uses: egor-tensin/setup-mingw@d54f5ef0ce585bd3fbb1f8a71b5b254f59dbf931 # v3.0 with: platform: x64 version: 12.2.0 # https://github.com/egor-tensin/setup-mingw/issues/14 From c451fac1bcbb6f03d62a169c25487ae442ea84da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:18:56 +0100 Subject: [PATCH 4/7] Bump step-security/harden-runner from 2.22.1 to 2.22.2 (#5962) Bumps [step-security/harden-runner](https://github.com/step-security/harden-runner) from 2.22.1 to 2.22.2. - [Release notes](https://github.com/step-security/harden-runner/releases) - [Commits](https://github.com/step-security/harden-runner/compare/ccd8616d44fd3846e67624a50d5aad6d37bf2d25...dc2769abcc0fcfbadfc1df54081c1608579ef6b2) --- updated-dependencies: - dependency-name: step-security/harden-runner dependency-version: 2.22.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/cancel_closed_pr_runs.yml | 2 +- .github/workflows/check_amalgamation.yml | 4 ++-- .github/workflows/check_docs_links.yml | 2 +- .github/workflows/cifuzz.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/comment_check_amalgamation.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/flawfinder.yml | 2 +- .github/workflows/json_view_benchmarks.yml | 2 +- .github/workflows/labeler.yml | 2 +- .github/workflows/publish_documentation.yml | 2 +- .github/workflows/scorecards.yml | 2 +- .github/workflows/semgrep.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/ubuntu.yml | 10 +++++----- 15 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/cancel_closed_pr_runs.yml b/.github/workflows/cancel_closed_pr_runs.yml index 99010d81b..93d65312d 100644 --- a/.github/workflows/cancel_closed_pr_runs.yml +++ b/.github/workflows/cancel_closed_pr_runs.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/check_amalgamation.yml b/.github/workflows/check_amalgamation.yml index 910eeef0f..66af126fa 100644 --- a/.github/workflows/check_amalgamation.yml +++ b/.github/workflows/check_amalgamation.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit @@ -46,7 +46,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/check_docs_links.yml b/.github/workflows/check_docs_links.yml index bf174c245..12f5ad074 100644 --- a/.github/workflows/check_docs_links.yml +++ b/.github/workflows/check_docs_links.yml @@ -20,7 +20,7 @@ jobs: timeout-minutes: 30 steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index 43efd0e16..f5feb9a8f 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 48b8db457..b89043b10 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -27,7 +27,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/comment_check_amalgamation.yml b/.github/workflows/comment_check_amalgamation.yml index fc96d5c18..04bfaae88 100644 --- a/.github/workflows/comment_check_amalgamation.yml +++ b/.github/workflows/comment_check_amalgamation.yml @@ -20,7 +20,7 @@ jobs: pull-requests: write steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 333546437..bd2d295a4 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/flawfinder.yml b/.github/workflows/flawfinder.yml index 0c6e5fa9c..84bc4d6ba 100644 --- a/.github/workflows/flawfinder.yml +++ b/.github/workflows/flawfinder.yml @@ -31,7 +31,7 @@ jobs: security-events: write steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/json_view_benchmarks.yml b/.github/workflows/json_view_benchmarks.yml index 305f1ef35..d87357c42 100644 --- a/.github/workflows/json_view_benchmarks.yml +++ b/.github/workflows/json_view_benchmarks.yml @@ -54,7 +54,7 @@ jobs: timeout-minutes: 60 steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 21cf6c99c..35549b4e0 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/publish_documentation.yml b/.github/workflows/publish_documentation.yml index 979f2937f..12c55a0d4 100644 --- a/.github/workflows/publish_documentation.yml +++ b/.github/workflows/publish_documentation.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index dbebf3e23..4900d0607 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -40,7 +40,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 203a1416f..6b8713b41 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -36,7 +36,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 10d527504..41928ab20 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 8e7d8e0b9..a8cd05766 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -80,7 +80,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit @@ -112,7 +112,7 @@ jobs: target: [ci_test_single_header, ci_cppcheck, ci_cpplint, ci_reproducible_tests, ci_non_git_tests, ci_offline_testdata, ci_reuse_compliance, ci_test_valgrind] steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit @@ -178,7 +178,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit @@ -433,7 +433,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit @@ -456,7 +456,7 @@ jobs: target: [ci_test_examples, ci_test_build_documentation, ci_test_documentation_mermaid] steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit From e2cb15c479404f0f4590139529d8b97b66c97162 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:19:08 +0100 Subject: [PATCH 5/7] Bump the codeql-action group across 1 directory with 4 updates (#5961) Bumps the codeql-action group with 4 updates in the / directory: [github/codeql-action/init](https://github.com/github/codeql-action), [github/codeql-action/autobuild](https://github.com/github/codeql-action), [github/codeql-action/analyze](https://github.com/github/codeql-action) and [github/codeql-action/upload-sarif](https://github.com/github/codeql-action). Updates `github/codeql-action/init` from 4.39.0 to 4.39.1 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee...9b3254085494b55a0f7c4c2b3d2d52cc712ce370) Updates `github/codeql-action/autobuild` from 4.39.0 to 4.39.1 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee...9b3254085494b55a0f7c4c2b3d2d52cc712ce370) Updates `github/codeql-action/analyze` from 4.39.0 to 4.39.1 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee...9b3254085494b55a0f7c4c2b3d2d52cc712ce370) Updates `github/codeql-action/upload-sarif` from 4.39.0 to 4.39.1 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee...9b3254085494b55a0f7c4c2b3d2d52cc712ce370) --- updated-dependencies: - dependency-name: github/codeql-action/analyze dependency-version: 4.39.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: codeql-action - dependency-name: github/codeql-action/autobuild dependency-version: 4.39.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: codeql-action - dependency-name: github/codeql-action/init dependency-version: 4.39.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: codeql-action - dependency-name: github/codeql-action/upload-sarif dependency-version: 4.39.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: codeql-action ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 6 +++--- .github/workflows/flawfinder.yml | 2 +- .github/workflows/scorecards.yml | 2 +- .github/workflows/semgrep.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index b89043b10..61036d79d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -38,14 +38,14 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 + uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: languages: c-cpp # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 + uses: github/codeql-action/autobuild@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 + uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 diff --git a/.github/workflows/flawfinder.yml b/.github/workflows/flawfinder.yml index 84bc4d6ba..b3e2511ae 100644 --- a/.github/workflows/flawfinder.yml +++ b/.github/workflows/flawfinder.yml @@ -48,6 +48,6 @@ jobs: output: 'flawfinder_results.sarif' - name: Upload analysis results to GitHub Security tab - uses: github/codeql-action/upload-sarif@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: sarif_file: ${{github.workspace}}/flawfinder_results.sarif diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 4900d0607..14b987200 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -80,6 +80,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: sarif_file: results.sarif diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 6b8713b41..8b1e6ed5e 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -65,7 +65,7 @@ jobs: # Upload SARIF file generated in previous step - name: Upload SARIF file - uses: github/codeql-action/upload-sarif@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: sarif_file: semgrep.sarif if: always() From ea03d601af66c03ce625f73d853968166be6eac4 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 30 Oct 2026 15:19:42 +0100 Subject: [PATCH 6/7] json_view: comparison without temporaries and a faster image float check (#5890) * Compare json_view values without a vector per container and a basic_json per number operator== of views (view/view, view/json, and editable documents) built two vectors of members for every object it entered, sorted them with stable_sort for every object, and converted each number to a basic_json value. It now: - keeps the members of all open objects in one vector (reserved with geometric growth, so that nested objects do not reallocate), and orders them by insertion sort for up to 16 members, or not at all when the keys ascend already; - compares numbers of one type directly, and builds a basic_json only for numbers of different types; - compares short texts (keys, numbers, most strings) with two overlapping loads (bytes_equal(), in node.hpp); - first walks both documents in lockstep, in the order of their text, which finds most pairs of equal values without the work of the comparison. The walk only enters an object if the comparison would visit its members in the same order (the object type keeps the order, the keys ascend, or all values are scalars), so a difference is never found later than before, and it records the containers it found different, so that the comparison does not walk them again; - rejects values of different types and arrays of different sizes before any of this, and, for an object type that keeps the order, objects whose first (up to four) members have different keys, which ends a comparison that differs near the start without preparing the members. Results are unchanged: numbers compare by value across types, duplicate keys resolve as parse() resolves them, and objects compare in document order for ordered_json and by key otherwise. Instructions of one comparison of a document with an equal copy, macOS arm64 (develop -> now), json_document / ordered_json_document / editable: twitter 16.1M -> 7.0M 17.7M -> 1.5M 16.6M -> 7.3M citm 45.0M -> 3.4M 47.0M -> 3.2M 45.7M -> 4.3M canada 111.4M -> 13.7M 111.4M -> 13.7M 113.3M -> 15.0M jeopardy 26.2M -> 3.7M 27.3M -> 3.7M 26.6M -> 12.1M and of a view with a basic_json value: twitter 11.4M -> 5.9M 11.2M -> 7.3M 11.6M -> 5.9M citm 33.9M -> 9.7M 32.6M -> 9.1M 34.6M -> 9.8M canada 92.9M -> 32.1M 92.9M -> 32.1M 93.9M -> 32.8M and Linux instruction counts (GCC 13 / Clang 18) of the same comparisons: document with an equal copy json_document ordered_json_document editable twitter -49 % / -52 % -89 % / -89 % -48 % / -50 % citm -90 % / -89 % -91 % / -90 % -88 % / -87 % jeopardy (4,000 questions) -85 % / -85 % -86 % / -86 % -46 % / -49 % view with a basic_json value twitter -41 % / -45 % -28 % / -35 % -45 % / -44 % citm -63 % / -65 % -63 % / -66 % -65 % / -64 % jeopardy -39 % / -43 % -14 % / -24 % -44 % / -43 % A sweep that places one difference at about 2,100 positions of eight documents (a token at an even spacing, the first and last tokens, the first, middle and last member of the root removed, one appended, the document wrapped) in five kinds of comparison (view/view of read-only and of editable documents, view/json, json_document, editable/view) finds the same results as before and no placement slower on the Mac (Clang); on Linux no placement is slower than 3 % (Clang and GCC). Signed-off-by: Niels Lohmann * Check the float tokens of a json_document image without a sort and a scan of every character load(image_check::full) checked every float token of an image by sorting the token ranges, scanning the token character by character, and comparing the digit layout with the node's. Now: - the ranges are sorted only if they are not sorted already (save() writes them in the order of the nodes, so they are); - a token of the shape [-]digits.digits whose digit counts are the ones the node records (nearly all of them) is accepted by checking its digits eight at a time, without the general scan; - the general scan skips runs of digits eight at a time. Damaged tokens are still rejected as before: the check accepts exactly the tokens that the general scan accepts. Instructions of load(image_check::full) (develop -> now): macOS arm64 canada 44.6M -> 30.9M (-31 %), jeopardy 11.3M -> 10.7M, citm 10.4M -> 10.0M, twitter 5.06M -> 4.94M Linux GCC 13 canada -50 %, jeopardy -43 %, twitter -31 %, citm -28 % Linux Clang 18 canada -42 %, jeopardy -48 %, twitter -35 %, citm -32 % image_check::bounds is unchanged (within 0.4 %). Signed-off-by: Niels Lohmann * Rename names that Flawfinder takes for the file function open() The comparison cursors' open() is now enter(), and a test's open and close strings are head and tail. Signed-off-by: Niels Lohmann * Fix the compiler and lint failures of the json_view comparison - all_digits(): the lambda declares its constant itself; MSVC does not let a lambda without captures read a constexpr local of the function - equal(): the frame's "in document order" index is a namespace-scope constant (document_order); the local constexpr npos in the default member initializers of the local struct crashed GCC 9 and MinGW GCC and was not found by MSVC - keys_differ_early() keeps the first keys in std::arrays (cpplint: variable-length arrays; clang-tidy: C arrays), and the basic_json side's first_keys() is not noexcept (key() may throw) - the numbers of one type are compared in an if chain instead of a nested conditional operator - the dump test avoids C arrays and temporary string concatenations View comparison instructions are unchanged (Apple clang, twitter, citm, canada: -0.1 % to +0.1 %). Signed-off-by: Niels Lohmann * Keep the byte comparison helpers in compare.hpp Address review comments: bytes_equal() and bytes_less() move from node.hpp to compare.hpp, their only user; key_less() no longer splits the comment of resolve_indices() from its function; and the loop counters of bytes_equal() and all_digits() are scoped to their loops. Signed-off-by: Niels Lohmann * Dispatch on the compile-time conditions of the json_view comparison MSVC (/W4 /WX) reported C4127 (conditional expression is constant) for the template parameter EA and the object type's ordering in an if condition; both are now tag-dispatched. Signed-off-by: Niels Lohmann * Remove a nested duplicate guard from the json_view image test clang-tidy reported the inner #if !defined(JSON_NOEXCEPTION) as redundant (the test case is in the same #if) and a find() != npos, which gets the NOLINT the other tests use. Signed-off-by: Niels Lohmann * Mark Infer's null dereference in materialize() as a false positive Infer (ci_infer, also red on develop since #5882) reports a null dereference in json_sax_dom_parser::null() called from materialize(). The member pointer the parser stores a value of an object in is set by the key() before it, so it is not null there. Signed-off-by: Niels Lohmann * Move the json_view comparison tests into their own file With the tests this branch adds, the object file of unit-json_view_dump.cpp got too large for the MinGW x64 assembler ("file too big"). The test case "json_view comparison" is now unit-json_view_compare.cpp, unchanged and also built with C++17. Signed-off-by: Niels Lohmann * Build the DOM parser with Visual Studio 2015 again Since #5882, the three VS 2015 jobs of AppVeyor fail on develop: - the detection of keys that can be indexed instantiated std::hash for every string type, which VS 2015 rejects with a static_assert for a type it does not hash (alt_string): std::hash is now looked at only for objects in a vector - the frame stack copied its frames with std::copy to a raw pointer, which VS 2015 reports as unsafe (C4996, an error with /WX): a loop now Signed-off-by: Niels Lohmann * json_view: no per-object allocations in ordered_json comparisons For an object type that keeps the order of its members (ordered_json), resolve_indices() allocated a vector per compared object, and resolve_members() two. They now use one working vector owned by equal(), whose capacity is reused, and resolve_indices() needs none when the keys are unique. resolve_members() sorts small objects by insertion, as the other paths do. Instructions per comparison (callgrind, GCC 13 and clang 18, x86-64): twitter, ordered_json view == ordered_json: 0.939 / 0.941; 20000 small objects with a duplicate key: 0.741-0.768; all other rows (json, citm_catalog, canada, view == view) within 0.988-1.003. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- include/nlohmann/detail/view/compare.hpp | 1022 +++++++++++- include/nlohmann/detail/view/image.hpp | 118 +- include/nlohmann/json_view.hpp | 4 + single_include/nlohmann/json_view.hpp | 1828 ++++++++++++++++------ tests/src/unit-json_view_compare.cpp | 364 +++++ tests/src/unit-json_view_dump.cpp | 95 -- tests/src/unit-json_view_image.cpp | 58 + 7 files changed, 2858 insertions(+), 631 deletions(-) create mode 100644 tests/src/unit-json_view_compare.cpp diff --git a/include/nlohmann/detail/view/compare.hpp b/include/nlohmann/detail/view/compare.hpp index 8ca6058f0..96a14efbb 100644 --- a/include/nlohmann/detail/view/compare.hpp +++ b/include/nlohmann/detail/view/compare.hpp @@ -8,14 +8,22 @@ #pragma once -#include // sort, stable_sort +#include // is_sorted, sort, stable_sort +#include // array #include // size_t +#include // int64_t, uint32_t, uint64_t +#include // memcpy +#include // equal_to #include // string -#include // move, pair +#include // move, pair, swap #include // vector #include +#include #include +#include +#include +#include NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -30,17 +38,355 @@ namespace view // value, at the position of the first occurrence), and in document order if // the object type keeps an order (ordered_json), by key otherwise. +/// whether n bytes are equal; the short texts of JSON (keys, numbers, most +/// strings) are compared by two overlapping loads instead of a library call +NLOHMANN_VIEW_ALWAYS_INLINE bool bytes_equal(const char* x, const char* y, std::size_t n) noexcept +{ + const auto load8 = [](const char* p) + { + std::uint64_t v = 0; + std::memcpy(&v, p, sizeof(v)); + return v; + }; + const auto load4 = [](const char* p) + { + std::uint32_t v = 0; + std::memcpy(&v, p, sizeof(v)); + return v; + }; + if (n >= 8) + { + for (std::size_t i = 0; i + 8 < n; i += 8) + { + if (load8(x + i) != load8(y + i)) + { + return false; + } + } + return load8(x + n - 8) == load8(y + n - 8); + } + if (n >= 4) + { + return load4(x) == load4(y) && load4(x + n - 4) == load4(y + n - 4); + } + for (std::size_t i = 0; i < n; ++i) + { + if (x[i] != y[i]) + { + return false; + } + } + return true; +} + +/// whether the n1 bytes at x are before the n2 bytes at y (memcmp() order, then the shorter first); +/// the first difference is found eight bytes at a time +NLOHMANN_VIEW_ALWAYS_INLINE bool bytes_less(const char* x, std::size_t n1, const char* y, std::size_t n2) noexcept +{ + const std::size_t n = n1 < n2 ? n1 : n2; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + const std::uint64_t u = read_eight_bytes(x + i); + const std::uint64_t v = read_eight_bytes(y + i); + if (u != v) + { + const auto shift = static_cast(count_trailing_zeros(u ^ v)) & ~7u; // (the first byte that differs) + return ((u >> shift) & 0xFFu) < ((v >> shift) & 0xFFu); + } + } + for (; i < n; ++i) + { + if (x[i] != y[i]) + { + return static_cast(x[i]) < static_cast(y[i]); + } + } + return n1 < n2; +} + +/// whether the key a is before the key b +template +NLOHMANN_VIEW_ALWAYS_INLINE bool key_less(const Key& a, const Key& b) noexcept +{ + return bytes_less(a.data(), a.size(), b.data(), b.size()); +} + +/// sorts the indices order[0..n) of the members m by key; equal keys stay in +/// document order (small objects with an insertion sort, which needs no memory) +template +void sort_by_key(std::uint32_t* order, std::size_t n, const Entry* m) +{ + for (std::size_t i = 0; i < n; ++i) + { + order[i] = static_cast(i); + } + if (n <= 16) + { + for (std::size_t i = 1; i < n; ++i) + { + const std::uint32_t x = order[i]; + std::size_t j = i; + for (; j > 0 && key_less(m[x].first, m[order[j - 1]].first); --j) + { + order[j] = order[j - 1]; + } + order[j] = x; + } + } + else + { + std::stable_sort(order, order + n, [m](std::uint32_t x, std::uint32_t y) + { + return key_less(m[x].first, m[y].first); + }); + } +} + +/// The members of an object as parse() keeps them: one per key, the last +/// value at the position of the first occurrence, in the order of the +/// comparison (sorted by key, or, if the object type keeps the order, the order +/// of first occurrence). The members m[0..n) are in document order; this +/// appends to out the indices of the member of each key that has its value +/// (its last one), in the order of the comparison. Most objects have unique +/// keys. scratch is working memory, kept by the caller so that its capacity +/// is reused. +template +void resolve_indices(const Entry* m, std::size_t n, bool ordered, std::vector& out, std::vector& scratch) +{ + const std::size_t start = out.size(); + out.resize(start + n); + std::uint32_t* const order = out.data() + start; + sort_by_key(order, n, m); + if (!ordered) + { + // the last of each run of equal keys + std::size_t w = 0; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && m[order[last + 1]].first == m[order[i]].first) + { + ++last; + } + order[w++] = order[last]; + i = last + 1; + } + out.resize(start + w); + return; + } + // (document order) the value of the last occurrence at the first occurrence of each key + constexpr std::uint32_t none = 0xFFFFFFFFu; + bool any = false; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && m[order[last + 1]].first == m[order[i]].first) + { + ++last; + } + if (last != i) + { + if (!any) + { + scratch.assign(n, none); + for (std::size_t k = 0; k < i; ++k) + { + scratch[order[k]] = order[k]; + } + any = true; + } + scratch[order[i]] = order[last]; // (stable: order[i] is the first occurrence) + } + else if (any) + { + scratch[order[i]] = order[i]; + } + i = last + 1; + } + if (!any) + { + // unique keys: document order + for (std::size_t i = 0; i < n; ++i) + { + order[i] = static_cast(i); + } + return; + } + std::size_t w = 0; + for (std::size_t i = 0; i < n; ++i) + { + if (scratch[i] != none) + { + order[w++] = scratch[i]; + } + } + out.resize(start + w); +} + +/// whether the keys of n members are the same, in the same order +template +bool same_keys(const EntryA* a, const EntryB* b, std::size_t n) +{ + for (std::size_t i = 0; i < n; ++i) + { + if (!(a[i].first == b[i].first)) + { + return false; + } + } + return true; +} + +/// The members of an object as parse() keeps them (one per key: the last +/// value at the position of the first occurrence), sorted by key or, if the +/// object type keeps the order, in the order of first occurrence. The +/// members are given in document order, at the end of m (from the position +/// base on); most objects have unique keys, in which case they are only sorted +/// (for small objects with an insertion sort, which keeps equal keys in order +/// and needs no memory). scratch is working memory, kept by the caller. +template +void resolve_members(std::vector>& m, std::size_t base, bool ordered, std::vector& scratch) +{ + using entry = std::pair; + const std::size_t n = m.size() - base; + entry* const first = m.data() + base; + if (ordered) + { + // (document order) later values of a key go to its first occurrence; + // scratch holds the order by key, then whether a member is dropped + constexpr std::uint32_t dropped = 1; + scratch.assign(2 * n, 0); + std::uint32_t* const order = scratch.data(); + std::uint32_t* const drop = order + n; + sort_by_key(order, n, first); + bool any = false; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && first[order[last + 1]].first == first[order[i]].first) + { + ++last; + } + if (last != i) + { + first[order[i]].second = first[order[last]].second; + for (std::size_t k = i + 1; k <= last; ++k) + { + drop[order[k]] = dropped; + } + any = true; + } + i = last + 1; + } + if (any) + { + std::size_t w = 0; + for (std::size_t i = 0; i < n; ++i) + { + if (drop[i] != dropped) + { + if (w != i) + { + first[w] = std::move(first[i]); + } + ++w; + } + } + m.erase(m.begin() + static_cast(base + w), m.end()); + } + return; + } + // sorted by key; equal keys stay in document order + if (n <= 16) + { + for (std::size_t i = 1; i < n; ++i) + { + entry x = std::move(first[i]); + std::size_t j = i; + for (; j > 0 && key_less(x.first, first[j - 1].first); --j) + { + first[j] = std::move(first[j - 1]); + } + first[j] = std::move(x); + } + } + else + { + std::stable_sort(first, first + n, [](const entry & x, const entry & y) + { + return key_less(x.first, y.first); + }); + } + // the first key of a run with the last value + std::size_t w = 0; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && first[last + 1].first == first[i].first) + { + ++last; + } + if (w != i || last != i) + { + first[w] = entry(first[i].first, std::move(first[last].second)); + } + ++w; + i = last + 1; + } + m.erase(m.begin() + static_cast(base + w), m.end()); +} + +/// room for k more entries, growing geometrically (reserve(size() + k) alone would +/// reallocate for each object nested in another) +template +void reserve_more(std::vector& v, std::size_t k) +{ + const std::size_t needed = v.size() + k; + if (needed > v.capacity()) + { + v.reserve((std::max)(needed, 2 * v.capacity())); + } +} + /// one side of a comparison: a view template class view_side { public: using string_view_t = typename View::string_view_t; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using nav = navigation; + + /// the position in the elements of an array + struct cursor + { + const node* pos = nullptr; + }; + + view_side() noexcept + : m_view() + {} explicit view_side(const View& v) noexcept : m_view(v) {} + /// the document and the node of the view (nullptr: a discarded view) + const document_data& document() const noexcept + { + return *m_view.m_doc; + } + + const node* value_node() const noexcept + { + return m_view.m_node; + } + + static constexpr bool is_editable = View::is_editable; + value_t type() const noexcept { return m_view.type(); @@ -56,6 +402,32 @@ class view_side return m_view.get_string(); } + bool boolean() const noexcept + { + return (m_view.m_node->flags & node_flags::is_true) != 0; + } + + number_integer_t integer() const noexcept + { + return static_cast(static_cast(integer_bits(*m_view.m_node))); + } + + number_unsigned_t unsigned_integer() const noexcept + { + return static_cast(integer_bits(*m_view.m_node)); + } + + number_float_t floating() const + { + return float_value(*m_view.m_doc, *m_view.m_node); + } + + /// the token of a float + string_view_t float_token() const noexcept + { + return string_view_t(m_view.m_doc->str(*m_view.m_node), m_view.m_node->len); + } + /// a number, boolean, or null as a basic_json value (no allocation) BasicJsonType scalar() const { @@ -80,60 +452,47 @@ class view_side } } - void elements(std::vector& out) const + /// start at the first element of this array + void enter(cursor& c) const noexcept { - out.reserve(m_view.size()); - for (const View e : m_view) + c.pos = nav::first(*m_view.m_doc, m_view.m_node); + } + + /// the element at c (and advance) + view_side take(cursor& c) const noexcept + { + const node* const v = nav::value(c.pos); + c.pos = document_data::after(c.pos); + return view_side(View(m_view.m_doc, v)); + } + + /// the keys of the first n members of this object (n <= its size), in document order + void first_keys(string_view_t* out, std::size_t n) const noexcept + { + const node* pos = nav::first(*m_view.m_doc, m_view.m_node); + for (std::size_t i = 0; i < n; ++i) { - out.emplace_back(e); + out[i] = string_view_t(m_view.m_doc->str(*pos), pos->len); + pos = document_data::after(pos + 1); } } - /// the members as parse() keeps them: one per key, the last value at the - /// position of the first occurrence; in that order, or sorted by key - void members(std::vector>& out, bool ordered) const + /// append the members to out, in document order; whether their keys ascend + /// (then they are unique, and in the order of the comparison of a std::map) + bool members(std::vector>& out, bool /*ordered*/) const { - struct member + const std::size_t base = out.size(); + reserve_more(out, m_view.m_node->len); + bool sorted = true; + const node* pos = nav::first(*m_view.m_doc, m_view.m_node); + for (std::size_t i = m_view.m_node->len; i != 0; --i) { - string_view_t key; - View value; - std::size_t position; - }; - std::vector all; - all.reserve(m_view.size()); - std::size_t position = 0; - for (auto it = m_view.begin(); it != m_view.end(); ++it) - { - all.push_back(member{it.key(), it.value(), position++}); - } - std::stable_sort(all.begin(), all.end(), [](const member & a, const member & b) - { - return a.key < b.key; - }); - std::vector unique; - unique.reserve(all.size()); - for (std::size_t i = 0; i < all.size();) - { - std::size_t last = i; - while (last + 1 < all.size() && all[last + 1].key == all[i].key) - { - ++last; - } - unique.push_back(member{all[i].key, all[last].value, all[i].position}); - i = last + 1; - } - if (ordered) - { - std::sort(unique.begin(), unique.end(), [](const member & a, const member & b) - { - return a.position < b.position; - }); - } - out.reserve(unique.size()); - for (const member& m : unique) - { - out.emplace_back(m.key, view_side(m.value)); + const string_view_t key(m_view.m_doc->str(*pos), pos->len); + sorted = sorted && (out.size() == base || key_less(out.back().first, key)); + out.emplace_back(key, view_side(View(m_view.m_doc, nav::value(pos + 1)))); + pos = document_data::after(pos + 1); } + return sorted; } private: @@ -146,6 +505,17 @@ class json_side { public: using string_view_t = StringView; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + + /// the position in the elements of an array + struct cursor + { + typename BasicJsonType::const_iterator it{}; + }; + + json_side() noexcept = default; explicit json_side(const BasicJsonType& j) noexcept : m_json(&j) @@ -167,45 +537,428 @@ class json_side return string_view_t(s.data(), s.size()); } + bool boolean() const noexcept + { + return *m_json->template get_ptr(); + } + + number_integer_t integer() const noexcept + { + return *m_json->template get_ptr(); + } + + number_unsigned_t unsigned_integer() const noexcept + { + return *m_json->template get_ptr(); + } + + number_float_t floating() const noexcept + { + return *m_json->template get_ptr(); + } + BasicJsonType scalar() const { return *m_json; } - void elements(std::vector& out) const + void enter(cursor& c) const { - out.reserve(m_json->size()); - for (const auto& e : *m_json) + c.it = m_json->cbegin(); + } + + json_side take(cursor& c) const + { + const json_side r(*c.it); + ++c.it; + return r; + } + + /// the keys of the first n members of this object (n <= its size), in document order + void first_keys(string_view_t* out, std::size_t n) const + { + auto it = m_json->cbegin(); + for (std::size_t i = 0; i < n; ++i, ++it) { - out.emplace_back(e); + out[i] = string_view_t(it.key().data(), it.key().size()); } } - void members(std::vector>& out, bool ordered) const + /// append the members to out, in the order of the comparison (the keys of a + /// basic_json are unique); true + bool members(std::vector>& out, bool ordered) const { - out.reserve(m_json->size()); + const std::size_t base = out.size(); + reserve_more(out, m_json->size()); for (auto it = m_json->cbegin(); it != m_json->cend(); ++it) { out.emplace_back(string_view_t(it.key().data(), it.key().size()), json_side(it.value())); } if (!ordered) { - std::sort(out.begin(), out.end(), [](const std::pair& a, const std::pair& b) + const auto less = [](const std::pair& a, const std::pair& b) { - return a.first < b.first; - }); + return key_less(a.first, b.first); + }; + const auto first = out.begin() + static_cast(base); + if (!std::is_sorted(first, out.end(), less)) + { + std::sort(first, out.end(), less); + } + } + return true; + } + + private: + const BasicJsonType* m_json = nullptr; +}; + +/*! +@brief the values at two nodes, found equal by walking both documents in the +same order + +Two values are equal if they have the same type, the same number token or +string, the same elements, and the same members with the same keys in the same +order: a value that matches this way is equal under the rules above whatever +the object type and duplicate keys (the last value of a key then wins on +both sides, and their order of first occurrence is the same). The walk does +not find every equal pair (numbers spelled differently, members in another +order, duplicate keys with other values), only the ones that are the same +text; it stops at the first difference and then says nothing. Containers +nested deeper than the budget are not walked. + +The walk goes through the document in its order, the comparison that follows +goes through the members of an object by key. So that the walk never reaches +a difference later than the comparison would, and never costs more than it, +it enters an object only if the two orders agree (the keys ascend, or the +object type keeps the order) or if the walk is cheaper than preparing the +comparison of the object (all values are scalars). +*/ +/// the containers that lockstep found different, from the outermost to the +/// innermost: the comparison that goes on from there does not walk them again +struct lockstep_path +{ + /// the nesting depth up to which lockstep walks (the recursion is bounded) + static constexpr unsigned budget() noexcept + { + return 48; + } + + /// a pair of containers that differ; for arrays, the elements where the walk stopped + /// (the ones before are equal) + struct entry + { + const node* a; + const node* b; + const node* element_a; + const node* element_b; + std::uint32_t index; + }; + + /// the number of entries that a path can hold (a container deeper than the + /// budget of the walk is not entered) + static constexpr std::size_t levels() noexcept + { + return 50; + } + + explicit lockstep_path(entry* storage) noexcept + : nodes(storage) + {} + + entry* nodes; ///< levels() entries, written before they are read + unsigned length = 0; ///< pairs recorded by the last walk + unsigned next = 0; ///< the pair that the comparison enters next + const entry* resume = nullptr; ///< the entry of the pair that was just looked at, if it is in the path +}; + +template +class lockstep +{ + public: + lockstep(const document_data& a, const document_data& b, lockstep_path& path, bool ordered) noexcept + : m_a(a), m_b(b), m_path(path), m_ordered(ordered) + {} + + /// whether the values at a and b are the same (the nodes of the values, + /// not entries of a sequence) + bool same(const node* a, const node* b, unsigned budget) noexcept + { + if (a->kind != b->kind) + { + return false; + } + switch (static_cast(a->kind)) + { + case value_t::null: + return true; + case value_t::boolean: + return ((a->flags ^ b->flags) & node_flags::is_true) == 0; + case value_t::string: + return a->len == b->len && bytes_equal(m_a.str(*a), m_b.str(*b), a->len); + case value_t::number_float: // (the token, and the digit layout that the value is read with: they only differ in a damaged image) + return a->extra == b->extra && a->len == b->len && bytes_equal(m_a.str(*a), m_b.str(*b), a->len); + case value_t::number_integer: + case value_t::number_unsigned: + return integer_bits(*a) == integer_bits(*b); + case value_t::array: + if (budget == 0 || a->len != b->len) + { + return false; + } + { + lockstep_path::entry stop{a, b, nullptr, nullptr, 0}; + if (elements(a, b, budget - 1, stop)) + { + return true; + } + record(stop, budget); + return false; + } + case value_t::object: + if (budget == 0 || a->len != b->len) + { + return false; + } + if (!visits_in_order(a) || !members(a, b, budget - 1)) + { + record(lockstep_path::entry{a, b, nullptr, nullptr, 0}, budget); + return false; + } + return true; + case value_t::binary: // LCOV_EXCL_LINE (not in a document) + case value_t::discarded: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + return false; // LCOV_EXCL_LINE } } private: - const BasicJsonType* m_json; + using nav_a = navigation; + using nav_b = navigation; + + /// note the containers a and b (at the nesting level that budget stands + /// for) as different + NLOHMANN_VIEW_NOINLINE void record(const lockstep_path::entry& stop, unsigned budget) const noexcept + { + const unsigned level = lockstep_path::budget() - budget; + m_path.nodes[level] = stop; + if (m_path.length == 0) + { + m_path.length = level + 1; // (the innermost container is the first to be left) + } + } + + /// whether the object @a a of a document that cannot be edited has scalar values only + static bool scalar_values_only(const node* a, std::false_type /*editable*/) noexcept + { + return a->next == 1 + (2 * static_cast(a->len)); + } + + /// (the nodes of an editable document can be elsewhere: not decided) + static bool scalar_values_only(const node* /*a*/, std::true_type /*editable*/) noexcept + { + return false; + } + + /// Whether the comparison that follows visits the members of the object a in the + /// order of the document: if the object type keeps the order, if the keys + /// ascend (it sorts by key), or if all values are scalars (then the walk costs + /// less than collecting and sorting the members, whichever member differs). + /// Otherwise a difference could be found by the walk later than by the + /// comparison, which is not worth the walk. + bool visits_in_order(const node* a) const noexcept + { + if (m_ordered || a->len < 2) + { + return true; + } + if (scalar_values_only(a, std::integral_constant {})) + { + return true; + } + const node* k = nav_a::first(m_a, a); + string_ref previous(m_a.str(*k), k->len); + for (std::uint32_t i = a->len - 1; i != 0; --i) + { + k = document_data::after(k + 1); + const string_ref key(m_a.str(*k), k->len); + if (!key_less(previous, key)) + { + return false; + } + previous = key; + } + return true; + } + + bool elements(const node* a, const node* b, unsigned budget, lockstep_path::entry& stop) noexcept + { + const node* pa = nav_a::first(m_a, a); + const node* pb = nav_b::first(m_b, b); + for (std::uint32_t i = a->len; i != 0; --i) + { + if (!same(nav_a::value(pa), nav_b::value(pb), budget)) + { + stop.element_a = pa; + stop.element_b = pb; + stop.index = a->len - i; + return false; + } + pa = document_data::after(pa); + pb = document_data::after(pb); + } + return true; + } + + bool members(const node* a, const node* b, unsigned budget) noexcept + { + const node* pa = nav_a::first(m_a, a); + const node* pb = nav_b::first(m_b, b); + for (std::uint32_t i = a->len; i != 0; --i) + { + if (pa->len != pb->len || !bytes_equal(m_a.str(*pa), m_b.str(*pb), pa->len)) + { + return false; + } + if (!same(nav_a::value(pa + 1), nav_b::value(pb + 1), budget)) + { + return false; + } + pa = document_data::after(pa + 1); + pb = document_data::after(pb + 1); + } + return true; + } + + const document_data& m_a; + const document_data& m_b; + lockstep_path& m_path; + bool m_ordered; ///< the object type keeps the order of the members }; +/// whether two views certainly have equal values (see lockstep); if not, the +/// path says where they differ +template +bool certainly_equal(const view_side& a, const view_side& b, lockstep_path& path, bool ordered) noexcept +{ + const node* const na = a.value_node(); + const node* const nb = b.value_node(); + if (na == nullptr || nb == nullptr) + { + return false; + } + if (na == nb) + { + return true; // (the same node of the same document: documents have distinct nodes) + } + path.resume = nullptr; + if (path.next < path.length && path.nodes[path.next].a == na && path.nodes[path.next].b == nb) + { + path.resume = &path.nodes[path.next]; + ++path.next; // (the walk found these different already) + return false; + } + path.length = 0; + const bool equal = lockstep::is_editable, view_side::is_editable>(a.document(), b.document(), path, ordered).same(na, nb, lockstep_path::budget()); + path.next = path.length != 0 ? 1 : 0; // (this pair is the first in the path) + path.resume = path.length != 0 ? &path.nodes[0] : nullptr; + return equal; +} + +/// (a view and a basic_json value: no walk) +template +bool certainly_equal(const A& /*a*/, const B& /*b*/, lockstep_path& /*path*/, bool /*ordered*/) noexcept +{ + return false; +} + +/// (arrays of two views) the elements where the walk stopped are the next pair: the ones before are equal +template +void resume_array(const lockstep_path& path, const view_side& a, const view_side& b, + typename view_side::cursor& cursor_a, typename view_side::cursor& cursor_b, std::size_t& next) noexcept +{ + if (path.resume != nullptr && path.resume->element_a != nullptr && path.resume->a == a.value_node() && path.resume->b == b.value_node()) + { + cursor_a.pos = path.resume->element_a; + cursor_b.pos = path.resume->element_b; + next = path.resume->index; + } +} + +template +void resume_array(const lockstep_path& /*path*/, const A& /*a*/, const B& /*b*/, CursorA& /*cursor_a*/, CursorB& /*cursor_b*/, std::size_t& /*next*/) noexcept +{} + +/// whether two floats are equal (the tokens of two views: equal text and digit layout is an equal value; they +/// only differ in a damaged image) +template +bool floats_equal(const view_side& a, const view_side& b) +{ + return (a.value_node()->extra == b.value_node()->extra && a.float_token() == b.float_token()) || std::equal_to()(a.floating(), b.floating()); +} + +template +bool floats_equal(const A& a, const B& b) +{ + return std::equal_to()(a.floating(), b.floating()); +} + +/// Whether two objects of a type that keeps the order are different because of the keys of +/// their first members, found without the preparation of the comparison: the comparison goes +/// through the members in the order of the first occurrence of their keys, so if the first +/// members have the same distinct keys and then one pair of keys differs, where neither is +/// one of the keys before it, the objects are different. (The keys that are the same as an +/// earlier key are the duplicates that the comparison drops, so those are not decided here.) +template +bool keys_differ_early(const SideA& a, const SideB& b) +{ + using string_view_t = typename SideA::string_view_t; + constexpr std::size_t limit = 4; + const std::size_t n = (std::min)((std::min)(a.size(), b.size()), limit); + std::array ka{}; + std::array kb{}; + a.first_keys(ka.data(), n); + b.first_keys(kb.data(), n); + for (std::size_t i = 0; i < n; ++i) + { + if (ka[i] != kb[i]) + { + for (std::size_t k = 0; k < i; ++k) + { + if (ka[i] == ka[k] || kb[i] == ka[k]) + { + return false; // (a duplicate: not decided) + } + } + return true; + } + } + return false; +} + +/// keys_differ_early() for a type that keeps the order of the members +template +bool keys_differ_early(const SideA& a, const SideB& b, std::true_type /*ordered*/) +{ + return keys_differ_early(a, b); +} + +/// (a type that sorts the members by key: not decided early) +template +bool keys_differ_early(const SideA& /*a*/, const SideB& /*b*/, std::false_type /*ordered*/) noexcept +{ + return false; +} + inline bool is_number(value_t t) noexcept { return t == value_t::number_integer || t == value_t::number_unsigned || t == value_t::number_float; } +/// the index_a/index_b of a frame of equal() whose members are compared in document order +constexpr std::size_t document_order = static_cast(-1); + /// whether two sides are equal; iterative, so that the nesting depth is /// limited by memory only template @@ -216,20 +969,52 @@ bool equal(const A& a0, const B& b0) struct frame { - std::vector elements_a{}; - std::vector elements_b{}; - std::vector> members_a{}; - std::vector> members_b{}; + A array_a{}; + B array_b{}; + typename A::cursor cursor_a{}; + typename B::cursor cursor_b{}; + std::size_t base_a = 0; ///< the members of an object: from this position in members_a + std::size_t base_b = 0; + std::size_t index_a = document_order; ///< their order in the comparison: from this position in indices (document_order: as they are) + std::size_t index_b = document_order; + std::size_t index_mark = 0; ///< the size of indices before this frame bool object = false; std::size_t next = 0; + std::size_t count = 0; }; std::vector stack; + std::vector indices; // the order of the members of the open objects, where it is not their order in the document + std::vector scratch; // working memory of resolve_indices() and resolve_members() + // the members of the open objects (the ones of an object follow those of the objects around it) + std::vector> members_a; + std::vector> members_b; + std::array path_storage; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + lockstep_path path(path_storage.data()); + { + // (values that differ in type or in the size of an array need no walk) + const value_t first = a0.type(); + const value_t second = b0.type(); + if (first != second && !(is_number(first) && is_number(second)) && first != value_t::discarded && second != value_t::discarded) + { + return false; + } + if (first == value_t::array && second == value_t::array && a0.size() != b0.size()) + { + return false; + } + } A a = a0; B b = b0; + // (pairs of views that are certainly equal need no further look) + if (certainly_equal(a, b, path, ordered)) + { + return true; + } for (;;) { const value_t ta = a.type(); const value_t tb = b.type(); + const bool numbers = is_number(ta) && is_number(tb); if (ta == value_t::discarded || tb == value_t::discarded) { // basic_json decides (JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON) @@ -240,7 +1025,7 @@ bool equal(const A& a0, const B& b0) } else { - if (ta != tb && !(is_number(ta) && is_number(tb))) // numbers of different types can be equal + if (ta != tb && !numbers) // numbers of different types can be equal { return false; } @@ -257,31 +1042,98 @@ bool equal(const A& a0, const B& b0) { return false; } - frame f; + if (ta == value_t::object && keys_differ_early(a, b, std::integral_constant::value> {})) + { + return false; + } + stack.emplace_back(); + frame& f = stack.back(); f.object = ta == value_t::object; if (f.object) { - a.members(f.members_a, ordered); - b.members(f.members_b, ordered); - if (f.members_a.size() != f.members_b.size()) + f.base_a = members_a.size(); + f.base_b = members_b.size(); + f.index_mark = indices.size(); + const bool sorted_a = a.members(members_a, ordered); + const bool sorted_b = b.members(members_b, ordered); + const std::size_t size_a = members_a.size() - f.base_a; + const std::size_t size_b = members_b.size() - f.base_b; + std::size_t count_a = size_a; + std::size_t count_b = size_b; + if (!sorted_a && !sorted_b && size_a == size_b && same_keys(members_a.data() + f.base_a, members_b.data() + f.base_b, size_a)) + { + // the same keys in the same order: one order of the comparison for both + f.index_a = indices.size(); + f.index_b = f.index_a; + resolve_indices(members_a.data() + f.base_a, size_a, ordered, indices, scratch); + count_a = indices.size() - f.index_a; + count_b = count_a; + } + else + { + if (!sorted_a) + { + resolve_members(members_a, f.base_a, ordered, scratch); + count_a = members_a.size() - f.base_a; + } + if (!sorted_b) + { + resolve_members(members_b, f.base_b, ordered, scratch); + count_b = members_b.size() - f.base_b; + } + } + f.count = count_a; + if (count_a != count_b) { return false; } } else { - a.elements(f.elements_a); - b.elements(f.elements_b); + f.array_a = a; + f.array_b = b; + a.enter(f.cursor_a); + b.enter(f.cursor_b); + f.count = a.size(); + resume_array(path, a, b, f.cursor_a, f.cursor_b, f.next); } - stack.push_back(std::move(f)); } - else if (a.scalar() != b.scalar()) // numbers (also of different types), null, boolean + else if (ta == value_t::boolean) + { + if (a.boolean() != b.boolean()) + { + return false; + } + } + else if (numbers && ta == tb) + { + // numbers of one type: as basic_json compares them + bool differ = false; + if (ta == value_t::number_integer) + { + differ = a.integer() != b.integer(); + } + else if (ta == value_t::number_unsigned) + { + differ = a.unsigned_integer() != b.unsigned_integer(); + } + else + { + differ = !floats_equal(a, b); + } + if (differ) + { + return false; + } + } + else if (numbers && !(a.scalar() == b.scalar())) // numbers of different types { return false; } + // (null equals null) } - // the next pair of values + // the next pair of values that is not certainly equal for (;;) { if (stack.empty()) @@ -289,28 +1141,38 @@ bool equal(const A& a0, const B& b0) return true; } frame& f = stack.back(); - const std::size_t count = f.object ? f.members_a.size() : f.elements_a.size(); - if (f.next == count) + if (f.next == f.count) { + if (f.object) + { + members_a.erase(members_a.begin() + static_cast(f.base_a), members_a.end()); + members_b.erase(members_b.begin() + static_cast(f.base_b), members_b.end()); + indices.resize(f.index_mark); + } stack.pop_back(); // (f dangles now, and is bound again after continue) continue; } if (f.object) { - if (!(f.members_a[f.next].first == f.members_b[f.next].first)) + const auto& member_a = members_a[f.base_a + (f.index_a == document_order ? f.next : indices[f.index_a + f.next])]; + const auto& member_b = members_b[f.base_b + (f.index_b == document_order ? f.next : indices[f.index_b + f.next])]; + if (!(member_a.first == member_b.first)) { return false; } - a = f.members_a[f.next].second; - b = f.members_b[f.next].second; + a = member_a.second; + b = member_b.second; } else { - a = f.elements_a[f.next]; - b = f.elements_b[f.next]; + a = f.array_a.take(f.cursor_a); + b = f.array_b.take(f.cursor_b); } ++f.next; - break; + if (!certainly_equal(a, b, path, ordered)) + { + break; + } } } } diff --git a/include/nlohmann/detail/view/image.hpp b/include/nlohmann/detail/view/image.hpp index 545e04a6a..9e57082de 100644 --- a/include/nlohmann/detail/view/image.hpp +++ b/include/nlohmann/detail/view/image.hpp @@ -258,6 +258,81 @@ struct number_token bool is_float; ///< a fraction or an exponent }; +/// advance over ASCII digits, eight bytes at a time +inline const unsigned char* digit_run(const unsigned char* p, const unsigned char* e) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + while (e - p >= 8) + { + // (the digits become 0..9; a byte above 9, or with bit 7 set, is no digit. A carry + // out of a byte with bit 7 set only affects the bytes above, after the first one + // that count_trailing_zeros finds.) + const std::uint64_t x = read_eight_bytes(p) ^ (ones * 0x30u); + const std::uint64_t not_digit = ((x + (ones * 0x76u)) | x) & (ones * 0x80u); + if (not_digit != 0) + { + return p + (static_cast(count_trailing_zeros(not_digit)) / 8); + } + p += 8; + } + while (p != e && is_digit(*p)) + { + ++p; + } + return p; +} + +/// whether the n bytes at p are ASCII digits (n > 0) +inline bool all_digits(const unsigned char* p, std::size_t n) noexcept +{ + const auto digits = [](const unsigned char* q) + { + constexpr std::uint64_t ones = 0x0101010101010101ull; + const std::uint64_t x = read_eight_bytes(q) ^ (ones * 0x30u); + return (((x + (ones * 0x76u)) | x) & (ones * 0x80u)) == 0; + }; + if (n >= 8) + { + for (std::size_t i = 0; i + 8 < n; i += 8) + { + if (!digits(p + i)) + { + return false; + } + } + return digits(p + n - 8); // (overlapping the bytes before: they are digits) + } + for (std::size_t i = 0; i < n; ++i) + { + if (!is_digit(p[i])) + { + return false; + } + } + return true; +} + +/// Whether a float token has the shape that the parser records for most of +/// them: [-]digits.digits, with the numbers of digits of the node's layout +/// (extra). Then the token is a number of that layout, and the general scan +/// that follows is not needed. +inline bool plain_float_token(const unsigned char* s, std::size_t len, std::uint16_t extra) noexcept +{ + const std::size_t int_digits = extra & 0xFFu; + const std::size_t frac_digits = extra >> 8u; + if (int_digits == 0 || int_digits >= 255 || frac_digits == 0 || frac_digits >= 255 || len == 0) + { + return false; + } + const std::size_t negative = *s == '-' ? 1 : 0; + if (len != negative + int_digits + 1 + frac_digits) + { + return false; + } + const unsigned char* const p = s + negative; + return !(int_digits > 1 && *p == '0') && p[int_digits] == '.' && all_digits(p, int_digits) && all_digits(p + int_digits + 1, frac_digits); +} + /// whether [s, s + len) is a JSON number (the grammar the parser accepts) inline bool scan_number_token(const unsigned char* s, std::size_t len, number_token& t) { @@ -276,11 +351,7 @@ inline bool scan_number_token(const unsigned char* s, std::size_t len, number_to } else if (*p >= '1' && *p <= '9') { - do - { - ++p; - } - while (p != e && is_digit(*p)); + p = digit_run(p + 1, e); } else { @@ -292,10 +363,7 @@ inline bool scan_number_token(const unsigned char* s, std::size_t len, number_to if (p != e && *p == '.') { const unsigned char* const f0 = ++p; - while (p != e && is_digit(*p)) - { - ++p; - } + p = digit_run(p, e); if (p == f0) { return false; @@ -406,10 +474,15 @@ struct float_range template bool check_distinct_ranges(std::vector& ranges, Check check) { - std::sort(ranges.begin(), ranges.end(), [](const byte_range & a, const byte_range & b) + // (save() writes the ranges in the order of the nodes, which is sorted already) + const auto before = [](const byte_range & a, const byte_range & b) { return a.off != b.off ? a.off < b.off : a.len < b.len; - }); + }; + if (!std::is_sorted(ranges.begin(), ranges.end(), before)) + { + std::sort(ranges.begin(), ranges.end(), before); + } std::size_t end = 0; for (std::size_t i = 0; i < ranges.size(); ++i) { @@ -431,7 +504,8 @@ bool check_distinct_ranges(std::vector& ranges, Check check) /// record layouts that match it), tokens must not overlap. inline bool check_float_ranges(std::vector& ranges, const unsigned char* text) { - std::sort(ranges.begin(), ranges.end(), [](const float_range & a, const float_range & b) + // (save() writes the tokens in the order of the nodes, which is sorted already) + const auto before = [](const float_range & a, const float_range & b) { if (a.off != b.off) { @@ -442,7 +516,11 @@ inline bool check_float_ranges(std::vector& ranges, const unsigned return a.len < b.len; } return a.extra < b.extra; - }); + }; + if (!std::is_sorted(ranges.begin(), ranges.end(), before)) + { + std::sort(ranges.begin(), ranges.end(), before); + } std::size_t end = 0; std::size_t i = 0; while (i < ranges.size()) @@ -452,14 +530,18 @@ inline bool check_float_ranges(std::vector& ranges, const unsigned { return false; } - number_token t{}; const unsigned char* const s = text + r.off; - if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t)) + std::uint16_t layout = r.extra; + if (!plain_float_token(s, r.len, r.extra)) { - return false; + number_token t{}; + if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t)) + { + return false; + } + // the digit layout the parser records (or "many", as compaction writes it) + layout = float_layout(t); } - // the digit layout the parser records (or "many", as compaction writes it) - const std::uint16_t layout = float_layout(t); for (; i < ranges.size() && ranges[i].off == r.off && ranges[i].len == r.len; ++i) { if (ranges[i].extra != layout && ranges[i].extra != 0xFFFFu) diff --git a/include/nlohmann/json_view.hpp b/include/nlohmann/json_view.hpp index d413a55b9..0c6c7410b 100644 --- a/include/nlohmann/json_view.hpp +++ b/include/nlohmann/json_view.hpp @@ -763,8 +763,12 @@ class basic_json_view template friend class basic_json_document; template friend class basic_json_view; template friend class detail::view::editor; + template friend class detail::view::view_side; friend iterator; + /// whether the view is one of an editable document + static constexpr bool is_editable = Editable; + basic_json_view(const document_data* d, const node* n) noexcept : m_doc(d), m_node(n) {} diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 20de9f188..c4636da14 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -2479,15 +2479,367 @@ NLOHMANN_JSON_NAMESPACE_END -#include // sort, stable_sort +#include // is_sorted, sort, stable_sort +#include // array #include // size_t +#include // int64_t, uint32_t, uint64_t +#include // memcpy +#include // equal_to #include // string -#include // move, pair +#include // move, pair, swap #include // vector // #include +// #include + // #include +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // size_t +#include // int64_t, uint64_t +#include // numeric_limits +#include // string +#include // integral_constant + +// #include +// #include + +// #include + +// #include + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ +namespace view +{ + +/*! +@brief locate the decimal point and the end of the mantissa of a float token + +Also checks that the token is a JSON number. Tokens of the parser and of edits +always are; an image loaded with image_check::bounds can hold any bytes, which +must not reach the conversion (it expects a well-formed token). +*/ +inline bool float_token_layout(const char* first, const char* last, std::size_t& dot, std::size_t& mantissa_end) noexcept +{ + const auto digit = [last](const char* q) + { + return q != last && is_digit(static_cast(*q)); + }; + const char* p = first; + p += (p != last && *p == '-') ? 1 : 0; + if (!digit(p) || (*p == '0' && digit(p + 1))) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + dot = std::string::npos; + if (p != last && *p == '.') + { + dot = static_cast(p - first); + if (!digit(++p)) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + } + mantissa_end = static_cast(p - first); + if (p != last && (*p == 'e' || *p == 'E')) + { + ++p; + p += (p != last && (*p == '+' || *p == '-')) ? 1 : 0; + if (!digit(p)) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + } + return p == last; +} + +/*! +@brief the value of the float token of a node, as parse() converts it + +Uses the lexer's conversion (detail::convert_float), so that the values are +bit-identical to parse(): float and double are converted without allocation +and independent of the locale. A token that is not a JSON number (only in a +damaged image loaded with image_check::bounds) yields 0. +*/ +template +NLOHMANN_VIEW_NOINLINE FloatType float_value(const char* first, const node& n) +{ + const char* const last = first + n.len; + std::size_t dot = 0; + std::size_t mantissa_end = 0; + if (NLOHMANN_VIEW_UNLIKELY(!float_token_layout(first, last, dot, mantissa_end))) + { + return FloatType{}; + } + return convert_float(first, last, dot, mantissa_end); +} + +/*! +@brief the digits of a float token with at most 19 digits, from its layout + +The digit layout recorded while parsing says where the integer digits, the +fraction digits, and the exponent are, so the digits are read eight at a +time without scanning. + +@param[in] p first character of the token +@param[in] e end of the token +@param[in] limit end of the readable memory (the source text) +*/ +NLOHMANN_VIEW_ALWAYS_INLINE float_significand layout_decimal(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept +{ + const bool negative = *p == '-'; + p += negative ? 1 : 0; + std::uint64_t w = parse_upto19(p, int_digits, limit); + p += int_digits; + std::int64_t q = 0; + if (frac_digits != 0) + { + w = (w * int_pow10(frac_digits)) + parse_upto19(p + 1, frac_digits, limit); + p += 1 + frac_digits; + q = -static_cast(frac_digits); + } + if (p != e) + { + // [eE][+-]digits; huge exponents saturate (the parser rejected + // overflow). The token is not read beyond e, and the digits are taken + // as unsigned, so that a token that is not well-formed (a damaged + // image loaded with image_check::bounds) yields a wrong value, but no + // overflow. + ++p; + const bool exp_negative = p != e && *p == '-'; + p += (p != e && (*p == '-' || *p == '+')) ? 1 : 0; + std::int64_t exp_value = 0; + for (; p != e; ++p) + { + if (exp_value < 0x10000000) + { + exp_value = (exp_value * 10) + static_cast(*p - '0'); + } + } + q += exp_negative ? -exp_value : exp_value; + } + + float_significand d; + d.w = w; + d.exponent = q; + d.negative = negative; + return d; +} + +/*! +@brief the value of a float token with at most 19 digits, from its layout + +The result is correctly rounded by the lexer's conversion +(detail::decimal_to_float(): Clinger's fast path where both operands are +exact, else the Eisel-Lemire algorithm, which needs no fallback for up to 19 +digits), so it is the value parse() produces. +*/ +template +NLOHMANN_VIEW_ALWAYS_INLINE FloatType layout_float(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept +{ + return decimal_to_float(layout_decimal(p, e, int_digits, frac_digits, limit)); +} + +/// the value of a float set by an edit: its token (the shortest round-trip +/// text, or "nan", "inf", "-inf") in the edit arena +template +NLOHMANN_VIEW_NOINLINE FloatType edited_float(const char* token, const node& n) +{ + if (token[0] == 'n') + { + return std::numeric_limits::quiet_NaN(); + } + if (token[0] == 'i' || (token[0] == '-' && token[1] == 'i')) + { + return token[0] == 'i' ? std::numeric_limits::infinity() : -std::numeric_limits::infinity(); + } + return float_value(token, n); +} + +/// the value of the float token of a node, as parse() converts it; floats and +/// doubles with at most 19 digits are converted from the digit layout +template +FloatType float_value(const document_data& d, const node& n) +{ + if (NLOHMANN_VIEW_UNLIKELY((n.flags & node_flags::storage) == node_flags::edited)) + { + return edited_float(d.str(n), n); + } + return float_value(d, n, std::integral_constant::value> {}); +} + +template +FloatType float_value(const document_data& d, const node& n, std::true_type /*binary32 or binary64*/) +{ + const unsigned int_digits = n.extra & 0xFFu; + const unsigned frac_digits = n.extra >> 8u; + if (NLOHMANN_VIEW_LIKELY(int_digits + frac_digits <= 19)) // (255 marks "many") + { + // (a float token not written by an edit is in the text) + const auto* const first = reinterpret_cast(d.src + n.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + return layout_float(first, first + n.len, int_digits, frac_digits, reinterpret_cast(d.src + d.size)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + } + return float_value(d.str(n), n); +} + +template +FloatType float_value(const document_data& d, const node& n, std::false_type /*other*/) +{ + return float_value(d.str(n), n); +} + +} // namespace view +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // min +#include // size_t +#include // memcmp, strlen +#include // basic_string + +// #include +// #include + + +#if NLOHMANN_VIEW_HAS_CPP_17 + #include // string_view +#endif +#ifndef JSON_NO_IO + #include // ostream +#endif + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ +namespace view +{ + +#if NLOHMANN_VIEW_HAS_CPP_17 +using string_ref = std::string_view; +#else +/// minimal C++11 stand-in for std::string_view +class string_ref +{ + public: + using size_type = std::size_t; + using const_iterator = const char*; + + string_ref() noexcept = default; + // s must be null-terminated, as for std::string_view(const char*) + // flawfinder: ignore + string_ref(const char* s) : m_data(s), m_size(std::strlen(s)) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + string_ref(const char* s, std::size_t n) noexcept : m_data(s), m_size(n) {} + template + string_ref(const std::basic_string& s) noexcept : m_data(s.data()), m_size(s.size()) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + + const char* data() const noexcept + { + return m_data; + } + std::size_t size() const noexcept + { + return m_size; + } + std::size_t length() const noexcept + { + return m_size; + } + bool empty() const noexcept + { + return m_size == 0; + } + const char* begin() const noexcept + { + return m_data; + } + const char* end() const noexcept + { + return m_data + m_size; + } + char operator[](std::size_t i) const noexcept + { + return m_data[i]; + } + + template + explicit operator std::basic_string() const + { + return std::basic_string(m_data, m_size); + } + + friend bool operator==(string_ref a, string_ref b) noexcept + { + return a.m_size == b.m_size && (a.m_size == 0 || std::memcmp(a.m_data, b.m_data, a.m_size) == 0); + } + friend bool operator!=(string_ref a, string_ref b) noexcept + { + return !(a == b); + } + friend bool operator<(string_ref a, string_ref b) noexcept + { + const int c = std::memcmp(a.m_data, b.m_data, (std::min)(a.m_size, b.m_size)); + return c != 0 ? c < 0 : a.m_size < b.m_size; + } +#ifndef JSON_NO_IO + friend std::ostream& operator<<(std::ostream& o, string_ref s) + { + return o.write(s.m_data, static_cast(s.m_size)); + } +#endif + + private: + const char* m_data = ""; + std::size_t m_size = 0; +}; +#endif + +} // namespace view +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -2502,17 +2854,355 @@ namespace view // value, at the position of the first occurrence), and in document order if // the object type keeps an order (ordered_json), by key otherwise. +/// whether n bytes are equal; the short texts of JSON (keys, numbers, most +/// strings) are compared by two overlapping loads instead of a library call +NLOHMANN_VIEW_ALWAYS_INLINE bool bytes_equal(const char* x, const char* y, std::size_t n) noexcept +{ + const auto load8 = [](const char* p) + { + std::uint64_t v = 0; + std::memcpy(&v, p, sizeof(v)); + return v; + }; + const auto load4 = [](const char* p) + { + std::uint32_t v = 0; + std::memcpy(&v, p, sizeof(v)); + return v; + }; + if (n >= 8) + { + for (std::size_t i = 0; i + 8 < n; i += 8) + { + if (load8(x + i) != load8(y + i)) + { + return false; + } + } + return load8(x + n - 8) == load8(y + n - 8); + } + if (n >= 4) + { + return load4(x) == load4(y) && load4(x + n - 4) == load4(y + n - 4); + } + for (std::size_t i = 0; i < n; ++i) + { + if (x[i] != y[i]) + { + return false; + } + } + return true; +} + +/// whether the n1 bytes at x are before the n2 bytes at y (memcmp() order, then the shorter first); +/// the first difference is found eight bytes at a time +NLOHMANN_VIEW_ALWAYS_INLINE bool bytes_less(const char* x, std::size_t n1, const char* y, std::size_t n2) noexcept +{ + const std::size_t n = n1 < n2 ? n1 : n2; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + const std::uint64_t u = read_eight_bytes(x + i); + const std::uint64_t v = read_eight_bytes(y + i); + if (u != v) + { + const auto shift = static_cast(count_trailing_zeros(u ^ v)) & ~7u; // (the first byte that differs) + return ((u >> shift) & 0xFFu) < ((v >> shift) & 0xFFu); + } + } + for (; i < n; ++i) + { + if (x[i] != y[i]) + { + return static_cast(x[i]) < static_cast(y[i]); + } + } + return n1 < n2; +} + +/// whether the key a is before the key b +template +NLOHMANN_VIEW_ALWAYS_INLINE bool key_less(const Key& a, const Key& b) noexcept +{ + return bytes_less(a.data(), a.size(), b.data(), b.size()); +} + +/// sorts the indices order[0..n) of the members m by key; equal keys stay in +/// document order (small objects with an insertion sort, which needs no memory) +template +void sort_by_key(std::uint32_t* order, std::size_t n, const Entry* m) +{ + for (std::size_t i = 0; i < n; ++i) + { + order[i] = static_cast(i); + } + if (n <= 16) + { + for (std::size_t i = 1; i < n; ++i) + { + const std::uint32_t x = order[i]; + std::size_t j = i; + for (; j > 0 && key_less(m[x].first, m[order[j - 1]].first); --j) + { + order[j] = order[j - 1]; + } + order[j] = x; + } + } + else + { + std::stable_sort(order, order + n, [m](std::uint32_t x, std::uint32_t y) + { + return key_less(m[x].first, m[y].first); + }); + } +} + +/// The members of an object as parse() keeps them: one per key, the last +/// value at the position of the first occurrence, in the order of the +/// comparison (sorted by key, or, if the object type keeps the order, the order +/// of first occurrence). The members m[0..n) are in document order; this +/// appends to out the indices of the member of each key that has its value +/// (its last one), in the order of the comparison. Most objects have unique +/// keys. scratch is working memory, kept by the caller so that its capacity +/// is reused. +template +void resolve_indices(const Entry* m, std::size_t n, bool ordered, std::vector& out, std::vector& scratch) +{ + const std::size_t start = out.size(); + out.resize(start + n); + std::uint32_t* const order = out.data() + start; + sort_by_key(order, n, m); + if (!ordered) + { + // the last of each run of equal keys + std::size_t w = 0; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && m[order[last + 1]].first == m[order[i]].first) + { + ++last; + } + order[w++] = order[last]; + i = last + 1; + } + out.resize(start + w); + return; + } + // (document order) the value of the last occurrence at the first occurrence of each key + constexpr std::uint32_t none = 0xFFFFFFFFu; + bool any = false; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && m[order[last + 1]].first == m[order[i]].first) + { + ++last; + } + if (last != i) + { + if (!any) + { + scratch.assign(n, none); + for (std::size_t k = 0; k < i; ++k) + { + scratch[order[k]] = order[k]; + } + any = true; + } + scratch[order[i]] = order[last]; // (stable: order[i] is the first occurrence) + } + else if (any) + { + scratch[order[i]] = order[i]; + } + i = last + 1; + } + if (!any) + { + // unique keys: document order + for (std::size_t i = 0; i < n; ++i) + { + order[i] = static_cast(i); + } + return; + } + std::size_t w = 0; + for (std::size_t i = 0; i < n; ++i) + { + if (scratch[i] != none) + { + order[w++] = scratch[i]; + } + } + out.resize(start + w); +} + +/// whether the keys of n members are the same, in the same order +template +bool same_keys(const EntryA* a, const EntryB* b, std::size_t n) +{ + for (std::size_t i = 0; i < n; ++i) + { + if (!(a[i].first == b[i].first)) + { + return false; + } + } + return true; +} + +/// The members of an object as parse() keeps them (one per key: the last +/// value at the position of the first occurrence), sorted by key or, if the +/// object type keeps the order, in the order of first occurrence. The +/// members are given in document order, at the end of m (from the position +/// base on); most objects have unique keys, in which case they are only sorted +/// (for small objects with an insertion sort, which keeps equal keys in order +/// and needs no memory). scratch is working memory, kept by the caller. +template +void resolve_members(std::vector>& m, std::size_t base, bool ordered, std::vector& scratch) +{ + using entry = std::pair; + const std::size_t n = m.size() - base; + entry* const first = m.data() + base; + if (ordered) + { + // (document order) later values of a key go to its first occurrence; + // scratch holds the order by key, then whether a member is dropped + constexpr std::uint32_t dropped = 1; + scratch.assign(2 * n, 0); + std::uint32_t* const order = scratch.data(); + std::uint32_t* const drop = order + n; + sort_by_key(order, n, first); + bool any = false; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && first[order[last + 1]].first == first[order[i]].first) + { + ++last; + } + if (last != i) + { + first[order[i]].second = first[order[last]].second; + for (std::size_t k = i + 1; k <= last; ++k) + { + drop[order[k]] = dropped; + } + any = true; + } + i = last + 1; + } + if (any) + { + std::size_t w = 0; + for (std::size_t i = 0; i < n; ++i) + { + if (drop[i] != dropped) + { + if (w != i) + { + first[w] = std::move(first[i]); + } + ++w; + } + } + m.erase(m.begin() + static_cast(base + w), m.end()); + } + return; + } + // sorted by key; equal keys stay in document order + if (n <= 16) + { + for (std::size_t i = 1; i < n; ++i) + { + entry x = std::move(first[i]); + std::size_t j = i; + for (; j > 0 && key_less(x.first, first[j - 1].first); --j) + { + first[j] = std::move(first[j - 1]); + } + first[j] = std::move(x); + } + } + else + { + std::stable_sort(first, first + n, [](const entry & x, const entry & y) + { + return key_less(x.first, y.first); + }); + } + // the first key of a run with the last value + std::size_t w = 0; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && first[last + 1].first == first[i].first) + { + ++last; + } + if (w != i || last != i) + { + first[w] = entry(first[i].first, std::move(first[last].second)); + } + ++w; + i = last + 1; + } + m.erase(m.begin() + static_cast(base + w), m.end()); +} + +/// room for k more entries, growing geometrically (reserve(size() + k) alone would +/// reallocate for each object nested in another) +template +void reserve_more(std::vector& v, std::size_t k) +{ + const std::size_t needed = v.size() + k; + if (needed > v.capacity()) + { + v.reserve((std::max)(needed, 2 * v.capacity())); + } +} + /// one side of a comparison: a view template class view_side { public: using string_view_t = typename View::string_view_t; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using nav = navigation; + + /// the position in the elements of an array + struct cursor + { + const node* pos = nullptr; + }; + + view_side() noexcept + : m_view() + {} explicit view_side(const View& v) noexcept : m_view(v) {} + /// the document and the node of the view (nullptr: a discarded view) + const document_data& document() const noexcept + { + return *m_view.m_doc; + } + + const node* value_node() const noexcept + { + return m_view.m_node; + } + + static constexpr bool is_editable = View::is_editable; + value_t type() const noexcept { return m_view.type(); @@ -2528,6 +3218,32 @@ class view_side return m_view.get_string(); } + bool boolean() const noexcept + { + return (m_view.m_node->flags & node_flags::is_true) != 0; + } + + number_integer_t integer() const noexcept + { + return static_cast(static_cast(integer_bits(*m_view.m_node))); + } + + number_unsigned_t unsigned_integer() const noexcept + { + return static_cast(integer_bits(*m_view.m_node)); + } + + number_float_t floating() const + { + return float_value(*m_view.m_doc, *m_view.m_node); + } + + /// the token of a float + string_view_t float_token() const noexcept + { + return string_view_t(m_view.m_doc->str(*m_view.m_node), m_view.m_node->len); + } + /// a number, boolean, or null as a basic_json value (no allocation) BasicJsonType scalar() const { @@ -2552,60 +3268,47 @@ class view_side } } - void elements(std::vector& out) const + /// start at the first element of this array + void enter(cursor& c) const noexcept { - out.reserve(m_view.size()); - for (const View e : m_view) + c.pos = nav::first(*m_view.m_doc, m_view.m_node); + } + + /// the element at c (and advance) + view_side take(cursor& c) const noexcept + { + const node* const v = nav::value(c.pos); + c.pos = document_data::after(c.pos); + return view_side(View(m_view.m_doc, v)); + } + + /// the keys of the first n members of this object (n <= its size), in document order + void first_keys(string_view_t* out, std::size_t n) const noexcept + { + const node* pos = nav::first(*m_view.m_doc, m_view.m_node); + for (std::size_t i = 0; i < n; ++i) { - out.emplace_back(e); + out[i] = string_view_t(m_view.m_doc->str(*pos), pos->len); + pos = document_data::after(pos + 1); } } - /// the members as parse() keeps them: one per key, the last value at the - /// position of the first occurrence; in that order, or sorted by key - void members(std::vector>& out, bool ordered) const + /// append the members to out, in document order; whether their keys ascend + /// (then they are unique, and in the order of the comparison of a std::map) + bool members(std::vector>& out, bool /*ordered*/) const { - struct member + const std::size_t base = out.size(); + reserve_more(out, m_view.m_node->len); + bool sorted = true; + const node* pos = nav::first(*m_view.m_doc, m_view.m_node); + for (std::size_t i = m_view.m_node->len; i != 0; --i) { - string_view_t key; - View value; - std::size_t position; - }; - std::vector all; - all.reserve(m_view.size()); - std::size_t position = 0; - for (auto it = m_view.begin(); it != m_view.end(); ++it) - { - all.push_back(member{it.key(), it.value(), position++}); - } - std::stable_sort(all.begin(), all.end(), [](const member & a, const member & b) - { - return a.key < b.key; - }); - std::vector unique; - unique.reserve(all.size()); - for (std::size_t i = 0; i < all.size();) - { - std::size_t last = i; - while (last + 1 < all.size() && all[last + 1].key == all[i].key) - { - ++last; - } - unique.push_back(member{all[i].key, all[last].value, all[i].position}); - i = last + 1; - } - if (ordered) - { - std::sort(unique.begin(), unique.end(), [](const member & a, const member & b) - { - return a.position < b.position; - }); - } - out.reserve(unique.size()); - for (const member& m : unique) - { - out.emplace_back(m.key, view_side(m.value)); + const string_view_t key(m_view.m_doc->str(*pos), pos->len); + sorted = sorted && (out.size() == base || key_less(out.back().first, key)); + out.emplace_back(key, view_side(View(m_view.m_doc, nav::value(pos + 1)))); + pos = document_data::after(pos + 1); } + return sorted; } private: @@ -2618,6 +3321,17 @@ class json_side { public: using string_view_t = StringView; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + + /// the position in the elements of an array + struct cursor + { + typename BasicJsonType::const_iterator it{}; + }; + + json_side() noexcept = default; explicit json_side(const BasicJsonType& j) noexcept : m_json(&j) @@ -2639,45 +3353,428 @@ class json_side return string_view_t(s.data(), s.size()); } + bool boolean() const noexcept + { + return *m_json->template get_ptr(); + } + + number_integer_t integer() const noexcept + { + return *m_json->template get_ptr(); + } + + number_unsigned_t unsigned_integer() const noexcept + { + return *m_json->template get_ptr(); + } + + number_float_t floating() const noexcept + { + return *m_json->template get_ptr(); + } + BasicJsonType scalar() const { return *m_json; } - void elements(std::vector& out) const + void enter(cursor& c) const { - out.reserve(m_json->size()); - for (const auto& e : *m_json) + c.it = m_json->cbegin(); + } + + json_side take(cursor& c) const + { + const json_side r(*c.it); + ++c.it; + return r; + } + + /// the keys of the first n members of this object (n <= its size), in document order + void first_keys(string_view_t* out, std::size_t n) const + { + auto it = m_json->cbegin(); + for (std::size_t i = 0; i < n; ++i, ++it) { - out.emplace_back(e); + out[i] = string_view_t(it.key().data(), it.key().size()); } } - void members(std::vector>& out, bool ordered) const + /// append the members to out, in the order of the comparison (the keys of a + /// basic_json are unique); true + bool members(std::vector>& out, bool ordered) const { - out.reserve(m_json->size()); + const std::size_t base = out.size(); + reserve_more(out, m_json->size()); for (auto it = m_json->cbegin(); it != m_json->cend(); ++it) { out.emplace_back(string_view_t(it.key().data(), it.key().size()), json_side(it.value())); } if (!ordered) { - std::sort(out.begin(), out.end(), [](const std::pair& a, const std::pair& b) + const auto less = [](const std::pair& a, const std::pair& b) { - return a.first < b.first; - }); + return key_less(a.first, b.first); + }; + const auto first = out.begin() + static_cast(base); + if (!std::is_sorted(first, out.end(), less)) + { + std::sort(first, out.end(), less); + } + } + return true; + } + + private: + const BasicJsonType* m_json = nullptr; +}; + +/*! +@brief the values at two nodes, found equal by walking both documents in the +same order + +Two values are equal if they have the same type, the same number token or +string, the same elements, and the same members with the same keys in the same +order: a value that matches this way is equal under the rules above whatever +the object type and duplicate keys (the last value of a key then wins on +both sides, and their order of first occurrence is the same). The walk does +not find every equal pair (numbers spelled differently, members in another +order, duplicate keys with other values), only the ones that are the same +text; it stops at the first difference and then says nothing. Containers +nested deeper than the budget are not walked. + +The walk goes through the document in its order, the comparison that follows +goes through the members of an object by key. So that the walk never reaches +a difference later than the comparison would, and never costs more than it, +it enters an object only if the two orders agree (the keys ascend, or the +object type keeps the order) or if the walk is cheaper than preparing the +comparison of the object (all values are scalars). +*/ +/// the containers that lockstep found different, from the outermost to the +/// innermost: the comparison that goes on from there does not walk them again +struct lockstep_path +{ + /// the nesting depth up to which lockstep walks (the recursion is bounded) + static constexpr unsigned budget() noexcept + { + return 48; + } + + /// a pair of containers that differ; for arrays, the elements where the walk stopped + /// (the ones before are equal) + struct entry + { + const node* a; + const node* b; + const node* element_a; + const node* element_b; + std::uint32_t index; + }; + + /// the number of entries that a path can hold (a container deeper than the + /// budget of the walk is not entered) + static constexpr std::size_t levels() noexcept + { + return 50; + } + + explicit lockstep_path(entry* storage) noexcept + : nodes(storage) + {} + + entry* nodes; ///< levels() entries, written before they are read + unsigned length = 0; ///< pairs recorded by the last walk + unsigned next = 0; ///< the pair that the comparison enters next + const entry* resume = nullptr; ///< the entry of the pair that was just looked at, if it is in the path +}; + +template +class lockstep +{ + public: + lockstep(const document_data& a, const document_data& b, lockstep_path& path, bool ordered) noexcept + : m_a(a), m_b(b), m_path(path), m_ordered(ordered) + {} + + /// whether the values at a and b are the same (the nodes of the values, + /// not entries of a sequence) + bool same(const node* a, const node* b, unsigned budget) noexcept + { + if (a->kind != b->kind) + { + return false; + } + switch (static_cast(a->kind)) + { + case value_t::null: + return true; + case value_t::boolean: + return ((a->flags ^ b->flags) & node_flags::is_true) == 0; + case value_t::string: + return a->len == b->len && bytes_equal(m_a.str(*a), m_b.str(*b), a->len); + case value_t::number_float: // (the token, and the digit layout that the value is read with: they only differ in a damaged image) + return a->extra == b->extra && a->len == b->len && bytes_equal(m_a.str(*a), m_b.str(*b), a->len); + case value_t::number_integer: + case value_t::number_unsigned: + return integer_bits(*a) == integer_bits(*b); + case value_t::array: + if (budget == 0 || a->len != b->len) + { + return false; + } + { + lockstep_path::entry stop{a, b, nullptr, nullptr, 0}; + if (elements(a, b, budget - 1, stop)) + { + return true; + } + record(stop, budget); + return false; + } + case value_t::object: + if (budget == 0 || a->len != b->len) + { + return false; + } + if (!visits_in_order(a) || !members(a, b, budget - 1)) + { + record(lockstep_path::entry{a, b, nullptr, nullptr, 0}, budget); + return false; + } + return true; + case value_t::binary: // LCOV_EXCL_LINE (not in a document) + case value_t::discarded: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + return false; // LCOV_EXCL_LINE } } private: - const BasicJsonType* m_json; + using nav_a = navigation; + using nav_b = navigation; + + /// note the containers a and b (at the nesting level that budget stands + /// for) as different + NLOHMANN_VIEW_NOINLINE void record(const lockstep_path::entry& stop, unsigned budget) const noexcept + { + const unsigned level = lockstep_path::budget() - budget; + m_path.nodes[level] = stop; + if (m_path.length == 0) + { + m_path.length = level + 1; // (the innermost container is the first to be left) + } + } + + /// whether the object @a a of a document that cannot be edited has scalar values only + static bool scalar_values_only(const node* a, std::false_type /*editable*/) noexcept + { + return a->next == 1 + (2 * static_cast(a->len)); + } + + /// (the nodes of an editable document can be elsewhere: not decided) + static bool scalar_values_only(const node* /*a*/, std::true_type /*editable*/) noexcept + { + return false; + } + + /// Whether the comparison that follows visits the members of the object a in the + /// order of the document: if the object type keeps the order, if the keys + /// ascend (it sorts by key), or if all values are scalars (then the walk costs + /// less than collecting and sorting the members, whichever member differs). + /// Otherwise a difference could be found by the walk later than by the + /// comparison, which is not worth the walk. + bool visits_in_order(const node* a) const noexcept + { + if (m_ordered || a->len < 2) + { + return true; + } + if (scalar_values_only(a, std::integral_constant {})) + { + return true; + } + const node* k = nav_a::first(m_a, a); + string_ref previous(m_a.str(*k), k->len); + for (std::uint32_t i = a->len - 1; i != 0; --i) + { + k = document_data::after(k + 1); + const string_ref key(m_a.str(*k), k->len); + if (!key_less(previous, key)) + { + return false; + } + previous = key; + } + return true; + } + + bool elements(const node* a, const node* b, unsigned budget, lockstep_path::entry& stop) noexcept + { + const node* pa = nav_a::first(m_a, a); + const node* pb = nav_b::first(m_b, b); + for (std::uint32_t i = a->len; i != 0; --i) + { + if (!same(nav_a::value(pa), nav_b::value(pb), budget)) + { + stop.element_a = pa; + stop.element_b = pb; + stop.index = a->len - i; + return false; + } + pa = document_data::after(pa); + pb = document_data::after(pb); + } + return true; + } + + bool members(const node* a, const node* b, unsigned budget) noexcept + { + const node* pa = nav_a::first(m_a, a); + const node* pb = nav_b::first(m_b, b); + for (std::uint32_t i = a->len; i != 0; --i) + { + if (pa->len != pb->len || !bytes_equal(m_a.str(*pa), m_b.str(*pb), pa->len)) + { + return false; + } + if (!same(nav_a::value(pa + 1), nav_b::value(pb + 1), budget)) + { + return false; + } + pa = document_data::after(pa + 1); + pb = document_data::after(pb + 1); + } + return true; + } + + const document_data& m_a; + const document_data& m_b; + lockstep_path& m_path; + bool m_ordered; ///< the object type keeps the order of the members }; +/// whether two views certainly have equal values (see lockstep); if not, the +/// path says where they differ +template +bool certainly_equal(const view_side& a, const view_side& b, lockstep_path& path, bool ordered) noexcept +{ + const node* const na = a.value_node(); + const node* const nb = b.value_node(); + if (na == nullptr || nb == nullptr) + { + return false; + } + if (na == nb) + { + return true; // (the same node of the same document: documents have distinct nodes) + } + path.resume = nullptr; + if (path.next < path.length && path.nodes[path.next].a == na && path.nodes[path.next].b == nb) + { + path.resume = &path.nodes[path.next]; + ++path.next; // (the walk found these different already) + return false; + } + path.length = 0; + const bool equal = lockstep::is_editable, view_side::is_editable>(a.document(), b.document(), path, ordered).same(na, nb, lockstep_path::budget()); + path.next = path.length != 0 ? 1 : 0; // (this pair is the first in the path) + path.resume = path.length != 0 ? &path.nodes[0] : nullptr; + return equal; +} + +/// (a view and a basic_json value: no walk) +template +bool certainly_equal(const A& /*a*/, const B& /*b*/, lockstep_path& /*path*/, bool /*ordered*/) noexcept +{ + return false; +} + +/// (arrays of two views) the elements where the walk stopped are the next pair: the ones before are equal +template +void resume_array(const lockstep_path& path, const view_side& a, const view_side& b, + typename view_side::cursor& cursor_a, typename view_side::cursor& cursor_b, std::size_t& next) noexcept +{ + if (path.resume != nullptr && path.resume->element_a != nullptr && path.resume->a == a.value_node() && path.resume->b == b.value_node()) + { + cursor_a.pos = path.resume->element_a; + cursor_b.pos = path.resume->element_b; + next = path.resume->index; + } +} + +template +void resume_array(const lockstep_path& /*path*/, const A& /*a*/, const B& /*b*/, CursorA& /*cursor_a*/, CursorB& /*cursor_b*/, std::size_t& /*next*/) noexcept +{} + +/// whether two floats are equal (the tokens of two views: equal text and digit layout is an equal value; they +/// only differ in a damaged image) +template +bool floats_equal(const view_side& a, const view_side& b) +{ + return (a.value_node()->extra == b.value_node()->extra && a.float_token() == b.float_token()) || std::equal_to()(a.floating(), b.floating()); +} + +template +bool floats_equal(const A& a, const B& b) +{ + return std::equal_to()(a.floating(), b.floating()); +} + +/// Whether two objects of a type that keeps the order are different because of the keys of +/// their first members, found without the preparation of the comparison: the comparison goes +/// through the members in the order of the first occurrence of their keys, so if the first +/// members have the same distinct keys and then one pair of keys differs, where neither is +/// one of the keys before it, the objects are different. (The keys that are the same as an +/// earlier key are the duplicates that the comparison drops, so those are not decided here.) +template +bool keys_differ_early(const SideA& a, const SideB& b) +{ + using string_view_t = typename SideA::string_view_t; + constexpr std::size_t limit = 4; + const std::size_t n = (std::min)((std::min)(a.size(), b.size()), limit); + std::array ka{}; + std::array kb{}; + a.first_keys(ka.data(), n); + b.first_keys(kb.data(), n); + for (std::size_t i = 0; i < n; ++i) + { + if (ka[i] != kb[i]) + { + for (std::size_t k = 0; k < i; ++k) + { + if (ka[i] == ka[k] || kb[i] == ka[k]) + { + return false; // (a duplicate: not decided) + } + } + return true; + } + } + return false; +} + +/// keys_differ_early() for a type that keeps the order of the members +template +bool keys_differ_early(const SideA& a, const SideB& b, std::true_type /*ordered*/) +{ + return keys_differ_early(a, b); +} + +/// (a type that sorts the members by key: not decided early) +template +bool keys_differ_early(const SideA& /*a*/, const SideB& /*b*/, std::false_type /*ordered*/) noexcept +{ + return false; +} + inline bool is_number(value_t t) noexcept { return t == value_t::number_integer || t == value_t::number_unsigned || t == value_t::number_float; } +/// the index_a/index_b of a frame of equal() whose members are compared in document order +constexpr std::size_t document_order = static_cast(-1); + /// whether two sides are equal; iterative, so that the nesting depth is /// limited by memory only template @@ -2688,20 +3785,52 @@ bool equal(const A& a0, const B& b0) struct frame { - std::vector elements_a{}; - std::vector elements_b{}; - std::vector> members_a{}; - std::vector> members_b{}; + A array_a{}; + B array_b{}; + typename A::cursor cursor_a{}; + typename B::cursor cursor_b{}; + std::size_t base_a = 0; ///< the members of an object: from this position in members_a + std::size_t base_b = 0; + std::size_t index_a = document_order; ///< their order in the comparison: from this position in indices (document_order: as they are) + std::size_t index_b = document_order; + std::size_t index_mark = 0; ///< the size of indices before this frame bool object = false; std::size_t next = 0; + std::size_t count = 0; }; std::vector stack; + std::vector indices; // the order of the members of the open objects, where it is not their order in the document + std::vector scratch; // working memory of resolve_indices() and resolve_members() + // the members of the open objects (the ones of an object follow those of the objects around it) + std::vector> members_a; + std::vector> members_b; + std::array path_storage; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + lockstep_path path(path_storage.data()); + { + // (values that differ in type or in the size of an array need no walk) + const value_t first = a0.type(); + const value_t second = b0.type(); + if (first != second && !(is_number(first) && is_number(second)) && first != value_t::discarded && second != value_t::discarded) + { + return false; + } + if (first == value_t::array && second == value_t::array && a0.size() != b0.size()) + { + return false; + } + } A a = a0; B b = b0; + // (pairs of views that are certainly equal need no further look) + if (certainly_equal(a, b, path, ordered)) + { + return true; + } for (;;) { const value_t ta = a.type(); const value_t tb = b.type(); + const bool numbers = is_number(ta) && is_number(tb); if (ta == value_t::discarded || tb == value_t::discarded) { // basic_json decides (JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON) @@ -2712,7 +3841,7 @@ bool equal(const A& a0, const B& b0) } else { - if (ta != tb && !(is_number(ta) && is_number(tb))) // numbers of different types can be equal + if (ta != tb && !numbers) // numbers of different types can be equal { return false; } @@ -2729,31 +3858,98 @@ bool equal(const A& a0, const B& b0) { return false; } - frame f; + if (ta == value_t::object && keys_differ_early(a, b, std::integral_constant::value> {})) + { + return false; + } + stack.emplace_back(); + frame& f = stack.back(); f.object = ta == value_t::object; if (f.object) { - a.members(f.members_a, ordered); - b.members(f.members_b, ordered); - if (f.members_a.size() != f.members_b.size()) + f.base_a = members_a.size(); + f.base_b = members_b.size(); + f.index_mark = indices.size(); + const bool sorted_a = a.members(members_a, ordered); + const bool sorted_b = b.members(members_b, ordered); + const std::size_t size_a = members_a.size() - f.base_a; + const std::size_t size_b = members_b.size() - f.base_b; + std::size_t count_a = size_a; + std::size_t count_b = size_b; + if (!sorted_a && !sorted_b && size_a == size_b && same_keys(members_a.data() + f.base_a, members_b.data() + f.base_b, size_a)) + { + // the same keys in the same order: one order of the comparison for both + f.index_a = indices.size(); + f.index_b = f.index_a; + resolve_indices(members_a.data() + f.base_a, size_a, ordered, indices, scratch); + count_a = indices.size() - f.index_a; + count_b = count_a; + } + else + { + if (!sorted_a) + { + resolve_members(members_a, f.base_a, ordered, scratch); + count_a = members_a.size() - f.base_a; + } + if (!sorted_b) + { + resolve_members(members_b, f.base_b, ordered, scratch); + count_b = members_b.size() - f.base_b; + } + } + f.count = count_a; + if (count_a != count_b) { return false; } } else { - a.elements(f.elements_a); - b.elements(f.elements_b); + f.array_a = a; + f.array_b = b; + a.enter(f.cursor_a); + b.enter(f.cursor_b); + f.count = a.size(); + resume_array(path, a, b, f.cursor_a, f.cursor_b, f.next); } - stack.push_back(std::move(f)); } - else if (a.scalar() != b.scalar()) // numbers (also of different types), null, boolean + else if (ta == value_t::boolean) + { + if (a.boolean() != b.boolean()) + { + return false; + } + } + else if (numbers && ta == tb) + { + // numbers of one type: as basic_json compares them + bool differ = false; + if (ta == value_t::number_integer) + { + differ = a.integer() != b.integer(); + } + else if (ta == value_t::number_unsigned) + { + differ = a.unsigned_integer() != b.unsigned_integer(); + } + else + { + differ = !floats_equal(a, b); + } + if (differ) + { + return false; + } + } + else if (numbers && !(a.scalar() == b.scalar())) // numbers of different types { return false; } + // (null equals null) } - // the next pair of values + // the next pair of values that is not certainly equal for (;;) { if (stack.empty()) @@ -2761,28 +3957,38 @@ bool equal(const A& a0, const B& b0) return true; } frame& f = stack.back(); - const std::size_t count = f.object ? f.members_a.size() : f.elements_a.size(); - if (f.next == count) + if (f.next == f.count) { + if (f.object) + { + members_a.erase(members_a.begin() + static_cast(f.base_a), members_a.end()); + members_b.erase(members_b.begin() + static_cast(f.base_b), members_b.end()); + indices.resize(f.index_mark); + } stack.pop_back(); // (f dangles now, and is bound again after continue) continue; } if (f.object) { - if (!(f.members_a[f.next].first == f.members_b[f.next].first)) + const auto& member_a = members_a[f.base_a + (f.index_a == document_order ? f.next : indices[f.index_a + f.next])]; + const auto& member_b = members_b[f.base_b + (f.index_b == document_order ? f.next : indices[f.index_b + f.next])]; + if (!(member_a.first == member_b.first)) { return false; } - a = f.members_a[f.next].second; - b = f.members_b[f.next].second; + a = member_a.second; + b = member_b.second; } else { - a = f.elements_a[f.next]; - b = f.elements_b[f.next]; + a = f.array_a.take(f.cursor_a); + b = f.array_b.take(f.cursor_b); } ++f.next; - break; + if (!certainly_equal(a, b, path, ordered)) + { + break; + } } } } @@ -4720,232 +5926,6 @@ NLOHMANN_JSON_NAMESPACE_END // #include // #include -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - - - -#include // size_t -#include // int64_t, uint64_t -#include // numeric_limits -#include // string -#include // integral_constant - -// #include -// #include - -// #include - -// #include - -// #include - - -NLOHMANN_JSON_NAMESPACE_BEGIN -namespace detail -{ -namespace view -{ - -/*! -@brief locate the decimal point and the end of the mantissa of a float token - -Also checks that the token is a JSON number. Tokens of the parser and of edits -always are; an image loaded with image_check::bounds can hold any bytes, which -must not reach the conversion (it expects a well-formed token). -*/ -inline bool float_token_layout(const char* first, const char* last, std::size_t& dot, std::size_t& mantissa_end) noexcept -{ - const auto digit = [last](const char* q) - { - return q != last && is_digit(static_cast(*q)); - }; - const char* p = first; - p += (p != last && *p == '-') ? 1 : 0; - if (!digit(p) || (*p == '0' && digit(p + 1))) - { - return false; - } - do - { - ++p; - } - while (digit(p)); - dot = std::string::npos; - if (p != last && *p == '.') - { - dot = static_cast(p - first); - if (!digit(++p)) - { - return false; - } - do - { - ++p; - } - while (digit(p)); - } - mantissa_end = static_cast(p - first); - if (p != last && (*p == 'e' || *p == 'E')) - { - ++p; - p += (p != last && (*p == '+' || *p == '-')) ? 1 : 0; - if (!digit(p)) - { - return false; - } - do - { - ++p; - } - while (digit(p)); - } - return p == last; -} - -/*! -@brief the value of the float token of a node, as parse() converts it - -Uses the lexer's conversion (detail::convert_float), so that the values are -bit-identical to parse(): float and double are converted without allocation -and independent of the locale. A token that is not a JSON number (only in a -damaged image loaded with image_check::bounds) yields 0. -*/ -template -NLOHMANN_VIEW_NOINLINE FloatType float_value(const char* first, const node& n) -{ - const char* const last = first + n.len; - std::size_t dot = 0; - std::size_t mantissa_end = 0; - if (NLOHMANN_VIEW_UNLIKELY(!float_token_layout(first, last, dot, mantissa_end))) - { - return FloatType{}; - } - return convert_float(first, last, dot, mantissa_end); -} - -/*! -@brief the digits of a float token with at most 19 digits, from its layout - -The digit layout recorded while parsing says where the integer digits, the -fraction digits, and the exponent are, so the digits are read eight at a -time without scanning. - -@param[in] p first character of the token -@param[in] e end of the token -@param[in] limit end of the readable memory (the source text) -*/ -NLOHMANN_VIEW_ALWAYS_INLINE float_significand layout_decimal(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept -{ - const bool negative = *p == '-'; - p += negative ? 1 : 0; - std::uint64_t w = parse_upto19(p, int_digits, limit); - p += int_digits; - std::int64_t q = 0; - if (frac_digits != 0) - { - w = (w * int_pow10(frac_digits)) + parse_upto19(p + 1, frac_digits, limit); - p += 1 + frac_digits; - q = -static_cast(frac_digits); - } - if (p != e) - { - // [eE][+-]digits; huge exponents saturate (the parser rejected - // overflow). The token is not read beyond e, and the digits are taken - // as unsigned, so that a token that is not well-formed (a damaged - // image loaded with image_check::bounds) yields a wrong value, but no - // overflow. - ++p; - const bool exp_negative = p != e && *p == '-'; - p += (p != e && (*p == '-' || *p == '+')) ? 1 : 0; - std::int64_t exp_value = 0; - for (; p != e; ++p) - { - if (exp_value < 0x10000000) - { - exp_value = (exp_value * 10) + static_cast(*p - '0'); - } - } - q += exp_negative ? -exp_value : exp_value; - } - - float_significand d; - d.w = w; - d.exponent = q; - d.negative = negative; - return d; -} - -/*! -@brief the value of a float token with at most 19 digits, from its layout - -The result is correctly rounded by the lexer's conversion -(detail::decimal_to_float(): Clinger's fast path where both operands are -exact, else the Eisel-Lemire algorithm, which needs no fallback for up to 19 -digits), so it is the value parse() produces. -*/ -template -NLOHMANN_VIEW_ALWAYS_INLINE FloatType layout_float(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept -{ - return decimal_to_float(layout_decimal(p, e, int_digits, frac_digits, limit)); -} - -/// the value of a float set by an edit: its token (the shortest round-trip -/// text, or "nan", "inf", "-inf") in the edit arena -template -NLOHMANN_VIEW_NOINLINE FloatType edited_float(const char* token, const node& n) -{ - if (token[0] == 'n') - { - return std::numeric_limits::quiet_NaN(); - } - if (token[0] == 'i' || (token[0] == '-' && token[1] == 'i')) - { - return token[0] == 'i' ? std::numeric_limits::infinity() : -std::numeric_limits::infinity(); - } - return float_value(token, n); -} - -/// the value of the float token of a node, as parse() converts it; floats and -/// doubles with at most 19 digits are converted from the digit layout -template -FloatType float_value(const document_data& d, const node& n) -{ - if (NLOHMANN_VIEW_UNLIKELY((n.flags & node_flags::storage) == node_flags::edited)) - { - return edited_float(d.str(n), n); - } - return float_value(d, n, std::integral_constant::value> {}); -} - -template -FloatType float_value(const document_data& d, const node& n, std::true_type /*binary32 or binary64*/) -{ - const unsigned int_digits = n.extra & 0xFFu; - const unsigned frac_digits = n.extra >> 8u; - if (NLOHMANN_VIEW_LIKELY(int_digits + frac_digits <= 19)) // (255 marks "many") - { - // (a float token not written by an edit is in the text) - const auto* const first = reinterpret_cast(d.src + n.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - return layout_float(first, first + n.len, int_digits, frac_digits, reinterpret_cast(d.src + d.size)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - } - return float_value(d.str(n), n); -} - -template -FloatType float_value(const document_data& d, const node& n, std::false_type /*other*/) -{ - return float_value(d.str(n), n); -} - -} // namespace view -} // namespace detail -NLOHMANN_JSON_NAMESPACE_END // #include @@ -5184,6 +6164,81 @@ struct number_token bool is_float; ///< a fraction or an exponent }; +/// advance over ASCII digits, eight bytes at a time +inline const unsigned char* digit_run(const unsigned char* p, const unsigned char* e) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + while (e - p >= 8) + { + // (the digits become 0..9; a byte above 9, or with bit 7 set, is no digit. A carry + // out of a byte with bit 7 set only affects the bytes above, after the first one + // that count_trailing_zeros finds.) + const std::uint64_t x = read_eight_bytes(p) ^ (ones * 0x30u); + const std::uint64_t not_digit = ((x + (ones * 0x76u)) | x) & (ones * 0x80u); + if (not_digit != 0) + { + return p + (static_cast(count_trailing_zeros(not_digit)) / 8); + } + p += 8; + } + while (p != e && is_digit(*p)) + { + ++p; + } + return p; +} + +/// whether the n bytes at p are ASCII digits (n > 0) +inline bool all_digits(const unsigned char* p, std::size_t n) noexcept +{ + const auto digits = [](const unsigned char* q) + { + constexpr std::uint64_t ones = 0x0101010101010101ull; + const std::uint64_t x = read_eight_bytes(q) ^ (ones * 0x30u); + return (((x + (ones * 0x76u)) | x) & (ones * 0x80u)) == 0; + }; + if (n >= 8) + { + for (std::size_t i = 0; i + 8 < n; i += 8) + { + if (!digits(p + i)) + { + return false; + } + } + return digits(p + n - 8); // (overlapping the bytes before: they are digits) + } + for (std::size_t i = 0; i < n; ++i) + { + if (!is_digit(p[i])) + { + return false; + } + } + return true; +} + +/// Whether a float token has the shape that the parser records for most of +/// them: [-]digits.digits, with the numbers of digits of the node's layout +/// (extra). Then the token is a number of that layout, and the general scan +/// that follows is not needed. +inline bool plain_float_token(const unsigned char* s, std::size_t len, std::uint16_t extra) noexcept +{ + const std::size_t int_digits = extra & 0xFFu; + const std::size_t frac_digits = extra >> 8u; + if (int_digits == 0 || int_digits >= 255 || frac_digits == 0 || frac_digits >= 255 || len == 0) + { + return false; + } + const std::size_t negative = *s == '-' ? 1 : 0; + if (len != negative + int_digits + 1 + frac_digits) + { + return false; + } + const unsigned char* const p = s + negative; + return !(int_digits > 1 && *p == '0') && p[int_digits] == '.' && all_digits(p, int_digits) && all_digits(p + int_digits + 1, frac_digits); +} + /// whether [s, s + len) is a JSON number (the grammar the parser accepts) inline bool scan_number_token(const unsigned char* s, std::size_t len, number_token& t) { @@ -5202,11 +6257,7 @@ inline bool scan_number_token(const unsigned char* s, std::size_t len, number_to } else if (*p >= '1' && *p <= '9') { - do - { - ++p; - } - while (p != e && is_digit(*p)); + p = digit_run(p + 1, e); } else { @@ -5218,10 +6269,7 @@ inline bool scan_number_token(const unsigned char* s, std::size_t len, number_to if (p != e && *p == '.') { const unsigned char* const f0 = ++p; - while (p != e && is_digit(*p)) - { - ++p; - } + p = digit_run(p, e); if (p == f0) { return false; @@ -5332,10 +6380,15 @@ struct float_range template bool check_distinct_ranges(std::vector& ranges, Check check) { - std::sort(ranges.begin(), ranges.end(), [](const byte_range & a, const byte_range & b) + // (save() writes the ranges in the order of the nodes, which is sorted already) + const auto before = [](const byte_range & a, const byte_range & b) { return a.off != b.off ? a.off < b.off : a.len < b.len; - }); + }; + if (!std::is_sorted(ranges.begin(), ranges.end(), before)) + { + std::sort(ranges.begin(), ranges.end(), before); + } std::size_t end = 0; for (std::size_t i = 0; i < ranges.size(); ++i) { @@ -5357,7 +6410,8 @@ bool check_distinct_ranges(std::vector& ranges, Check check) /// record layouts that match it), tokens must not overlap. inline bool check_float_ranges(std::vector& ranges, const unsigned char* text) { - std::sort(ranges.begin(), ranges.end(), [](const float_range & a, const float_range & b) + // (save() writes the tokens in the order of the nodes, which is sorted already) + const auto before = [](const float_range & a, const float_range & b) { if (a.off != b.off) { @@ -5368,7 +6422,11 @@ inline bool check_float_ranges(std::vector& ranges, const unsigned return a.len < b.len; } return a.extra < b.extra; - }); + }; + if (!std::is_sorted(ranges.begin(), ranges.end(), before)) + { + std::sort(ranges.begin(), ranges.end(), before); + } std::size_t end = 0; std::size_t i = 0; while (i < ranges.size()) @@ -5378,14 +6436,18 @@ inline bool check_float_ranges(std::vector& ranges, const unsigned { return false; } - number_token t{}; const unsigned char* const s = text + r.off; - if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t)) + std::uint16_t layout = r.extra; + if (!plain_float_token(s, r.len, r.extra)) { - return false; + number_token t{}; + if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t)) + { + return false; + } + // the digit layout the parser records (or "many", as compaction writes it) + layout = float_layout(t); } - // the digit layout the parser records (or "many", as compaction writes it) - const std::uint16_t layout = float_layout(t); for (; i < ranges.size() && ranges[i].off == r.off && ranges[i].len == r.len; ++i) { if (ranges[i].extra != layout && ranges[i].extra != 0xFFFFu) @@ -7910,120 +8972,6 @@ class view_serializer NLOHMANN_JSON_NAMESPACE_END // #include -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - - - -#include // min -#include // size_t -#include // memcmp, strlen -#include // basic_string - -// #include -// #include - - -#if NLOHMANN_VIEW_HAS_CPP_17 - #include // string_view -#endif -#ifndef JSON_NO_IO - #include // ostream -#endif - -NLOHMANN_JSON_NAMESPACE_BEGIN -namespace detail -{ -namespace view -{ - -#if NLOHMANN_VIEW_HAS_CPP_17 -using string_ref = std::string_view; -#else -/// minimal C++11 stand-in for std::string_view -class string_ref -{ - public: - using size_type = std::size_t; - using const_iterator = const char*; - - string_ref() noexcept = default; - // s must be null-terminated, as for std::string_view(const char*) - // flawfinder: ignore - string_ref(const char* s) : m_data(s), m_size(std::strlen(s)) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) - string_ref(const char* s, std::size_t n) noexcept : m_data(s), m_size(n) {} - template - string_ref(const std::basic_string& s) noexcept : m_data(s.data()), m_size(s.size()) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) - - const char* data() const noexcept - { - return m_data; - } - std::size_t size() const noexcept - { - return m_size; - } - std::size_t length() const noexcept - { - return m_size; - } - bool empty() const noexcept - { - return m_size == 0; - } - const char* begin() const noexcept - { - return m_data; - } - const char* end() const noexcept - { - return m_data + m_size; - } - char operator[](std::size_t i) const noexcept - { - return m_data[i]; - } - - template - explicit operator std::basic_string() const - { - return std::basic_string(m_data, m_size); - } - - friend bool operator==(string_ref a, string_ref b) noexcept - { - return a.m_size == b.m_size && (a.m_size == 0 || std::memcmp(a.m_data, b.m_data, a.m_size) == 0); - } - friend bool operator!=(string_ref a, string_ref b) noexcept - { - return !(a == b); - } - friend bool operator<(string_ref a, string_ref b) noexcept - { - const int c = std::memcmp(a.m_data, b.m_data, (std::min)(a.m_size, b.m_size)); - return c != 0 ? c < 0 : a.m_size < b.m_size; - } -#ifndef JSON_NO_IO - friend std::ostream& operator<<(std::ostream& o, string_ref s) - { - return o.write(s.m_data, static_cast(s.m_size)); - } -#endif - - private: - const char* m_data = ""; - std::size_t m_size = 0; -}; -#endif - -} // namespace view -} // namespace detail -NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ @@ -8856,8 +9804,12 @@ class basic_json_view template friend class basic_json_document; template friend class basic_json_view; template friend class detail::view::editor; + template friend class detail::view::view_side; friend iterator; + /// whether the view is one of an editable document + static constexpr bool is_editable = Editable; + basic_json_view(const document_data* d, const node* n) noexcept : m_doc(d), m_node(n) {} diff --git a/tests/src/unit-json_view_compare.cpp b/tests/src/unit-json_view_compare.cpp new file mode 100644 index 000000000..48527adb4 --- /dev/null +++ b/tests/src/unit-json_view_compare.cpp @@ -0,0 +1,364 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; +using nlohmann::ordered_json; +using nlohmann::json_document; +using nlohmann::ordered_json_editable_document; +using nlohmann::json_view; +using nlohmann::ordered_json_document; + +#include "json_view_test_helpers.hpp" +using json_view_test::generator; + +#include +#include +#include +#include + +// These tests were split off unit-json_view_dump.cpp, whose object file got too +// large for the MinGW assembler ("file too big"). Like that file, this one is +// also built with C++17, as it mentions JSON_HAS_CPP_17. + +TEST_CASE("json_view comparison") +{ + SECTION("equality of the values parse() produces") + { + generator g; + std::vector texts; + for (int i = 0; i < 600; ++i) + { + std::string text; + g.value(text, 0); + texts.push_back(text); + // the same value written differently: sorted keys, canonical numbers + texts.push_back(json::parse(text).dump(1)); + } + for (std::size_t i = 0; i + 2 < texts.size(); ++i) + { + for (std::size_t k = i; k < i + 3; ++k) + { + CAPTURE(texts[i]) + CAPTURE(texts[k]) + const json_document a = json_document::parse(texts[i]); + const json_document b = json_document::parse(texts[k]); + const json ja = json::parse(texts[i]); + const json jb = json::parse(texts[k]); + CHECK((a.root() == b.root()) == (ja == jb)); + CHECK((a.root() != b.root()) == (ja != jb)); + CHECK((a.root() == jb) == (ja == jb)); + CHECK((jb == a.root()) == (ja == jb)); + CHECK((a.root() != jb) == (ja != jb)); + CHECK((jb != a.root()) == (ja != jb)); + + // ordered_json compares members in order + const ordered_json_document oa = ordered_json_document::parse(texts[i]); + const ordered_json_document ob = ordered_json_document::parse(texts[k]); + const ordered_json oja = ordered_json::parse(texts[i]); + const ordered_json ojb = ordered_json::parse(texts[k]); + CHECK((oa.root() == ob.root()) == (oja == ojb)); + CHECK((oa.root() == ojb) == (oja == ojb)); + } + } + } + + SECTION("objects whose first keys differ or repeat") + { + const std::array texts = + { + { + R"({"a":1,"b":2})", R"({"a":1,"c":2})", R"({"b":1,"a":2})", R"({"a":1,"a":2,"b":3})", R"({"a":2,"b":3})", + R"({"a":1,"b":1,"a":2})", R"({"a":2,"b":1})", R"({"a":1,"a":2})", R"({"b":1})", R"({"a":1,"b":2,"c":3,"d":4,"e":5})", + R"({"a":1,"b":2,"c":3,"d":4,"f":5})", R"({"a":1,"b":2,"c":3,"a":4,"e":5})", R"({"a":4,"b":2,"c":3,"e":5})", + R"({"\u0061":1,"b":2})", R"({})", R"([{"a":1,"b":2}])", R"([{"a":1,"c":2}])" + } + }; + for (const char* const x : texts) + { + for (const char* const y : texts) + { + CAPTURE(x) + CAPTURE(y) + const ordered_json_document dx = ordered_json_document::parse(x); + const ordered_json_document dy = ordered_json_document::parse(y); + const ordered_json_editable_document ex = ordered_json_editable_document::parse(x); + const ordered_json jx = ordered_json::parse(x); + const ordered_json jy = ordered_json::parse(y); + CHECK((dx.root() == dy.root()) == (jx == jy)); + CHECK((dx.root() == jy) == (jx == jy)); + CHECK((ex.root() == dy.root()) == (jx == jy)); + } + } + } + + SECTION("objects with duplicate keys, small and large") + { + // n members with keys in descending order; with duplicates, some keys + // repeat later (the first after unique ones in key order); in reverse, + // the members are in ascending key order + const auto object = [](int n, bool duplicates, bool reverse, int value) + { + std::string s = "{"; + for (int i = 0; i < n; ++i) + { + const int k = reverse ? i : n - 1 - i; + s += (i != 0 ? ",\"k" : "\"k") + std::to_string(100 + k) + "\":" + std::to_string(value + k); + } + if (duplicates) + { + for (int k = n / 2; k < n; k += 3) + { + s += ",\"k" + std::to_string(100 + k) + "\":" + std::to_string(value + k); + } + } + return s + "}"; + }; + std::vector texts; + for (const int n : + { + 3, 16, 17, 40 + }) + { + for (const int flags : + { + 0, 1, 2, 3 + }) + { + texts.push_back(object(n, (flags & 1) != 0, (flags & 2) != 0, 0)); + } + texts.push_back(object(n, true, false, 1)); // other values + } + // many objects in one comparison (working memory reused across objects) + std::string array = "["; + std::string array_other = "["; + for (std::size_t i = 0; i < texts.size(); ++i) + { + array += (i != 0 ? "," : "") + texts[i]; + array_other += (i != 0 ? "," : "") + texts[texts.size() - 1 - i]; + } + texts.push_back(array + "]"); + texts.push_back(array_other + "]"); + for (const std::string& x : texts) + { + for (const std::string& y : texts) + { + CAPTURE(x) + CAPTURE(y) + const ordered_json_document dx = ordered_json_document::parse(x); + const ordered_json_document dy = ordered_json_document::parse(y); + const ordered_json jx = ordered_json::parse(x); + const ordered_json jy = ordered_json::parse(y); + CHECK((dx.root() == dy.root()) == (jx == jy)); + CHECK((dx.root() == jy) == (jx == jy)); + const json_document ux = json_document::parse(x); + const json uy = json::parse(y); + CHECK((ux.root() == uy) == (json::parse(x) == uy)); + } + } + } + + SECTION("numbers, duplicate keys, member order") + { + const auto same = [](const char* x, const char* y) + { + const json_document dx = json_document::parse(x); + const json_document dy = json_document::parse(y); + return dx.root() == dy.root(); + }; + CHECK(same("1", "1.0")); + CHECK(same("[1, -1, 2.5]", "[1.0, -1.0, 25e-1]")); + CHECK(!same("1", "1.5")); + CHECK(same("18446744073709551615", "18446744073709551615")); + CHECK(same(R"({"a": 1, "a": 2})", R"({"a": 2})")); + CHECK(!same(R"({"a": 1, "a": 2})", R"({"a": 1})")); + CHECK(same(R"({"a": 1, "b": 2})", R"({"b": 2, "a": 1})")); + CHECK(!same(R"({"a": 1})", R"({"a": 1, "b": 2})")); + CHECK(!same("[1, 2]", "[2, 1]")); + CHECK(!same("\"a\"", "\"b\"")); + CHECK(same("\"\\u00e9\"", "\"\xc3\xa9\"")); + CHECK(!same("null", "false")); + CHECK(!same("[]", "{}")); + const ordered_json_document dup = ordered_json_document::parse(R"({"a": 1, "b": 2, "a": 3})"); + const ordered_json_document last = ordered_json_document::parse(R"({"a": 3, "b": 2})"); + CHECK(dup.root() == last.root()); + // (== compares as parse() resolves duplicates, a lookup finds the first) + CHECK(dup.root()["a"].materialize() == 1); + CHECK(dup.root()["a"] != last.root()["a"]); + const ordered_json_document ab = ordered_json_document::parse(R"({"a": 1, "b": 2})"); + const ordered_json_document ba = ordered_json_document::parse(R"({"b": 2, "a": 1})"); + CHECK(ab.root() != ba.root()); + + // discarded values compare as basic_json's do + const json discarded(json::value_t::discarded); + CHECK((json_view() == json_view()) == (discarded == discarded)); // NOLINT(readability-container-size-empty): operator== is tested + CHECK((json_view() == discarded) == (discarded == discarded)); // NOLINT(readability-container-size-empty) + const json_document null_document = json_document::parse("null"); + CHECK(!(json_view() == null_document.root())); // NOLINT(readability-container-size-empty) + CHECK(!(null_document.root() == discarded)); + } + + SECTION("values that are the same text, or the same value written differently, at any depth") + { + // nested arrays and objects around depths where the walk over equal text gives up + for (const int depth : + { + 1, 5, 47, 48, 49, 50, 60, 300 + }) + { + CAPTURE(depth) + std::string head; + std::string tail; + for (int i = 0; i < depth; ++i) + { + head += i % 2 == 0 ? R"({"k":[1,"s",{"x":true},)" : R"([null,2.5,{"y":)"; + tail.insert(0, i % 2 == 0 ? "]}" : "}]"); + } + // the same text, one number changed, and one number written differently + const auto with_number = [&](const char* number) + { + std::string result = head; + result += number; + result += tail; + return result; + }; + const std::string text = with_number("7"); + const std::string changed = with_number("8"); + const std::string spelled = with_number("7.0"); + const json_document a = json_document::parse(text); + const json_document b = json_document::parse(text); + const json_document c = json_document::parse(changed); + const json_document d = json_document::parse(spelled); + CHECK(a.root() == b.root()); + CHECK(a.root() != c.root()); + CHECK(c.root() != a.root()); + CHECK(a.root() == d.root()); + CHECK(d.root() == a.root()); + CHECK(a.root() == json::parse(text)); + CHECK(a.root() != json::parse(changed)); + CHECK(json::parse(spelled) == a.root()); + const nlohmann::json_editable_document e = nlohmann::json_editable_document::parse(text); + CHECK(a.root() == e.root()); + CHECK(e.root() == a.root()); + CHECK(c.root() != e.root()); + CHECK(e.root() == e.root()); + const ordered_json_document oa = ordered_json_document::parse(text); + const ordered_json_document oc = ordered_json_document::parse(changed); + CHECK(oa.root() != oc.root()); + CHECK(oa.root() == ordered_json::parse(text)); + CHECK(oc.root() != ordered_json::parse(text)); + } + + // not the same text, but the same value + const auto same = [](const char* x, const char* y) + { + const json_document dx = json_document::parse(x); + const json_document dy = json_document::parse(y); + const bool equal = dx.root() == dy.root(); + CHECK(equal == (json::parse(x) == json::parse(y))); + CHECK(equal == (dx.root() == json::parse(y))); + return equal; + }; + CHECK(same(R"({"a":1,"a":2})", R"({"a":3,"a":2})")); + CHECK(same(R"({"a":1,"b":[1,{"c":2,"c":3}]})", R"({"b":[1,{"c":9,"c":3}],"a":1})")); + CHECK(!same(R"({"a":1,"a":2})", R"({"a":2,"a":3})")); + CHECK(same("[1.0,-0,0,1e2]", "[1.00,0,-0,100.0]")); + CHECK(same(R"(["\u00e9","\n"])", "[\"\xc3\xa9\",\"\\n\"]")); + CHECK(!same(R"({"a":[1,2,{"b":null}]})", R"({"a":[1,2,{"b":false}]})")); + CHECK(!same(R"({"a":[1,2,{"b":null}]})", R"({"a":[1,2,{"c":null}]})")); + CHECK(!same(R"({"a":[1,2,{"b":null}]})", R"({"a":[1,2,{"b":null,"c":1}]})")); + + // edited documents: moved sequences, links, and values of new nodes + nlohmann::json_editable_document edited = nlohmann::json_editable_document::parse(R"({"a":[1,2,3],"b":{"c":"d"},"e":0.5})"); + const json_document plain = json_document::parse(R"({"a":[1,2,3,4],"b":{"c":"x","y":null},"e":0.5,"z":[]})"); + edited.push_back(edited.root()["a"], 4); + edited.set(edited.root()["b"], "c", "x"); + edited.set(edited.root()["b"], "y", nullptr); + edited.set(edited.root(), "z", json::array()); + CHECK(edited.root() == plain.root()); + CHECK(plain.root() == edited.root()); + edited.erase(edited.root()["a"], 0); + CHECK(edited.root() != plain.root()); + CHECK(edited.root()["a"] == json::parse("[2,3,4]")); + const json_document tail = json_document::parse("[2,3,4]"); + CHECK(edited.root()["a"] == tail.root()); + } + + SECTION("large documents, with a difference at any place") + { + const auto make = [](std::size_t count, std::size_t changed, bool swap_members) + { + std::string text = R"({"b":1,"list":[)"; + for (std::size_t i = 0; i < count; ++i) + { + text += (i != 0 ? "," : ""); + const std::string id = std::to_string(i == changed ? i + 1 : i); + if (swap_members && i % 3 == 0) + { + text += R"({"v":[1.5,"s",{"k":)"; + text += id; + text += R"(}],"id":)"; + text += id; + text += "}"; + } + else + { + text += R"({"id":)"; + text += id; + text += R"(,"v":[1.5,"s",{"k":)"; + text += id; + text += "}]}"; + } + } + return text + R"(],"a":{"y":2,"x":3}})"; + }; + const std::size_t count = 3000; + for (const std::size_t changed : + { + count, std::size_t{0}, std::size_t{1}, count / 2, count - 1 + }) + { + for (const bool swap : + { + false, true + }) + { + CAPTURE(changed) + CAPTURE(swap) + const std::string text_a = make(count, count, false); + const std::string text_b = make(count, changed, swap); + const json_document a = json_document::parse(text_a); + const json_document b = json_document::parse(text_b); + const bool equal = changed == count; + CHECK((a.root() == b.root()) == equal); + CHECK((b.root() == a.root()) == equal); + CHECK((a.root() == json::parse(text_b)) == equal); + CHECK((json::parse(text_b) == a.root()) == equal); + CHECK((a.root()["list"] == b.root()["list"]) == equal); + const ordered_json_document oa = ordered_json_document::parse(text_a); + const ordered_json_document ob = ordered_json_document::parse(text_b); + CHECK((oa.root() == ob.root()) == (equal && !swap)); + CHECK((oa.root() == ordered_json::parse(text_b)) == (equal && !swap)); + } + } + } + + SECTION("deep nesting") + { + const std::string deep = std::string(100000, '[') + std::string(100000, ']'); + const json_document a = json_document::parse(deep); + const json_document b = json_document::parse(deep); + CHECK(a.root() == b.root()); + CHECK(a.root() == json::parse(deep)); + const std::string other = std::string(100000, '[') + "1" + std::string(100000, ']'); + const json_document c = json_document::parse(other); + CHECK(a.root() != c.root()); + } +} diff --git a/tests/src/unit-json_view_dump.cpp b/tests/src/unit-json_view_dump.cpp index f3ccc004e..6470ab680 100644 --- a/tests/src/unit-json_view_dump.cpp +++ b/tests/src/unit-json_view_dump.cpp @@ -432,101 +432,6 @@ TEST_CASE("json_view dump") } } -TEST_CASE("json_view comparison") -{ - SECTION("equality of the values parse() produces") - { - generator g; - std::vector texts; - for (int i = 0; i < 600; ++i) - { - std::string text; - g.value(text, 0); - texts.push_back(text); - // the same value written differently: sorted keys, canonical numbers - texts.push_back(json::parse(text).dump(1)); - } - for (std::size_t i = 0; i + 2 < texts.size(); ++i) - { - for (std::size_t k = i; k < i + 3; ++k) - { - CAPTURE(texts[i]) - CAPTURE(texts[k]) - const json_document a = json_document::parse(texts[i]); - const json_document b = json_document::parse(texts[k]); - const json ja = json::parse(texts[i]); - const json jb = json::parse(texts[k]); - CHECK((a.root() == b.root()) == (ja == jb)); - CHECK((a.root() != b.root()) == (ja != jb)); - CHECK((a.root() == jb) == (ja == jb)); - CHECK((jb == a.root()) == (ja == jb)); - CHECK((a.root() != jb) == (ja != jb)); - CHECK((jb != a.root()) == (ja != jb)); - - // ordered_json compares members in order - const ordered_json_document oa = ordered_json_document::parse(texts[i]); - const ordered_json_document ob = ordered_json_document::parse(texts[k]); - const ordered_json oja = ordered_json::parse(texts[i]); - const ordered_json ojb = ordered_json::parse(texts[k]); - CHECK((oa.root() == ob.root()) == (oja == ojb)); - CHECK((oa.root() == ojb) == (oja == ojb)); - } - } - } - - SECTION("numbers, duplicate keys, member order") - { - const auto same = [](const char* x, const char* y) - { - const json_document dx = json_document::parse(x); - const json_document dy = json_document::parse(y); - return dx.root() == dy.root(); - }; - CHECK(same("1", "1.0")); - CHECK(same("[1, -1, 2.5]", "[1.0, -1.0, 25e-1]")); - CHECK(!same("1", "1.5")); - CHECK(same("18446744073709551615", "18446744073709551615")); - CHECK(same(R"({"a": 1, "a": 2})", R"({"a": 2})")); - CHECK(!same(R"({"a": 1, "a": 2})", R"({"a": 1})")); - CHECK(same(R"({"a": 1, "b": 2})", R"({"b": 2, "a": 1})")); - CHECK(!same(R"({"a": 1})", R"({"a": 1, "b": 2})")); - CHECK(!same("[1, 2]", "[2, 1]")); - CHECK(!same("\"a\"", "\"b\"")); - CHECK(same("\"\\u00e9\"", "\"\xc3\xa9\"")); - CHECK(!same("null", "false")); - CHECK(!same("[]", "{}")); - const ordered_json_document dup = ordered_json_document::parse(R"({"a": 1, "b": 2, "a": 3})"); - const ordered_json_document last = ordered_json_document::parse(R"({"a": 3, "b": 2})"); - CHECK(dup.root() == last.root()); - // (== compares as parse() resolves duplicates, a lookup finds the first) - CHECK(dup.root()["a"].materialize() == 1); - CHECK(dup.root()["a"] != last.root()["a"]); - const ordered_json_document ab = ordered_json_document::parse(R"({"a": 1, "b": 2})"); - const ordered_json_document ba = ordered_json_document::parse(R"({"b": 2, "a": 1})"); - CHECK(ab.root() != ba.root()); - - // discarded values compare as basic_json's do - const json discarded(json::value_t::discarded); - CHECK((json_view() == json_view()) == (discarded == discarded)); // NOLINT(readability-container-size-empty): operator== is tested - CHECK((json_view() == discarded) == (discarded == discarded)); // NOLINT(readability-container-size-empty) - const json_document null_document = json_document::parse("null"); - CHECK(!(json_view() == null_document.root())); // NOLINT(readability-container-size-empty) - CHECK(!(null_document.root() == discarded)); - } - - SECTION("deep nesting") - { - const std::string deep = std::string(100000, '[') + std::string(100000, ']'); - const json_document a = json_document::parse(deep); - const json_document b = json_document::parse(deep); - CHECK(a.root() == b.root()); - CHECK(a.root() == json::parse(deep)); - const std::string other = std::string(100000, '[') + "1" + std::string(100000, ']'); - const json_document c = json_document::parse(other); - CHECK(a.root() != c.root()); - } -} - TEST_CASE("json_view large objects") { // objects with 128 members or more are looked up with a hash index diff --git a/tests/src/unit-json_view_image.cpp b/tests/src/unit-json_view_image.cpp index 4f3e6319f..5e392559d 100644 --- a/tests/src/unit-json_view_image.cpp +++ b/tests/src/unit-json_view_image.cpp @@ -701,6 +701,64 @@ TEST_CASE("json_view images: ownership") #if !defined(JSON_NOEXCEPTION) TEST_CASE("json_view images: errors") { + SECTION("float tokens: a changed byte is accepted if the text is still a number with the digits the node records") + { + for (const char* const token : + { + "-12.375", "0.5", "123456.0", "-0.0", "9999999999999999999.5", "0.000000000000000000000001", "1.25e10", "5E-3" + }) + { + CAPTURE(token) + const std::string text = std::string("[") + token + "]"; + const json_document d = json_document::parse(text); + const std::vector image = d.save(); + const std::size_t at = text_at(image) + 1; // the first byte of the token + const std::uint16_t recorded = node_at(image, 1).extra; + CHECK(loaded_dump(image) == json::parse(text).dump()); + for (std::size_t pos = 0; token[pos] != '\0'; ++pos) + { + for (unsigned int byte = 0; byte < 256; ++byte) + { + std::vector changed = image; + changed[at + pos] = static_cast(byte); + std::string t = token; + t[pos] = static_cast(byte); + // the digits of the integer part and of the fraction of the changed token + std::size_t integer = 0; + std::size_t fraction = 0; + std::size_t k = (t[0] == '-') ? 1 : 0; + for (; k < t.size() && t[k] >= '0' && t[k] <= '9'; ++k) + { + ++integer; + } + if (k < t.size() && t[k] == '.') + { + for (++k; k < t.size() && t[k] >= '0' && t[k] <= '9'; ++k) + { + ++fraction; + } + } + // (as a JSON text, the token may have white space around it) + const bool float_token = json::accept(t) && t.find_first_of(std::string(" \t\r\n\0", 5)) == std::string::npos && (t.find('.') != std::string::npos || t.find_first_of("eE") != std::string::npos) // NOLINT(abseil-string-find-str-contains) + && ((integer | (fraction << 8U)) == recorded); + CAPTURE(pos) + CAPTURE(byte) + std::string result; + try + { + result = loaded_dump(changed); + } + catch (const json::parse_error& e) + { + CHECK(std::string(e.what()) == check_failed); + result = "rejected"; + } + CHECK((result != "rejected") == float_token); + } + } + } + } + SECTION("invalid UTF-8 of a decoded string: the compact dump writes it, the others throw") { const json_document d = json_document::parse(R"({"k\n":"\u00e9x"})"); From d20c4dcd894728cd6a25e8e96533adf85fd17223 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 30 Oct 2026 15:20:12 +0100 Subject: [PATCH 7/7] json_view benchmarks: compare fairly and state what each row means (#5914) - bench_view: add "(editable)" rows for json_editable_document and a parse+dump workload, so the cost of lazy float conversion shows next to the parse row (the traverse workload already includes the parse). Check that all engines' dumps agree where outputs are comparable. - compare.py: add --aligned, which also builds and runs with aligned code and reports both tables, so code-layout artifacts become visible. Off by default; the default output is unchanged. Not run end to end yet. - README: say which engines are like-for-like peers (yyjson, simdjson DOM) and which are mutable reference points (Boost.JSON, json::parse, the editable json_view); explain the lazy float conversion and the eager prototype that was measured and dropped. Signed-off-by: Niels Lohmann --- tests/benchmarks/json_view/README.md | 28 +++++- tests/benchmarks/json_view/bench_view.cpp | 116 +++++++++++++++++----- tests/benchmarks/json_view/compare.py | 72 +++++++++----- 3 files changed, 166 insertions(+), 50 deletions(-) diff --git a/tests/benchmarks/json_view/README.md b/tests/benchmarks/json_view/README.md index a0bd2afbc..8f56c2693 100644 --- a/tests/benchmarks/json_view/README.md +++ b/tests/benchmarks/json_view/README.md @@ -21,6 +21,10 @@ python3 tests/benchmarks/json_view/compare.py --data - The other libraries come from the system: pkg-config, or Homebrew (`brew install yyjson simdjson boost`). With `--download`, pinned releases are downloaded instead and checked against their SHA-256. Without Boost headers (or with `--no-boost`), the Boost.JSON columns are skipped, and the results say so. +- `--aligned` additionally builds and runs everything with all functions aligned to 64 bytes (`-mllvm + -align-all-functions=6` for Clang, `-falign-functions=64 -falign-loops=64` for GCC), and reports both result + tables as "default build" and "aligned build". Code layout alone can move a result by a few percent; a difference + that appears in only one of the two tables is a layout artifact. Off by default, which doubles the run time. - `--corpus file...` adds files to the corpus benchmark, e.g. those of [simdjson-data](https://github.com/simdjson/simdjson-data) or the [yyjson benchmark](https://github.com/ibireme/yyjson_benchmark). @@ -39,7 +43,7 @@ where `json_view` stands on another architecture; they are not meant for publica ## What is measured -`bench_view.cpp` runs four workloads on twitter, citm_catalog, canada, jeopardy, a single tweet (`status`), and a +`bench_view.cpp` runs these workloads on twitter, citm_catalog, canada, jeopardy, a single tweet (`status`), and a JSON-RPC request (`rpc`): | workload | what it does | @@ -48,6 +52,7 @@ JSON-RPC request (`rpc`): | traverse | parse, then visit every value, convert every number, touch every string and key | | select | parse, then read a few fields per record (e.g. id, user name, and retweet count of each tweet) | | dump | serialize a parsed document (compact) | +| parse+dump | build a fresh document, then serialize it (compact), in one timed call | `bench_corpus.cpp` runs parse, traverse, and dump on any list of files, so that no library is tuned to a handful of documents. Its dump also writes the numbers as they are in the input: `json_view` with `number_format::source`, and @@ -72,11 +77,32 @@ call follows an untimed call of the same engine: otherwise the engine after `jso cleaning up the tens of thousands of nodes `json::parse` just freed (with glibc, this made `json_view` look 1.7 times slower on citm_catalog traverse). +`json_view` is an index into the source text, not a tree of owning nodes. Its like-for-like peers are yyjson and +simdjson DOM (immutable, indexed, no per-node allocation). Boost.JSON and `json::parse` build mutable, owning DOMs and +are reference points for what that costs, not equals; simdjson On-Demand is lazy and forward-only, so it is the other +way round: it has no random access to pay for. All engines validate UTF-8 and syntax by default. + +Number conversion differs: `json_view` converts integers while parsing but converts floating-point numbers lazily, +when they are read or dumped; yyjson (default flags) and simdjson DOM convert every number while parsing. The cost of +float conversion therefore appears in the traverse and dump rows for `json_view`, and in the parse row for the others. +For float-heavy documents (canada), the fair reading is a sum of parse and use: the `traverse` workload already +includes the parse, and `parse+dump` measures a fresh document and its dump in one timed call; the `dump` row alone +leaves the conversion in `json_view` and out of the others. Eager conversion at parse time was prototyped as an opt-in +and measured: canada traverse (with its parse) was 4-7 % faster, but parse+dump 11-30 % slower and parse 36-90 % slower. It was +dropped, so lazy conversion is the only mode. + +`json_view` can also be edited: `json_editable_document` has the same read API, and the `(editable)` rows in +`bench_view` measure it on the same workloads. They are the fair counterpart of Boost.JSON and `json::parse`, which are +mutable too; the plain `json_view` rows are the counterpart of yyjson and simdjson DOM, which are not. `bench_edit` adds +the read-modify-write workloads. Read the tables with that in mind: an immutable-only library is not compared with an +editable `json_view` unless it says so. + The engines do not all offer the same features, which the numbers should be read with: | engine | document | random access | editable | notes | |---|---|---|---|---| | `json_view` | immutable index into the text | yes | no | a fresh document per parse; "reused" parses into the same document | +| `json_view` (editable) | index into the text plus edits | yes | yes, in place | `json_editable_document`; keeps member order and number spellings | | yyjson | immutable (`yyjson_read`) | yes | via a mutable copy | | | simdjson DOM | immutable, parser reused | yes | no | "fresh" uses a new parser per parse | | simdjson On-Demand | none: forward-only, lazy | no | no | only traverse and select | diff --git a/tests/benchmarks/json_view/bench_view.cpp b/tests/benchmarks/json_view/bench_view.cpp index 32d038649..71e19d68b 100644 --- a/tests/benchmarks/json_view/bench_view.cpp +++ b/tests/benchmarks/json_view/bench_view.cpp @@ -8,8 +8,9 @@ // Same-feature-set benchmark: read-only JSON documents with random access. // -// json_view nlohmann/json_view.hpp (fresh document per parse / reused) -// yyjson yyjson_read(): immutable document, random access +// json_view nlohmann/json_view.hpp (fresh document per parse / reused; +// "(editable)": json_editable_document, same read API) +// yyjson yyjson_read(): immutable document, random access // simdjson DOM dom::parser (reused, as recommended; "fresh": a new parser // per parse): immutable, random access // references (different feature sets): @@ -19,7 +20,11 @@ // // Workloads: parse (build + free), traverse (visit everything, convert every // number, touch every string and key), select (a few fields per document), -// dump (compact serialization of the parsed document). +// dump (compact serialization of the parsed document), and parse+dump (a fresh +// document, then its dump, in one timed call). json_view converts floating-point +// numbers lazily (when read or dumped), yyjson and simdjson DOM while parsing: +// for float-heavy documents, traverse (which includes the parse) and parse+dump +// are the fair reading of the parse and dump rows. // All engines run interleaved in every round, each timed call after an untimed // one of the same engine; the best round is reported. #include @@ -43,6 +48,8 @@ using nlohmann::json; using nlohmann::json_document; +using nlohmann::json_editable_document; +using nlohmann::json_editable_view; using nlohmann::json_view; static volatile double g_sink; @@ -55,7 +62,8 @@ struct stats // ---------------- traversal ---------------- -static void walk(json_view v, stats& st) +template +static void walk_view(V v, stats& st) { ++st.nodes; switch (v.type()) @@ -64,32 +72,42 @@ static void walk(json_view v, stats& st) for (auto it = v.begin(); it != v.end(); ++it) { st.str += it.key().size(); - walk(*it, st); + walk_view(*it, st); } break; case json::value_t::array: - for (const json_view e : v) + for (const V e : v) { - walk(e, st); + walk_view(e, st); } break; case json::value_t::string: st.str += v.get_string().size(); break; case json::value_t::number_integer: - st.num += static_cast(v.get()); + st.num += static_cast(v.template get()); break; case json::value_t::number_unsigned: - st.num += static_cast(v.get()); + st.num += static_cast(v.template get()); break; case json::value_t::number_float: - st.num += v.get(); + st.num += v.template get(); break; default: break; } } +static void walk(json_view v, stats& st) +{ + walk_view(v, st); +} + +static void walk(json_editable_view v, stats& st) +{ + walk_view(v, st); +} + static void walk(const json& j, stats& st) { ++st.nodes; @@ -300,20 +318,21 @@ static void walk(const boost::json::value& v, stats& st) // status (one tweet): id, user.screen_name, retweet_count // rpc: method, params.minuend, id -static double pick(const std::string& name, json_view r) +template +static double pick_view(const std::string& name, V r) { double acc = 0; if (name == "twitter" || name == "status") { - auto one = [&](json_view s) + auto one = [&](V s) { - acc += static_cast(s["id"].get()); + acc += static_cast(s["id"].template get()); acc += static_cast(s["user"]["screen_name"].get_string().size()); - acc += static_cast(s["retweet_count"].get()); + acc += static_cast(s["retweet_count"].template get()); }; if (name == "twitter") { - for (const json_view s : r["statuses"]) + for (const V s : r["statuses"]) { one(s); } @@ -325,20 +344,20 @@ static double pick(const std::string& name, json_view r) } else if (name == "citm_catalog") { - for (const json_view p : r["performances"]) + for (const V p : r["performances"]) { - acc += static_cast(p["id"].get() + p["eventId"].get() + p["seatCategories"].size()); + acc += static_cast(p["id"].template get() + p["eventId"].template get() + p["seatCategories"].size()); } acc += static_cast(r["events"].size()); } else if (name == "canada") { - const json_view g = r["features"][0]["geometry"]; + const V g = r["features"][0]["geometry"]; acc += static_cast(r["type"].get_string().size() + g["type"].get_string().size() + g["coordinates"].size()); } else if (name == "jeopardy") { - for (const json_view q : r) + for (const V q : r) { acc += q["round"].get_string() == "Final Jeopardy!" ? 1 : 0; acc += static_cast(q["category"].get_string().size()); @@ -347,11 +366,21 @@ static double pick(const std::string& name, json_view r) else if (name == "rpc") { acc += static_cast(r["method"].get_string().size()); - acc += static_cast(r["params"]["minuend"].get() + r["id"].get()); + acc += static_cast(r["params"]["minuend"].template get() + r["id"].template get()); } return acc; } +static double pick(const std::string& name, json_view r) +{ + return pick_view(name, r); +} + +static double pick(const std::string& name, json_editable_view r) +{ + return pick_view(name, r); +} + static double pick(const std::string& name, const json& r) { double acc = 0; @@ -634,15 +663,35 @@ int main(int argc, char** argv) walk(json::parse(dc.text), a); auto d = json_document::parse(dc.text); walk(d.root(), b); + auto ed = json_editable_document::parse(dc.text); + stats e; + walk(ed.root(), e); yyjson_doc* y = yyjson_read(dc.text.data(), dc.text.size(), 0); walk(yyjson_doc_get_root(y), c); simdjson::dom::parser p; walk(p.parse(dc.text).value(), dd); - const bool ok = a.nodes == b.nodes && a.nodes == c.nodes && a.nodes == dd.nodes && a.str == b.str && a.str == c.str && a.str == dd.str + const bool ok = a.nodes == b.nodes && a.nodes == e.nodes && a.str == e.str && std::fabs(a.num - e.num) <= 1e-9 * std::fabs(a.num) && a.nodes == c.nodes && a.nodes == dd.nodes && a.str == b.str && a.str == c.str && a.str == dd.str && std::fabs(a.num - b.num) <= 1e-9 * std::fabs(a.num) && std::fabs(a.num - c.num) <= 1e-9 * std::fabs(a.num); - const double pa = pick(dc.name, json::parse(dc.text)), pb = pick(dc.name, d.root()), pc = pick(dc.name, yyjson_doc_get_root(y)), pd = pick(dc.name, p.parse(dc.text).value()); - std::printf("check %-13s traverse %s select %s\n", dc.name.c_str(), ok ? "OK" : "MISMATCH", (pa == pb && pa == pc && pa == pd) ? "OK" : "MISMATCH"); + const double pa = pick(dc.name, json::parse(dc.text)), pb = pick(dc.name, d.root()), pe = pick(dc.name, ed.root()), pc = pick(dc.name, yyjson_doc_get_root(y)), pd = pick(dc.name, p.parse(dc.text).value()); + std::printf("check %-13s traverse %s select %s\n", dc.name.c_str(), ok ? "OK" : "MISMATCH", (pa == pb && pa == pe && pa == pc && pa == pd) ? "OK" : "MISMATCH"); yyjson_doc_free(y); + + // the dumps of the engines may format floats differently, so only: every dump is non-empty, and the + // dump of json_view (also editable) and of json::parse parse back to the same number of nodes + const std::string dj = json::parse(dc.text).dump(); + const std::string dv = d.root().dump(); + const std::string de = ed.root().dump(); + std::size_t yn = 0; + yyjson_doc* y2 = yyjson_read(dc.text.data(), dc.text.size(), 0); + char* yo = yyjson_write(y2, 0, &yn); + const bool dumps_ok = !dj.empty() && !dv.empty() && !de.empty() && yo != nullptr && yn > 0 && !simdjson::to_string(p.parse(dc.text).value()).empty(); + std::free(yo); + yyjson_doc_free(y2); + stats sj_nodes, sv_nodes, se_nodes; + walk(json::parse(dj), sj_nodes); + walk(json::parse(dv), sv_nodes); + walk(json::parse(de), se_nodes); + std::printf("check %-13s dump %s\n", dc.name.c_str(), (dumps_ok && sj_nodes.nodes == a.nodes && sv_nodes.nodes == a.nodes && se_nodes.nodes == a.nodes) ? "OK" : "MISMATCH"); } std::FILE* csv = std::fopen("bench_view.csv", "w"); @@ -669,6 +718,7 @@ int main(int argc, char** argv) workloads.push_back({"parse", { {"json_view", [&] { auto d = json_document::parse(s); g_sink = static_cast(d.node_count()); }}, {"json_view (reused)", [&] { reused.read(s); g_sink = static_cast(reused.node_count()); }}, + {"json_view (editable)", [&] { auto d = json_editable_document::parse(s); g_sink = static_cast(d.root().size()); }}, {"yyjson", [&] { yyjson_doc* d = yyjson_read(s.data(), s.size(), 0); g_sink = static_cast(yyjson_doc_get_val_count(d)); yyjson_doc_free(d); }}, {"simdjson DOM", [&] { auto e = sj.parse(ps).value_unsafe(); g_sink = e.is_object(); }}, {"simdjson DOM (fresh)", [&] { simdjson::dom::parser p; auto e = p.parse(ps).value_unsafe(); g_sink = e.is_object(); }}, @@ -680,6 +730,7 @@ int main(int argc, char** argv) workloads.push_back({"traverse", { {"json_view", [&] { auto d = json_document::parse(s); stats st; walk(d.root(), st); g_sink = st.num; }}, {"json_view (reused)", [&] { reused.read(s); stats st; walk(reused.root(), st); g_sink = st.num; }}, + {"json_view (editable)", [&] { auto d = json_editable_document::parse(s); stats st; walk(d.root(), st); g_sink = st.num; }}, {"yyjson", [&] { yyjson_doc* d = yyjson_read(s.data(), s.size(), 0); stats st; walk(yyjson_doc_get_root(d), st); g_sink = st.num; yyjson_doc_free(d); }}, {"simdjson DOM", [&] { stats st; walk(sj.parse(ps).value_unsafe(), st); g_sink = st.num; }}, {"simdjson OD", [&] { auto d = od.iterate(ps).value_unsafe(); stats st; walk_od(d.get_value().value_unsafe(), st); g_sink = st.num; }}, @@ -691,15 +742,31 @@ int main(int argc, char** argv) workloads.push_back({"select", { {"json_view", [&] { auto d = json_document::parse(s); g_sink = pick(name, d.root()); }}, {"json_view (reused)", [&] { reused.read(s); g_sink = pick(name, reused.root()); }}, + {"json_view (editable)", [&] { auto d = json_editable_document::parse(s); g_sink = pick(name, d.root()); }}, {"yyjson", [&] { yyjson_doc* d = yyjson_read(s.data(), s.size(), 0); g_sink = pick(name, yyjson_doc_get_root(d)); yyjson_doc_free(d); }}, {"simdjson DOM", [&] { g_sink = pick(name, sj.parse(ps).value_unsafe()); }}, {"simdjson OD", [&] { auto d = od.iterate(ps).value_unsafe(); g_sink = pick_od(name, d); }}, {"json::parse", [&] { json j = json::parse(s); g_sink = pick(name, j); }}, }}); + // pipeline: a fresh document, then a dump of it, in one timed call + // (simdjson DOM reuses its parser, as in the parse row); the traverse + // workload above is already parse+traverse + workloads.push_back({"parse+dump", { + {"json_view", [&] { auto d = json_document::parse(s); std::string o = d.root().dump(); g_sink = static_cast(o.size()); }}, + {"json_view (editable)", [&] { auto d = json_editable_document::parse(s); std::string o = d.root().dump(); g_sink = static_cast(o.size()); }}, + {"yyjson", [&] { yyjson_doc* d = yyjson_read(s.data(), s.size(), 0); std::size_t n = 0; char* o = yyjson_write(d, 0, &n); g_sink = static_cast(n); std::free(o); yyjson_doc_free(d); }}, + {"simdjson DOM", [&] { std::string o = simdjson::to_string(sj.parse(ps).value_unsafe()); g_sink = static_cast(o.size()); }}, +#if JSON_VIEW_BENCH_BOOST + {"Boost.JSON", [&] { boost::json::monotonic_resource mr; auto v = boost::json::parse(s, &mr); std::string o = boost::json::serialize(v); g_sink = static_cast(o.size()); }}, +#endif + {"json::parse", [&] { json j = json::parse(s); std::string o = j.dump(); g_sink = static_cast(o.size()); }}, + }}); { // serialization of an already parsed document static json_document vd; vd.read(s); + static json_editable_document ed = json_editable_document::parse(s); + ed = json_editable_document::parse(s); static yyjson_doc* yd = nullptr; if (yd) { @@ -713,6 +780,7 @@ int main(int argc, char** argv) jd = json::parse(s); workloads.push_back({"dump", { {"json_view", [&] { std::string o = vd.root().dump(); g_sink = static_cast(o.size()); }}, + {"json_view (editable)", [&] { std::string o = ed.root().dump(); g_sink = static_cast(o.size()); }}, {"yyjson", [&] { std::size_t n = 0; char* o = yyjson_write(yd, 0, &n); g_sink = static_cast(n); std::free(o); }}, {"simdjson DOM", [&] { std::string o = simdjson::to_string(se); g_sink = static_cast(o.size()); }}, {"json::parse", [&] { std::string o = jd.dump(); g_sink = static_cast(o.size()); }}, @@ -740,7 +808,7 @@ int main(int argc, char** argv) } } const double ref = best[0]; - std::printf("%-13s %-9s", dc.name.c_str(), wl.first.c_str()); + std::printf("%-13s %-14s", dc.name.c_str(), wl.first.c_str()); for (std::size_t k = 0; k < wl.second.size(); ++k) { const double us = best[k] / 1e3; diff --git a/tests/benchmarks/json_view/compare.py b/tests/benchmarks/json_view/compare.py index 6573c88af..1e8ae2494 100755 --- a/tests/benchmarks/json_view/compare.py +++ b/tests/benchmarks/json_view/compare.py @@ -239,6 +239,8 @@ def main(): ap.add_argument('--download', action='store_true', help='use pinned downloads instead of system libraries') ap.add_argument('--no-boost', action='store_true', help='skip Boost.JSON') ap.add_argument('--native', action='store_true', help='compile for this CPU (-march=native / -mcpu=native)') + ap.add_argument('--aligned', action='store_true', + help='also build and run with all functions (and loops) aligned, to make code layout artifacts visible') ap.add_argument('--rounds', type=int, default=30, help='rounds of bench_view (default: 30)') ap.add_argument('--corpus', nargs='*', default=[], help='more files for bench_corpus') ap.add_argument('--build-dir', default=os.path.join(HERE, 'build'), help='where to build (default: build/ next to this script)') @@ -282,32 +284,46 @@ def main(): + ['-I' + d for d in lib.include]) objects.append(obj) - binaries = {} - for bench in ['bench_view', 'bench_corpus', 'bench_edit', 'bench_decode']: - exe = os.path.join(args.build_dir, bench) - run(cxx + flags + include + [os.path.join(HERE, bench + '.cpp')] + objects + link + ['-o', exe]) - binaries[bench] = exe + cxx_version = output(cxx + ['--version']) + is_clang = 'clang' in cxx_version.lower() + # alignment flags: functions and (gcc) loops on 64-byte boundaries + align_flags = ['-mllvm', '-align-all-functions=6'] if is_clang else ['-falign-functions=64', '-falign-loops=64'] + + # the builds: the default one and, with --aligned, one with aligned code + variants = [('default build', args.build_dir, [])] + if args.aligned: + variants.append(('aligned build', os.path.join(args.build_dir, 'aligned'), align_flags)) - # run: bench_view on its documents, bench_corpus on those and the given files corpus = [os.path.join(args.data, f) for f in DEFAULT_CORPUS] + args.corpus - outputs = {} - outputs['bench_view'] = run([binaries['bench_view'], args.data, str(args.rounds)], cwd=args.build_dir, - capture_output=True, text=True).stdout - outputs['bench_corpus'] = run([binaries['bench_corpus']] + corpus, cwd=args.build_dir, - capture_output=True, text=True).stdout - outputs['bench_edit'] = run([binaries['bench_edit'], args.data, str(max(1, args.rounds // 2))], cwd=args.build_dir, - capture_output=True, text=True).stdout - outputs['bench_decode'] = run([binaries['bench_decode'], args.data, str(max(1, args.rounds // 2))], cwd=args.build_dir, - capture_output=True, text=True).stdout - for name, text in outputs.items(): - print(text) + # label -> {benchmark: output} + all_outputs = {} + for label, build_dir, extra in variants: + os.makedirs(build_dir, exist_ok=True) + binaries = {} + for bench in ['bench_view', 'bench_corpus', 'bench_edit', 'bench_decode']: + exe = os.path.join(build_dir, bench) + run(cxx + flags + extra + include + [os.path.join(HERE, bench + '.cpp')] + objects + link + ['-o', exe]) + binaries[bench] = exe + + # run: bench_view on its documents, bench_corpus on those and the given files + outputs = {} + outputs['bench_view'] = run([binaries['bench_view'], args.data, str(args.rounds)], cwd=build_dir, + capture_output=True, text=True).stdout + outputs['bench_corpus'] = run([binaries['bench_corpus']] + corpus, cwd=build_dir, + capture_output=True, text=True).stdout + outputs['bench_edit'] = run([binaries['bench_edit'], args.data, str(max(1, args.rounds // 2))], cwd=build_dir, + capture_output=True, text=True).stdout + outputs['bench_decode'] = run([binaries['bench_decode'], args.data, str(max(1, args.rounds // 2))], + cwd=build_dir, capture_output=True, text=True).stdout + for name, text in outputs.items(): + print(text) + all_outputs[label] = outputs # results with their metadata now = datetime.datetime.now() host = re.sub(r'[^A-Za-z0-9-]+', '-', platform.node().split('.')[0]) or 'host' stem = os.path.join(HERE, 'results', f'{now:%Y-%m-%d}-{host}') os.makedirs(os.path.dirname(stem), exist_ok=True) - cxx_version = output(cxx + ['--version']) compiler_version = cxx_version.splitlines()[0] if cxx_version else ' '.join(cxx) meta = [ ('date', f'{now:%Y-%m-%d %H:%M}'), @@ -316,6 +332,7 @@ def main(): ('OS', f'{platform.system()} {platform.release()} ({platform.machine()})'), ('compiler', compiler_version), ('flags', ' '.join(flags)), + ('aligned results', ('included (flags: ' + ' '.join(align_flags) + ')') if args.aligned else 'not included'), ('yyjson', libs['yyjson'].version), ('simdjson', libs['simdjson'].version), ('Boost.JSON', libs['boost'].version if with_boost else 'skipped (not found)'), @@ -328,15 +345,20 @@ def main(): f.write('| | |\n|---|---|\n') for key, value in meta: f.write(f'| {key} | {value} |\n') - for name, text in outputs.items(): - f.write(f'\n## {name}\n\n```\n{text.rstrip()}\n```\n') + for label, outputs in all_outputs.items(): + # (without --aligned there is only the default build, and no extra heading level) + prefix = f'{label}: ' if args.aligned else '' + for name, text in outputs.items(): + f.write(f'\n## {prefix}{name}\n\n```\n{text.rstrip()}\n```\n') with open(stem + '.csv', 'w', encoding='utf-8') as out: out.write(''.join(f'# {key}: {value}\n' for key, value in meta)) - for name in ['bench_view', 'bench_corpus', 'bench_edit', 'bench_decode']: - path = os.path.join(args.build_dir, name + '.csv') - if os.path.isfile(path): - with open(path, encoding='utf-8') as f: - out.write(f'# {name}\n' + f.read()) + for label, build_dir, _ in variants: + for name in ['bench_view', 'bench_corpus', 'bench_edit', 'bench_decode']: + path = os.path.join(build_dir, name + '.csv') + if os.path.isfile(path): + with open(path, encoding='utf-8') as f: + # (the default build keeps its plain section names) + out.write(f'# {name}' + (f' ({label})' if args.aligned else '') + '\n' + f.read()) print(f'results: {stem}.md, {stem}.csv')