From 3fb7d9c71050d67fd8b619b792e8ebd97a38eaec Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 21 Oct 2026 22:58:51 +0200 Subject: [PATCH] Check the float tokens of a json_document image without a sort and a scan of every character load(image_check::full) checked every float token of an image by sorting the token ranges, scanning the token character by character, and comparing the digit layout with the node's. Now: - the ranges are sorted only if they are not sorted already (save() writes them in the order of the nodes, so they are); - a token of the shape [-]digits.digits whose digit counts are the ones the node records (nearly all of them) is accepted by checking its digits eight at a time, without the general scan; - the general scan skips runs of digits eight at a time. Damaged tokens are still rejected as before: the check accepts exactly the tokens that the general scan accepts. Instructions of load(image_check::full) (develop -> now): macOS arm64 canada 44.6M -> 30.9M (-31 %), jeopardy 11.3M -> 10.7M, citm 10.4M -> 10.0M, twitter 5.06M -> 4.94M Linux GCC 13 canada -50 %, jeopardy -43 %, twitter -31 %, citm -28 % Linux Clang 18 canada -42 %, jeopardy -48 %, twitter -35 %, citm -32 % image_check::bounds is unchanged (within 0.4 %). Signed-off-by: Niels Lohmann --- include/nlohmann/detail/view/image.hpp | 119 +++++++++++++++++++++---- single_include/nlohmann/json_view.hpp | 119 +++++++++++++++++++++---- tests/src/unit-json_view_image.cpp | 60 +++++++++++++ 3 files changed, 262 insertions(+), 36 deletions(-) diff --git a/include/nlohmann/detail/view/image.hpp b/include/nlohmann/detail/view/image.hpp index 702a55530..14ab6609a 100644 --- a/include/nlohmann/detail/view/image.hpp +++ b/include/nlohmann/detail/view/image.hpp @@ -255,6 +255,82 @@ struct number_token bool is_float; ///< a fraction or an exponent }; +/// advance over ASCII digits, eight bytes at a time +inline const unsigned char* digit_run(const unsigned char* p, const unsigned char* e) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + while (e - p >= 8) + { + // (the digits become 0..9; a byte above 9, or with bit 7 set, is no digit. A carry + // out of a byte with bit 7 set only affects the bytes above, after the first one + // that count_trailing_zeros finds.) + const std::uint64_t x = read_eight_bytes(p) ^ (ones * 0x30u); + const std::uint64_t not_digit = ((x + (ones * 0x76u)) | x) & (ones * 0x80u); + if (not_digit != 0) + { + return p + (static_cast(count_trailing_zeros(not_digit)) / 8); + } + p += 8; + } + while (p != e && is_digit(*p)) + { + ++p; + } + return p; +} + +/// whether the n bytes at p are ASCII digits (n > 0) +inline bool all_digits(const unsigned char* p, std::size_t n) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + const auto digits = [](const unsigned char* q) + { + const std::uint64_t x = read_eight_bytes(q) ^ (ones * 0x30u); + return (((x + (ones * 0x76u)) | x) & (ones * 0x80u)) == 0; + }; + if (n >= 8) + { + std::size_t i = 0; + for (; i + 8 < n; i += 8) + { + if (!digits(p + i)) + { + return false; + } + } + return digits(p + n - 8); // (overlapping the bytes before: they are digits) + } + for (std::size_t i = 0; i < n; ++i) + { + if (!is_digit(p[i])) + { + return false; + } + } + return true; +} + +/// Whether a float token has the shape that the parser records for most of +/// them: [-]digits.digits, with the numbers of digits of the node's layout +/// (extra). Then the token is a number of that layout, and the general scan +/// that follows is not needed. +inline bool plain_float_token(const unsigned char* s, std::size_t len, std::uint16_t extra) noexcept +{ + const std::size_t int_digits = extra & 0xFFu; + const std::size_t frac_digits = extra >> 8u; + if (int_digits == 0 || int_digits >= 255 || frac_digits == 0 || frac_digits >= 255 || len == 0) + { + return false; + } + const std::size_t negative = *s == '-' ? 1 : 0; + if (len != negative + int_digits + 1 + frac_digits) + { + return false; + } + const unsigned char* const p = s + negative; + return !(int_digits > 1 && *p == '0') && p[int_digits] == '.' && all_digits(p, int_digits) && all_digits(p + int_digits + 1, frac_digits); +} + /// whether [s, s + len) is a JSON number (the grammar the parser accepts) inline bool scan_number_token(const unsigned char* s, std::size_t len, number_token& t) { @@ -273,11 +349,7 @@ inline bool scan_number_token(const unsigned char* s, std::size_t len, number_to } else if (*p >= '1' && *p <= '9') { - do - { - ++p; - } - while (p != e && is_digit(*p)); + p = digit_run(p + 1, e); } else { @@ -289,10 +361,7 @@ inline bool scan_number_token(const unsigned char* s, std::size_t len, number_to if (p != e && *p == '.') { const unsigned char* const f0 = ++p; - while (p != e && is_digit(*p)) - { - ++p; - } + p = digit_run(p, e); if (p == f0) { return false; @@ -403,10 +472,15 @@ struct float_range template bool check_distinct_ranges(std::vector& ranges, Check check) { - std::sort(ranges.begin(), ranges.end(), [](const byte_range & a, const byte_range & b) + // (save() writes the ranges in the order of the nodes, which is sorted already) + const auto before = [](const byte_range & a, const byte_range & b) { return a.off != b.off ? a.off < b.off : a.len < b.len; - }); + }; + if (!std::is_sorted(ranges.begin(), ranges.end(), before)) + { + std::sort(ranges.begin(), ranges.end(), before); + } std::size_t end = 0; for (std::size_t i = 0; i < ranges.size(); ++i) { @@ -428,7 +502,8 @@ bool check_distinct_ranges(std::vector& ranges, Check check) /// record layouts that match it), tokens must not overlap. inline bool check_float_ranges(std::vector& ranges, const unsigned char* text) { - std::sort(ranges.begin(), ranges.end(), [](const float_range & a, const float_range & b) + // (save() writes the tokens in the order of the nodes, which is sorted already) + const auto before = [](const float_range & a, const float_range & b) { if (a.off != b.off) { @@ -439,7 +514,11 @@ inline bool check_float_ranges(std::vector& ranges, const unsigned return a.len < b.len; } return a.extra < b.extra; - }); + }; + if (!std::is_sorted(ranges.begin(), ranges.end(), before)) + { + std::sort(ranges.begin(), ranges.end(), before); + } std::size_t end = 0; std::size_t i = 0; while (i < ranges.size()) @@ -449,14 +528,18 @@ inline bool check_float_ranges(std::vector& ranges, const unsigned { return false; } - number_token t{}; const unsigned char* const s = text + r.off; - if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t)) + std::uint16_t layout = r.extra; + if (!plain_float_token(s, r.len, r.extra)) { - return false; + number_token t{}; + if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t)) + { + return false; + } + // the digit layout the parser records (or "many", as compaction writes it) + layout = float_layout(t); } - // the digit layout the parser records (or "many", as compaction writes it) - const std::uint16_t layout = float_layout(t); for (; i < ranges.size() && ranges[i].off == r.off && ranges[i].len == r.len; ++i) { if (ranges[i].extra != layout && ranges[i].extra != 0xFFFFu) diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index c36adf457..cc9fe4ae1 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -5712,6 +5712,82 @@ struct number_token bool is_float; ///< a fraction or an exponent }; +/// advance over ASCII digits, eight bytes at a time +inline const unsigned char* digit_run(const unsigned char* p, const unsigned char* e) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + while (e - p >= 8) + { + // (the digits become 0..9; a byte above 9, or with bit 7 set, is no digit. A carry + // out of a byte with bit 7 set only affects the bytes above, after the first one + // that count_trailing_zeros finds.) + const std::uint64_t x = read_eight_bytes(p) ^ (ones * 0x30u); + const std::uint64_t not_digit = ((x + (ones * 0x76u)) | x) & (ones * 0x80u); + if (not_digit != 0) + { + return p + (static_cast(count_trailing_zeros(not_digit)) / 8); + } + p += 8; + } + while (p != e && is_digit(*p)) + { + ++p; + } + return p; +} + +/// whether the n bytes at p are ASCII digits (n > 0) +inline bool all_digits(const unsigned char* p, std::size_t n) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + const auto digits = [](const unsigned char* q) + { + const std::uint64_t x = read_eight_bytes(q) ^ (ones * 0x30u); + return (((x + (ones * 0x76u)) | x) & (ones * 0x80u)) == 0; + }; + if (n >= 8) + { + std::size_t i = 0; + for (; i + 8 < n; i += 8) + { + if (!digits(p + i)) + { + return false; + } + } + return digits(p + n - 8); // (overlapping the bytes before: they are digits) + } + for (std::size_t i = 0; i < n; ++i) + { + if (!is_digit(p[i])) + { + return false; + } + } + return true; +} + +/// Whether a float token has the shape that the parser records for most of +/// them: [-]digits.digits, with the numbers of digits of the node's layout +/// (extra). Then the token is a number of that layout, and the general scan +/// that follows is not needed. +inline bool plain_float_token(const unsigned char* s, std::size_t len, std::uint16_t extra) noexcept +{ + const std::size_t int_digits = extra & 0xFFu; + const std::size_t frac_digits = extra >> 8u; + if (int_digits == 0 || int_digits >= 255 || frac_digits == 0 || frac_digits >= 255 || len == 0) + { + return false; + } + const std::size_t negative = *s == '-' ? 1 : 0; + if (len != negative + int_digits + 1 + frac_digits) + { + return false; + } + const unsigned char* const p = s + negative; + return !(int_digits > 1 && *p == '0') && p[int_digits] == '.' && all_digits(p, int_digits) && all_digits(p + int_digits + 1, frac_digits); +} + /// whether [s, s + len) is a JSON number (the grammar the parser accepts) inline bool scan_number_token(const unsigned char* s, std::size_t len, number_token& t) { @@ -5730,11 +5806,7 @@ inline bool scan_number_token(const unsigned char* s, std::size_t len, number_to } else if (*p >= '1' && *p <= '9') { - do - { - ++p; - } - while (p != e && is_digit(*p)); + p = digit_run(p + 1, e); } else { @@ -5746,10 +5818,7 @@ inline bool scan_number_token(const unsigned char* s, std::size_t len, number_to if (p != e && *p == '.') { const unsigned char* const f0 = ++p; - while (p != e && is_digit(*p)) - { - ++p; - } + p = digit_run(p, e); if (p == f0) { return false; @@ -5860,10 +5929,15 @@ struct float_range template bool check_distinct_ranges(std::vector& ranges, Check check) { - std::sort(ranges.begin(), ranges.end(), [](const byte_range & a, const byte_range & b) + // (save() writes the ranges in the order of the nodes, which is sorted already) + const auto before = [](const byte_range & a, const byte_range & b) { return a.off != b.off ? a.off < b.off : a.len < b.len; - }); + }; + if (!std::is_sorted(ranges.begin(), ranges.end(), before)) + { + std::sort(ranges.begin(), ranges.end(), before); + } std::size_t end = 0; for (std::size_t i = 0; i < ranges.size(); ++i) { @@ -5885,7 +5959,8 @@ bool check_distinct_ranges(std::vector& ranges, Check check) /// record layouts that match it), tokens must not overlap. inline bool check_float_ranges(std::vector& ranges, const unsigned char* text) { - std::sort(ranges.begin(), ranges.end(), [](const float_range & a, const float_range & b) + // (save() writes the tokens in the order of the nodes, which is sorted already) + const auto before = [](const float_range & a, const float_range & b) { if (a.off != b.off) { @@ -5896,7 +5971,11 @@ inline bool check_float_ranges(std::vector& ranges, const unsigned return a.len < b.len; } return a.extra < b.extra; - }); + }; + if (!std::is_sorted(ranges.begin(), ranges.end(), before)) + { + std::sort(ranges.begin(), ranges.end(), before); + } std::size_t end = 0; std::size_t i = 0; while (i < ranges.size()) @@ -5906,14 +5985,18 @@ inline bool check_float_ranges(std::vector& ranges, const unsigned { return false; } - number_token t{}; const unsigned char* const s = text + r.off; - if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t)) + std::uint16_t layout = r.extra; + if (!plain_float_token(s, r.len, r.extra)) { - return false; + number_token t{}; + if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t)) + { + return false; + } + // the digit layout the parser records (or "many", as compaction writes it) + layout = float_layout(t); } - // the digit layout the parser records (or "many", as compaction writes it) - const std::uint16_t layout = float_layout(t); for (; i < ranges.size() && ranges[i].off == r.off && ranges[i].len == r.len; ++i) { if (ranges[i].extra != layout && ranges[i].extra != 0xFFFFu) diff --git a/tests/src/unit-json_view_image.cpp b/tests/src/unit-json_view_image.cpp index 50023fe63..fa9676617 100644 --- a/tests/src/unit-json_view_image.cpp +++ b/tests/src/unit-json_view_image.cpp @@ -682,6 +682,66 @@ TEST_CASE("json_view images: ownership") #if !defined(JSON_NOEXCEPTION) TEST_CASE("json_view images: errors") { +#if !defined(JSON_NOEXCEPTION) + SECTION("float tokens: a changed byte is accepted if the text is still a number with the digits the node records") + { + for (const char* const token : + { + "-12.375", "0.5", "123456.0", "-0.0", "9999999999999999999.5", "0.000000000000000000000001", "1.25e10", "5E-3" + }) + { + CAPTURE(token) + const std::string text = std::string("[") + token + "]"; + const json_document d = json_document::parse(text); + const std::vector image = d.save(); + const std::size_t at = text_at(image) + 1; // the first byte of the token + const std::uint16_t recorded = node_at(image, 1).extra; + CHECK(loaded_dump(image) == json::parse(text).dump()); + for (std::size_t pos = 0; token[pos] != '\0'; ++pos) + { + for (unsigned int byte = 0; byte < 256; ++byte) + { + std::vector changed = image; + changed[at + pos] = static_cast(byte); + std::string t = token; + t[pos] = static_cast(byte); + // the digits of the integer part and of the fraction of the changed token + std::size_t integer = 0; + std::size_t fraction = 0; + std::size_t k = (t[0] == '-') ? 1 : 0; + for (; k < t.size() && t[k] >= '0' && t[k] <= '9'; ++k) + { + ++integer; + } + if (k < t.size() && t[k] == '.') + { + for (++k; k < t.size() && t[k] >= '0' && t[k] <= '9'; ++k) + { + ++fraction; + } + } + // (as a JSON text, the token may have white space around it) + const bool float_token = json::accept(t) && t.find_first_of(std::string(" \t\r\n\0", 5)) == std::string::npos && (t.find('.') != std::string::npos || t.find_first_of("eE") != std::string::npos) + && ((integer | (fraction << 8U)) == recorded); + CAPTURE(pos) + CAPTURE(byte) + std::string result; + try + { + result = loaded_dump(changed); + } + catch (const json::parse_error& e) + { + CHECK(std::string(e.what()) == check_failed); + result = "rejected"; + } + CHECK((result != "rejected") == float_token); + } + } + } + } +#endif + SECTION("a literal as the root: dump() after loading") { for (const char* text :