From e7d245db7e20bcd01f3277adf8caf27ee12423a5 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 25 Oct 2026 06:17:50 +0100 Subject: [PATCH] json_view: dump canada.json faster than yyjson (batched float decisions) Dump of float tokens of 16 or 17 digits without an exponent (SIMD targets, little-endian), in batches of up to 256 floats ahead of the walk, one loop per step: - the digits (16 bytes at a time) and the token's last four digits; - locate_token(): Eisel-Lemire's product gives the double x, and x's position relative to the token is exact in a 64-bit fixed-point number (56 bits after the point of the token's last digit), with the radius of its interval; - token_digits_1/2(), one loop per kind (g): the multiple of 10^g nearest to x if it is in the interval, else the multiple of 10^(g - 1) nearest to x, the digits that dump() (Zmij) writes. Exact ties between two candidates (like 754570942.33984373, whose double is ...33984375) are left to the full conversion; - the output is the token with its last four digits replaced; a rounding that carries further goes to the writer of Zmij's results. The batch storage is on the heap (allocated by the first conversion), and a pair of floats like [x, y] in a read-only document is written without a frame. Reading: get() converts such tokens with a lean Eisel-Lemire (coordinate_float()). Output is byte-identical to the previous writer: 10^9 generated tokens in one array and 3 * 10^8 in arrays of 1 to 9 elements on Apple M1, 2.5 * 10^8 and 10^8 on x86-64 (all exponents, subnormals, 15-18 digits, exact binary fractions and ties, near-short decimals); get() equals json::parse on 5 * 10^7 values. compare.py --download, Apple M1 Max, clang 21, 30 rounds, two runs (and two with aligned code: -falign-loops=64 -mllvm -align-all-functions=6); json_view, and the factor of yyjson 0.13.0 over it: - canada dump 3636 -> 1623 us (0.47x -> 1.06x; aligned 1.05x) - canada traverse 2287 -> 1817 us (0.85x -> 1.07x; aligned 0.99x) - canada patch 4628 -> 2735 us (0.80x -> 1.36x) - canada update 4763 -> 2911 us (0.78x -> 1.28x) Linux x86-64 (Haswell VPS, GCC 13, one run): canada dump 0.53x -> 0.72x, traverse 0.89x -> 0.95x, patch 1.01x -> 1.17x, update 0.96x -> 1.13x. Open: canada traverse in the aligned build (0.99x) and canada dump on x86-64 (0.72x) are still behind yyjson. Canada parse and select are about 2 % slower in bench_view in both builds, with identical parser code (likely layout; the corpus parse row is not slower). In the plain build only, rpc, status, and twitter dump are 2-4 % slower (0-6 % faster in the aligned build). Signed-off-by: Niels Lohmann --- include/nlohmann/detail/view/number.hpp | 81 ++++ include/nlohmann/detail/view/serializer.hpp | 405 ++++++++++------ single_include/nlohmann/json_view.hpp | 486 ++++++++++++++------ tests/src/unit-json_view_dump.cpp | 15 +- 4 files changed, 721 insertions(+), 266 deletions(-) diff --git a/include/nlohmann/detail/view/number.hpp b/include/nlohmann/detail/view/number.hpp index c11e80fe2..ecae4c1d2 100644 --- a/include/nlohmann/detail/view/number.hpp +++ b/include/nlohmann/detail/view/number.hpp @@ -259,6 +259,80 @@ NLOHMANN_VIEW_NOINLINE FloatType edited_float(const char* token, const node& n) return float_value(token, n); } +#if NLOHMANN_VIEW_VECTOR +/// eisel_lemire() for the token w * 10^q of coordinate_float() +template +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t coordinate_bits(std::int64_t q, std::uint64_t w, Format /*format*/) noexcept +{ + return eisel_lemire(q, w); +} + +/*! +@brief (binary64) eisel_lemire() without the cases that tokens of 16 or 17 +digits with 5 or more after the point cannot reach: a value of 0, a +subnormal, an infinity, or a tie (q < -4) +*/ +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t coordinate_bits(std::int64_t q, std::uint64_t w, ieee_binary_format<53> format) noexcept +{ + if (NLOHMANN_VIEW_UNLIKELY(w == 0 || q > -5)) + { + return eisel_lemire>(q, w); + } + static_cast(format); + const int lz = count_leading_zeros(w); + const std::uint64_t normalized = w << static_cast(lz); + const auto index = static_cast(2 * (q - pow5_128_smallest_power)); + const uint128_parts product = full_multiplication(normalized, pow5_128()[index]); + std::uint64_t high = product.high; + if (NLOHMANN_VIEW_UNLIKELY((high & 0x1FFu) == 0x1FFu)) + { + // (the lower bits may carry into the result) + const std::uint64_t second = full_multiplication(normalized, pow5_128()[index + 1]).high; + high += product.low + second < product.low ? 1u : 0u; + } + const auto upperbit = static_cast(high >> 63u); + std::uint64_t mantissa = high >> (upperbit + 9); + std::uint64_t power2 = static_cast(((217706 * q) >> 16) + 63 + static_cast(upperbit) - lz + 1023); + mantissa = (mantissa + (mantissa & 1u)) >> 1u; + const bool carry = mantissa >= (std::uint64_t{2} << 52u); + mantissa = carry ? (std::uint64_t{1} << 52u) : mantissa; + power2 += carry ? 1u : 0u; + return (mantissa & ~(std::uint64_t{1} << 52u)) | (power2 << 52u); +} + +/*! +@brief (binary32 and binary64) the value of a float token of the source with +16 or 17 digits, a point, and no exponent, the shape of most coordinates +written with full precision; whether the token of n has this shape + +These tokens skip the general layout of layout_decimal(): no digits beyond +the 17th, no exponent (and Eisel-Lemire directly, as in layout_float()). +*/ +template +NLOHMANN_VIEW_ALWAYS_INLINE bool coordinate_float(const document_data& d, const node& n, FloatType& value) noexcept +{ + const unsigned int_digits = n.extra & 0xFFu; + const unsigned frac_digits = n.extra >> 8u; + const unsigned count = int_digits + frac_digits; + const auto* const first = reinterpret_cast(d.src + n.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const unsigned negative = first[0] == '-' ? 1u : 0u; + if (count - 16u > 1u || frac_digits == 0 || n.len != negative + count + 1 || d.size - n.off < 33) + { + return false; + } + const unsigned char* const digits = first + negative; + const std::uint64_t w = sixteen_digits(digits, int_digits); + // (the 17th digit follows the point) + const std::uint64_t w17 = (w * 10) + static_cast(digits[17] - '0'); + using format = ieee_binary_format::digits>; + native_float_t result{}; + float_from_bits(coordinate_bits(-static_cast(frac_digits), count == 17 ? w17 : w, format{}) | (static_cast(negative) << format::sign_bit()), result); + value = result; + return true; +} + +#endif + /// the value of the float token of a node, as parse() converts it; floats and /// doubles with at most 19 digits are converted from the digit layout template @@ -274,6 +348,13 @@ FloatType float_value(const document_data& d, const node& n) template FloatType float_value(const document_data& d, const node& n, std::true_type /*binary32 or binary64*/) { +#if NLOHMANN_VIEW_VECTOR + FloatType value {}; + if (coordinate_float(d, n, value)) + { + return value; + } +#endif const unsigned int_digits = n.extra & 0xFFu; const unsigned frac_digits = n.extra >> 8u; if (NLOHMANN_VIEW_LIKELY(int_digits + frac_digits <= 19)) // (255 marks "many") diff --git a/include/nlohmann/detail/view/serializer.hpp b/include/nlohmann/detail/view/serializer.hpp index d32f1e999..109685666 100644 --- a/include/nlohmann/detail/view/serializer.hpp +++ b/include/nlohmann/detail/view/serializer.hpp @@ -16,6 +16,7 @@ #include // memcpy, memset #include // less #include // numeric_limits +#include // unique_ptr #include // integral_constant #include // vector @@ -386,89 +387,124 @@ struct depth_counter }; /*! -@brief the digits of the double nearest to a token w * 10^q (with the given -bits, from eisel_lemire()) as dump() writes them (in units of 10^q), for -10^15 <= w < 10^17 and -17 <= q <= -1; 0 if they are not decided here +@brief where the double x nearest to a token w * 10^q lies, for 10^15 <= w < +10^17 and -17 <= q <= -1 (Eisel-Lemire, then a fixed-point position) -dump() writes the shortest decimal in the rounding interval of the double x -(the closest one if there are several, see zmij::to_shortest()). Its digits -are those of a multiple of 10^g of the token's last digit, with g = dec_exp + -1 - q for x's decimal exponent dec_exp (at most 2 here): the multiple of 10^g -nearest to x if it lies in the interval (at most one does), else the multiple -of 10^(g - 1) nearest to x. For g <= 0, the token itself is in the interval. +dump() writes the shortest decimal in the rounding interval of x (the closest +one if there are several, see zmij::to_shortest()). Its digits are those of a +multiple of 10^g of the token's last digit, with g = dec_exp + 1 - q for x's +decimal exponent dec_exp: the multiple of 10^g nearest to x if it lies in the +interval (at most one does), else the multiple of 10^(g - 1) nearest to x +(see token_digits_1() and token_digits_2()). For g <= 0, the token itself is +in the interval. -All of it is exact in a 64-bit fixed-point number with 56 bits after the point -(of the token's last digit): x / 10^q = 2c * 5^-q / 2^t for x = c * 2^e2 and t -= q + 1 - e2, with at most 56 bits after the point here, and |x / 10^q - w| is -at most half the spacing of the doubles, below 12 units. So only the low 64 -bits of 2c * 5^-q are needed, one multiplication beyond Eisel-Lemire's. An -exact tie between two candidates and a power of two (whose interval is not -symmetric) are left to the full conversion. +The position is exact in a 64-bit fixed-point number with 56 bits after the +point (of the token's last digit): x / 10^q = 2c * 5^-q / 2^t for x = c * 2^e2 +and t = q + 1 - e2, with at most 56 bits after the point here, and |x / 10^q - +w| is at most half the spacing of the doubles, below 12 units. So only the low +64 bits of 2c * 5^-q are needed. c and e2 come from Eisel-Lemire's product +directly (as eisel_lemire() rounds it: no tie is possible for q < -4, and its +second product is added where it can carry into the result). + +@param[out] offset x / 10^q - w + 32, with 56 bits after the point +@param[out] radius half the spacing of the doubles at x, likewise (+ 1 if + the interval is closed) +@return g; 3 if the digits are not decided here (a power of two, whose + interval is not symmetric, or out of range) */ -NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t shortest_token_digits(std::uint64_t w, int q, std::uint64_t bits) noexcept +NLOHMANN_VIEW_ALWAYS_INLINE int locate_token(std::uint64_t w, int q, std::uint64_t& offset, std::uint64_t& radius) noexcept { - const std::uint64_t fraction = bits & ((std::uint64_t{1} << 52u) - 1); - const std::uint64_t c = fraction | (std::uint64_t{1} << 52u); - const int e2 = static_cast(bits >> 52u) - 1075; + const int lz = count_leading_zeros(w); + const std::uint64_t normalized = w << static_cast(lz); + const auto index = static_cast(2 * (q - pow5_128_smallest_power)); + const uint128_parts product = full_multiplication(normalized, pow5_128()[index]); + std::uint64_t high = product.high; + if (NLOHMANN_VIEW_UNLIKELY((high & 0x1FFu) == 0x1FFu)) + { + // (as eisel_lemire(): the lower bits may carry into the result) + const std::uint64_t second = full_multiplication(normalized, pow5_128()[index + 1]).high; + high += product.low + second < product.low ? 1u : 0u; + } + const auto upperbit = static_cast(high >> 63u); + const std::uint64_t mantissa = high >> (upperbit + 9); + const std::uint64_t c = (mantissa + (mantissa & 1u)) >> 1u; + // e2 as eisel_lemire() computes the exponent (log2(10) ~ 217706 / 2^16) + const int e2 = static_cast((217706 * static_cast(q)) >> 16) + 63 + static_cast(upperbit) - lz - 52; const int t = q + 1 - e2; const int g = ((e2 * 315653) >> 20) + 1 - q; // (dec_exp as in zmij::to_shortest(), for a regular x) - if (NLOHMANN_VIEW_UNLIKELY(fraction == 0 || t < 1 || t > 56 || g > 2)) - { - return 0; - } - if (g <= 0) - { - return w; - } - const unsigned s = static_cast(56 - t); - const unsigned minus_q = static_cast(-q); - const std::uint64_t five = ::nlohmann::detail::dtoa_impl::powers_of_ten()[minus_q] >> minus_q; // 5^-q - constexpr std::uint64_t one = std::uint64_t{1} << 56u; - // positions are kept 32 units above their value, so that they are not - // negative; x and the candidates are at most 112 units apart - constexpr std::uint64_t bias = 32; - const std::uint64_t x = (2 * c * five) << s; // x / 10^q modulo 256 units - const std::uint64_t half = five << s; // half the spacing of the doubles (the interval's radius) - const bool even = (c & 1u) == 0; // (the interval is closed) - // (the choices are masks, not conditions, which compilers could make - // branches that these data mispredict) - const std::uint64_t g1 = 0 - static_cast(g == 1); - const std::uint64_t r10 = w - ((w / 10) * 10); - const std::uint64_t r100 = w - ((w / 100) * 100); - // the multiple of G = 10^g nearest to x: a0 or a neighbor - const std::uint64_t grid = (10u & g1) | (100u & ~g1); - const std::uint64_t r = (r10 & g1) | (r100 & ~g1); - const std::uint64_t at = x - ((w - r - bias) << 56u); // x - a0, biased - const std::uint64_t up = 0 - static_cast(at > (bias * one) + (grid * (one / 2))); - const std::uint64_t down = 0 - static_cast(at + (grid * (one / 2)) < bias * one); - const std::uint64_t step = (grid & up) - (grid & down); - const std::uint64_t center = (bias + step) * one; - const std::uint64_t below = 0 - static_cast(at < center); - const std::uint64_t dist = ((center - at) & below) | ((at - center) & ~below); - const std::uint64_t shorter = 0 - static_cast(dist < half + (even ? 1u : 0u)); - const std::uint64_t a = w - r + step; - // else the multiple of 1 (g == 1) or 10 (g == 2) nearest to x - const std::uint64_t rl = r10 & ~g1; - const std::uint64_t pos = x - ((w - rl - bias) << 56u) + ((one / 2) & g1) + ((5 * one) & ~g1); // x - b0 + half a step, biased - const std::uint64_t whole = pos >> 56u; // in [20, 59] - const std::uint64_t m = whole + 20 - bias; // (a multiple of 10 above whole - bias) - const std::uint64_t tens = (m * 205) >> 11u; // m / 10 - const std::uint64_t tie = 0 - static_cast((pos & (one - 1)) == 0 && (g == 1 || m == tens * 10)); - const std::uint64_t b = ((w + whole - bias) & g1) | ((w - rl + (tens * 10) - 20) & ~g1); - return (a & shorter) | (b & ~tie & ~shorter); + static const std::array fives = {{1u, 5u, 25u, 125u, 625u, 3125u, 15625u, 78125u, 390625u, 1953125u, 9765625u, 48828125u, 244140625u, 1220703125u, 6103515625u, 30517578125u, 152587890625u}}; + const std::uint64_t five = fives[static_cast(-q)]; // 5^-q + const unsigned s = static_cast(56 - t) & 63u; + offset = ((2 * c * five) << s) - (w << 56u) + (std::uint64_t{32} << 56u); + radius = (five << s) + ((c & 1u) == 0 ? 1u : 0u); + // (c = 2^52: a power of two; c = 2^53: rounded up to one. With 8 to 16 + // fraction digits, as the batch's tokens have, t is in [15, 40] and g at + // most 2. x lies halfway between two candidates only if its distance from + // the token is a whole unit (g == 2) or a half (g == 1): such tokens are + // left to the full conversion) + const bool decided = c - ((std::uint64_t{1} << 52u) + 1) < (std::uint64_t{1} << 52u) - 1 + && (offset & ((std::uint64_t{1} << 55u) - 1)) != 0; + return decided ? g : 3; } -/// the eight ASCII digits of x < 10^8, the first in the lowest byte (as -/// little-endian memory holds them) -NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t eight_digit_text(std::uint64_t x) noexcept +/// the distance of a position (see locate_token()) from the biased point 32 +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t biased_distance(std::uint64_t position) noexcept { - const std::uint64_t abcd = (x * 109951163u) >> 40u; // x / 10000 - std::uint64_t v = abcd | ((x - (abcd * 10000u)) << 32u); - const std::uint64_t ab = ((v * 10486u) >> 20u) & 0x0000007F0000007Fu; // / 100 per 32-bit lane - v = ab | ((v - (ab * 100u)) << 16u); - const std::uint64_t a = ((v * 103u) >> 10u) & 0x000F000F000F000Fu; // / 10 per 16-bit lane + constexpr std::uint64_t center = std::uint64_t{32} << 56u; + return position >= center ? position - center : center - position; +} + +/*! +@brief (g == 1) the digits dump() writes for a token, from the position of its +double (see locate_token()): the multiple of 10 nearest to x if it is in the +interval, else w rounded to x (an exact tie between two candidates, like +754570942.33984373, whose double is 754570942.33984375, is excluded by +locate_token()) + +With |x - w| at most the radius, which is below half the spacing of the +candidates, the multiple of 10 nearest to x is w - r10 or the one above it. +*/ +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t token_digits_1(std::uint64_t w, std::uint64_t offset, std::uint64_t radius, std::uint64_t r10) noexcept +{ + constexpr std::uint64_t one = std::uint64_t{1} << 56u; + const std::uint64_t from = offset + (r10 * one); // x - (w - r10) (+ 32) + const bool up = from > 37 * one; // (nearer to the multiple above) + const bool shorter = biased_distance(from - (up ? 10 * one : 0)) < radius; + const std::uint64_t a = w - r10 + (up ? 10u : 0u); + const std::uint64_t b = w + ((offset + (one / 2)) >> 56u) - 32; // w rounded to x + return shorter ? a : b; +} + +/*! +@brief (g == 2) the digits dump() writes for a token, as token_digits_1(): the +multiple of 100 nearest to x if it is in the interval, else the multiple of 10 +nearest to x +*/ +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t token_digits_2(std::uint64_t w, std::uint64_t offset, std::uint64_t radius, std::uint64_t r10, std::uint64_t r100) noexcept +{ + constexpr std::uint64_t one = std::uint64_t{1} << 56u; + const std::uint64_t from = offset + (r100 * one); + const bool up = from > 82 * one; + const bool shorter = biased_distance(from - (up ? 100 * one : 0)) < radius; + const std::uint64_t a = w - r100 + (up ? 100u : 0u); + // x - (w - r10), in [-12, 21]: the multiple of 10 nearest to x is that one, + // or one or two above, or one below + const std::uint64_t y = offset + (r10 * one); + const std::uint64_t steps = (y > 37 * one ? 1u : 0u) + (y > 47 * one ? 1u : 0u); + const bool down = y < 27 * one; + const std::uint64_t b = w - r10 + (10 * steps) - (down ? 10u : 0u); + return shorter ? a : b; +} + +/// the four ASCII digits of x < 10^4, the first in the lowest byte (as +/// little-endian memory holds them) +NLOHMANN_VIEW_ALWAYS_INLINE std::uint32_t four_digit_text(std::uint32_t x) noexcept +{ + const std::uint32_t ab = (x * 5243u) >> 19u; // x / 100 + std::uint32_t v = ab | ((x - (ab * 100u)) << 16u); + const std::uint32_t a = ((v * 103u) >> 10u) & 0x000F000Fu; // / 10 per 16-bit lane v = a | ((v - (a * 10u)) << 8u); - return v + 0x3030303030303030u; + return v + 0x30303030u; } /// how the view's dump() writes a value @@ -938,7 +974,7 @@ class view_serializer else if (enabled(fast_double::value)) { room(64); - w = write_double_ahead(w, n, batch); + w = write_double_ahead(w, src, n, batch); } else { @@ -963,6 +999,20 @@ class view_serializer case value_t::array: { const bool object = n->kind == static_cast(value_t::object); + // (read-only documents, compact output) a pair of floats, + // like the coordinates of GeoJSON, without a frame + if (!Editable && !Pretty && !SourceNumbers && enabled(fast_double::value) && n->next == 3 && n->len == 2 + && n[1].kind == static_cast(value_t::number_float) && n[2].kind == static_cast(value_t::number_float)) + { + room(2 * 64); + *w++ = '['; + w = write_double_ahead(w, src, n + 1, batch); + *w++ = ','; + w = write_double_ahead(w, src, n + 2, batch); + *w++ = ']'; + n += 2; // (the element before the next node) + break; + } room(8); if (n->len == 0) { @@ -1072,13 +1122,31 @@ class view_serializer } } + /// (float_batch) the mark of digits whose rounding carried beyond the + /// token's last four (see mark()) + static constexpr std::uint64_t carried() noexcept + { + return std::uint64_t{1} << 63u; + } + /// floats of 16 or 17 digits ahead of the walk, converted together (see - /// convert_ahead()); the digits are those of shortest_token_digits(), 0 - /// for a float the batch does not write + /// convert_ahead()), with their digits as mark() keeps them struct float_batch { - std::array at; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read - std::array digits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + static constexpr std::size_t capacity() noexcept + { + return 256; + } + /// (on the heap, allocated by the first conversion: a frame of more + /// than a page made every dump of a small document slower) + struct entries + { + std::array at; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array digits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + }; + std::unique_ptr storage{}; + const node** at = nullptr; + std::uint64_t* digits = nullptr; std::size_t count = 0; std::size_t next = 0; const node* scanned = nullptr; ///< where the last scan of the node array ended @@ -1086,21 +1154,24 @@ class view_serializer /// (doubles) the float at n as dump() writes it, at w (64 bytes of room): /// from the batch if it holds n - NLOHMANN_VIEW_ALWAYS_INLINE char* write_double_ahead(char* w, const node* n, float_batch& batch) + NLOHMANN_VIEW_ALWAYS_INLINE char* write_double_ahead(char* w, const char* src, const node* n, float_batch& batch) { -#if NLOHMANN_VIEW_NEON - if (batch.next != batch.count && batch.at[batch.next] == n && batch.digits[batch.next] != 0) +#if NLOHMANN_VIEW_VECTOR && NLOHMANN_VIEW_LITTLE_ENDIAN + // (digits 0: not written by the batch; carried(): rounding carried + // beyond the last four digits) + if (batch.next != batch.count && batch.at[batch.next] == n && batch.digits[batch.next] - 1 < carried() - 1) { - return write_token_digits(w, m_doc.src + n->off, n->len, n->extra >> 8u, batch.digits[batch.next++]); + return write_token_digits(w, src, *n, batch.digits[batch.next++]); } return write_double_beside(w, n, batch); #else + static_cast(src); static_cast(batch); return write_double_at(w, *n); #endif } -#if NLOHMANN_VIEW_NEON +#if NLOHMANN_VIEW_VECTOR && NLOHMANN_VIEW_LITTLE_ENDIAN /// write_double_ahead() for a float that is not the batch's next one (out /// of line: the walk is faster with less code in its loop) NLOHMANN_VIEW_NOINLINE char* write_double_beside(char* w, const node* n, float_batch& batch) @@ -1114,9 +1185,13 @@ class view_serializer if ((batch.next != batch.count && batch.at[batch.next] == n) || convert_ahead(n, batch)) { const std::uint64_t digits = batch.digits[batch.next++]; + if ((digits & carried()) != 0) + { + return write_carried_digits(w, *n, digits & ~carried()); + } if (digits != 0) { - return write_token_digits(w, m_doc.src + n->off, n->len, n->extra >> 8u, digits); + return write_token_digits(w, m_doc.src, *n, digits); } } return write_double_at(w, *n); @@ -1127,16 +1202,14 @@ class view_serializer array at once; whether n is the first of them Such tokens (like the coordinates of GeoJSON, written with 17 digits) are - written as the token with its last digits replaced (write_token_digits()). - Each step (the digits, Eisel-Lemire, shortest_token_digits()) is a long - chain of dependent multiplications; one loop per step over many floats lets - the processor overlap the floats, which it does not across the steps of - one float (on Apple M1, the dump of canada.json takes 30 % less time than - with a conversion of each float, of which 10 points are this). Each node of the array + written as the token with its last four digits replaced + (write_token_digits()). Each step (the digits, locate_token(), + token_digits_1/2()) is a long chain of dependent operations; one loop per + step over many floats lets the processor overlap the floats, which it does + not across the steps of one float (on Apple M1, one loop for the digits and + locate_token() made the dump of canada.json 12 % slower). Each node of the array is scanned once; a node that the walk does not reach in order (in an - editable document) is written without the batch. (AArch64 only: on x86-64 - (Haswell, GCC 13 and Clang 18), the batch was not faster than the - conversion of each float.) + editable document) is written without the batch. */ NLOHMANN_VIEW_NOINLINE bool convert_ahead(const node* n, float_batch& batch) { @@ -1148,72 +1221,144 @@ class view_serializer } const char* const src = m_doc.src; const char* const src_end = src + m_doc.size; - const node* const stop = last - n > 128 ? n + 128 : last; + const node* const stop = last - n > 1024 ? n + 1024 : last; batch.count = 0; batch.next = 0; + // the digits of each token (and its last four from the text) + std::array digits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array exponent; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array last_four; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read const node* p = n; - for (; p != stop && batch.count != batch.at.size(); ++p) + if (!batch.storage) + { + batch.storage.reset(new typename float_batch::entries); // NOLINT(cppcoreguidelines-owning-memory,modernize-make-unique): C++11 + batch.at = batch.storage->at.data(); + batch.digits = batch.storage->digits.data(); + } + for (; p != stop && batch.count != float_batch::capacity(); ++p) { // a token of the source: 16 or 17 digits, at least 8 after the - // point, no exponent, no leading zero, 32 bytes to read + // point, no exponent, 32 bytes to read const unsigned frac_digits = p->extra >> 8u; - const unsigned count = (p->extra & 0xFFu) + frac_digits; + const unsigned int_digits = p->extra & 0xFFu; + const unsigned count = int_digits + frac_digits; if (p->kind == static_cast(value_t::number_float) && (p->flags & node_flags::storage) == 0 && count - 16u <= 1u && frac_digits >= 8 && src_end - (src + p->off) >= 33) { - const char* const token = src + p->off; + const auto* const token = reinterpret_cast(src + p->off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) const unsigned sign = token[0] == '-' ? 1 : 0; - if (p->len == sign + count + 1 && token[sign] != '0') - { - batch.at[batch.count++] = p; - } + const unsigned char* const first = token + sign; + const std::uint64_t w = sixteen_digits(first, int_digits); + // (the 17th digit follows the point; a token with an exponent + // gets digits outside the range of locate_token(), and is not + // written by the batch. A token 0.xxx with 16 significant + // digits is: its double is written as "0." and its digits too) + const std::uint64_t w17 = (w * 10) + static_cast(first[17] - '0'); + const std::size_t i = batch.count++; + batch.at[i] = p; + digits[i] = p->len != sign + count + 1 ? 0 : (count == 17 ? w17 : w); + exponent[i] = -static_cast(frac_digits); + // (byte 0: the last digit, byte 1: the last two, bits 16-29: the last four) + std::uint32_t four = 0; + std::memcpy(&four, token + p->len - 4, 4); + four -= 0x30303030u; + const std::uint32_t pairs = ((four & 0x0F0F0F0Fu) * 2561u) >> 8u; // (byte 0, 2: the pairs) + last_four[i] = (four >> 24u) | ((pairs >> 16u & 0xFFu) << 8u) | ((((pairs & 0x00FF00FFu) * 6553601u) >> 16u) << 16u); } } batch.scanned = p; + // the position of each double, and the floats of either kind + std::array offset; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array radius; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array kind_1; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array kind_2; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::size_t count_1 = 0; + std::size_t count_2 = 0; for (std::size_t i = 0; i != batch.count; ++i) { - const node& f = *batch.at[i]; - const auto* digits = reinterpret_cast(src + f.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - digits += *digits == '-' ? 1 : 0; - const std::uint64_t w = sixteen_digits(digits, f.extra & 0xFFu); - // (the 17th digit follows the point) - batch.digits[i] = (f.extra & 0xFFu) + (f.extra >> 8u) == 17 ? (w * 10) + static_cast(digits[17] - '0') : w; + const std::uint64_t w = digits[i]; + // (the range excludes the values of bytes that are not digits, only + // in a damaged image loaded with image_check::bounds) + const int g = w - 1000000000000000u < 99000000000000000u ? locate_token(w, exponent[i], offset[i], radius[i]) : 3; + batch.digits[i] = g <= 0 ? (last_four[i] >> 16u) + 1 : 0; + kind_1[count_1] = static_cast(i); + count_1 += g == 1 ? 1 : 0; + kind_2[count_2] = static_cast(i); + count_2 += g == 2 ? 1 : 0; } - std::array bits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read - for (std::size_t i = 0; i != batch.count; ++i) + // the digits to write, in one loop per kind (see mark()) + for (std::size_t k = 0; k != count_1; ++k) { - bits[i] = eisel_lemire>(-static_cast(batch.at[i]->extra >> 8u), batch.digits[i]); + const std::size_t i = kind_1[k]; + const std::uint64_t s = token_digits_1(digits[i], offset[i], radius[i], last_four[i] & 0xFFu); + batch.digits[i] = mark(digits[i], s, last_four[i] >> 16u); } - for (std::size_t i = 0; i != batch.count; ++i) + for (std::size_t k = 0; k != count_2; ++k) { - const std::uint64_t w = batch.digits[i]; - // (the range also excludes the values of bytes that are not - // digits, only in a damaged image loaded with image_check::bounds) - const std::uint64_t s = w - 1000000000000000u < 99000000000000000u ? shortest_token_digits(w, -static_cast(batch.at[i]->extra >> 8u), bits[i]) : 0; - // (only the last eight digits are written over the token's) - batch.digits[i] = s / 100000000u == w / 100000000u ? s : 0; + const std::size_t i = kind_2[k]; + const std::uint64_t s = token_digits_2(digits[i], offset[i], radius[i], last_four[i] & 0xFFu, (last_four[i] >> 8u) & 0xFFu); + batch.digits[i] = mark(digits[i], s, last_four[i] >> 16u); } return batch.count != 0 && batch.at[0] == n; } - /// a float token of 16 or 17 digits as dump() writes it, at w (64 bytes - /// of room): the token with its last eight digits replaced by those of - /// digits (see shortest_token_digits(); the others are the token's), and - /// without trailing zeros after the point, but one - static char* write_token_digits(char* w, const char* token, std::uint32_t len, unsigned frac_digits, std::uint64_t digits) noexcept + /*! + @brief a float token of 16 or 17 digits as dump() writes it, at w (64 bytes + of room), from the digits of its double (see locate_token()) whose + rounding carried beyond the token's last four digits + + (like 43.42027299999999 -> 43.420273): the digits go to the writer of + Zmij's results as its 15 or 16 leading digits and one more, which omits + the trailing zeros. + */ + NLOHMANN_VIEW_NOINLINE char* write_carried_digits(char* w, const node& n, std::uint64_t digits) const noexcept { + const int exponent = -static_cast(n.extra >> 8u); + const std::uint64_t leading = digits / 10; + const auto last = static_cast(digits - (leading * 10)); + const bool seventeen = digits >= 10000000000000000u; + *w = '-'; + w += m_doc.src[n.off] == '-' ? 1 : 0; + return ::nlohmann::detail::dtoa_impl::write_shortest(w, seventeen ? ::nlohmann::detail::zmij::shortest_decimal{leading, exponent, last, last != 0} + : ::nlohmann::detail::zmij::shortest_decimal{digits, exponent - 1, 0, false}); + } + + /// the digits s of a token with the digits w and the last four digits + /// r4, as the batch keeps them: the new last four digits + 1 (only those + /// are written over the token's), or s marked if it differs from w beyond + /// them (0 stays for a float the batch does not write) + static std::uint64_t mark(std::uint64_t w, std::uint64_t s, std::uint64_t r4) noexcept + { + const std::uint64_t last = s - (w - r4); + return last < 10000 ? last + 1 : s | carried(); + } + + /*! + @brief a float token of 16 or 17 digits as dump() writes it, at w (64 bytes + of room): the token with its last four digits replaced by digits - 1 (see + mark(); the others are the token's), and without trailing zeros after the + point, but one + */ + char* write_token_digits(char* w, const char* src, const node& n, std::uint64_t digits) + { + const char* const token = src + n.off; std::memcpy(w, token, 32); - char* const end = w + len; - const std::uint64_t last = eight_digit_text(digits % 100000000u); - std::memcpy(end - 8, &last, 8); + char* const end = w + n.len; + const std::uint32_t last = four_digit_text(static_cast(digits - 1)); + std::memcpy(end - 4, &last, 4); // (the token's text before the new digits, from the source: reading // the output back would wait for the stores) std::uint64_t before = 0; - std::memcpy(&before, token + len - 16, 8); - const std::uint64_t last_zeros = last ^ 0x3030303030303030u; + std::memcpy(&before, token + n.len - 12, 8); + const std::uint32_t last_zeros = last ^ 0x30303030u; const std::uint64_t before_zeros = before ^ 0x3030303030303030u; - const unsigned zeros = last_zeros != 0 ? static_cast(count_leading_zeros(last_zeros)) / 8 - : 8 + (before_zeros != 0 ? static_cast(count_leading_zeros(before_zeros)) / 8 : 8); + if (NLOHMANN_VIEW_UNLIKELY(last_zeros == 0 && before_zeros == 0)) + { + return write_double_at(w, n); // (twelve or more trailing zeros) + } + const unsigned zeros = last_zeros != 0 ? static_cast(count_leading_zeros(last_zeros)) / 8 - 4 + : 4 + (static_cast(count_leading_zeros(before_zeros)) / 8); + const unsigned frac_digits = n.extra >> 8u; return end - (zeros < frac_digits ? zeros : frac_digits - 1); } #endif diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 4fa4ab020..34a67470e 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -4846,6 +4846,80 @@ NLOHMANN_VIEW_NOINLINE FloatType edited_float(const char* token, const node& n) return float_value(token, n); } +#if NLOHMANN_VIEW_VECTOR +/// eisel_lemire() for the token w * 10^q of coordinate_float() +template +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t coordinate_bits(std::int64_t q, std::uint64_t w, Format /*format*/) noexcept +{ + return eisel_lemire(q, w); +} + +/*! +@brief (binary64) eisel_lemire() without the cases that tokens of 16 or 17 +digits with 5 or more after the point cannot reach: a value of 0, a +subnormal, an infinity, or a tie (q < -4) +*/ +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t coordinate_bits(std::int64_t q, std::uint64_t w, ieee_binary_format<53> format) noexcept +{ + if (NLOHMANN_VIEW_UNLIKELY(w == 0 || q > -5)) + { + return eisel_lemire>(q, w); + } + static_cast(format); + const int lz = count_leading_zeros(w); + const std::uint64_t normalized = w << static_cast(lz); + const auto index = static_cast(2 * (q - pow5_128_smallest_power)); + const uint128_parts product = full_multiplication(normalized, pow5_128()[index]); + std::uint64_t high = product.high; + if (NLOHMANN_VIEW_UNLIKELY((high & 0x1FFu) == 0x1FFu)) + { + // (the lower bits may carry into the result) + const std::uint64_t second = full_multiplication(normalized, pow5_128()[index + 1]).high; + high += product.low + second < product.low ? 1u : 0u; + } + const auto upperbit = static_cast(high >> 63u); + std::uint64_t mantissa = high >> (upperbit + 9); + std::uint64_t power2 = static_cast(((217706 * q) >> 16) + 63 + static_cast(upperbit) - lz + 1023); + mantissa = (mantissa + (mantissa & 1u)) >> 1u; + const bool carry = mantissa >= (std::uint64_t{2} << 52u); + mantissa = carry ? (std::uint64_t{1} << 52u) : mantissa; + power2 += carry ? 1u : 0u; + return (mantissa & ~(std::uint64_t{1} << 52u)) | (power2 << 52u); +} + +/*! +@brief (binary32 and binary64) the value of a float token of the source with +16 or 17 digits, a point, and no exponent, the shape of most coordinates +written with full precision; whether the token of n has this shape + +These tokens skip the general layout of layout_decimal(): no digits beyond +the 17th, no exponent (and Eisel-Lemire directly, as in layout_float()). +*/ +template +NLOHMANN_VIEW_ALWAYS_INLINE bool coordinate_float(const document_data& d, const node& n, FloatType& value) noexcept +{ + const unsigned int_digits = n.extra & 0xFFu; + const unsigned frac_digits = n.extra >> 8u; + const unsigned count = int_digits + frac_digits; + const auto* const first = reinterpret_cast(d.src + n.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const unsigned negative = first[0] == '-' ? 1u : 0u; + if (count - 16u > 1u || frac_digits == 0 || n.len != negative + count + 1 || d.size - n.off < 33) + { + return false; + } + const unsigned char* const digits = first + negative; + const std::uint64_t w = sixteen_digits(digits, int_digits); + // (the 17th digit follows the point) + const std::uint64_t w17 = (w * 10) + static_cast(digits[17] - '0'); + using format = ieee_binary_format::digits>; + native_float_t result{}; + float_from_bits(coordinate_bits(-static_cast(frac_digits), count == 17 ? w17 : w, format{}) | (static_cast(negative) << format::sign_bit()), result); + value = result; + return true; +} + +#endif + /// the value of the float token of a node, as parse() converts it; floats and /// doubles with at most 19 digits are converted from the digit layout template @@ -4861,6 +4935,13 @@ FloatType float_value(const document_data& d, const node& n) template FloatType float_value(const document_data& d, const node& n, std::true_type /*binary32 or binary64*/) { +#if NLOHMANN_VIEW_VECTOR + FloatType value {}; + if (coordinate_float(d, n, value)) + { + return value; + } +#endif const unsigned int_digits = n.extra & 0xFFu; const unsigned frac_digits = n.extra >> 8u; if (NLOHMANN_VIEW_LIKELY(int_digits + frac_digits <= 19)) // (255 marks "many") @@ -6434,6 +6515,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // memcpy, memset #include // less #include // numeric_limits +#include // unique_ptr #include // integral_constant #include // vector @@ -6809,89 +6891,124 @@ struct depth_counter }; /*! -@brief the digits of the double nearest to a token w * 10^q (with the given -bits, from eisel_lemire()) as dump() writes them (in units of 10^q), for -10^15 <= w < 10^17 and -17 <= q <= -1; 0 if they are not decided here +@brief where the double x nearest to a token w * 10^q lies, for 10^15 <= w < +10^17 and -17 <= q <= -1 (Eisel-Lemire, then a fixed-point position) -dump() writes the shortest decimal in the rounding interval of the double x -(the closest one if there are several, see zmij::to_shortest()). Its digits -are those of a multiple of 10^g of the token's last digit, with g = dec_exp + -1 - q for x's decimal exponent dec_exp (at most 2 here): the multiple of 10^g -nearest to x if it lies in the interval (at most one does), else the multiple -of 10^(g - 1) nearest to x. For g <= 0, the token itself is in the interval. +dump() writes the shortest decimal in the rounding interval of x (the closest +one if there are several, see zmij::to_shortest()). Its digits are those of a +multiple of 10^g of the token's last digit, with g = dec_exp + 1 - q for x's +decimal exponent dec_exp: the multiple of 10^g nearest to x if it lies in the +interval (at most one does), else the multiple of 10^(g - 1) nearest to x +(see token_digits_1() and token_digits_2()). For g <= 0, the token itself is +in the interval. -All of it is exact in a 64-bit fixed-point number with 56 bits after the point -(of the token's last digit): x / 10^q = 2c * 5^-q / 2^t for x = c * 2^e2 and t -= q + 1 - e2, with at most 56 bits after the point here, and |x / 10^q - w| is -at most half the spacing of the doubles, below 12 units. So only the low 64 -bits of 2c * 5^-q are needed, one multiplication beyond Eisel-Lemire's. An -exact tie between two candidates and a power of two (whose interval is not -symmetric) are left to the full conversion. +The position is exact in a 64-bit fixed-point number with 56 bits after the +point (of the token's last digit): x / 10^q = 2c * 5^-q / 2^t for x = c * 2^e2 +and t = q + 1 - e2, with at most 56 bits after the point here, and |x / 10^q - +w| is at most half the spacing of the doubles, below 12 units. So only the low +64 bits of 2c * 5^-q are needed. c and e2 come from Eisel-Lemire's product +directly (as eisel_lemire() rounds it: no tie is possible for q < -4, and its +second product is added where it can carry into the result). + +@param[out] offset x / 10^q - w + 32, with 56 bits after the point +@param[out] radius half the spacing of the doubles at x, likewise (+ 1 if + the interval is closed) +@return g; 3 if the digits are not decided here (a power of two, whose + interval is not symmetric, or out of range) */ -NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t shortest_token_digits(std::uint64_t w, int q, std::uint64_t bits) noexcept +NLOHMANN_VIEW_ALWAYS_INLINE int locate_token(std::uint64_t w, int q, std::uint64_t& offset, std::uint64_t& radius) noexcept { - const std::uint64_t fraction = bits & ((std::uint64_t{1} << 52u) - 1); - const std::uint64_t c = fraction | (std::uint64_t{1} << 52u); - const int e2 = static_cast(bits >> 52u) - 1075; + const int lz = count_leading_zeros(w); + const std::uint64_t normalized = w << static_cast(lz); + const auto index = static_cast(2 * (q - pow5_128_smallest_power)); + const uint128_parts product = full_multiplication(normalized, pow5_128()[index]); + std::uint64_t high = product.high; + if (NLOHMANN_VIEW_UNLIKELY((high & 0x1FFu) == 0x1FFu)) + { + // (as eisel_lemire(): the lower bits may carry into the result) + const std::uint64_t second = full_multiplication(normalized, pow5_128()[index + 1]).high; + high += product.low + second < product.low ? 1u : 0u; + } + const auto upperbit = static_cast(high >> 63u); + const std::uint64_t mantissa = high >> (upperbit + 9); + const std::uint64_t c = (mantissa + (mantissa & 1u)) >> 1u; + // e2 as eisel_lemire() computes the exponent (log2(10) ~ 217706 / 2^16) + const int e2 = static_cast((217706 * static_cast(q)) >> 16) + 63 + static_cast(upperbit) - lz - 52; const int t = q + 1 - e2; const int g = ((e2 * 315653) >> 20) + 1 - q; // (dec_exp as in zmij::to_shortest(), for a regular x) - if (NLOHMANN_VIEW_UNLIKELY(fraction == 0 || t < 1 || t > 56 || g > 2)) - { - return 0; - } - if (g <= 0) - { - return w; - } - const unsigned s = static_cast(56 - t); - const unsigned minus_q = static_cast(-q); - const std::uint64_t five = ::nlohmann::detail::dtoa_impl::powers_of_ten()[minus_q] >> minus_q; // 5^-q - constexpr std::uint64_t one = std::uint64_t{1} << 56u; - // positions are kept 32 units above their value, so that they are not - // negative; x and the candidates are at most 112 units apart - constexpr std::uint64_t bias = 32; - const std::uint64_t x = (2 * c * five) << s; // x / 10^q modulo 256 units - const std::uint64_t half = five << s; // half the spacing of the doubles (the interval's radius) - const bool even = (c & 1u) == 0; // (the interval is closed) - // (the choices are masks, not conditions, which compilers could make - // branches that these data mispredict) - const std::uint64_t g1 = 0 - static_cast(g == 1); - const std::uint64_t r10 = w - ((w / 10) * 10); - const std::uint64_t r100 = w - ((w / 100) * 100); - // the multiple of G = 10^g nearest to x: a0 or a neighbor - const std::uint64_t grid = (10u & g1) | (100u & ~g1); - const std::uint64_t r = (r10 & g1) | (r100 & ~g1); - const std::uint64_t at = x - ((w - r - bias) << 56u); // x - a0, biased - const std::uint64_t up = 0 - static_cast(at > (bias * one) + (grid * (one / 2))); - const std::uint64_t down = 0 - static_cast(at + (grid * (one / 2)) < bias * one); - const std::uint64_t step = (grid & up) - (grid & down); - const std::uint64_t center = (bias + step) * one; - const std::uint64_t below = 0 - static_cast(at < center); - const std::uint64_t dist = ((center - at) & below) | ((at - center) & ~below); - const std::uint64_t shorter = 0 - static_cast(dist < half + (even ? 1u : 0u)); - const std::uint64_t a = w - r + step; - // else the multiple of 1 (g == 1) or 10 (g == 2) nearest to x - const std::uint64_t rl = r10 & ~g1; - const std::uint64_t pos = x - ((w - rl - bias) << 56u) + ((one / 2) & g1) + ((5 * one) & ~g1); // x - b0 + half a step, biased - const std::uint64_t whole = pos >> 56u; // in [20, 59] - const std::uint64_t m = whole + 20 - bias; // (a multiple of 10 above whole - bias) - const std::uint64_t tens = (m * 205) >> 11u; // m / 10 - const std::uint64_t tie = 0 - static_cast((pos & (one - 1)) == 0 && (g == 1 || m == tens * 10)); - const std::uint64_t b = ((w + whole - bias) & g1) | ((w - rl + (tens * 10) - 20) & ~g1); - return (a & shorter) | (b & ~tie & ~shorter); + static const std::array fives = {{1u, 5u, 25u, 125u, 625u, 3125u, 15625u, 78125u, 390625u, 1953125u, 9765625u, 48828125u, 244140625u, 1220703125u, 6103515625u, 30517578125u, 152587890625u}}; + const std::uint64_t five = fives[static_cast(-q)]; // 5^-q + const unsigned s = static_cast(56 - t) & 63u; + offset = ((2 * c * five) << s) - (w << 56u) + (std::uint64_t{32} << 56u); + radius = (five << s) + ((c & 1u) == 0 ? 1u : 0u); + // (c = 2^52: a power of two; c = 2^53: rounded up to one. With 8 to 16 + // fraction digits, as the batch's tokens have, t is in [15, 40] and g at + // most 2. x lies halfway between two candidates only if its distance from + // the token is a whole unit (g == 2) or a half (g == 1): such tokens are + // left to the full conversion) + const bool decided = c - ((std::uint64_t{1} << 52u) + 1) < (std::uint64_t{1} << 52u) - 1 + && (offset & ((std::uint64_t{1} << 55u) - 1)) != 0; + return decided ? g : 3; } -/// the eight ASCII digits of x < 10^8, the first in the lowest byte (as -/// little-endian memory holds them) -NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t eight_digit_text(std::uint64_t x) noexcept +/// the distance of a position (see locate_token()) from the biased point 32 +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t biased_distance(std::uint64_t position) noexcept { - const std::uint64_t abcd = (x * 109951163u) >> 40u; // x / 10000 - std::uint64_t v = abcd | ((x - (abcd * 10000u)) << 32u); - const std::uint64_t ab = ((v * 10486u) >> 20u) & 0x0000007F0000007Fu; // / 100 per 32-bit lane - v = ab | ((v - (ab * 100u)) << 16u); - const std::uint64_t a = ((v * 103u) >> 10u) & 0x000F000F000F000Fu; // / 10 per 16-bit lane + constexpr std::uint64_t center = std::uint64_t{32} << 56u; + return position >= center ? position - center : center - position; +} + +/*! +@brief (g == 1) the digits dump() writes for a token, from the position of its +double (see locate_token()): the multiple of 10 nearest to x if it is in the +interval, else w rounded to x (an exact tie between two candidates, like +754570942.33984373, whose double is 754570942.33984375, is excluded by +locate_token()) + +With |x - w| at most the radius, which is below half the spacing of the +candidates, the multiple of 10 nearest to x is w - r10 or the one above it. +*/ +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t token_digits_1(std::uint64_t w, std::uint64_t offset, std::uint64_t radius, std::uint64_t r10) noexcept +{ + constexpr std::uint64_t one = std::uint64_t{1} << 56u; + const std::uint64_t from = offset + (r10 * one); // x - (w - r10) (+ 32) + const bool up = from > 37 * one; // (nearer to the multiple above) + const bool shorter = biased_distance(from - (up ? 10 * one : 0)) < radius; + const std::uint64_t a = w - r10 + (up ? 10u : 0u); + const std::uint64_t b = w + ((offset + (one / 2)) >> 56u) - 32; // w rounded to x + return shorter ? a : b; +} + +/*! +@brief (g == 2) the digits dump() writes for a token, as token_digits_1(): the +multiple of 100 nearest to x if it is in the interval, else the multiple of 10 +nearest to x +*/ +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t token_digits_2(std::uint64_t w, std::uint64_t offset, std::uint64_t radius, std::uint64_t r10, std::uint64_t r100) noexcept +{ + constexpr std::uint64_t one = std::uint64_t{1} << 56u; + const std::uint64_t from = offset + (r100 * one); + const bool up = from > 82 * one; + const bool shorter = biased_distance(from - (up ? 100 * one : 0)) < radius; + const std::uint64_t a = w - r100 + (up ? 100u : 0u); + // x - (w - r10), in [-12, 21]: the multiple of 10 nearest to x is that one, + // or one or two above, or one below + const std::uint64_t y = offset + (r10 * one); + const std::uint64_t steps = (y > 37 * one ? 1u : 0u) + (y > 47 * one ? 1u : 0u); + const bool down = y < 27 * one; + const std::uint64_t b = w - r10 + (10 * steps) - (down ? 10u : 0u); + return shorter ? a : b; +} + +/// the four ASCII digits of x < 10^4, the first in the lowest byte (as +/// little-endian memory holds them) +NLOHMANN_VIEW_ALWAYS_INLINE std::uint32_t four_digit_text(std::uint32_t x) noexcept +{ + const std::uint32_t ab = (x * 5243u) >> 19u; // x / 100 + std::uint32_t v = ab | ((x - (ab * 100u)) << 16u); + const std::uint32_t a = ((v * 103u) >> 10u) & 0x000F000Fu; // / 10 per 16-bit lane v = a | ((v - (a * 10u)) << 8u); - return v + 0x3030303030303030u; + return v + 0x30303030u; } /// how the view's dump() writes a value @@ -7361,7 +7478,7 @@ class view_serializer else if (enabled(fast_double::value)) { room(64); - w = write_double_ahead(w, n, batch); + w = write_double_ahead(w, src, n, batch); } else { @@ -7386,6 +7503,20 @@ class view_serializer case value_t::array: { const bool object = n->kind == static_cast(value_t::object); + // (read-only documents, compact output) a pair of floats, + // like the coordinates of GeoJSON, without a frame + if (!Editable && !Pretty && !SourceNumbers && enabled(fast_double::value) && n->next == 3 && n->len == 2 + && n[1].kind == static_cast(value_t::number_float) && n[2].kind == static_cast(value_t::number_float)) + { + room(2 * 64); + *w++ = '['; + w = write_double_ahead(w, src, n + 1, batch); + *w++ = ','; + w = write_double_ahead(w, src, n + 2, batch); + *w++ = ']'; + n += 2; // (the element before the next node) + break; + } room(8); if (n->len == 0) { @@ -7495,13 +7626,31 @@ class view_serializer } } + /// (float_batch) the mark of digits whose rounding carried beyond the + /// token's last four (see mark()) + static constexpr std::uint64_t carried() noexcept + { + return std::uint64_t{1} << 63u; + } + /// floats of 16 or 17 digits ahead of the walk, converted together (see - /// convert_ahead()); the digits are those of shortest_token_digits(), 0 - /// for a float the batch does not write + /// convert_ahead()), with their digits as mark() keeps them struct float_batch { - std::array at; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read - std::array digits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + static constexpr std::size_t capacity() noexcept + { + return 256; + } + /// (on the heap, allocated by the first conversion: a frame of more + /// than a page made every dump of a small document slower) + struct entries + { + std::array at; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array digits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + }; + std::unique_ptr storage{}; + const node** at = nullptr; + std::uint64_t* digits = nullptr; std::size_t count = 0; std::size_t next = 0; const node* scanned = nullptr; ///< where the last scan of the node array ended @@ -7509,21 +7658,24 @@ class view_serializer /// (doubles) the float at n as dump() writes it, at w (64 bytes of room): /// from the batch if it holds n - NLOHMANN_VIEW_ALWAYS_INLINE char* write_double_ahead(char* w, const node* n, float_batch& batch) + NLOHMANN_VIEW_ALWAYS_INLINE char* write_double_ahead(char* w, const char* src, const node* n, float_batch& batch) { -#if NLOHMANN_VIEW_NEON - if (batch.next != batch.count && batch.at[batch.next] == n && batch.digits[batch.next] != 0) +#if NLOHMANN_VIEW_VECTOR && NLOHMANN_VIEW_LITTLE_ENDIAN + // (digits 0: not written by the batch; carried(): rounding carried + // beyond the last four digits) + if (batch.next != batch.count && batch.at[batch.next] == n && batch.digits[batch.next] - 1 < carried() - 1) { - return write_token_digits(w, m_doc.src + n->off, n->len, n->extra >> 8u, batch.digits[batch.next++]); + return write_token_digits(w, src, *n, batch.digits[batch.next++]); } return write_double_beside(w, n, batch); #else + static_cast(src); static_cast(batch); return write_double_at(w, *n); #endif } -#if NLOHMANN_VIEW_NEON +#if NLOHMANN_VIEW_VECTOR && NLOHMANN_VIEW_LITTLE_ENDIAN /// write_double_ahead() for a float that is not the batch's next one (out /// of line: the walk is faster with less code in its loop) NLOHMANN_VIEW_NOINLINE char* write_double_beside(char* w, const node* n, float_batch& batch) @@ -7537,9 +7689,13 @@ class view_serializer if ((batch.next != batch.count && batch.at[batch.next] == n) || convert_ahead(n, batch)) { const std::uint64_t digits = batch.digits[batch.next++]; + if ((digits & carried()) != 0) + { + return write_carried_digits(w, *n, digits & ~carried()); + } if (digits != 0) { - return write_token_digits(w, m_doc.src + n->off, n->len, n->extra >> 8u, digits); + return write_token_digits(w, m_doc.src, *n, digits); } } return write_double_at(w, *n); @@ -7550,16 +7706,14 @@ class view_serializer array at once; whether n is the first of them Such tokens (like the coordinates of GeoJSON, written with 17 digits) are - written as the token with its last digits replaced (write_token_digits()). - Each step (the digits, Eisel-Lemire, shortest_token_digits()) is a long - chain of dependent multiplications; one loop per step over many floats lets - the processor overlap the floats, which it does not across the steps of - one float (on Apple M1, the dump of canada.json takes 30 % less time than - with a conversion of each float, of which 10 points are this). Each node of the array + written as the token with its last four digits replaced + (write_token_digits()). Each step (the digits, locate_token(), + token_digits_1/2()) is a long chain of dependent operations; one loop per + step over many floats lets the processor overlap the floats, which it does + not across the steps of one float (on Apple M1, one loop for the digits and + locate_token() made the dump of canada.json 12 % slower). Each node of the array is scanned once; a node that the walk does not reach in order (in an - editable document) is written without the batch. (AArch64 only: on x86-64 - (Haswell, GCC 13 and Clang 18), the batch was not faster than the - conversion of each float.) + editable document) is written without the batch. */ NLOHMANN_VIEW_NOINLINE bool convert_ahead(const node* n, float_batch& batch) { @@ -7571,72 +7725,144 @@ class view_serializer } const char* const src = m_doc.src; const char* const src_end = src + m_doc.size; - const node* const stop = last - n > 128 ? n + 128 : last; + const node* const stop = last - n > 1024 ? n + 1024 : last; batch.count = 0; batch.next = 0; + // the digits of each token (and its last four from the text) + std::array digits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array exponent; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array last_four; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read const node* p = n; - for (; p != stop && batch.count != batch.at.size(); ++p) + if (!batch.storage) + { + batch.storage.reset(new typename float_batch::entries); // NOLINT(cppcoreguidelines-owning-memory,modernize-make-unique): C++11 + batch.at = batch.storage->at.data(); + batch.digits = batch.storage->digits.data(); + } + for (; p != stop && batch.count != float_batch::capacity(); ++p) { // a token of the source: 16 or 17 digits, at least 8 after the - // point, no exponent, no leading zero, 32 bytes to read + // point, no exponent, 32 bytes to read const unsigned frac_digits = p->extra >> 8u; - const unsigned count = (p->extra & 0xFFu) + frac_digits; + const unsigned int_digits = p->extra & 0xFFu; + const unsigned count = int_digits + frac_digits; if (p->kind == static_cast(value_t::number_float) && (p->flags & node_flags::storage) == 0 && count - 16u <= 1u && frac_digits >= 8 && src_end - (src + p->off) >= 33) { - const char* const token = src + p->off; + const auto* const token = reinterpret_cast(src + p->off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) const unsigned sign = token[0] == '-' ? 1 : 0; - if (p->len == sign + count + 1 && token[sign] != '0') - { - batch.at[batch.count++] = p; - } + const unsigned char* const first = token + sign; + const std::uint64_t w = sixteen_digits(first, int_digits); + // (the 17th digit follows the point; a token with an exponent + // gets digits outside the range of locate_token(), and is not + // written by the batch. A token 0.xxx with 16 significant + // digits is: its double is written as "0." and its digits too) + const std::uint64_t w17 = (w * 10) + static_cast(first[17] - '0'); + const std::size_t i = batch.count++; + batch.at[i] = p; + digits[i] = p->len != sign + count + 1 ? 0 : (count == 17 ? w17 : w); + exponent[i] = -static_cast(frac_digits); + // (byte 0: the last digit, byte 1: the last two, bits 16-29: the last four) + std::uint32_t four = 0; + std::memcpy(&four, token + p->len - 4, 4); + four -= 0x30303030u; + const std::uint32_t pairs = ((four & 0x0F0F0F0Fu) * 2561u) >> 8u; // (byte 0, 2: the pairs) + last_four[i] = (four >> 24u) | ((pairs >> 16u & 0xFFu) << 8u) | ((((pairs & 0x00FF00FFu) * 6553601u) >> 16u) << 16u); } } batch.scanned = p; + // the position of each double, and the floats of either kind + std::array offset; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array radius; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array kind_1; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array kind_2; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::size_t count_1 = 0; + std::size_t count_2 = 0; for (std::size_t i = 0; i != batch.count; ++i) { - const node& f = *batch.at[i]; - const auto* digits = reinterpret_cast(src + f.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - digits += *digits == '-' ? 1 : 0; - const std::uint64_t w = sixteen_digits(digits, f.extra & 0xFFu); - // (the 17th digit follows the point) - batch.digits[i] = (f.extra & 0xFFu) + (f.extra >> 8u) == 17 ? (w * 10) + static_cast(digits[17] - '0') : w; + const std::uint64_t w = digits[i]; + // (the range excludes the values of bytes that are not digits, only + // in a damaged image loaded with image_check::bounds) + const int g = w - 1000000000000000u < 99000000000000000u ? locate_token(w, exponent[i], offset[i], radius[i]) : 3; + batch.digits[i] = g <= 0 ? (last_four[i] >> 16u) + 1 : 0; + kind_1[count_1] = static_cast(i); + count_1 += g == 1 ? 1 : 0; + kind_2[count_2] = static_cast(i); + count_2 += g == 2 ? 1 : 0; } - std::array bits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read - for (std::size_t i = 0; i != batch.count; ++i) + // the digits to write, in one loop per kind (see mark()) + for (std::size_t k = 0; k != count_1; ++k) { - bits[i] = eisel_lemire>(-static_cast(batch.at[i]->extra >> 8u), batch.digits[i]); + const std::size_t i = kind_1[k]; + const std::uint64_t s = token_digits_1(digits[i], offset[i], radius[i], last_four[i] & 0xFFu); + batch.digits[i] = mark(digits[i], s, last_four[i] >> 16u); } - for (std::size_t i = 0; i != batch.count; ++i) + for (std::size_t k = 0; k != count_2; ++k) { - const std::uint64_t w = batch.digits[i]; - // (the range also excludes the values of bytes that are not - // digits, only in a damaged image loaded with image_check::bounds) - const std::uint64_t s = w - 1000000000000000u < 99000000000000000u ? shortest_token_digits(w, -static_cast(batch.at[i]->extra >> 8u), bits[i]) : 0; - // (only the last eight digits are written over the token's) - batch.digits[i] = s / 100000000u == w / 100000000u ? s : 0; + const std::size_t i = kind_2[k]; + const std::uint64_t s = token_digits_2(digits[i], offset[i], radius[i], last_four[i] & 0xFFu, (last_four[i] >> 8u) & 0xFFu); + batch.digits[i] = mark(digits[i], s, last_four[i] >> 16u); } return batch.count != 0 && batch.at[0] == n; } - /// a float token of 16 or 17 digits as dump() writes it, at w (64 bytes - /// of room): the token with its last eight digits replaced by those of - /// digits (see shortest_token_digits(); the others are the token's), and - /// without trailing zeros after the point, but one - static char* write_token_digits(char* w, const char* token, std::uint32_t len, unsigned frac_digits, std::uint64_t digits) noexcept + /*! + @brief a float token of 16 or 17 digits as dump() writes it, at w (64 bytes + of room), from the digits of its double (see locate_token()) whose + rounding carried beyond the token's last four digits + + (like 43.42027299999999 -> 43.420273): the digits go to the writer of + Zmij's results as its 15 or 16 leading digits and one more, which omits + the trailing zeros. + */ + NLOHMANN_VIEW_NOINLINE char* write_carried_digits(char* w, const node& n, std::uint64_t digits) const noexcept { + const int exponent = -static_cast(n.extra >> 8u); + const std::uint64_t leading = digits / 10; + const auto last = static_cast(digits - (leading * 10)); + const bool seventeen = digits >= 10000000000000000u; + *w = '-'; + w += m_doc.src[n.off] == '-' ? 1 : 0; + return ::nlohmann::detail::dtoa_impl::write_shortest(w, seventeen ? ::nlohmann::detail::zmij::shortest_decimal{leading, exponent, last, last != 0} + : ::nlohmann::detail::zmij::shortest_decimal{digits, exponent - 1, 0, false}); + } + + /// the digits s of a token with the digits w and the last four digits + /// r4, as the batch keeps them: the new last four digits + 1 (only those + /// are written over the token's), or s marked if it differs from w beyond + /// them (0 stays for a float the batch does not write) + static std::uint64_t mark(std::uint64_t w, std::uint64_t s, std::uint64_t r4) noexcept + { + const std::uint64_t last = s - (w - r4); + return last < 10000 ? last + 1 : s | carried(); + } + + /*! + @brief a float token of 16 or 17 digits as dump() writes it, at w (64 bytes + of room): the token with its last four digits replaced by digits - 1 (see + mark(); the others are the token's), and without trailing zeros after the + point, but one + */ + char* write_token_digits(char* w, const char* src, const node& n, std::uint64_t digits) + { + const char* const token = src + n.off; std::memcpy(w, token, 32); - char* const end = w + len; - const std::uint64_t last = eight_digit_text(digits % 100000000u); - std::memcpy(end - 8, &last, 8); + char* const end = w + n.len; + const std::uint32_t last = four_digit_text(static_cast(digits - 1)); + std::memcpy(end - 4, &last, 4); // (the token's text before the new digits, from the source: reading // the output back would wait for the stores) std::uint64_t before = 0; - std::memcpy(&before, token + len - 16, 8); - const std::uint64_t last_zeros = last ^ 0x3030303030303030u; + std::memcpy(&before, token + n.len - 12, 8); + const std::uint32_t last_zeros = last ^ 0x30303030u; const std::uint64_t before_zeros = before ^ 0x3030303030303030u; - const unsigned zeros = last_zeros != 0 ? static_cast(count_leading_zeros(last_zeros)) / 8 - : 8 + (before_zeros != 0 ? static_cast(count_leading_zeros(before_zeros)) / 8 : 8); + if (NLOHMANN_VIEW_UNLIKELY(last_zeros == 0 && before_zeros == 0)) + { + return write_double_at(w, n); // (twelve or more trailing zeros) + } + const unsigned zeros = last_zeros != 0 ? static_cast(count_leading_zeros(last_zeros)) / 8 - 4 + : 4 + (static_cast(count_leading_zeros(before_zeros)) / 8); + const unsigned frac_digits = n.extra >> 8u; return end - (zeros < frac_digits ? zeros : frac_digits - 1); } #endif diff --git a/tests/src/unit-json_view_dump.cpp b/tests/src/unit-json_view_dump.cpp index 6bdad3fa3..18b8508b6 100644 --- a/tests/src/unit-json_view_dump.cpp +++ b/tests/src/unit-json_view_dump.cpp @@ -151,10 +151,12 @@ TEST_CASE("json_view dump") } // tokens of 16 and 17 digits without an exponent (like GeoJSON - // coordinates), which dump() converts in batches: random digits, and - // digits near a shorter decimal (...99999997, ...00000003), whose - // rounding carries; in pairs and among other values, so that the - // batches are interrupted; the last token near the end of the coordinates + // coordinates), which dump() converts in batches: doubles exactly + // halfway between two candidates for the shortest digits, random + // digits, and digits near a shorter decimal (...99999997, + // ...00000003), whose rounding carries; in pairs and among other + // values, so that the batches are interrupted; the last token near the + // end of the text { std::mt19937_64 tokens(5627); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) const auto draw = [&tokens](std::size_t n) @@ -162,7 +164,8 @@ TEST_CASE("json_view dump") const std::uint64_t r = tokens() % n; return static_cast(r); }; - std::string coordinates = "["; + std::string coordinates = "[754570942.33984373,257769445.56835938,858106901.94921875,97944964.876953125," + "[0.5,1e-7],[1.5,-0.0,2.5],[1E2,12.5,0.1,3.25,4.5,5.75,6.125,7.0,8.5],[],[1.5],[1,2.5],[2.5,\"x\"],[[1.5],2.5],"; for (int i = 0; i < 20000; ++i) { const std::size_t length = 16 + draw(2); @@ -191,7 +194,7 @@ TEST_CASE("json_view dump") { const ordered_json& e = j[i++]; CHECK(v.dump() == e.dump()); - if (v.is_array()) + if (v.is_array() && v.size() != 0 && v[0].is_number()) { CHECK(v[0].get() == e[0].get()); }