diff --git a/include/nlohmann/detail/conversions/to_chars.hpp b/include/nlohmann/detail/conversions/to_chars.hpp index db50795b5..48f86cf91 100644 --- a/include/nlohmann/detail/conversions/to_chars.hpp +++ b/include/nlohmann/detail/conversions/to_chars.hpp @@ -1342,6 +1342,66 @@ inline char* write_shortest(char* first, const zmij::shortest_decimal d) noexcep return end + (three ? 5 : 4); } +/// the powers of ten up to 10^19 (all that fit in std::uint64_t) +inline const std::array& powers_of_ten() noexcept +{ + static const std::array powers = + { + { + 1u, + 10u, + 100u, + 1000u, + 10000u, + 100000u, + 1000000u, + 10000000u, + 100000000u, + 1000000000u, + 10000000000u, + 100000000000u, + 1000000000000u, + 10000000000000u, + 100000000000000u, + 1000000000000000u, + 10000000000000000u, + 100000000000000000u, + 1000000000000000000u, + 10000000000000000000u + } + }; + return powers; +} + +/*! +@brief digits * 10^exp, as write_decimal() writes it, for the digits of a +double that need no conversion (count digits, at most 15, the first not 0; +trailing zeros allowed): extended to 16 digits and written by write_shortest() + +@return a pointer past the text; up to 41 bytes at @a first are written + (some beyond the returned end) +*/ +JSON_HEDLEY_NON_NULL(1) +JSON_HEDLEY_RETURNS_NON_NULL +inline char* write_short_decimal(char* first, std::uint64_t digits, int count, int exp) noexcept +{ + JSON_ASSERT(digits >= powers_of_ten()[static_cast(count - 1)] && count <= 15); + const int scale = 16 - count; + return write_shortest(first, zmij::shortest_decimal{digits * powers_of_ten()[static_cast(scale)], exp - scale - 1, 0, false}); +} + +/// as write_short_decimal(), counting the digits (not 0, less than 10^15) +JSON_HEDLEY_NON_NULL(1) +JSON_HEDLEY_RETURNS_NON_NULL +inline char* write_short_decimal(char* first, std::uint64_t digits, int exp) noexcept +{ + JSON_ASSERT(digits != 0 && digits < 1000000000000000u); + // floor(log10(2^bits)) + 1 digits, or one less + const int log2_bound = ((64 - count_leading_zeros(digits)) * 1233) >> 12; + const int count = log2_bound + (digits >= powers_of_ten()[static_cast(log2_bound)] ? 1 : 0); + return write_short_decimal(first, digits, count, exp); +} + /*! @brief whether FloatType is an IEEE 754 binary64 type (a double, or a long double that has the same format, as with MSVC and on Apple's Arm CPUs) diff --git a/include/nlohmann/detail/view/scan.hpp b/include/nlohmann/detail/view/scan.hpp index c6139c8e5..e230afbdb 100644 --- a/include/nlohmann/detail/view/scan.hpp +++ b/include/nlohmann/detail/view/scan.hpp @@ -175,32 +175,7 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_digits(const unsigned char /// powers of ten up to 10^19 as integers inline std::uint64_t int_pow10(unsigned k) noexcept { - static const std::array table = - { - { - 1u, - 10u, - 100u, - 1000u, - 10000u, - 100000u, - 1000000u, - 10000000u, - 100000000u, - 1000000000u, - 10000000000u, - 100000000000u, - 1000000000000u, - 10000000000000u, - 100000000000000u, - 1000000000000000u, - 10000000000000000u, - 100000000000000000u, - 1000000000000000000u, - 10000000000000000000u - } - }; - return table[k]; + return ::nlohmann::detail::dtoa_impl::powers_of_ten()[k]; } /// value of 0 < k < 8 digits at p in one step if [p, p + 8) lies below diff --git a/include/nlohmann/detail/view/serializer.hpp b/include/nlohmann/detail/view/serializer.hpp index 6a6719a65..5d1074820 100644 --- a/include/nlohmann/detail/view/serializer.hpp +++ b/include/nlohmann/detail/view/serializer.hpp @@ -31,7 +31,11 @@ namespace view { /// append-only output buffer: writes through a raw pointer into a string that -/// is resized ahead, and trimmed by finish() +/// is resized ahead, and trimmed by finish(). The estimate is reserved, and the +/// string grows in steps of 64 KiB within it: resize() fills the new bytes +/// with zeros (before C++23, a string cannot grow without), and a small step +/// is filled while the writer is about to use it, in the cache, instead of +/// filling the whole estimate in memory first. template class output_buffer { @@ -83,17 +87,49 @@ class output_buffer m_pos += n; } + /// the write position and the end of the writable space, for a writer + /// that keeps the position in a local variable (set_cursor() hands it back) + char* cursor() const noexcept + { + return m_pos; + } + + char* limit() const noexcept + { + return m_end; + } + + void set_cursor(char* p) noexcept + { + m_pos = p; + } + private: + /// the size of a growth step (a function: std::min() takes a reference, + /// which a static constexpr member does not have before C++17) + static constexpr std::size_t step() noexcept + { + return 0x10000; + } + static StringType& sized(StringType& out, std::size_t estimate) { - out.resize((std::max)(estimate, static_cast(64))); + out.reserve(estimate); + out.resize((std::min)((std::max)(estimate, static_cast(64)), step())); return out; } NLOHMANN_VIEW_NOINLINE void grow(std::size_t n) { const auto used = static_cast(m_pos - m_out.data()); - m_out.resize((std::max)(m_out.size() * 2, used + n + 256)); + // (a step does not go beyond the reserved estimate, so that a good + // estimate is never copied to a larger allocation) + const std::size_t size = (std::max)((std::min)(m_out.size() + step(), m_out.capacity()), used + n + 256); + if (size > m_out.capacity()) + { + m_out.reserve((std::max)(m_out.capacity() * 2, size)); + } + m_out.resize(size); m_pos = &m_out[0] + used; m_end = &m_out[0] + m_out.size(); } @@ -103,6 +139,117 @@ class output_buffer char* m_end; }; +/// The length of the run at s that dump() writes unchanged without +/// ensure_ascii: all bytes but quotes, backslashes, and control characters. +/// Unlike detail::string_bulk_run(), non-ASCII bytes are not validated: the +/// strings of a document are valid UTF-8 (a damaged image loaded with +/// image_check::bounds can have others, which are then written unchanged). +inline std::size_t plain_output_run(const unsigned char* s, std::size_t n) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + const std::uint64_t v = read_eight_bytes(s + i); + const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' + const std::uint64_t stop = (((q - ones) & ~q) | ((b - ones) & ~b) | ((v - 0x2020202020202020ull) & ~v)) & high; + if (stop != 0) + { + // the lowest flagged byte is the first stop: borrows only flag bytes above a true one + return i + (static_cast(count_trailing_zeros(stop)) / 8); + } + } + for (; i < n; ++i) + { + if (s[i] == '"' || s[i] == '\\' || s[i] < 0x20) + { + return i; + } + } + return n; +} + +/// A stack that starts in a buffer of the caller (a local array) and moves to +/// the heap (a vector of the caller) only when that is full, so that dumps of +/// shallow documents need no allocation. The top is a pointer, as in +/// std::vector. grow() is static and out of line: it gets the vector and the +/// used range and returns the new storage, so the address of the stack never +/// reaches a call. Otherwise the compiler would have to keep the three +/// pointers in memory and load and store them on every push_back() and +/// pop_back(), not just when growing. +template +class small_stack +{ + public: + small_stack(T* buffer, std::size_t capacity, std::vector& heap) noexcept + : m_begin(buffer), m_top(buffer), m_end(buffer + capacity), m_heap(heap) + {} + small_stack(const small_stack&) = delete; + small_stack(small_stack&&) = delete; + small_stack& operator=(const small_stack&) = delete; + small_stack& operator=(small_stack&&) = delete; + ~small_stack() = default; + + NLOHMANN_VIEW_ALWAYS_INLINE void push_back(const T& x) + { + if (NLOHMANN_VIEW_UNLIKELY(m_top == m_end)) + { + const std::size_t used = size(); + const std::size_t capacity = 2 * static_cast(m_end - m_begin); + m_begin = grow(m_heap, m_begin, used, capacity); + m_top = m_begin + used; + m_end = m_begin + capacity; + } + *m_top++ = x; + } + + NLOHMANN_VIEW_ALWAYS_INLINE T& back() noexcept + { + return m_top[-1]; + } + + NLOHMANN_VIEW_ALWAYS_INLINE void pop_back() noexcept + { + --m_top; + } + + NLOHMANN_VIEW_ALWAYS_INLINE bool empty() const noexcept + { + return m_top == m_begin; + } + + NLOHMANN_VIEW_ALWAYS_INLINE std::size_t size() const noexcept + { + return static_cast(m_top - m_begin); + } + + private: + /// the used entries moved to heap storage of the given capacity + NLOHMANN_VIEW_NOINLINE static T* grow(std::vector& heap, const T* begin, std::size_t used, std::size_t capacity) + { + std::vector bigger(capacity); + std::copy(begin, begin + used, bigger.begin()); + heap.swap(bigger); + return heap.data(); + } + + T* m_begin; + T* m_top; + T* m_end; + std::vector& m_heap; +}; + +/// a string literal at w, without its terminating NUL (after room()) +/// @return a pointer past it +template +NLOHMANN_VIEW_ALWAYS_INLINE char* put_literal(char* w, const char (&s)[N]) noexcept // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) +{ + std::memcpy(w, s, N - 1); + return w + N - 1; +} + /// how the view's dump() writes a value struct dump_style { @@ -129,6 +276,8 @@ class view_serializer using nav = navigation; using string_t = typename BasicJsonType::string_t; using number_float_t = typename BasicJsonType::number_float_t; + /// doubles that are IEEE 754 binary64 are written from their bits (Zmij) + using fast_double = std::integral_constant < bool, std::is_same::value && ::nlohmann::detail::dtoa_impl::is_binary64::value >; public: view_serializer(const document_data& d, string_t& out, std::size_t estimate, const dump_style& style) @@ -137,6 +286,18 @@ class view_serializer void dump(const node* root) { + if (!m_style.pretty && !m_style.ensure_ascii) + { + if (m_style.source_numbers) + { + dump_compact(root); + } + else + { + dump_compact(root); + } + return; + } struct frame { const node* pos; ///< next element, or key of the next member @@ -144,7 +305,9 @@ class view_serializer bool object; bool first; ///< nothing written yet }; - std::vector stack; + std::array buffer; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::vector heap; + small_stack stack(buffer.data(), buffer.size(), heap); const node* n = root; for (;;) { @@ -215,6 +378,293 @@ class view_serializer } private: + /// a string that is not a plain string of the source (decoded, or written + /// by an edit), without ensure_ascii: runs without characters to escape + /// are copied + NLOHMANN_VIEW_NOINLINE void write_decoded(const node& n) + { + const auto* const s = reinterpret_cast(m_doc.str(n)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + m_out.put('"'); + for (std::size_t i = 0; i < n.len;) + { + const std::size_t run = plain_output_run(s + i, n.len - i); + if (run != 0) + { + m_out.put(reinterpret_cast(s + i), run); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + i += run; + continue; + } + write_codepoint(s[i], s + i, 1); // a quote, a backslash, or a control character + ++i; + } + m_out.put('"'); + } + + /// the copies of dump_compact() that are not fixed-size moves (long + /// strings, or near the end of the source); out of line, so that the + /// compiler does not merge the fixed-size moves into this call + NLOHMANN_VIEW_NOINLINE static void copy_long(char* to, const char* from, std::size_t n) noexcept + { + std::memcpy(to, from, n); + } + + /// a compile-time option as a runtime condition: testing the template + /// argument directly makes a condition like `Editable && c` constant when + /// the option is off, which MSVC reports as C4127 + static NLOHMANN_VIEW_ALWAYS_INLINE bool enabled(bool option) noexcept + { + return option; + } + + /*! + @brief the compact output without ensure_ascii (the default dump()) + + The same walk as dump(), with the write position in a local variable + (stores through char pointers would otherwise force a reload of the + buffer's members after each one), and with strings and number tokens of + the source copied by fixed-size moves of 32 bytes where the source has + that many bytes left, instead of a library call per token. The buffer + keeps 64 bytes of slack for the overshoot. + */ + template + void dump_compact(const node* root) + { + struct frame + { + const node* pos; ///< (editable documents) next element, or key of the next member + const node* end; + bool object; + }; + std::array buffer; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::vector heap; + small_stack stack(buffer.data(), buffer.size(), heap); + const char* const src = m_doc.src; + const char* const src_end = src + m_doc.size; + char* w = m_out.cursor(); + char* lim = m_out.limit(); + // room for n bytes and the slack + const auto room = [&](std::size_t n) + { + if (NLOHMANN_VIEW_UNLIKELY(static_cast(lim - w) < n + 64)) + { + m_out.set_cursor(w); + m_out.reserve(n + 64); + w = m_out.cursor(); + lim = m_out.limit(); + } + }; + // copy n bytes of the source (after room(n)) + const auto copy = [&](const char* from, std::size_t n) + { + if (n <= 32 && static_cast(src_end - from) >= 32) + { + std::memcpy(w, from, 32); + } + else if (n <= 256 && static_cast(src_end - from) >= n + 32) + { + for (std::size_t i = 0; i < n; i += 32) + { + std::memcpy(w + i, from + i, 32); + } + } + else + { + copy_long(w, from, n); + } + w += n; + }; + // a string that is not a plain string of the source (out of line, so + // that the cursor stays in a register here) + const auto escaped = [&](const node & n) + { + m_out.set_cursor(w); + write_decoded(n); + w = m_out.cursor(); + lim = m_out.limit(); + }; + + // Read-only documents: the elements of a container follow it in the + // node array, so the walk goes through the array in order, and a + // frame only needs the end of its container. Editable documents: the + // elements of a moved container live elsewhere, so a frame keeps the + // position of the next element (see navigation). + // The innermost open container is kept in registers (cur; end == + // nullptr: none), the stack holds the ones around it. + frame cur{nullptr, nullptr, false}; + const node* n = root; + for (;;) + { + // write the value at n (read-only documents: and advance n) + bool opened = false; + // false positive: n comes from nav::value(), which never returns null for a valid index + // @infer-ignore NULLPTR_DEREFERENCE + switch (static_cast(n->kind)) + { + case value_t::string: + if ((n->flags & node_flags::storage) == 0) + { + room(n->len + 2); + *w++ = '"'; + copy(src + n->off, n->len); + *w++ = '"'; + } + else + { + escaped(*n); + } + break; + case value_t::number_integer: + case value_t::number_unsigned: + { + const std::uint32_t len = number_length(*n); + room(len); + if (enabled(Editable) && (n->flags & node_flags::storage) != 0) + { + copy_long(w, m_doc.str(*n), len); // a canonical token written by an edit + w += len; + break; + } + const char* const token = src + n->off; + if (!enabled(SourceNumbers) && NLOHMANN_VIEW_UNLIKELY(len == 2 && token[0] == '-' && token[1] == '0')) + { + *w++ = '0'; // parse() reads -0 as the integer 0 + } + else + { + copy(token, len); + } + break; + } + case value_t::number_float: + if (enabled(SourceNumbers) && (n->flags & node_flags::storage) != node_flags::edited) + { + room(n->len); + copy(src + n->off, n->len); + } + else if (enabled(fast_double::value)) + { + room(64); + w = write_double_at(w, *n); + } + else + { + m_out.set_cursor(w); + write_float_node(*n); + w = m_out.cursor(); + lim = m_out.limit(); + } + break; + case value_t::boolean: + room(8); + if ((n->flags & node_flags::is_true) != 0) + { + w = put_literal(w, "true"); + } + else + { + w = put_literal(w, "false"); + } + break; + case value_t::object: + case value_t::array: + { + const bool object = n->kind == static_cast(value_t::object); + room(8); + if (n->len == 0) + { + w = object ? put_literal(w, "{}") : put_literal(w, "[]"); + } + else + { + *w++ = object ? '{' : '['; + stack.push_back(cur); + if (Editable) + { + cur = frame{nav::first(m_doc, n), nav::end(m_doc, n), object}; + } + else + { + cur = frame{nullptr, n + n->next, object}; + } + opened = true; + } + break; + } + case value_t::null: + room(8); + w = put_literal(w, "null"); + break; + case value_t::binary: // LCOV_EXCL_LINE (not in a document) + case value_t::discarded: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + break; // LCOV_EXCL_LINE + } + if (!Editable) + { + ++n; // the next node: the first element of an opened container, or the node after a scalar + } + + // go to the next value: close finished containers, then separate + // (a container just opened has an element) + if (!opened) + { + for (;;) + { + if (cur.end == nullptr) + { + m_out.set_cursor(w); + m_out.finish(); + return; + } + if ((Editable ? cur.pos : n) != cur.end) + { + break; + } + room(1); + *w++ = cur.object ? '}' : ']'; + cur = stack.back(); + stack.pop_back(); + } + room(1); + *w++ = ','; + } + const node* const at = Editable ? cur.pos : n; + if (cur.object) + { + const node& key = *at; + if ((key.flags & node_flags::storage) == 0) + { + room(key.len + 3); + *w++ = '"'; + copy(src + key.off, key.len); + w[0] = '"'; + w[1] = ':'; + w += 2; + } + else + { + escaped(key); + room(1); + *w++ = ':'; + } + if (Editable) + { + n = nav::value(at + 1); + cur.pos = document_data::after(at + 1); + } + else + { + ++n; + } + } + else if (Editable) + { + n = nav::value(at); + cur.pos = document_data::after(at); + } + } + } + void newline(std::size_t level) { if (m_style.pretty) @@ -266,7 +716,7 @@ class view_serializer } else { - write_float(float_value(m_doc, n)); + write_float_node(n); } break; case value_t::object: // LCOV_EXCL_LINE (containers are written by dump()) @@ -278,6 +728,88 @@ class view_serializer } } + /// a float node as dump() writes it + void write_float_node(const node& n) + { + write_float_node(n, fast_double {}); + } + + void write_float_node(const node& n, std::false_type /*other*/) + { + write_float(float_value(m_doc, n)); + } + + void write_float_node(const node& n, std::true_type /*double*/) + { + m_out.reserve(64); + m_out.set_cursor(write_double_at(m_out.cursor(), n)); + } + + /*! + @brief (doubles) the float at n as dump() writes it, at w (64 bytes of room) + + A token of at most 15 significant digits is written from its digits, + without a conversion: two decimals of at most 15 digits are farther + apart than the rounding interval of a (normal) double (the argument + behind DBL_DIG), so the token's digits are the shortest ones of its + double, which the library's conversion writes (Zmij). Other tokens are + converted from the digits already read. + */ + char* write_double_at(char* w, const node& n) + { + const unsigned int_digits = n.extra & 0xFFu; + const unsigned frac_digits = n.extra >> 8u; + if ((n.flags & node_flags::storage) != node_flags::edited && int_digits + frac_digits <= 19) + { + const auto* const first = reinterpret_cast(m_doc.src + n.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const float_significand d = layout_decimal(first, first + n.len, int_digits, frac_digits, reinterpret_cast(m_doc.src + m_doc.size)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + // (the exponent keeps the value far from subnormals and overflow) + if (d.w != 0 && d.w < 1000000000000000u && d.exponent >= -290 && d.exponent <= 290) + { + *w = '-'; + w += d.negative ? 1 : 0; + // (without leading zeros, all digits of the token count; the + // check also keeps an image that was only checked for bounds, + // whose token may not be made of digits, from the counted + // overload) + const auto& powers = ::nlohmann::detail::dtoa_impl::powers_of_ten(); + const unsigned count = int_digits + frac_digits; + return count - 1u < 15u && d.w >= powers[count - 1u] && d.w < powers[count] + ? ::nlohmann::detail::dtoa_impl::write_short_decimal(w, d.w, static_cast(count), static_cast(d.exponent)) + : ::nlohmann::detail::dtoa_impl::write_short_decimal(w, d.w, static_cast(d.exponent)); + } + return write_double_value_at(w, decimal_to_float(d)); // (without reading the token again) + } + return write_double_value_at(w, static_cast(float_value(m_doc, n))); + } + + /// n bytes of text at w + static char* write_text_at(char* w, const char* text, std::size_t n) noexcept + { + std::memcpy(w, text, n); + return w + n; + } + + /// a double as dump() writes it, at w (64 bytes of room) + static char* write_double_value_at(char* w, double x) + { + // (from the bits: without the checks of to_chars()) + std::uint64_t bits = 0; + std::memcpy(&bits, &x, sizeof(bits)); + if (NLOHMANN_VIEW_UNLIKELY((bits & 0x7FF0000000000000u) == 0x7FF0000000000000u)) + { + return write_text_at(w, "null", 4); + } + *w = '-'; + w += bits >> 63u; + bits &= ~(std::uint64_t{1} << 63u); + if (bits == 0) + { + return write_text_at(w, "0.0", 3); + } + return ::nlohmann::detail::dtoa_impl::write_shortest(w, ::nlohmann::detail::zmij::to_shortest(bits)); + } + /// as serializer::dump_float() void write_float(number_float_t x) { diff --git a/include/nlohmann/json_view.hpp b/include/nlohmann/json_view.hpp index fd43d3990..f0fc58dc9 100644 --- a/include/nlohmann/json_view.hpp +++ b/include/nlohmann/json_view.hpp @@ -623,8 +623,10 @@ class basic_json_view style.indent_char = indent_char; style.ensure_ascii = ensure_ascii; style.source_numbers = numbers == number_format::source; - // the compact text is about as long as the source text of the value - const std::size_t estimate = source_extent() + (style.pretty ? source_extent() / 2 : 0) + 64; + // the compact text is about as long as the source text of the value; + // the compact writer keeps 64 bytes of slack, so that it does not grow + // the buffer just before the end + const std::size_t estimate = source_extent() + (style.pretty ? source_extent() / 2 : 0) + 160; detail::view::view_serializer(*m_doc, out, estimate, style).dump(m_node); return out; } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index a12fdb632..0df0fa83d 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -28382,6 +28382,66 @@ inline char* write_shortest(char* first, const zmij::shortest_decimal d) noexcep return end + (three ? 5 : 4); } +/// the powers of ten up to 10^19 (all that fit in std::uint64_t) +inline const std::array& powers_of_ten() noexcept +{ + static const std::array powers = + { + { + 1u, + 10u, + 100u, + 1000u, + 10000u, + 100000u, + 1000000u, + 10000000u, + 100000000u, + 1000000000u, + 10000000000u, + 100000000000u, + 1000000000000u, + 10000000000000u, + 100000000000000u, + 1000000000000000u, + 10000000000000000u, + 100000000000000000u, + 1000000000000000000u, + 10000000000000000000u + } + }; + return powers; +} + +/*! +@brief digits * 10^exp, as write_decimal() writes it, for the digits of a +double that need no conversion (count digits, at most 15, the first not 0; +trailing zeros allowed): extended to 16 digits and written by write_shortest() + +@return a pointer past the text; up to 41 bytes at @a first are written + (some beyond the returned end) +*/ +JSON_HEDLEY_NON_NULL(1) +JSON_HEDLEY_RETURNS_NON_NULL +inline char* write_short_decimal(char* first, std::uint64_t digits, int count, int exp) noexcept +{ + JSON_ASSERT(digits >= powers_of_ten()[static_cast(count - 1)] && count <= 15); + const int scale = 16 - count; + return write_shortest(first, zmij::shortest_decimal{digits * powers_of_ten()[static_cast(scale)], exp - scale - 1, 0, false}); +} + +/// as write_short_decimal(), counting the digits (not 0, less than 10^15) +JSON_HEDLEY_NON_NULL(1) +JSON_HEDLEY_RETURNS_NON_NULL +inline char* write_short_decimal(char* first, std::uint64_t digits, int exp) noexcept +{ + JSON_ASSERT(digits != 0 && digits < 1000000000000000u); + // floor(log10(2^bits)) + 1 digits, or one less + const int log2_bound = ((64 - count_leading_zeros(digits)) * 1233) >> 12; + const int count = log2_bound + (digits >= powers_of_ten()[static_cast(log2_bound)] ? 1 : 0); + return write_short_decimal(first, digits, count, exp); +} + /*! @brief whether FloatType is an IEEE 754 binary64 type (a double, or a long double that has the same format, as with MSVC and on Apple's Arm CPUs) diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 0682c08be..dfefa2633 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -1136,32 +1136,7 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_digits(const unsigned char /// powers of ten up to 10^19 as integers inline std::uint64_t int_pow10(unsigned k) noexcept { - static const std::array table = - { - { - 1u, - 10u, - 100u, - 1000u, - 10000u, - 100000u, - 1000000u, - 10000000u, - 100000000u, - 1000000000u, - 10000000000u, - 100000000000u, - 1000000000000u, - 10000000000000u, - 100000000000000u, - 1000000000000000u, - 10000000000000000u, - 100000000000000000u, - 1000000000000000000u, - 10000000000000000000u - } - }; - return table[k]; + return ::nlohmann::detail::dtoa_impl::powers_of_ten()[k]; } /// value of 0 < k < 8 digits at p in one step if [p, p + 8) lies below @@ -6117,7 +6092,11 @@ namespace view { /// append-only output buffer: writes through a raw pointer into a string that -/// is resized ahead, and trimmed by finish() +/// is resized ahead, and trimmed by finish(). The estimate is reserved, and the +/// string grows in steps of 64 KiB within it: resize() fills the new bytes +/// with zeros (before C++23, a string cannot grow without), and a small step +/// is filled while the writer is about to use it, in the cache, instead of +/// filling the whole estimate in memory first. template class output_buffer { @@ -6169,17 +6148,49 @@ class output_buffer m_pos += n; } + /// the write position and the end of the writable space, for a writer + /// that keeps the position in a local variable (set_cursor() hands it back) + char* cursor() const noexcept + { + return m_pos; + } + + char* limit() const noexcept + { + return m_end; + } + + void set_cursor(char* p) noexcept + { + m_pos = p; + } + private: + /// the size of a growth step (a function: std::min() takes a reference, + /// which a static constexpr member does not have before C++17) + static constexpr std::size_t step() noexcept + { + return 0x10000; + } + static StringType& sized(StringType& out, std::size_t estimate) { - out.resize((std::max)(estimate, static_cast(64))); + out.reserve(estimate); + out.resize((std::min)((std::max)(estimate, static_cast(64)), step())); return out; } NLOHMANN_VIEW_NOINLINE void grow(std::size_t n) { const auto used = static_cast(m_pos - m_out.data()); - m_out.resize((std::max)(m_out.size() * 2, used + n + 256)); + // (a step does not go beyond the reserved estimate, so that a good + // estimate is never copied to a larger allocation) + const std::size_t size = (std::max)((std::min)(m_out.size() + step(), m_out.capacity()), used + n + 256); + if (size > m_out.capacity()) + { + m_out.reserve((std::max)(m_out.capacity() * 2, size)); + } + m_out.resize(size); m_pos = &m_out[0] + used; m_end = &m_out[0] + m_out.size(); } @@ -6189,6 +6200,117 @@ class output_buffer char* m_end; }; +/// The length of the run at s that dump() writes unchanged without +/// ensure_ascii: all bytes but quotes, backslashes, and control characters. +/// Unlike detail::string_bulk_run(), non-ASCII bytes are not validated: the +/// strings of a document are valid UTF-8 (a damaged image loaded with +/// image_check::bounds can have others, which are then written unchanged). +inline std::size_t plain_output_run(const unsigned char* s, std::size_t n) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + const std::uint64_t v = read_eight_bytes(s + i); + const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' + const std::uint64_t stop = (((q - ones) & ~q) | ((b - ones) & ~b) | ((v - 0x2020202020202020ull) & ~v)) & high; + if (stop != 0) + { + // the lowest flagged byte is the first stop: borrows only flag bytes above a true one + return i + (static_cast(count_trailing_zeros(stop)) / 8); + } + } + for (; i < n; ++i) + { + if (s[i] == '"' || s[i] == '\\' || s[i] < 0x20) + { + return i; + } + } + return n; +} + +/// A stack that starts in a buffer of the caller (a local array) and moves to +/// the heap (a vector of the caller) only when that is full, so that dumps of +/// shallow documents need no allocation. The top is a pointer, as in +/// std::vector. grow() is static and out of line: it gets the vector and the +/// used range and returns the new storage, so the address of the stack never +/// reaches a call. Otherwise the compiler would have to keep the three +/// pointers in memory and load and store them on every push_back() and +/// pop_back(), not just when growing. +template +class small_stack +{ + public: + small_stack(T* buffer, std::size_t capacity, std::vector& heap) noexcept + : m_begin(buffer), m_top(buffer), m_end(buffer + capacity), m_heap(heap) + {} + small_stack(const small_stack&) = delete; + small_stack(small_stack&&) = delete; + small_stack& operator=(const small_stack&) = delete; + small_stack& operator=(small_stack&&) = delete; + ~small_stack() = default; + + NLOHMANN_VIEW_ALWAYS_INLINE void push_back(const T& x) + { + if (NLOHMANN_VIEW_UNLIKELY(m_top == m_end)) + { + const std::size_t used = size(); + const std::size_t capacity = 2 * static_cast(m_end - m_begin); + m_begin = grow(m_heap, m_begin, used, capacity); + m_top = m_begin + used; + m_end = m_begin + capacity; + } + *m_top++ = x; + } + + NLOHMANN_VIEW_ALWAYS_INLINE T& back() noexcept + { + return m_top[-1]; + } + + NLOHMANN_VIEW_ALWAYS_INLINE void pop_back() noexcept + { + --m_top; + } + + NLOHMANN_VIEW_ALWAYS_INLINE bool empty() const noexcept + { + return m_top == m_begin; + } + + NLOHMANN_VIEW_ALWAYS_INLINE std::size_t size() const noexcept + { + return static_cast(m_top - m_begin); + } + + private: + /// the used entries moved to heap storage of the given capacity + NLOHMANN_VIEW_NOINLINE static T* grow(std::vector& heap, const T* begin, std::size_t used, std::size_t capacity) + { + std::vector bigger(capacity); + std::copy(begin, begin + used, bigger.begin()); + heap.swap(bigger); + return heap.data(); + } + + T* m_begin; + T* m_top; + T* m_end; + std::vector& m_heap; +}; + +/// a string literal at w, without its terminating NUL (after room()) +/// @return a pointer past it +template +NLOHMANN_VIEW_ALWAYS_INLINE char* put_literal(char* w, const char (&s)[N]) noexcept // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) +{ + std::memcpy(w, s, N - 1); + return w + N - 1; +} + /// how the view's dump() writes a value struct dump_style { @@ -6215,6 +6337,8 @@ class view_serializer using nav = navigation; using string_t = typename BasicJsonType::string_t; using number_float_t = typename BasicJsonType::number_float_t; + /// doubles that are IEEE 754 binary64 are written from their bits (Zmij) + using fast_double = std::integral_constant < bool, std::is_same::value && ::nlohmann::detail::dtoa_impl::is_binary64::value >; public: view_serializer(const document_data& d, string_t& out, std::size_t estimate, const dump_style& style) @@ -6223,6 +6347,18 @@ class view_serializer void dump(const node* root) { + if (!m_style.pretty && !m_style.ensure_ascii) + { + if (m_style.source_numbers) + { + dump_compact(root); + } + else + { + dump_compact(root); + } + return; + } struct frame { const node* pos; ///< next element, or key of the next member @@ -6230,7 +6366,9 @@ class view_serializer bool object; bool first; ///< nothing written yet }; - std::vector stack; + std::array buffer; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::vector heap; + small_stack stack(buffer.data(), buffer.size(), heap); const node* n = root; for (;;) { @@ -6301,6 +6439,293 @@ class view_serializer } private: + /// a string that is not a plain string of the source (decoded, or written + /// by an edit), without ensure_ascii: runs without characters to escape + /// are copied + NLOHMANN_VIEW_NOINLINE void write_decoded(const node& n) + { + const auto* const s = reinterpret_cast(m_doc.str(n)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + m_out.put('"'); + for (std::size_t i = 0; i < n.len;) + { + const std::size_t run = plain_output_run(s + i, n.len - i); + if (run != 0) + { + m_out.put(reinterpret_cast(s + i), run); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + i += run; + continue; + } + write_codepoint(s[i], s + i, 1); // a quote, a backslash, or a control character + ++i; + } + m_out.put('"'); + } + + /// the copies of dump_compact() that are not fixed-size moves (long + /// strings, or near the end of the source); out of line, so that the + /// compiler does not merge the fixed-size moves into this call + NLOHMANN_VIEW_NOINLINE static void copy_long(char* to, const char* from, std::size_t n) noexcept + { + std::memcpy(to, from, n); + } + + /// a compile-time option as a runtime condition: testing the template + /// argument directly makes a condition like `Editable && c` constant when + /// the option is off, which MSVC reports as C4127 + static NLOHMANN_VIEW_ALWAYS_INLINE bool enabled(bool option) noexcept + { + return option; + } + + /*! + @brief the compact output without ensure_ascii (the default dump()) + + The same walk as dump(), with the write position in a local variable + (stores through char pointers would otherwise force a reload of the + buffer's members after each one), and with strings and number tokens of + the source copied by fixed-size moves of 32 bytes where the source has + that many bytes left, instead of a library call per token. The buffer + keeps 64 bytes of slack for the overshoot. + */ + template + void dump_compact(const node* root) + { + struct frame + { + const node* pos; ///< (editable documents) next element, or key of the next member + const node* end; + bool object; + }; + std::array buffer; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::vector heap; + small_stack stack(buffer.data(), buffer.size(), heap); + const char* const src = m_doc.src; + const char* const src_end = src + m_doc.size; + char* w = m_out.cursor(); + char* lim = m_out.limit(); + // room for n bytes and the slack + const auto room = [&](std::size_t n) + { + if (NLOHMANN_VIEW_UNLIKELY(static_cast(lim - w) < n + 64)) + { + m_out.set_cursor(w); + m_out.reserve(n + 64); + w = m_out.cursor(); + lim = m_out.limit(); + } + }; + // copy n bytes of the source (after room(n)) + const auto copy = [&](const char* from, std::size_t n) + { + if (n <= 32 && static_cast(src_end - from) >= 32) + { + std::memcpy(w, from, 32); + } + else if (n <= 256 && static_cast(src_end - from) >= n + 32) + { + for (std::size_t i = 0; i < n; i += 32) + { + std::memcpy(w + i, from + i, 32); + } + } + else + { + copy_long(w, from, n); + } + w += n; + }; + // a string that is not a plain string of the source (out of line, so + // that the cursor stays in a register here) + const auto escaped = [&](const node & n) + { + m_out.set_cursor(w); + write_decoded(n); + w = m_out.cursor(); + lim = m_out.limit(); + }; + + // Read-only documents: the elements of a container follow it in the + // node array, so the walk goes through the array in order, and a + // frame only needs the end of its container. Editable documents: the + // elements of a moved container live elsewhere, so a frame keeps the + // position of the next element (see navigation). + // The innermost open container is kept in registers (cur; end == + // nullptr: none), the stack holds the ones around it. + frame cur{nullptr, nullptr, false}; + const node* n = root; + for (;;) + { + // write the value at n (read-only documents: and advance n) + bool opened = false; + // false positive: n comes from nav::value(), which never returns null for a valid index + // @infer-ignore NULLPTR_DEREFERENCE + switch (static_cast(n->kind)) + { + case value_t::string: + if ((n->flags & node_flags::storage) == 0) + { + room(n->len + 2); + *w++ = '"'; + copy(src + n->off, n->len); + *w++ = '"'; + } + else + { + escaped(*n); + } + break; + case value_t::number_integer: + case value_t::number_unsigned: + { + const std::uint32_t len = number_length(*n); + room(len); + if (enabled(Editable) && (n->flags & node_flags::storage) != 0) + { + copy_long(w, m_doc.str(*n), len); // a canonical token written by an edit + w += len; + break; + } + const char* const token = src + n->off; + if (!enabled(SourceNumbers) && NLOHMANN_VIEW_UNLIKELY(len == 2 && token[0] == '-' && token[1] == '0')) + { + *w++ = '0'; // parse() reads -0 as the integer 0 + } + else + { + copy(token, len); + } + break; + } + case value_t::number_float: + if (enabled(SourceNumbers) && (n->flags & node_flags::storage) != node_flags::edited) + { + room(n->len); + copy(src + n->off, n->len); + } + else if (enabled(fast_double::value)) + { + room(64); + w = write_double_at(w, *n); + } + else + { + m_out.set_cursor(w); + write_float_node(*n); + w = m_out.cursor(); + lim = m_out.limit(); + } + break; + case value_t::boolean: + room(8); + if ((n->flags & node_flags::is_true) != 0) + { + w = put_literal(w, "true"); + } + else + { + w = put_literal(w, "false"); + } + break; + case value_t::object: + case value_t::array: + { + const bool object = n->kind == static_cast(value_t::object); + room(8); + if (n->len == 0) + { + w = object ? put_literal(w, "{}") : put_literal(w, "[]"); + } + else + { + *w++ = object ? '{' : '['; + stack.push_back(cur); + if (Editable) + { + cur = frame{nav::first(m_doc, n), nav::end(m_doc, n), object}; + } + else + { + cur = frame{nullptr, n + n->next, object}; + } + opened = true; + } + break; + } + case value_t::null: + room(8); + w = put_literal(w, "null"); + break; + case value_t::binary: // LCOV_EXCL_LINE (not in a document) + case value_t::discarded: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + break; // LCOV_EXCL_LINE + } + if (!Editable) + { + ++n; // the next node: the first element of an opened container, or the node after a scalar + } + + // go to the next value: close finished containers, then separate + // (a container just opened has an element) + if (!opened) + { + for (;;) + { + if (cur.end == nullptr) + { + m_out.set_cursor(w); + m_out.finish(); + return; + } + if ((Editable ? cur.pos : n) != cur.end) + { + break; + } + room(1); + *w++ = cur.object ? '}' : ']'; + cur = stack.back(); + stack.pop_back(); + } + room(1); + *w++ = ','; + } + const node* const at = Editable ? cur.pos : n; + if (cur.object) + { + const node& key = *at; + if ((key.flags & node_flags::storage) == 0) + { + room(key.len + 3); + *w++ = '"'; + copy(src + key.off, key.len); + w[0] = '"'; + w[1] = ':'; + w += 2; + } + else + { + escaped(key); + room(1); + *w++ = ':'; + } + if (Editable) + { + n = nav::value(at + 1); + cur.pos = document_data::after(at + 1); + } + else + { + ++n; + } + } + else if (Editable) + { + n = nav::value(at); + cur.pos = document_data::after(at); + } + } + } + void newline(std::size_t level) { if (m_style.pretty) @@ -6352,7 +6777,7 @@ class view_serializer } else { - write_float(float_value(m_doc, n)); + write_float_node(n); } break; case value_t::object: // LCOV_EXCL_LINE (containers are written by dump()) @@ -6364,6 +6789,88 @@ class view_serializer } } + /// a float node as dump() writes it + void write_float_node(const node& n) + { + write_float_node(n, fast_double {}); + } + + void write_float_node(const node& n, std::false_type /*other*/) + { + write_float(float_value(m_doc, n)); + } + + void write_float_node(const node& n, std::true_type /*double*/) + { + m_out.reserve(64); + m_out.set_cursor(write_double_at(m_out.cursor(), n)); + } + + /*! + @brief (doubles) the float at n as dump() writes it, at w (64 bytes of room) + + A token of at most 15 significant digits is written from its digits, + without a conversion: two decimals of at most 15 digits are farther + apart than the rounding interval of a (normal) double (the argument + behind DBL_DIG), so the token's digits are the shortest ones of its + double, which the library's conversion writes (Zmij). Other tokens are + converted from the digits already read. + */ + char* write_double_at(char* w, const node& n) + { + const unsigned int_digits = n.extra & 0xFFu; + const unsigned frac_digits = n.extra >> 8u; + if ((n.flags & node_flags::storage) != node_flags::edited && int_digits + frac_digits <= 19) + { + const auto* const first = reinterpret_cast(m_doc.src + n.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const float_significand d = layout_decimal(first, first + n.len, int_digits, frac_digits, reinterpret_cast(m_doc.src + m_doc.size)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + // (the exponent keeps the value far from subnormals and overflow) + if (d.w != 0 && d.w < 1000000000000000u && d.exponent >= -290 && d.exponent <= 290) + { + *w = '-'; + w += d.negative ? 1 : 0; + // (without leading zeros, all digits of the token count; the + // check also keeps an image that was only checked for bounds, + // whose token may not be made of digits, from the counted + // overload) + const auto& powers = ::nlohmann::detail::dtoa_impl::powers_of_ten(); + const unsigned count = int_digits + frac_digits; + return count - 1u < 15u && d.w >= powers[count - 1u] && d.w < powers[count] + ? ::nlohmann::detail::dtoa_impl::write_short_decimal(w, d.w, static_cast(count), static_cast(d.exponent)) + : ::nlohmann::detail::dtoa_impl::write_short_decimal(w, d.w, static_cast(d.exponent)); + } + return write_double_value_at(w, decimal_to_float(d)); // (without reading the token again) + } + return write_double_value_at(w, static_cast(float_value(m_doc, n))); + } + + /// n bytes of text at w + static char* write_text_at(char* w, const char* text, std::size_t n) noexcept + { + std::memcpy(w, text, n); + return w + n; + } + + /// a double as dump() writes it, at w (64 bytes of room) + static char* write_double_value_at(char* w, double x) + { + // (from the bits: without the checks of to_chars()) + std::uint64_t bits = 0; + std::memcpy(&bits, &x, sizeof(bits)); + if (NLOHMANN_VIEW_UNLIKELY((bits & 0x7FF0000000000000u) == 0x7FF0000000000000u)) + { + return write_text_at(w, "null", 4); + } + *w = '-'; + w += bits >> 63u; + bits &= ~(std::uint64_t{1} << 63u); + if (bits == 0) + { + return write_text_at(w, "0.0", 3); + } + return ::nlohmann::detail::dtoa_impl::write_shortest(w, ::nlohmann::detail::zmij::to_shortest(bits)); + } + /// as serializer::dump_float() void write_float(number_float_t x) { @@ -7333,8 +7840,10 @@ class basic_json_view style.indent_char = indent_char; style.ensure_ascii = ensure_ascii; style.source_numbers = numbers == number_format::source; - // the compact text is about as long as the source text of the value - const std::size_t estimate = source_extent() + (style.pretty ? source_extent() / 2 : 0) + 64; + // the compact text is about as long as the source text of the value; + // the compact writer keeps 64 bytes of slack, so that it does not grow + // the buffer just before the end + const std::size_t estimate = source_extent() + (style.pretty ? source_extent() / 2 : 0) + 160; detail::view::view_serializer(*m_doc, out, estimate, style).dump(m_node); return out; } diff --git a/tests/src/json_view_test_helpers.hpp b/tests/src/json_view_test_helpers.hpp index 4a2481e47..b5af61798 100644 --- a/tests/src/json_view_test_helpers.hpp +++ b/tests/src/json_view_test_helpers.hpp @@ -15,6 +15,7 @@ #include #include // any_of +#include // function #include // mt19937 #include // string @@ -86,4 +87,20 @@ inline bool has_duplicate_keys(const nlohmann::ordered_json_view& v) return e.is_structured() && has_duplicate_keys(e); }); } + +#if !defined(JSON_NOEXCEPTION) +// the exception a call throws, or "" if it throws none +inline std::string exception_of_call(const std::function& f) +{ + try + { + f(); + } + catch (const nlohmann::json::exception& e) + { + return e.what(); + } + return ""; +} +#endif } // namespace json_view_test diff --git a/tests/src/unit-json_view_dump.cpp b/tests/src/unit-json_view_dump.cpp index fd04d397f..ea6e4e9cc 100644 --- a/tests/src/unit-json_view_dump.cpp +++ b/tests/src/unit-json_view_dump.cpp @@ -94,6 +94,58 @@ TEST_CASE("json_view dump") CHECK(d.root().dump() == json::parse(text).dump()); CHECK(d.root().dump() == "[1.5,100.0,0,-0.0,1.2345678901234568e+29,18446744073709551615,-9223372036854775808,0.1,1e-07,5e-324]"); CHECK(d.root().dump(-1, ' ', false, json_view::number_format::source) == "[1.50,1E2,-0,-0.0,123456789012345678901234567890,18446744073709551615,-9223372036854775808,0.1,1e-7,5e-324]"); + // also indented, and with ensure_ascii + CHECK(d.root().dump(0, ' ', false, json_view::number_format::source) == "[\n1.50,\n1E2,\n-0,\n-0.0,\n123456789012345678901234567890,\n18446744073709551615,\n-9223372036854775808,\n0.1,\n1e-7,\n5e-324\n]"); + CHECK(d.root().dump(-1, ' ', true, json_view::number_format::source) == "[1.50,1E2,-0,-0.0,123456789012345678901234567890,18446744073709551615,-9223372036854775808,0.1,1e-7,5e-324]"); + + // float tokens of up to 17 significant digits in every spelling: those + // of at most 15 digits are written from their digits, the others + // through the conversion; both as dump() writes them + { + std::mt19937_64 tokens(1170); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) + // a number below n; the remainder is a std::uint64_t, which is + // std::size_t on some platforms and wider on others + const auto draw = [&tokens](std::size_t n) + { + const std::uint64_t r = tokens() % n; + return static_cast(r); + }; + std::string many_tokens = "["; + for (int i = 0; i < 20000; ++i) + { + const std::size_t length = 1 + draw(17); + std::string digits(1, static_cast('1' + draw(9))); + for (std::size_t k = 1; k < length; ++k) + { + digits += static_cast('0' + draw(10)); + } + digits += std::string(draw(4), '0'); // trailing zeros + std::string token = draw(3) == 0 ? "-" : ""; + const std::size_t point = draw(digits.size() + 1); + if (point == 0) + { + token += "0." + std::string(draw(5), '0') + digits; + } + else + { + token += digits.substr(0, point) + (point < digits.size() ? "." + digits.substr(point) : ""); + } + // an exponent that keeps the value between about 1e-320 and 1e300 + const int exponent = static_cast(draw(600)) - 300 - static_cast(point); + if (draw(4) != 0) + { + token += (draw(2) == 0 ? "e" : "E") + std::string(exponent >= 0 && draw(2) == 0 ? "+" : "") + std::to_string(exponent); + } + else if (point == digits.size()) + { + token += ".0"; // (a float, not an integer) + } + many_tokens += (i != 0 ? "," : "") + token; + } + many_tokens += ']'; + const json_document many_doc = json_document::parse(many_tokens); + CHECK(many_doc.root().dump() == json::parse(many_tokens).dump()); + } // random doubles, written as parse() and dump() would std::mt19937_64 rng(1170); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) diff --git a/tests/src/unit-json_view_edit.cpp b/tests/src/unit-json_view_edit.cpp index eaca359c6..5c3b4396a 100644 --- a/tests/src/unit-json_view_edit.cpp +++ b/tests/src/unit-json_view_edit.cpp @@ -19,13 +19,18 @@ using nlohmann::ordered_json_editable_document; using nlohmann::ordered_json_editable_view; using ptr_t = ordered_json::json_pointer; +#include "json_view_test_helpers.hpp" +#if !defined(JSON_NOEXCEPTION) + using json_view_test::exception_of_call; +#endif + #include #include #include #include -#include #include #include +#include #include #include #include @@ -512,25 +517,6 @@ TEST_CASE("json_view edits: differential") } } -namespace -{ -#if !defined(JSON_NOEXCEPTION) -// the exception a call throws, or "" if it throws none -std::string exception_of_call(const std::function& f) -{ - try - { - f(); - } - catch (const json::exception& e) - { - return e.what(); - } - return ""; -} -#endif -} // namespace - TEST_CASE("json_view edits: errors") { SECTION("an empty document") @@ -586,316 +572,7 @@ TEST_CASE("json_view edits: errors") static_cast(other); } -TEST_CASE("json_view edits: views and values") -{ - SECTION("a value that is no longer part of the document") - { - json_editable_document d = json_editable_document::parse("[[[1,2]]]"); - const json_editable_view inner = d.root()[0][0]; - d.set(d.root()[0], json::array({7})); - d.set(inner, 5); - CHECK(d.root().dump() == "[[7]]"); - CHECK(inner.get() == 5); - } - - SECTION("views keep referring to their value") - { - json_editable_document d = json_editable_document::parse(R"({"a": [10, 20, 30], "b": {"c": "text"}})"); - const json_editable_view a = d.root()["a"]; - const json_editable_view twenty = a[1]; - const json_editable_view c = d.root()["b"]["c"]; - d.insert(a, 0, 5); - d.push_back(a, 40); - CHECK(twenty.get() == 20); - CHECK(a[2].get() == 20); - d.erase(a, 2); - CHECK(twenty.get() == 20); // an erased value keeps its last value - d.set(c, 7); - CHECK(c.get() == 7); // a held view sees an assignment - d.set(d.root()["b"], json::array({1, 2})); - CHECK(d.root()["b"].dump() == "[1,2]"); - CHECK(d.root().dump() == R"({"a":[5,10,30,40],"b":[1,2]})"); - CHECK(d.root()["a"][0].source_offset() == static_cast(-1)); // a new value - CHECK(d.root()["a"][1].source_offset() != static_cast(-1)); - } - - SECTION("strings stay valid while more edits come") - { - json_editable_document d = json_editable_document::parse("[]"); - const auto first = d.push_back(d.root(), std::string(100, 'x')).get_string(); - for (int i = 0; i < 1000; ++i) - { - d.push_back(d.root(), std::string(static_cast(i % 50), 'y')); - } - CHECK(std::string(first.data(), first.size()) == std::string(100, 'x')); - CHECK(d.root().size() == 1001); - } - - SECTION("numbers") - { - json_editable_document d = json_editable_document::parse(R"([1.50, 1E2, 3])"); - d.set(d.root(), 2, 0.1); - d.push_back(d.root(), std::numeric_limits::quiet_NaN()); - d.push_back(d.root(), -std::numeric_limits::infinity()); - d.push_back(d.root(), (std::numeric_limits::max)()); - d.push_back(d.root(), (std::numeric_limits::min)()); - CHECK(d.root().dump() == "[1.5,100.0,0.1,null,null,18446744073709551615,-9223372036854775808]"); - CHECK(d.root().dump(-1, ' ', false, json_editable_view::number_format::source) == "[1.50,1E2,0.1,null,null,18446744073709551615,-9223372036854775808]"); - CHECK(std::isnan(d.root()[3].get())); - CHECK(std::isinf(d.root()[4].get())); - CHECK(d.root()[2].number_token() == "0.1"); - CHECK(d.root()[5].get() == 18446744073709551615u); - CHECK(d.root()[6].number_token() == "-9223372036854775808"); - CHECK(d.root().materialize().dump() == json::parse(R"([1.5, 100.0, 0.1, null, null, 18446744073709551615, -9223372036854775808])").dump()); - } - - SECTION("nulls become containers, and the root can be replaced") - { - json_editable_document d = json_editable_document::parse("[null, null]"); - d.set(d.root()[0], "k", 1); - d.push_back(d.root()[1], true); - CHECK(d.root().dump() == R"([{"k":1},[true]])"); - d.set(d.root(), "scalar"); - CHECK(d.root().dump() == R"("scalar")"); - d.set(d.root(), json{{"x", {1, 2}}}); - d.set(json::json_pointer("/x/-"), 3); - d.set(json::json_pointer("/x/3"), 4); // the size of the array appends too - d.set(json::json_pointer("/y"), false); - CHECK(d.root().dump() == R"({"x":[1,2,3,4],"y":false})"); - CHECK(d.erase(json::json_pointer("/x/0")) == 1); - CHECK(d.erase(json::json_pointer("/y")) == 1); - CHECK(d.erase(json::json_pointer("/nothing")) == 0); - CHECK(d.root().dump() == R"({"x":[2,3,4]})"); - } - - SECTION("duplicate keys") - { - json_editable_document d = json_editable_document::parse(R"({"a": 1, "b": 2, "a": 3})"); - d.set(d.root(), "a", 4); // the first member is assigned, the others dropped - CHECK(d.root().dump() == R"({"a":4,"b":2})"); - d = json_editable_document::parse(R"({"a": 1, "b": 2, "a": 3})"); - CHECK(d.erase(d.root(), "a") == 2); - CHECK(d.root().dump() == R"({"b":2})"); - } - - SECTION("values from other documents") - { - const json_document source = json_document::parse(R"({"list": [1, "two", {"three": 3.5}], "text": "a\nb"})"); - json_editable_document edited = json_editable_document::parse("[0]"); - edited.set(edited.root(), 0, json{{"inner", {1, 2}}}); - json_editable_document d = json_editable_document::parse("{}"); - d.set(d.root(), "copy", source.root()["list"]); - d.set(d.root(), "text", source.root()["text"]); - d.set(d.root(), "edited", edited.root()[0]); - d.set(d.root(), "self", d.root()["copy"]); - // (a raw string with a backslash must not be a macro argument: MSVC C2017) - const std::string expected = R"({"copy":[1,"two",{"three":3.5}],"text":"a\nb","edited":{"inner":[1,2]},"self":[1,"two",{"three":3.5}]})"; - CHECK(d.root().dump() == expected); - CHECK(d.root()["copy"] == source.root()["list"]); - CHECK(source.root()["list"] == d.root()["self"]); - CHECK(d.root() != source.root()); - } - - SECTION("large objects") - { - std::string text = "{"; - for (int i = 0; i < 200; ++i) - { - text += (i != 0 ? ",\"k" : "\"k") + std::to_string(i) + "\":" + std::to_string(i); - } - text += '}'; - json_editable_document d = json_editable_document::parse(text); - d.set(d.root(), "k7", "seven"); // assigned in place: the index stays in use - CHECK(d.root()["k7"].get_string() == "seven"); - d.set(d.root(), "new", 1); // appended: the members move, the lookup is linear - CHECK(d.root()["new"].get() == 1); - CHECK(d.root()["k199"].get() == 199); - d.erase(d.root(), "k0"); - CHECK(!d.root().contains("k0")); - CHECK(d.root().size() == 200); - } - - SECTION("reuse and memory") - { - json_editable_document d = json_editable_document::parse("[1, 2, 3]"); - const std::size_t before = d.memory_usage(); - for (int i = 0; i < 100; ++i) - { - d.push_back(d.root(), "some text"); - } - CHECK(d.memory_usage() > before); - const json_editable_view first = d.root()[0]; - d.shrink_to_fit(); // (with edits, the index stays in place) - CHECK(first.get() == 1); - d.read(std::string("[true]")); - CHECK(d.root().dump() == "[true]"); - d.push_back(d.root(), false); - CHECK(d.root().dump() == "[true,false]"); - } -} - -TEST_CASE("json_view edits: deeply nested values") -{ - // copying a value into a document must not recurse per nesting level - const std::size_t depth = 100000; - const std::string brackets = std::string(depth, '[') + std::string(depth, ']'); - std::string braces; - for (std::size_t i = 0; i < depth; ++i) - { - braces += "{\"a\":"; - } - braces += '1'; - braces += std::string(depth, '}'); - - SECTION("a view of a read-only document") - { - const json_document source = json_document::parse(brackets); - json_editable_document d = json_editable_document::parse("[]"); - d.push_back(d.root(), source.root()); - CHECK(d.root().dump() == "[" + brackets + "]"); - } - - SECTION("a view of an editable document") - { - const json_editable_document source = json_editable_document::parse(braces); - json_editable_document d = json_editable_document::parse("{}"); - d.set(d.root(), "deep", source.root()); - CHECK(d.root().dump() == "{\"deep\":" + braces + "}"); - } - - SECTION("a view of an edited document (values behind links)") - { - const json_document source = json_document::parse(brackets); - json_editable_document edited = json_editable_document::parse("[[]]"); - edited.push_back(edited.root()[0], source.root()); - edited.push_back(edited.root(), source.root()); - json_editable_document d = json_editable_document::parse("null"); - d.set(d.root(), edited.root()); - CHECK(d.root().dump() == "[[" + brackets + "]," + brackets + "]"); - } - - SECTION("a basic_json value") - { - json deep = json::array(); - json* inner = &deep; - for (std::size_t i = 1; i < depth; ++i) - { - inner->push_back(json::array()); - inner = &inner->back(); - } - json_editable_document d = json_editable_document::parse("[]"); - d.push_back(d.root(), deep); - CHECK(d.root().dump() == "[" + brackets + "]"); - } - - SECTION("a basic_json value with objects") - { - json deep = 1; - for (std::size_t i = 0; i < depth; ++i) - { - json outer = json::object(); - outer["a"] = std::move(deep); - deep = std::move(outer); - } - json_editable_document d = json_editable_document::parse("{}"); - d.set(d.root(), "deep", deep); - CHECK(d.root().dump() == "{\"deep\":" + braces + "}"); - } -} - #if !defined(JSON_NOEXCEPTION) -TEST_CASE("json_view edits: pointers below a null value") -{ - // a null value on the way becomes what basic_json makes of it: an array - // for "-" and for digits, an object otherwise - struct test_case - { - const char* document; - const char* pointer; - }; - const std::array cases = - { - { - {R"({"a":null})", "/a/0"}, - {R"({"a":null})", "/a/-"}, - {R"({"a":null})", "/a/3"}, - {R"({"a":null})", "/a/x"}, - {R"({"a":null})", "/a/+1"}, - {R"({"a":null})", "/a/01"}, - {R"({"a":null})", "/a/"}, - {R"({"a":{"b":null}})", "/a/b/1"}, - {R"({"a":{"b":null}})", "/a/b/-"}, - {R"({"a":[null]})", "/a/0/0"}, - {R"({"a":[null,null]})", "/a/1/k"}, - {R"([null])", "/0"}, - {"null", "/0"}, - {"null", "/-"}, - {"null", "/k"}, - {"null", ""}, - } - }; - for (const test_case& c : cases) - { - CAPTURE(c.document) - CAPTURE(c.pointer) - json expected = json::parse(c.document); - const std::string error = exception_of_call([&] - { - expected[json::json_pointer(c.pointer)] = 1; - }); - json_editable_document d = json_editable_document::parse(c.document); - if (error.empty()) - { - d.set(json::json_pointer(c.pointer), 1); - CHECK(d.root().dump() == expected.dump()); - CHECK(d.root().materialize() == expected); - } - else - { - // the same error, and the document is not changed - CHECK(exception_of_call([&] { d.set(json::json_pointer(c.pointer), 1); }) == error); - CHECK(d.root().dump() == json::parse(c.document).dump()); - } - } -} - -TEST_CASE("json_view edits: strings of other documents are checked") -{ - // A document borrows the text it was parsed from, and sees later changes - // of the text: a way to get ill-formed UTF-8 into a view. Copying it into - // an editable document is an error, as for any other string. - std::string text = R"({"key":"abc","list":["abc"]})"; - const json_document source = json_document::parse(text); - const auto message_of = [](const std::string & bad) - { - return exception_of_call([&] - { - const std::string dumped = json(bad).dump(); - static_cast(dumped); - }); - }; - json_editable_document d = json_editable_document::parse("[1]"); - d.push_back(d.root(), source.root()); - CHECK(d.root().dump() == R"([1,{"key":"abc","list":["abc"]}])"); - - text[text.find("abc") + 1] = '\xC3'; // "a\xC3c" - text[text.rfind("abc") + 1] = '\xC3'; - const std::string bad_value = message_of(std::string("a\xC3" "c")); - CHECK(!bad_value.empty()); - CHECK(exception_of_call([&] { d.push_back(d.root(), source.root()["key"]); }) == bad_value); - CHECK(exception_of_call([&] { d.set(d.root()[0], source.root()["list"][0]); }) == bad_value); - CHECK(exception_of_call([&] { d.set(d.root(), 0, source.root()["list"]); }) == bad_value); - CHECK(exception_of_call([&] { d.set(d.root(), 0, source.root()); }) == bad_value); - - text[text.find("key") + 1] = '\xC3'; // a key is checked as well - const std::string bad_key = message_of(std::string("k\xC3" "y")); - CHECK(exception_of_call([&] { d.set(d.root(), 0, source.root()); }) == bad_key); - CHECK(exception_of_call([&] { d.push_back(d.root(), source.root()); }) == bad_key); - - // nothing of the failed edits is visible - CHECK(d.root().dump() == R"([1,{"key":"abc","list":["abc"]}])"); -} - TEST_CASE("json_view edits: the size of a text arena") { using nlohmann::detail::view::text_capacity; diff --git a/tests/src/unit-json_view_edit_values.cpp b/tests/src/unit-json_view_edit_values.cpp new file mode 100644 index 000000000..59aee3a9d --- /dev/null +++ b/tests/src/unit-json_view_edit_values.cpp @@ -0,0 +1,357 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; +using nlohmann::json_document; +using nlohmann::json_editable_document; +using nlohmann::json_editable_view; + +#include "json_view_test_helpers.hpp" +#if !defined(JSON_NOEXCEPTION) + using json_view_test::exception_of_call; +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// These tests were split off unit-json_view_edit.cpp, whose object file got too +// large for the MinGW linker (see json_view_test_helpers.hpp). + +TEST_CASE("json_view edits: views and values") +{ + SECTION("a value that is no longer part of the document") + { + json_editable_document d = json_editable_document::parse("[[[1,2]]]"); + const json_editable_view inner = d.root()[0][0]; + d.set(d.root()[0], json::array({7})); + d.set(inner, 5); + CHECK(d.root().dump() == "[[7]]"); + CHECK(inner.get() == 5); + } + + SECTION("views keep referring to their value") + { + json_editable_document d = json_editable_document::parse(R"({"a": [10, 20, 30], "b": {"c": "text"}})"); + const json_editable_view a = d.root()["a"]; + const json_editable_view twenty = a[1]; + const json_editable_view c = d.root()["b"]["c"]; + d.insert(a, 0, 5); + d.push_back(a, 40); + CHECK(twenty.get() == 20); + CHECK(a[2].get() == 20); + d.erase(a, 2); + CHECK(twenty.get() == 20); // an erased value keeps its last value + d.set(c, 7); + CHECK(c.get() == 7); // a held view sees an assignment + d.set(d.root()["b"], json::array({1, 2})); + CHECK(d.root()["b"].dump() == "[1,2]"); + CHECK(d.root().dump() == R"({"a":[5,10,30,40],"b":[1,2]})"); + CHECK(d.root()["a"][0].source_offset() == static_cast(-1)); // a new value + CHECK(d.root()["a"][1].source_offset() != static_cast(-1)); + } + + SECTION("strings stay valid while more edits come") + { + json_editable_document d = json_editable_document::parse("[]"); + const auto first = d.push_back(d.root(), std::string(100, 'x')).get_string(); + for (int i = 0; i < 1000; ++i) + { + d.push_back(d.root(), std::string(static_cast(i % 50), 'y')); + } + CHECK(std::string(first.data(), first.size()) == std::string(100, 'x')); + CHECK(d.root().size() == 1001); + } + + SECTION("numbers") + { + json_editable_document d = json_editable_document::parse(R"([1.50, 1E2, 3])"); + d.set(d.root(), 2, 0.1); + d.push_back(d.root(), std::numeric_limits::quiet_NaN()); + d.push_back(d.root(), -std::numeric_limits::infinity()); + d.push_back(d.root(), (std::numeric_limits::max)()); + d.push_back(d.root(), (std::numeric_limits::min)()); + CHECK(d.root().dump() == "[1.5,100.0,0.1,null,null,18446744073709551615,-9223372036854775808]"); + CHECK(d.root().dump(-1, ' ', false, json_editable_view::number_format::source) == "[1.50,1E2,0.1,null,null,18446744073709551615,-9223372036854775808]"); + CHECK(std::isnan(d.root()[3].get())); + CHECK(std::isinf(d.root()[4].get())); + CHECK(d.root()[2].number_token() == "0.1"); + CHECK(d.root()[5].get() == 18446744073709551615u); + CHECK(d.root()[6].number_token() == "-9223372036854775808"); + CHECK(d.root().materialize().dump() == json::parse(R"([1.5, 100.0, 0.1, null, null, 18446744073709551615, -9223372036854775808])").dump()); + } + + SECTION("numbers of other float types") + { + // doubles have their own path to the output; other float types are + // written as basic_json writes them, non-finite values as null + using json_float = nlohmann::basic_json; + using document_float = nlohmann::basic_json_document; + document_float d = document_float::parse("[1.5]"); + d.push_back(d.root(), std::numeric_limits::quiet_NaN()); + d.push_back(d.root(), -std::numeric_limits::infinity()); + CHECK(d.root().dump() == "[1.5,null,null]"); + CHECK(d.root().dump(2) == json_float::parse("[1.5, null, null]").dump(2)); + } + + SECTION("nulls become containers, and the root can be replaced") + { + json_editable_document d = json_editable_document::parse("[null, null]"); + d.set(d.root()[0], "k", 1); + d.push_back(d.root()[1], true); + CHECK(d.root().dump() == R"([{"k":1},[true]])"); + d.set(d.root(), "scalar"); + CHECK(d.root().dump() == R"("scalar")"); + d.set(d.root(), json{{"x", {1, 2}}}); + d.set(json::json_pointer("/x/-"), 3); + d.set(json::json_pointer("/x/3"), 4); // the size of the array appends too + d.set(json::json_pointer("/y"), false); + CHECK(d.root().dump() == R"({"x":[1,2,3,4],"y":false})"); + CHECK(d.erase(json::json_pointer("/x/0")) == 1); + CHECK(d.erase(json::json_pointer("/y")) == 1); + CHECK(d.erase(json::json_pointer("/nothing")) == 0); + CHECK(d.root().dump() == R"({"x":[2,3,4]})"); + } + + SECTION("duplicate keys") + { + json_editable_document d = json_editable_document::parse(R"({"a": 1, "b": 2, "a": 3})"); + d.set(d.root(), "a", 4); // the first member is assigned, the others dropped + CHECK(d.root().dump() == R"({"a":4,"b":2})"); + d = json_editable_document::parse(R"({"a": 1, "b": 2, "a": 3})"); + CHECK(d.erase(d.root(), "a") == 2); + CHECK(d.root().dump() == R"({"b":2})"); + } + + SECTION("values from other documents") + { + const json_document source = json_document::parse(R"({"list": [1, "two", {"three": 3.5}], "text": "a\nb"})"); + json_editable_document edited = json_editable_document::parse("[0]"); + edited.set(edited.root(), 0, json{{"inner", {1, 2}}}); + json_editable_document d = json_editable_document::parse("{}"); + d.set(d.root(), "copy", source.root()["list"]); + d.set(d.root(), "text", source.root()["text"]); + d.set(d.root(), "edited", edited.root()[0]); + d.set(d.root(), "self", d.root()["copy"]); + // (a raw string with a backslash must not be a macro argument: MSVC C2017) + const std::string expected = R"({"copy":[1,"two",{"three":3.5}],"text":"a\nb","edited":{"inner":[1,2]},"self":[1,"two",{"three":3.5}]})"; + CHECK(d.root().dump() == expected); + CHECK(d.root()["copy"] == source.root()["list"]); + CHECK(source.root()["list"] == d.root()["self"]); + CHECK(d.root() != source.root()); + } + + SECTION("large objects") + { + std::string text = "{"; + for (int i = 0; i < 200; ++i) + { + text += (i != 0 ? ",\"k" : "\"k") + std::to_string(i) + "\":" + std::to_string(i); + } + text += '}'; + json_editable_document d = json_editable_document::parse(text); + d.set(d.root(), "k7", "seven"); // assigned in place: the index stays in use + CHECK(d.root()["k7"].get_string() == "seven"); + d.set(d.root(), "new", 1); // appended: the members move, the lookup is linear + CHECK(d.root()["new"].get() == 1); + CHECK(d.root()["k199"].get() == 199); + d.erase(d.root(), "k0"); + CHECK(!d.root().contains("k0")); + CHECK(d.root().size() == 200); + } + + SECTION("reuse and memory") + { + json_editable_document d = json_editable_document::parse("[1, 2, 3]"); + const std::size_t before = d.memory_usage(); + for (int i = 0; i < 100; ++i) + { + d.push_back(d.root(), "some text"); + } + CHECK(d.memory_usage() > before); + const json_editable_view first = d.root()[0]; + d.shrink_to_fit(); // (with edits, the index stays in place) + CHECK(first.get() == 1); + d.read(std::string("[true]")); + CHECK(d.root().dump() == "[true]"); + d.push_back(d.root(), false); + CHECK(d.root().dump() == "[true,false]"); + } +} + +TEST_CASE("json_view edits: deeply nested values") +{ + // copying a value into a document must not recurse per nesting level + const std::size_t depth = 100000; + const std::string brackets = std::string(depth, '[') + std::string(depth, ']'); + std::string braces; + for (std::size_t i = 0; i < depth; ++i) + { + braces += "{\"a\":"; + } + braces += '1'; + braces += std::string(depth, '}'); + + SECTION("a view of a read-only document") + { + const json_document source = json_document::parse(brackets); + json_editable_document d = json_editable_document::parse("[]"); + d.push_back(d.root(), source.root()); + CHECK(d.root().dump() == "[" + brackets + "]"); + } + + SECTION("a view of an editable document") + { + const json_editable_document source = json_editable_document::parse(braces); + json_editable_document d = json_editable_document::parse("{}"); + d.set(d.root(), "deep", source.root()); + CHECK(d.root().dump() == "{\"deep\":" + braces + "}"); + } + + SECTION("a view of an edited document (values behind links)") + { + const json_document source = json_document::parse(brackets); + json_editable_document edited = json_editable_document::parse("[[]]"); + edited.push_back(edited.root()[0], source.root()); + edited.push_back(edited.root(), source.root()); + json_editable_document d = json_editable_document::parse("null"); + d.set(d.root(), edited.root()); + CHECK(d.root().dump() == "[[" + brackets + "]," + brackets + "]"); + } + + SECTION("a basic_json value") + { + json deep = json::array(); + json* inner = &deep; + for (std::size_t i = 1; i < depth; ++i) + { + inner->push_back(json::array()); + inner = &inner->back(); + } + json_editable_document d = json_editable_document::parse("[]"); + d.push_back(d.root(), deep); + CHECK(d.root().dump() == "[" + brackets + "]"); + } + + SECTION("a basic_json value with objects") + { + json deep = 1; + for (std::size_t i = 0; i < depth; ++i) + { + json outer = json::object(); + outer["a"] = std::move(deep); + deep = std::move(outer); + } + json_editable_document d = json_editable_document::parse("{}"); + d.set(d.root(), "deep", deep); + CHECK(d.root().dump() == "{\"deep\":" + braces + "}"); + } +} + +#if !defined(JSON_NOEXCEPTION) +TEST_CASE("json_view edits: pointers below a null value") +{ + // a null value on the way becomes what basic_json makes of it: an array + // for "-" and for digits, an object otherwise + struct test_case + { + const char* document; + const char* pointer; + }; + const std::array cases = + { + { + {R"({"a":null})", "/a/0"}, + {R"({"a":null})", "/a/-"}, + {R"({"a":null})", "/a/3"}, + {R"({"a":null})", "/a/x"}, + {R"({"a":null})", "/a/+1"}, + {R"({"a":null})", "/a/01"}, + {R"({"a":null})", "/a/"}, + {R"({"a":{"b":null}})", "/a/b/1"}, + {R"({"a":{"b":null}})", "/a/b/-"}, + {R"({"a":[null]})", "/a/0/0"}, + {R"({"a":[null,null]})", "/a/1/k"}, + {R"([null])", "/0"}, + {"null", "/0"}, + {"null", "/-"}, + {"null", "/k"}, + {"null", ""}, + } + }; + for (const test_case& c : cases) + { + CAPTURE(c.document) + CAPTURE(c.pointer) + json expected = json::parse(c.document); + const std::string error = exception_of_call([&] + { + expected[json::json_pointer(c.pointer)] = 1; + }); + json_editable_document d = json_editable_document::parse(c.document); + if (error.empty()) + { + d.set(json::json_pointer(c.pointer), 1); + CHECK(d.root().dump() == expected.dump()); + CHECK(d.root().materialize() == expected); + } + else + { + // the same error, and the document is not changed + CHECK(exception_of_call([&] { d.set(json::json_pointer(c.pointer), 1); }) == error); + CHECK(d.root().dump() == json::parse(c.document).dump()); + } + } +} + +TEST_CASE("json_view edits: strings of other documents are checked") +{ + // A document borrows the text it was parsed from, and sees later changes + // of the text: a way to get ill-formed UTF-8 into a view. Copying it into + // an editable document is an error, as for any other string. + std::string text = R"({"key":"abc","list":["abc"]})"; + const json_document source = json_document::parse(text); + const auto message_of = [](const std::string & bad) + { + return exception_of_call([&] + { + const std::string dumped = json(bad).dump(); + static_cast(dumped); + }); + }; + json_editable_document d = json_editable_document::parse("[1]"); + d.push_back(d.root(), source.root()); + CHECK(d.root().dump() == R"([1,{"key":"abc","list":["abc"]}])"); + + text[text.find("abc") + 1] = '\xC3'; // "a\xC3c" + text[text.rfind("abc") + 1] = '\xC3'; + const std::string bad_value = message_of(std::string("a\xC3" "c")); + CHECK(!bad_value.empty()); + CHECK(exception_of_call([&] { d.push_back(d.root(), source.root()["key"]); }) == bad_value); + CHECK(exception_of_call([&] { d.set(d.root()[0], source.root()["list"][0]); }) == bad_value); + CHECK(exception_of_call([&] { d.set(d.root(), 0, source.root()["list"]); }) == bad_value); + CHECK(exception_of_call([&] { d.set(d.root(), 0, source.root()); }) == bad_value); + + text[text.find("key") + 1] = '\xC3'; // a key is checked as well + const std::string bad_key = message_of(std::string("k\xC3" "y")); + CHECK(exception_of_call([&] { d.set(d.root(), 0, source.root()); }) == bad_key); + CHECK(exception_of_call([&] { d.push_back(d.root(), source.root()); }) == bad_key); + + // nothing of the failed edits is visible + CHECK(d.root().dump() == R"([1,{"key":"abc","list":["abc"]}])"); +} +#endif diff --git a/tests/src/unit-json_view_image.cpp b/tests/src/unit-json_view_image.cpp index 994bd211c..50023fe63 100644 --- a/tests/src/unit-json_view_image.cpp +++ b/tests/src/unit-json_view_image.cpp @@ -1036,6 +1036,31 @@ TEST_CASE("json_view images: check") } } + SECTION("float tokens with fewer digits than the layout records") + { + // The bytes of the token are not digits (they read as zeros or as + // other values), so the value has fewer (or more) digits than the + // layout says: dump() must still write a number. + for (const auto& source : std::vector> + { + {"123456789012345678.5", std::string("@") + std::string(16, '0') + "1.1"}, // 19 digits + {"1234.5", "@001.1"}, // 5 digits + {"1234.5", "9??.??"} // more than 5 digits + }) + { + CAPTURE(source.second) + const std::vector img = json_document::parse("[" + source.first + "]").save(); + const node n = node_at(img, 1); + REQUIRE(source.second.size() == n.len); + std::vector b = img; + std::memcpy(b.data() + text_at(img) + n.off, source.second.data(), n.len); + const json_document d = json_document::load(b, image_check::bounds); + const std::string dumped = d.root().dump(); + CAPTURE(dumped) + CHECK(json::parse(dumped)[0].is_number()); + } + } + SECTION("nodes that share a range") { // [big string, then n strings made to point to the big string]: every diff --git a/tests/src/unit-to_chars.cpp b/tests/src/unit-to_chars.cpp index 95c5f2bd0..fa24477e0 100644 --- a/tests/src/unit-to_chars.cpp +++ b/tests/src/unit-to_chars.cpp @@ -846,3 +846,85 @@ TEST_CASE("choice of the conversion") DOCTEST_MSVC_SUPPRESS_WARNING_POP } } + +TEST_CASE("short decimals") +{ + // write_short_decimal() writes digits * 10^exp for the digits of a double + // that need no conversion (at most 15, the first not 0): as to_chars() + // writes the (positive) double that has these digits + const auto written = [](std::uint64_t digits, int exp) -> std::string + { + std::array buf{}; // (up to 41 bytes are written) + char* const end = nlohmann::detail::dtoa_impl::write_short_decimal(buf.data(), digits, exp); + return {buf.data(), end}; + }; + const auto written_counted = [](std::uint64_t digits, int count, int exp) -> std::string + { + std::array buf{}; + char* const end = nlohmann::detail::dtoa_impl::write_short_decimal(buf.data(), digits, count, exp); + return {buf.data(), end}; + }; + const auto expected = [](std::uint64_t digits, int exp) -> std::string + { + const double value = std::strtod((std::to_string(digits) + "e" + std::to_string(exp)).c_str(), nullptr); + std::array buf{}; + char* const end = nlohmann::detail::to_chars(buf.data(), buf.data() + 32, value); + return {buf.data(), end}; + }; + + SECTION("powers of ten") + { + const auto& powers = nlohmann::detail::dtoa_impl::powers_of_ten(); + CHECK(powers.front() == 1u); + for (std::size_t i = 1; i < powers.size(); ++i) + { + CHECK(powers[i] == powers[i - 1] * 10u); + } + CHECK(powers.back() == 10000000000000000000u); + } + + SECTION("examples") + { + CHECK(written(1, 0) == "1.0"); + CHECK(written(15, -1) == "1.5"); + CHECK(written(125, -2) == "1.25"); + CHECK(written(1, 22) == "1e+22"); + CHECK(written(123456789012345, -2) == "1234567890123.45"); + CHECK(written(999999999999999, -15) == "0.999999999999999"); + CHECK(written(5, -324) == "5e-324"); + CHECK(written_counted(1, 1, 0) == "1.0"); + CHECK(written_counted(125, 3, -2) == "1.25"); + CHECK(written_counted(100, 3, -2) == "1.0"); + CHECK(written_counted(999999999999999, 15, -15) == "0.999999999999999"); + } + + SECTION("random digits, exponents and trailing zeros") + { + std::mt19937_64 rng(1170); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) + for (int i = 0; i < 100000; ++i) + { + // 1 to 15 digits, the first not 0, and up to 14 of them trailing zeros + std::uint64_t count = 1 + (rng() % 15u); + std::uint64_t power = 1; + for (std::uint64_t k = 1; k < count; ++k) + { + power *= 10; + } + std::uint64_t digits = power + (rng() % (9 * power)); + const std::uint64_t zeros = (rng() % 3u == 0) ? (rng() % count) : 0; + for (std::uint64_t k = 0; k < zeros; ++k) + { + digits = (digits / 10) * 10; + } + // (a value between 1e-300 and 1e300) + const int exp = static_cast(rng() % 560u) - 300 - static_cast(count); + + CAPTURE(digits) + CAPTURE(count) + CAPTURE(exp) + const std::string want = expected(digits, exp); + CHECK(written(digits, exp) == want); + CHECK(written_counted(digits, static_cast(count), exp) == want); + } + } +}