diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 02f42bfb7..c14e6eaf1 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -3546,6 +3546,107 @@ class binary_reader return unexpected_byte("invalid byte", "value"); } + /*! + @brief read the elements of an optimized array of numbers (type marker + @a type) that are in the input, in one loop, without the + per-element bookkeeping of @ref parse_ubjson_internal + + Values are emitted as @ref get_ubjson_value emits them. Types other than + the fixed-size integers and floats are left to @ref get_ubjson_value, as + are the elements the input ends before. + + @param[in,out] remaining the number of elements not yet read + @param[in] type the type marker of the elements + @return whether the SAX parser accepted the values + */ + JSON_HEDLEY_NEVER_INLINE + bool parse_ubjson_typed_run(std::size_t& remaining, const char_int_type type, std::true_type /*bulk*/) + { + const bool bjdata = input_format == input_format_t::bjdata; + switch (type) + { + case 'U': + return parse_ubjson_typed_run(remaining); + case 'i': + return parse_ubjson_typed_run(remaining); + case 'I': + return parse_ubjson_typed_run(remaining); + case 'l': + return parse_ubjson_typed_run(remaining); + case 'L': + return parse_ubjson_typed_run(remaining); + case 'u': + return !bjdata || parse_ubjson_typed_run(remaining); + case 'm': + return !bjdata || parse_ubjson_typed_run(remaining); + case 'M': + return !bjdata || parse_ubjson_typed_run(remaining); + case 'd': + return parse_ubjson_typed_run(remaining); + case 'D': + return parse_ubjson_typed_run(remaining); + default: + return true; + } + } + + /// input that is not contiguous: every element is read by get_ubjson_value + bool parse_ubjson_typed_run(std::size_t& /*remaining*/, const char_int_type /*type*/, std::false_type /*bulk*/) const noexcept + { + return true; + } + + template + bool parse_ubjson_typed_run(std::size_t& remaining) + { + constexpr std::size_t size = sizeof(NumberType); + const std::size_t count = (std::min)(remaining, ia.bulk_remaining() / size); + if (count == 0) + { + return true; + } + const auto* const data = reinterpret_cast(ia.bulk_data()); + // UBJSON stores numbers big-endian, BJData little-endian + const bool swap = is_little_endian != (input_format == input_format_t::bjdata); + std::size_t read = 0; + bool accepted = true; + while (read < count && accepted) + { + NumberType number{}; + std::memcpy(&number, data + (read * size), size); + if (swap) + { + byte_swap(number); + } + ++read; + accepted = emit_ubjson_number(number); + } + // as if read by get_number(): current is the last byte read + ia.bulk_skip(read * size); + chars_read += read * size; + current = data[(read * size) - 1]; + remaining -= read; + return accepted; + } + + template::value, int> = 0> + bool emit_ubjson_number(const NumberType number) + { + return emit_float(number); + } + + template < typename NumberType, enable_if_t < std::is_integral::value&& std::is_signed::value, int > = 0 > + bool emit_ubjson_number(const NumberType number) + { + return emit_signed(number); + } + + template < typename NumberType, enable_if_t < std::is_integral::value&& !std::is_signed::value, int > = 0 > + bool emit_ubjson_number(const NumberType number) + { + return emit_unsigned(number); + } + /*! @return whether array creation completed */ @@ -3581,7 +3682,15 @@ class binary_reader return false; } - for (std::size_t i = 0; i < size_and_type.first; ++i) + // the numbers that are in the input, in one loop; the rest (if + // the input ends early) one by one below + std::size_t remaining = size_and_type.first; + if (!nd_is_char && JSON_HEDLEY_UNLIKELY(!parse_ubjson_typed_run(remaining, size_and_type.second, std::integral_constant {}))) + { + return false; + } + + for (std::size_t i = size_and_type.first - remaining; i < size_and_type.first; ++i) { if (nd_is_char) { @@ -3641,7 +3750,8 @@ class binary_reader container_stack.back().remaining = 0; } - return true; + return size_and_type.second == 0 + || parse_ubjson_typed_run(container_stack.back().remaining, size_and_type.second, std::integral_constant {}); } return enter_array(detail::unknown_size()); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index d08ea5ee0..5090995b4 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -21323,6 +21323,107 @@ class binary_reader return unexpected_byte("invalid byte", "value"); } + /*! + @brief read the elements of an optimized array of numbers (type marker + @a type) that are in the input, in one loop, without the + per-element bookkeeping of @ref parse_ubjson_internal + + Values are emitted as @ref get_ubjson_value emits them. Types other than + the fixed-size integers and floats are left to @ref get_ubjson_value, as + are the elements the input ends before. + + @param[in,out] remaining the number of elements not yet read + @param[in] type the type marker of the elements + @return whether the SAX parser accepted the values + */ + JSON_HEDLEY_NEVER_INLINE + bool parse_ubjson_typed_run(std::size_t& remaining, const char_int_type type, std::true_type /*bulk*/) + { + const bool bjdata = input_format == input_format_t::bjdata; + switch (type) + { + case 'U': + return parse_ubjson_typed_run(remaining); + case 'i': + return parse_ubjson_typed_run(remaining); + case 'I': + return parse_ubjson_typed_run(remaining); + case 'l': + return parse_ubjson_typed_run(remaining); + case 'L': + return parse_ubjson_typed_run(remaining); + case 'u': + return !bjdata || parse_ubjson_typed_run(remaining); + case 'm': + return !bjdata || parse_ubjson_typed_run(remaining); + case 'M': + return !bjdata || parse_ubjson_typed_run(remaining); + case 'd': + return parse_ubjson_typed_run(remaining); + case 'D': + return parse_ubjson_typed_run(remaining); + default: + return true; + } + } + + /// input that is not contiguous: every element is read by get_ubjson_value + bool parse_ubjson_typed_run(std::size_t& /*remaining*/, const char_int_type /*type*/, std::false_type /*bulk*/) const noexcept + { + return true; + } + + template + bool parse_ubjson_typed_run(std::size_t& remaining) + { + constexpr std::size_t size = sizeof(NumberType); + const std::size_t count = (std::min)(remaining, ia.bulk_remaining() / size); + if (count == 0) + { + return true; + } + const auto* const data = reinterpret_cast(ia.bulk_data()); + // UBJSON stores numbers big-endian, BJData little-endian + const bool swap = is_little_endian != (input_format == input_format_t::bjdata); + std::size_t read = 0; + bool accepted = true; + while (read < count && accepted) + { + NumberType number{}; + std::memcpy(&number, data + (read * size), size); + if (swap) + { + byte_swap(number); + } + ++read; + accepted = emit_ubjson_number(number); + } + // as if read by get_number(): current is the last byte read + ia.bulk_skip(read * size); + chars_read += read * size; + current = data[(read * size) - 1]; + remaining -= read; + return accepted; + } + + template::value, int> = 0> + bool emit_ubjson_number(const NumberType number) + { + return emit_float(number); + } + + template < typename NumberType, enable_if_t < std::is_integral::value&& std::is_signed::value, int > = 0 > + bool emit_ubjson_number(const NumberType number) + { + return emit_signed(number); + } + + template < typename NumberType, enable_if_t < std::is_integral::value&& !std::is_signed::value, int > = 0 > + bool emit_ubjson_number(const NumberType number) + { + return emit_unsigned(number); + } + /*! @return whether array creation completed */ @@ -21358,7 +21459,15 @@ class binary_reader return false; } - for (std::size_t i = 0; i < size_and_type.first; ++i) + // the numbers that are in the input, in one loop; the rest (if + // the input ends early) one by one below + std::size_t remaining = size_and_type.first; + if (!nd_is_char && JSON_HEDLEY_UNLIKELY(!parse_ubjson_typed_run(remaining, size_and_type.second, std::integral_constant {}))) + { + return false; + } + + for (std::size_t i = size_and_type.first - remaining; i < size_and_type.first; ++i) { if (nd_is_char) { @@ -21418,7 +21527,8 @@ class binary_reader container_stack.back().remaining = 0; } - return true; + return size_and_type.second == 0 + || parse_ubjson_typed_run(container_stack.back().remaining, size_and_type.second, std::integral_constant {}); } return enter_array(detail::unknown_size()); diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index e4dffb6d0..c7deda0ba 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -26,11 +26,14 @@ using nlohmann::ordered_json; #include #include #include +#include #include #include "make_test_data_available.hpp" #include "test_utils.hpp" #include "sax_countdown.hpp" +#include "sax_event_loggers.hpp" using utils::SaxCountdown; +using utils::SaxEventLogger; // trait_test_arg and the "value_in_range_of trait" TEST_CASE_TEMPLATE_DEFINE @@ -4746,3 +4749,75 @@ TEST_CASE("BJData optimized container type marker is validated for count 0") CHECK(json::from_bjdata(vb) == json::binary({})); } } + +TEST_CASE("BJData optimized arrays from contiguous and non-contiguous input") +{ + // contiguous input reads the numbers of an array with a type marker in + // one loop, other input number by number; both must give the same events, + // value, or error for the complete document and for every truncation of + // it, and stop at every event a SAX handler rejects + json document = + { + {"u8", {0, 200, 255}}, + {"i8", {-5, 100, -128}}, + {"i16", {-30000, 1000, 30000}}, + {"i32", {-2000000000, 5, 70000}}, + {"i64", {-9000000000000000000LL, 1, 5000000000LL}}, + {"f64", {1.5, -0.1, 2.5}}, + {"pairs", {{1.25, -3.75}, {1.25, -3.75}, json::array()}}, + {"mixed", {1, "a", 2.5, nullptr}}, + {"u16", {40000, 1, 65535}}, + {"u32", {4000000000U, 1, 70000}}, + {"u64", {18446744073709551615ULL, 1, 5000000000ULL}}, + {"nd", {{"_ArrayType_", "double"}, {"_ArraySize_", {2, 2}}, {"_ArrayData_", {1.5, -2.5, 3.5, -4.5}}}}, + {"nd8", {{"_ArrayType_", "uint8"}, {"_ArraySize_", {3}}, {"_ArrayData_", {1, 2, 3}}}}, + }; + const std::vector encoded = json::to_bjdata(document, true, true); + CHECK(json::from_bjdata(encoded) == document); + + for (std::size_t length = 0; length <= encoded.size(); ++length) + { + CAPTURE(length) + const std::vector contiguous(encoded.begin(), encoded.begin() + static_cast(length)); + const std::list bytewise(contiguous.begin(), contiguous.end()); + + CHECK(json::from_bjdata(contiguous, true, false).dump() == json::from_bjdata(bytewise.begin(), bytewise.end(), true, false).dump()); +#if !defined(JSON_NOEXCEPTION) + std::string contiguous_error; + std::string bytewise_error; + try + { + static_cast(json::from_bjdata(contiguous)); + } + catch (const json::exception& e) + { + contiguous_error = e.what(); + } + try + { + static_cast(json::from_bjdata(bytewise.begin(), bytewise.end())); + } + catch (const json::exception& e) + { + bytewise_error = e.what(); + } + CHECK(contiguous_error == bytewise_error); +#endif + + SaxEventLogger from_contiguous; + SaxEventLogger from_bytewise; + CHECK(json::sax_parse(contiguous, &from_contiguous, json::input_format_t::bjdata) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::bjdata)); + CHECK(from_contiguous.events == from_bytewise.events); + } + + for (int events = 0; events <= 80; ++events) + { + CAPTURE(events) + const std::list bytewise(encoded.begin(), encoded.end()); + SaxCountdown from_contiguous(events); + SaxCountdown from_bytewise(events); + CHECK(json::sax_parse(encoded, &from_contiguous, json::input_format_t::bjdata) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::bjdata)); + } +} diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index 5c6378216..2552dde48 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -21,12 +21,15 @@ using nlohmann::json; #include #include +#include #include #include "make_test_data_available.hpp" #include "round_trip_corpus.hpp" #include "test_utils.hpp" #include "sax_countdown.hpp" +#include "sax_event_loggers.hpp" using utils::SaxCountdown; +using utils::SaxEventLogger; TEST_CASE("UBJSON") @@ -3400,3 +3403,70 @@ TEST_CASE("UBJSON optimized container type marker is validated for count 0") } } } + +TEST_CASE("UBJSON optimized arrays from contiguous and non-contiguous input") +{ + // contiguous input reads the numbers of an array with a type marker in + // one loop, other input number by number; both must give the same events, + // value, or error for the complete document and for every truncation of + // it, and stop at every event a SAX handler rejects + json document = + { + {"u8", {0, 200, 255}}, + {"i8", {-5, 100, -128}}, + {"i16", {-30000, 1000, 30000}}, + {"i32", {-2000000000, 5, 70000}}, + {"i64", {-9000000000000000000LL, 1, 5000000000LL}}, + {"f64", {1.5, -0.1, 2.5}}, + {"pairs", {{1.25, -3.75}, {1.25, -3.75}, json::array()}}, + {"mixed", {1, "a", 2.5, nullptr}}, + }; + const std::vector encoded = json::to_ubjson(document, true, true); + CHECK(json::from_ubjson(encoded) == document); + + for (std::size_t length = 0; length <= encoded.size(); ++length) + { + CAPTURE(length) + const std::vector contiguous(encoded.begin(), encoded.begin() + static_cast(length)); + const std::list bytewise(contiguous.begin(), contiguous.end()); + + CHECK(json::from_ubjson(contiguous, true, false).dump() == json::from_ubjson(bytewise.begin(), bytewise.end(), true, false).dump()); +#if !defined(JSON_NOEXCEPTION) + std::string contiguous_error; + std::string bytewise_error; + try + { + static_cast(json::from_ubjson(contiguous)); + } + catch (const json::exception& e) + { + contiguous_error = e.what(); + } + try + { + static_cast(json::from_ubjson(bytewise.begin(), bytewise.end())); + } + catch (const json::exception& e) + { + bytewise_error = e.what(); + } + CHECK(contiguous_error == bytewise_error); +#endif + + SaxEventLogger from_contiguous; + SaxEventLogger from_bytewise; + CHECK(json::sax_parse(contiguous, &from_contiguous, json::input_format_t::ubjson) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::ubjson)); + CHECK(from_contiguous.events == from_bytewise.events); + } + + for (int events = 0; events <= 80; ++events) + { + CAPTURE(events) + const std::list bytewise(encoded.begin(), encoded.end()); + SaxCountdown from_contiguous(events); + SaxCountdown from_bytewise(events); + CHECK(json::sax_parse(encoded, &from_contiguous, json::input_format_t::ubjson) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::ubjson)); + } +}