diff --git a/docs/mkdocs/docs/api/basic_json/from_ubjson.md b/docs/mkdocs/docs/api/basic_json/from_ubjson.md index cbe53eabd..4c77161f9 100644 --- a/docs/mkdocs/docs/api/basic_json/from_ubjson.md +++ b/docs/mkdocs/docs/api/basic_json/from_ubjson.md @@ -89,9 +89,10 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va - Throws [out_of_range.406](../../home/exceptions.md#jsonexceptionout_of_range406) if a high-precision number, or a float64 when `number_float_t` is `float`, does not fit into `number_float_t` - Throws [out_of_range.408](../../home/exceptions.md#jsonexceptionout_of_range408) if the count of an optimized - container exceeds the capacity of the container (or of `std::size_t`), if an optimized array of `Z`, `T`, or `F` has - more than 1,048,576 elements, or if the optimized arrays of `Z`, `T`, and `F` in the input together have more than - 1,048,576 elements plus 4 for every byte read so far + container exceeds the capacity of the container (or of `std::size_t`; an array of type `N` is empty, so only the + latter applies to it), if an optimized array of `Z`, `T`, or `F` has more than 1,048,576 elements, or if the + optimized arrays of `Z`, `T`, and `F` in the input together have more than 1,048,576 elements plus 4 for every byte + read so far ## Complexity @@ -133,6 +134,8 @@ Linear in the size of the input. - Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0. - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0. - Added `error_handler` parameter in version 3.13.0. +- Report `start_array(0)` to a SAX handler for an optimized array of type `N` (`[$N#n`), whatever its count, in version + 3.13.0. - Limited the total number of elements of optimized arrays of `Z`, `T`, and `F` in one input in version 3.13.0. !!! warning "Deprecation" diff --git a/docs/mkdocs/docs/features/binary_formats/ubjson.md b/docs/mkdocs/docs/features/binary_formats/ubjson.md index 948c7b55c..ac9baeee4 100644 --- a/docs/mkdocs/docs/features/binary_formats/ubjson.md +++ b/docs/mkdocs/docs/features/binary_formats/ubjson.md @@ -131,6 +131,10 @@ The library uses the following mapping from JSON values types to UBJSON types ac a `$Z` annotation. Version 3.12.0 writes such a row in 5 bytes (6 above 255 elements), so rows of 20 or fewer such values are never rejected. Objects and untyped counted arrays are not subject to this limit. + An optimized array of type `N` (no-op) is empty whatever its count is, because a no-op is not a value. Its count is + not passed on: `sax_parse` reports `start_array(0)` (versions before 3.13.0 reported the declared count), and the DOM + parser reserves nothing for it. + !!! info "Binary values" If the JSON data contains the binary type, the value stored is a list of integers, as suggested by the UBJSON diff --git a/docs/mkdocs/docs/features/parsing/untrusted_input.md b/docs/mkdocs/docs/features/parsing/untrusted_input.md index f8f8463dd..bea4503e0 100644 --- a/docs/mkdocs/docs/features/parsing/untrusted_input.md +++ b/docs/mkdocs/docs/features/parsing/untrusted_input.md @@ -72,17 +72,22 @@ For the binary formats, an announced size is never trusted outright: (reported as a parse error) instead of triggering an oversized allocation. - When an array announces its number of elements and the array container supports `reserve()` (as `#!cpp std::vector`, the default, does), the library reserves storage for at most 16384 of them upfront, regardless of how large the - announced count is; further elements still grow the container normally as they are read. + announced count is; further elements still grow the container normally as they are read. A truncated input whose + count is larger than its remaining bytes therefore reserves at most that much before the parse error is reported. + Every array element takes at least one byte of input, except in the [UBJSON](../binary_formats/ubjson.md) arrays of + type `Z`, `T` or `F` described below. - An announced array or object size that exceeds what the target container could ever hold (its `max_size()`) is rejected immediately as [`out_of_range.408`](../../home/exceptions.md#jsonexceptionout_of_range408), without attempting to allocate anything. -- A [UBJSON](../binary_formats/ubjson.md) array of type `Z`, `T` or `F` has no payload, so its announced count is not - bounded by the input. One array may announce at most 1,048,576 elements, and all such arrays of one input together - may contain at most 1,048,576 elements plus 4 for every byte read so far (checked when each array's header has been - read); beyond that the input is rejected as - [`out_of_range.408`](../../home/exceptions.md#jsonexceptionout_of_range408). For a DOM that bounds the memory these +- A UBJSON array of the no-op type (`[$N#`) is empty whatever count it declares: a SAX handler is told + `start_array(0)` and the DOM parser reserves nothing. +- A UBJSON array of type `Z`, `T` or `F` has no payload, so its announced count is not bounded by the input. One array + may announce at most 1,048,576 elements, and all such arrays of one input together may contain at most 1,048,576 + elements plus 4 for every byte read so far (checked when each array's header has been read); beyond that the input is + rejected as [`out_of_range.408`](../../home/exceptions.md#jsonexceptionout_of_range408). For a DOM that bounds the memory these arrays can need to about 80 bytes per byte of input plus about 33 MB for the first 1,048,576 elements; a SAX handler - receives at most that many events. A UBJSON object member always costs a key, so it needs input too. + receives at most that many events. Every other optimized array element, and every UBJSON object member (it has a + key), takes at least one byte of input. ## Strings diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index a3d00378b..170f62409 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -3861,18 +3861,15 @@ class binary_reader } } - if (JSON_HEDLEY_UNLIKELY(!enter_array(size_and_type.first, size_and_type.second))) + // a no-op is not a value, so an array of them is empty whatever + // count it declares: the SAX parser is told so, instead of being + // made to reserve storage for a count that no input backs + const std::size_t announced = size_and_type.second == 'N' ? 0 : size_and_type.first; + if (JSON_HEDLEY_UNLIKELY(!enter_array(announced, size_and_type.second))) { return false; } - if (size_and_type.second == 'N') - { - // a no-op is not a value, so a container of them holds none; - // the declared size has already been passed to the SAX parser - container_stack.back().remaining = 0; - } - return size_and_type.second == 0 || parse_ubjson_typed_run(container_stack.back().remaining, size_and_type.second, std::integral_constant {}); } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index c310fa57a..8e9d828b2 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -22140,18 +22140,15 @@ class binary_reader } } - if (JSON_HEDLEY_UNLIKELY(!enter_array(size_and_type.first, size_and_type.second))) + // a no-op is not a value, so an array of them is empty whatever + // count it declares: the SAX parser is told so, instead of being + // made to reserve storage for a count that no input backs + const std::size_t announced = size_and_type.second == 'N' ? 0 : size_and_type.first; + if (JSON_HEDLEY_UNLIKELY(!enter_array(announced, size_and_type.second))) { return false; } - if (size_and_type.second == 'N') - { - // a no-op is not a value, so a container of them holds none; - // the declared size has already been passed to the SAX parser - container_stack.back().remaining = 0; - } - return size_and_type.second == 0 || parse_ubjson_typed_run(container_stack.back().remaining, size_and_type.second, std::integral_constant {}); } diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index 01cab51c7..cabb6be96 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -2691,6 +2691,90 @@ TEST_CASE("UBJSON the valueless elements of a whole input are bounded") } } +TEST_CASE("UBJSON an optimized array of no-ops is empty whatever its count") +{ + // A no-op ('N') is not a value, so `[$N#` is an array without elements + // for every n. The count used to be passed on to the SAX parser, which + // told it to expect n elements, and the DOM parser reserved storage for + // them: 16384 values (256 KB) for a count of 2^31 in 9 bytes of input. + SECTION("the SAX parser is told that the array is empty") + { + for (const std::vector& input : + { + std::vector {'[', '$', 'N', '#', 'i', 2}, + std::vector {'[', '$', 'N', '#', 'I', 0x7F, 0xFF}, + std::vector {'[', '$', 'N', '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF}, + std::vector {'[', '$', 'N', '#', 'i', 0} + }) + { + SaxEventLogger logger; + CHECK(json::sax_parse(input, &logger, json::input_format_t::ubjson)); + CHECK(logger.events == std::vector({"start_array(0)", "end_array()"})); + } + + // between other values, the next value is read as usual + const std::vector nested = {'[', '[', '$', 'N', '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF, 'T', '[', '$', 'N', '#', 'i', 3, ']'}; + SaxEventLogger logger; + CHECK(json::sax_parse(nested, &logger, json::input_format_t::ubjson)); + CHECK(logger.events == std::vector({"start_array()", "start_array(0)", "end_array()", "boolean(true)", "start_array(0)", "end_array()", "end_array()"})); + } + + SECTION("the DOM parser reserves nothing") + { + // 2000 arrays of 9 bytes that announce 2^31 elements each: with a + // reservation of 16384 values per array this took about 500 MB + std::vector input = {'['}; + for (int i = 0; i < 2000; ++i) + { + input.insert(input.end(), {'[', '$', 'N', '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF}); + } + input.push_back(']'); + + const json j = json::from_ubjson(input); + REQUIRE(j.size() == 2000); + std::size_t capacity = 0; + for (const auto& element : j) + { + CHECK(element.is_array()); + CHECK(element.empty()); + capacity += element.get_ref().capacity(); + } + CHECK(capacity == 0); + + // the 7-byte form inside an optimized array of arrays + std::vector typed = {'[', '$', '[', '#', 'I', 0x07, 0xD0}; + for (int i = 0; i < 2000; ++i) + { + typed.insert(typed.end(), {'$', 'N', '#', 'I', 0x7F, 0xFF}); + } + const json k = json::from_ubjson(typed); + REQUIRE(k.size() == 2000); + capacity = 0; + for (const auto& element : k) + { + CHECK(element.empty()); + capacity += element.get_ref().capacity(); + } + CHECK(capacity == 0); + } + + SECTION("a count that no container could hold is no longer an error") + { + // the array is empty, so nothing has to fit: 0x7FFFFFFF used to be + // reported as out_of_range.408 where it exceeds the maximum size of + // an array, and is accepted now + const std::vector input = {'[', '$', 'N', '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF}; + CHECK(json::from_ubjson(input) == json::array()); + + // a count that does not fit into std::size_t still is +#if SIZE_MAX == 0xffffffff + const std::vector too_large = {'[', '$', 'N', '#', 'L', 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00}; + json _; + CHECK_THROWS_AS(_ = json::from_ubjson(too_large), json::out_of_range&); +#endif + } +} + TEST_CASE("issue #5405 - array reserve for definite-length UBJSON arrays") { #if !defined(JSON_NOEXCEPTION)