diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9318838bb..fd69a5f33 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -118,7 +118,7 @@ endif() if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") # avoid stack overflow, see https://github.com/nlohmann/json/issues/2955 - json_test_set_test_options("test-bon8;test-cbor;test-msgpack;test-ubjson;test-bjdata;test-binary_formats" LINK_OPTIONS /STACK:4000000) + json_test_set_test_options("test-bon8;test-cbor;test-msgpack;test-ubjson;test-bjdata;test-bjdata_round_trip_draft2;test-bjdata_round_trip_draft3;test-binary_formats" LINK_OPTIONS /STACK:4000000) endif() # disable exceptions for test-disabled_exceptions diff --git a/tests/src/bjdata_round_trip.hpp b/tests/src/bjdata_round_trip.hpp new file mode 100644 index 000000000..4fdde833b --- /dev/null +++ b/tests/src/bjdata_round_trip.hpp @@ -0,0 +1,103 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#pragma once + +#include // uint8_t +#include // vector + +#include "doctest_compatibility.h" + +#include + +#include "round_trip_corpus.hpp" + +// The BJData round-trip invariants, split by the BJData version that turns a +// corpus value into j1 (unit-bjdata_round_trip_draft2.cpp and +// unit-bjdata_round_trip_draft3.cpp). They were part of unit-bjdata.cpp, but +// took 80 % of its time; as separate tests, both halves run in parallel and +// stay below the ctest timeout under Valgrind. +namespace utils +{ + +// This checks what the parse_bjdata_fuzzer driver checks (see +// tests/src/fuzzer-parse_bjdata.cpp), so that a regression shows up in CI +// rather than as an OSS-Fuzz report: every value from_bjdata() returns +// (j1) can be serialized with any combination of options, the result can +// be parsed back (j2), and serializing j2 again with the same options +// yields a value-equal result. +// +// Beyond the driver, this also checks that j2 equals j1 and that +// serializing j2 reproduces the exact bytes, both except for values that +// contain a binary value: a binary value is only written as a binary +// value with Draft 3's optimized binary array, and otherwise read back as +// an array of integers, for which the writer may choose different (but +// equally valid) type markers when it is serialized again (see #5494). +// +// Values are compared with dump() rather than operator==, because a NaN +// never compares equal to itself. +// +// Only the corpus values written with @a initial_version become j1; each j1 +// is then serialized with every combination of options, both versions +// included. +inline void check_bjdata_round_trip(const nlohmann::json::bjdata_version_t initial_version) +{ + using nlohmann::json; + + struct options + { + bool use_size; + bool use_type; + json::bjdata_version_t version; + }; + const std::vector all_options = + { + {false, false, json::bjdata_version_t::draft2}, + {true, false, json::bjdata_version_t::draft2}, + {true, true, json::bjdata_version_t::draft2}, + {false, false, json::bjdata_version_t::draft3}, + {true, false, json::bjdata_version_t::draft3}, + {true, true, json::bjdata_version_t::draft3}, + }; + + for (const auto& j0 : utils::round_trip_corpus::values()) + { + // turn the corpus value into a value as from_bjdata() returns it + for (const auto& initial : all_options) + { + if (initial.version != initial_version) + { + continue; + } + + const json j1 = json::from_bjdata(json::to_bjdata(j0, initial.use_size, initial.use_type, initial.version)); + const bool has_binary = utils::round_trip_corpus::contains_binary(j1); + + for (const auto& o : all_options) + { + INFO("j1 = " << j1.dump() << ", use_size = " << o.use_size << ", use_type = " << o.use_type + << ", draft3 = " << (o.version == json::bjdata_version_t::draft3)); + + const std::vector vec = json::to_bjdata(j1, o.use_size, o.use_type, o.version); + json j2; + // anything the library writes must be parsable by the library + REQUIRE_NOTHROW(j2 = json::from_bjdata(vec)); + const std::vector vec2 = json::to_bjdata(j2, o.use_size, o.use_type, o.version); + CHECK(json::from_bjdata(vec2).dump() == j2.dump()); + + if (!has_binary) + { + CHECK(j2.dump() == j1.dump()); + CHECK(vec2 == vec); + } + } + } + } +} + +} // namespace utils diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index 2518f3386..2f91462da 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -28,7 +28,6 @@ using ordered_json = nlohmann::ordered_json; #include #include #include "make_test_data_available.hpp" -#include "round_trip_corpus.hpp" #include "test_utils.hpp" #include "sax_countdown.hpp" using utils::SaxCountdown; @@ -4441,70 +4440,6 @@ TEST_CASE("BJData use_type requires use_size") } } -TEST_CASE("BJData round-trip invariants") -{ - // This checks what the parse_bjdata_fuzzer driver checks (see - // tests/src/fuzzer-parse_bjdata.cpp), so that a regression shows up in CI - // rather than as an OSS-Fuzz report: every value from_bjdata() returns - // (j1) can be serialized with any combination of options, the result can - // be parsed back (j2), and serializing j2 again with the same options - // yields a value-equal result. - // - // Beyond the driver, this also checks that j2 equals j1 and that - // serializing j2 reproduces the exact bytes, both except for values that - // contain a binary value: a binary value is only written as a binary - // value with Draft 3's optimized binary array, and otherwise read back as - // an array of integers, for which the writer may choose different (but - // equally valid) type markers when it is serialized again (see #5494). - // - // Values are compared with dump() rather than operator==, because a NaN - // never compares equal to itself. - struct options - { - bool use_size; - bool use_type; - json::bjdata_version_t version; - }; - const std::vector all_options = - { - {false, false, json::bjdata_version_t::draft2}, - {true, false, json::bjdata_version_t::draft2}, - {true, true, json::bjdata_version_t::draft2}, - {false, false, json::bjdata_version_t::draft3}, - {true, false, json::bjdata_version_t::draft3}, - {true, true, json::bjdata_version_t::draft3}, - }; - - for (const auto& j0 : utils::round_trip_corpus::values()) - { - // turn the corpus value into a value as from_bjdata() returns it - for (const auto& initial : all_options) - { - const json j1 = json::from_bjdata(json::to_bjdata(j0, initial.use_size, initial.use_type, initial.version)); - const bool has_binary = utils::round_trip_corpus::contains_binary(j1); - - for (const auto& o : all_options) - { - INFO("j1 = " << j1.dump() << ", use_size = " << o.use_size << ", use_type = " << o.use_type - << ", draft3 = " << (o.version == json::bjdata_version_t::draft3)); - - const std::vector vec = json::to_bjdata(j1, o.use_size, o.use_type, o.version); - json j2; - // anything the library writes must be parsable by the library - REQUIRE_NOTHROW(j2 = json::from_bjdata(vec)); - const std::vector vec2 = json::to_bjdata(j2, o.use_size, o.use_type, o.version); - CHECK(json::from_bjdata(vec2).dump() == j2.dump()); - - if (!has_binary) - { - CHECK(j2.dump() == j1.dump()); - CHECK(vec2 == vec); - } - } - } - } -} - TEST_CASE("BJData round trip of a binary value is value-stable, not byte-stable") { // OSS-Fuzz issue 474480402: a Draft 3 optimized binary array is read as a diff --git a/tests/src/unit-bjdata_round_trip_draft2.cpp b/tests/src/unit-bjdata_round_trip_draft2.cpp new file mode 100644 index 000000000..6bbe6ea06 --- /dev/null +++ b/tests/src/unit-bjdata_round_trip_draft2.cpp @@ -0,0 +1,22 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include "bjdata_round_trip.hpp" + +// split from unit-bjdata.cpp, see bjdata_round_trip.hpp; the other half is in +// unit-bjdata_round_trip_draft3.cpp + +TEST_CASE("BJData round-trip invariants (j1 written with Draft 2)") +{ + utils::check_bjdata_round_trip(json::bjdata_version_t::draft2); +} diff --git a/tests/src/unit-bjdata_round_trip_draft3.cpp b/tests/src/unit-bjdata_round_trip_draft3.cpp new file mode 100644 index 000000000..97e7517b6 --- /dev/null +++ b/tests/src/unit-bjdata_round_trip_draft3.cpp @@ -0,0 +1,22 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include "bjdata_round_trip.hpp" + +// split from unit-bjdata.cpp, see bjdata_round_trip.hpp; the other half is in +// unit-bjdata_round_trip_draft2.cpp + +TEST_CASE("BJData round-trip invariants (j1 written with Draft 3)") +{ + utils::check_bjdata_round_trip(json::bjdata_version_t::draft3); +} diff --git a/tests/src/unit-large_json.cpp b/tests/src/unit-large_json.cpp index ecd6f9f81..ed988c242 100644 --- a/tests/src/unit-large_json.cpp +++ b/tests/src/unit-large_json.cpp @@ -13,8 +13,6 @@ using nlohmann::json; #include #include -#include -#include TEST_CASE("tests on very large JSONs") { @@ -55,24 +53,6 @@ const json* innermost_value(const json& j, std::size_t& depth) return current; } -// The text of a value nested depth levels deep around the number 0. Level i is -// an array if pattern[i % pattern.size()] is '[', and otherwise an object with -// the single member "a", which every object type enumerates in the same order. -std::string nested_text(std::size_t depth, const std::string& pattern) -{ - std::string text; - std::string closing; - for (std::size_t i = 0; i < depth; ++i) - { - const bool array = pattern[i % pattern.size()] == '['; - text += array ? "[" : "{\"a\":"; - closing += array ? ']' : '}'; - } - text += '0'; - text.append(closing.rbegin(), closing.rend()); - return text; -} - } // namespace TEST_CASE("tests on deeply nested JSONs") @@ -244,385 +224,4 @@ TEST_CASE("tests on deeply nested JSONs") CHECK(*innermost_value(j, unused) == 0); } } - - SECTION("issue #5650 - stack overflow converting between specializations") - { - const std::vector patterns = {"[", "{", "[{"}; - - SECTION("json to ordered_json") - { - for (const auto& pattern : patterns) - { - CAPTURE(pattern) - const std::string text = nested_text(depth, pattern); - const json j = json::parse(text); - - const nlohmann::ordered_json converted = j; - CHECK(converted.dump() == text); - } - } - - SECTION("ordered_json to json") - { - for (const auto& pattern : patterns) - { - CAPTURE(pattern) - const std::string text = nested_text(depth, pattern); - const nlohmann::ordered_json o = nlohmann::ordered_json::parse(text); - - const json converted = o; - CHECK(converted.dump() == text); - } - } - - SECTION("get()") - { - for (const auto& pattern : patterns) - { - CAPTURE(pattern) - const std::string text = nested_text(depth, pattern); - const json j = json::parse(text); - - CHECK(j.get().dump() == text); - } - } - - SECTION("depths around the bound of the recursive descent") - { - for (std::size_t d = 1; d <= 300; ++d) - { - CAPTURE(d) - for (const auto& pattern : patterns) - { - CAPTURE(pattern) - const std::string text = nested_text(d, pattern); - const json j = json::parse(text); - - const nlohmann::ordered_json converted = j; - CHECK(converted.dump() == text); - const json back = converted; - CHECK(back.dump() == text); - } - } - } - - SECTION("values below the bound are converted as values above it") - { - // Bury a value below the bound, where it is converted without the - // call stack, and compare it with the same value converted on its - // own by the containers' range constructors. Its objects have - // members that the two object types enumerate in different orders. - const auto bury = [](nlohmann::ordered_json value) - { - for (std::size_t i = 0; i < 200; ++i) - { - value = nlohmann::ordered_json::array({std::move(value)}); - } - return value; - }; - const auto dig = [](const json & value) - { - const json* current = &value; - for (std::size_t i = 0; i < 200; ++i) - { - current = ¤t->at(0); - } - return current; - }; - - nlohmann::ordered_json value = nlohmann::ordered_json::object(); - value["z"] = {1, -2, 3U, 4.5, true, nullptr, "six", nlohmann::ordered_json::binary({7, 8}, 9), - nlohmann::ordered_json::binary({10}), nlohmann::ordered_json::array(), nlohmann::ordered_json::object() - }; - value["y"] = {{"x", {{"w", 1}, {"v", 2}}}, {"u", {3, {{"t", 4}, {"s", 5}}}}}; - value["r"] = nlohmann::ordered_json::array({nlohmann::ordered_json(nlohmann::ordered_json::value_t::discarded)}); - - const json converted_above = value; - const json buried = bury(value); - const json& converted_below = *dig(buried); - - CHECK(converted_below.dump() == converted_above.dump()); - CHECK(converted_below.at("z").at(7).get_binary().subtype() == 9); - CHECK_FALSE(converted_below.at("z").at(8).get_binary().has_subtype()); - CHECK(converted_below.at("r").at(0).is_discarded()); - - // a discarded value is never equal to anything, so compare the rest - value.erase("r"); - const json without_discarded_above = value; - const json without_discarded_buried = bury(value); - CHECK(*dig(without_discarded_buried) == without_discarded_above); - } - } } - -namespace -{ -json nested_array(const std::size_t depth, json leaf) -{ - json j = std::move(leaf); - for (std::size_t i = 0; i < depth; ++i) - { - json a = json::array(); - a.push_back(std::move(j)); - j = std::move(a); - } - return j; -} - -json nested_object(const std::size_t depth, json leaf) -{ - json j = std::move(leaf); - for (std::size_t i = 0; i < depth; ++i) - { - json o = json::object(); - o["k"] = std::move(j); - j = std::move(o); - } - return j; -} -} // namespace - -TEST_CASE("issue #5392 - binary writers on deeply nested values") -{ - // 200 is past the point where the writers stop recursing, and still - // shallow enough that from_* and operator== (which still recurse) are fine. - const json deep_array = nested_array(200, json(0)); - const json deep_object = nested_object(200, json("x")); - const json empty_array = nested_array(200, json::array()); - const json empty_object = nested_object(200, json::object()); - const json mixed = nested_object(80, nested_array(80, json(true))); - - SECTION("roundtrip past the recursion bound") - { - CHECK(json::from_cbor(json::to_cbor(deep_array)) == deep_array); - CHECK(json::from_msgpack(json::to_msgpack(deep_array)) == deep_array); - CHECK(json::from_ubjson(json::to_ubjson(deep_array)) == deep_array); - CHECK(json::from_ubjson(json::to_ubjson(deep_array, true, false)) == deep_array); - CHECK(json::from_ubjson(json::to_ubjson(deep_array, true, true)) == deep_array); - CHECK(json::from_bjdata(json::to_bjdata(deep_array)) == deep_array); - CHECK(json::from_bon8(json::to_bon8(deep_array)) == deep_array); - - CHECK(json::from_cbor(json::to_cbor(deep_object)) == deep_object); - CHECK(json::from_msgpack(json::to_msgpack(deep_object)) == deep_object); - CHECK(json::from_ubjson(json::to_ubjson(deep_object)) == deep_object); - CHECK(json::from_ubjson(json::to_ubjson(deep_object, true, true)) == deep_object); - CHECK(json::from_bjdata(json::to_bjdata(deep_object)) == deep_object); - CHECK(json::from_bon8(json::to_bon8(deep_object)) == deep_object); - - CHECK(json::from_cbor(json::to_cbor(empty_array)) == empty_array); - CHECK(json::from_msgpack(json::to_msgpack(empty_array)) == empty_array); - CHECK(json::from_ubjson(json::to_ubjson(empty_array)) == empty_array); - CHECK(json::from_ubjson(json::to_ubjson(empty_array, true, true)) == empty_array); - CHECK(json::from_bon8(json::to_bon8(empty_array)) == empty_array); - - CHECK(json::from_cbor(json::to_cbor(empty_object)) == empty_object); - CHECK(json::from_msgpack(json::to_msgpack(empty_object)) == empty_object); - CHECK(json::from_ubjson(json::to_ubjson(empty_object)) == empty_object); - CHECK(json::from_bon8(json::to_bon8(empty_object)) == empty_object); - - CHECK(json::from_cbor(json::to_cbor(mixed)) == mixed); - CHECK(json::from_msgpack(json::to_msgpack(mixed)) == mixed); - CHECK(json::from_ubjson(json::to_ubjson(mixed)) == mixed); - CHECK(json::from_bjdata(json::to_bjdata(mixed)) == mixed); - CHECK(json::from_bon8(json::to_bon8(mixed)) == mixed); - } - - SECTION("the two ways of writing a value meet at the bound") - { - for (std::size_t depth = 120; depth <= 140; ++depth) - { - CAPTURE(depth) - - const json array = nested_array(depth, json(7)); - CHECK(json::from_cbor(json::to_cbor(array)) == array); - CHECK(json::from_msgpack(json::to_msgpack(array)) == array); - CHECK(json::from_ubjson(json::to_ubjson(array, true, true)) == array); - CHECK(json::from_bon8(json::to_bon8(array)) == array); - - const json object = nested_object(depth, json(7)); - CHECK(json::from_cbor(json::to_cbor(object)) == object); - CHECK(json::from_msgpack(json::to_msgpack(object)) == object); - CHECK(json::from_bjdata(json::to_bjdata(object)) == object); - CHECK(json::from_bon8(json::to_bon8(object)) == object); - } - } - - SECTION("a BJData ndarray below the bound is still an ndarray") - { - const json ndarray = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}}); - const json invalid = json({{"_ArrayType_", "nope"}, {"_ArraySize_", {1}}, {"_ArrayData_", {1}}}); - - const json deep_ndarray = nested_array(140, ndarray); - const json deep_invalid = nested_array(140, invalid); - - CHECK(json::from_bjdata(json::to_bjdata(deep_ndarray)) == deep_ndarray); - CHECK(json::from_bjdata(json::to_bjdata(deep_invalid)) == deep_invalid); - CHECK(json::from_bjdata(json::to_bjdata(ndarray)) == ndarray); - } - - SECTION("byte-exact across the switch-over") - { - // nested one-element arrays around the recursion bound: the exact - // bytes a writer produces do not depend on whether it stayed on the - // call stack or moved to the heap one partway through - for (const std::size_t depth : - { - nlohmann::detail::recursion_depth_limit() - 1, nlohmann::detail::recursion_depth_limit(), - nlohmann::detail::recursion_depth_limit() + 1, nlohmann::detail::recursion_depth_limit() + 2 - }) - { - CAPTURE(depth) - const json array = nested_array(depth, json(0)); - - std::vector expected_cbor(depth, 0x81); - expected_cbor.push_back(0x00); - CHECK(json::to_cbor(array) == expected_cbor); - - std::vector expected_msgpack(depth, 0x91); - expected_msgpack.push_back(0x00); - CHECK(json::to_msgpack(array) == expected_msgpack); - - std::string expected_ubjson(depth, '['); - expected_ubjson += "i"; - expected_ubjson += '\0'; - expected_ubjson.append(depth, ']'); - const auto packed_ubjson = json::to_ubjson(array); - CHECK(std::string(packed_ubjson.begin(), packed_ubjson.end()) == expected_ubjson); - - std::vector expected_bon8(depth, 0x81); - expected_bon8.push_back(0x90); - CHECK(json::to_bon8(array) == expected_bon8); - } - } - - SECTION("a deep object, and a BJData ndarray, past the recursion bound") - { - const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50; - - const json object = nested_object(depth, json(42)); - CHECK(json::from_cbor(json::to_cbor(object)) == object); - CHECK(json::from_msgpack(json::to_msgpack(object)) == object); - CHECK(json::from_ubjson(json::to_ubjson(object, true, true)) == object); - CHECK(json::from_bjdata(json::to_bjdata(object)) == object); - CHECK(json::from_bon8(json::to_bon8(object)) == object); - - const json ndarray = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}}); - const json deep_ndarray = nested_array(depth, ndarray); - CHECK(json::from_bjdata(json::to_bjdata(deep_ndarray)) == deep_ndarray); - } - - SECTION("a discarded value past the recursion bound still throws type_error.321") - { - const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50; - const json discarded_leaf(json::value_t::discarded); - const json deep_discarded = nested_array(depth, discarded_leaf); - - // with diagnostics, the message names the path to the discarded leaf -#if JSON_DIAGNOSTICS - std::string path; - for (std::size_t i = 0; i < depth; ++i) - { - path += "/0"; - } - const std::string prefix = "[json.exception.type_error.321] (" + path + ") "; -#else - const std::string prefix = "[json.exception.type_error.321] "; -#endif - - CHECK_THROWS_WITH_AS(json::to_cbor(deep_discarded), (prefix + "cannot serialize discarded value to CBOR").c_str(), json::type_error); - CHECK_THROWS_WITH_AS(json::to_msgpack(deep_discarded), (prefix + "cannot serialize discarded value to MessagePack").c_str(), json::type_error); - CHECK_THROWS_WITH_AS(json::to_ubjson(deep_discarded), (prefix + "cannot serialize discarded value to UBJSON").c_str(), json::type_error); - CHECK_THROWS_WITH_AS(json::to_bjdata(deep_discarded), (prefix + "cannot serialize discarded value to BJData").c_str(), json::type_error); - } - - SECTION("BON8 past the recursion bound: discarded values and errors") - { - const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50; - - // BON8 rejects a discarded value (type_error.321), deep or not - const json deep_discarded = nested_array(depth, json(json::value_t::discarded)); - CHECK_THROWS_AS(json::to_bon8(deep_discarded), json::type_error); - - // errors are thrown from below the bound as from above it - const json deep_invalid_string = nested_array(depth, json(std::string("\x80"))); - CHECK_THROWS_AS(json::to_bon8(deep_invalid_string), json::type_error); - - json deep_invalid_key = json::object(); - deep_invalid_key[std::string("\x80")] = 1; - deep_invalid_key = nested_object(depth, deep_invalid_key); - CHECK_THROWS_AS(json::to_bon8(deep_invalid_key), json::type_error); - - const json deep_too_large = nested_array(depth, json(9223372036854775808u)); - CHECK_THROWS_AS(json::to_bon8(deep_too_large), json::out_of_range); - } - - SECTION("does not overflow the C++ stack") - { - const std::size_t depth = 100000; - const json j = json::parse(std::string(depth, '[') + "0" + std::string(depth, ']')); - - std::vector packed; - CHECK_NOTHROW(packed = json::to_cbor(j)); - CHECK(json::from_cbor(packed) == j); - - CHECK_NOTHROW(packed = json::to_msgpack(j)); - CHECK(json::from_msgpack(packed) == j); - - CHECK_NOTHROW(packed = json::to_ubjson(j)); - CHECK(json::from_ubjson(packed) == j); - - CHECK_NOTHROW(packed = json::to_ubjson(j, true, false)); - CHECK(json::from_ubjson(packed) == j); - - CHECK_NOTHROW(packed = json::to_bjdata(j)); - CHECK(json::from_bjdata(packed) == j); - - CHECK_NOTHROW(packed = json::to_bon8(j)); - CHECK(json::from_bon8(packed) == j); - } - - SECTION("BON8 containers with more than four elements past the recursion bound") - { - // such containers are closed with 0xFE, which the iterative writer - // must emit when it leaves them; strings next to each other must - // still be separated with 0xFF - const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50; - json wide_array = json::array({1, "a", "b", json::array(), 5}); - json wide_object = json({{"k1", "v"}, {"k2", 2}, {"k3", ""}, {"k4", json::object()}, {"k5", nullptr}}); - for (std::size_t i = 0; i < depth; ++i) - { - wide_array = json::array({"s", wide_array, "t", wide_object, true}); - wide_object = json({{"a", wide_object}, {"b", "x"}, {"c", "y"}, {"d", wide_array.size()}, {"e", "z"}}); - } - - CHECK(json::from_bon8(json::to_bon8(wide_array)) == wide_array); - CHECK(json::from_bon8(json::to_bon8(wide_object)) == wide_object); - } - - SECTION("regression test for https://issues.oss-fuzz.com/issues/566583014") - { - // 200000 nested one-element CBOR arrays, the innermost holding null; - // round-tripping this used to recurse once per level on the way back - // out through to_cbor(), deep enough to overflow the stack - std::vector v(200000, 0x81); - v.push_back(0xf6); - const json j = json::from_cbor(v); - CHECK(json::to_cbor(j) == v); - - // the MessagePack analogue: fixarray of 1 nesting down to nil - std::vector v_msgpack(200000, 0x91); - v_msgpack.push_back(0xc0); - const json j_msgpack = json::from_msgpack(v_msgpack); - CHECK(json::to_msgpack(j_msgpack) == v_msgpack); - } - - SECTION("regression test for https://issues.oss-fuzz.com/issues/572238015") - { - // the BON8 analogue: 200000 nested one-element arrays down to null - std::vector v(200000, 0x81); - v.push_back(0xFA); - const json j = json::from_bon8(v); - CHECK(json::to_bon8(j) == v); - } -} - diff --git a/tests/src/unit-large_json_binary_writers.cpp b/tests/src/unit-large_json_binary_writers.cpp new file mode 100644 index 000000000..339c62135 --- /dev/null +++ b/tests/src/unit-large_json_binary_writers.cpp @@ -0,0 +1,294 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include +#include +#include +#include +#include + +// split from unit-large_json.cpp so that both run in parallel and stay below +// the ctest timeout under Valgrind + +namespace +{ +json nested_array(const std::size_t depth, json leaf) +{ + json j = std::move(leaf); + for (std::size_t i = 0; i < depth; ++i) + { + json a = json::array(); + a.push_back(std::move(j)); + j = std::move(a); + } + return j; +} + +json nested_object(const std::size_t depth, json leaf) +{ + json j = std::move(leaf); + for (std::size_t i = 0; i < depth; ++i) + { + json o = json::object(); + o["k"] = std::move(j); + j = std::move(o); + } + return j; +} +} // namespace + +TEST_CASE("issue #5392 - binary writers on deeply nested values") +{ + // 200 is past the point where the writers stop recursing, and still + // shallow enough that from_* and operator== (which still recurse) are fine. + const json deep_array = nested_array(200, json(0)); + const json deep_object = nested_object(200, json("x")); + const json empty_array = nested_array(200, json::array()); + const json empty_object = nested_object(200, json::object()); + const json mixed = nested_object(80, nested_array(80, json(true))); + + SECTION("roundtrip past the recursion bound") + { + CHECK(json::from_cbor(json::to_cbor(deep_array)) == deep_array); + CHECK(json::from_msgpack(json::to_msgpack(deep_array)) == deep_array); + CHECK(json::from_ubjson(json::to_ubjson(deep_array)) == deep_array); + CHECK(json::from_ubjson(json::to_ubjson(deep_array, true, false)) == deep_array); + CHECK(json::from_ubjson(json::to_ubjson(deep_array, true, true)) == deep_array); + CHECK(json::from_bjdata(json::to_bjdata(deep_array)) == deep_array); + CHECK(json::from_bon8(json::to_bon8(deep_array)) == deep_array); + + CHECK(json::from_cbor(json::to_cbor(deep_object)) == deep_object); + CHECK(json::from_msgpack(json::to_msgpack(deep_object)) == deep_object); + CHECK(json::from_ubjson(json::to_ubjson(deep_object)) == deep_object); + CHECK(json::from_ubjson(json::to_ubjson(deep_object, true, true)) == deep_object); + CHECK(json::from_bjdata(json::to_bjdata(deep_object)) == deep_object); + CHECK(json::from_bon8(json::to_bon8(deep_object)) == deep_object); + + CHECK(json::from_cbor(json::to_cbor(empty_array)) == empty_array); + CHECK(json::from_msgpack(json::to_msgpack(empty_array)) == empty_array); + CHECK(json::from_ubjson(json::to_ubjson(empty_array)) == empty_array); + CHECK(json::from_ubjson(json::to_ubjson(empty_array, true, true)) == empty_array); + CHECK(json::from_bon8(json::to_bon8(empty_array)) == empty_array); + + CHECK(json::from_cbor(json::to_cbor(empty_object)) == empty_object); + CHECK(json::from_msgpack(json::to_msgpack(empty_object)) == empty_object); + CHECK(json::from_ubjson(json::to_ubjson(empty_object)) == empty_object); + CHECK(json::from_bon8(json::to_bon8(empty_object)) == empty_object); + + CHECK(json::from_cbor(json::to_cbor(mixed)) == mixed); + CHECK(json::from_msgpack(json::to_msgpack(mixed)) == mixed); + CHECK(json::from_ubjson(json::to_ubjson(mixed)) == mixed); + CHECK(json::from_bjdata(json::to_bjdata(mixed)) == mixed); + CHECK(json::from_bon8(json::to_bon8(mixed)) == mixed); + } + + SECTION("the two ways of writing a value meet at the bound") + { + for (std::size_t depth = 120; depth <= 140; ++depth) + { + CAPTURE(depth) + + const json array = nested_array(depth, json(7)); + CHECK(json::from_cbor(json::to_cbor(array)) == array); + CHECK(json::from_msgpack(json::to_msgpack(array)) == array); + CHECK(json::from_ubjson(json::to_ubjson(array, true, true)) == array); + CHECK(json::from_bon8(json::to_bon8(array)) == array); + + const json object = nested_object(depth, json(7)); + CHECK(json::from_cbor(json::to_cbor(object)) == object); + CHECK(json::from_msgpack(json::to_msgpack(object)) == object); + CHECK(json::from_bjdata(json::to_bjdata(object)) == object); + CHECK(json::from_bon8(json::to_bon8(object)) == object); + } + } + + SECTION("a BJData ndarray below the bound is still an ndarray") + { + const json ndarray = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}}); + const json invalid = json({{"_ArrayType_", "nope"}, {"_ArraySize_", {1}}, {"_ArrayData_", {1}}}); + + const json deep_ndarray = nested_array(140, ndarray); + const json deep_invalid = nested_array(140, invalid); + + CHECK(json::from_bjdata(json::to_bjdata(deep_ndarray)) == deep_ndarray); + CHECK(json::from_bjdata(json::to_bjdata(deep_invalid)) == deep_invalid); + CHECK(json::from_bjdata(json::to_bjdata(ndarray)) == ndarray); + } + + SECTION("byte-exact across the switch-over") + { + // nested one-element arrays around the recursion bound: the exact + // bytes a writer produces do not depend on whether it stayed on the + // call stack or moved to the heap one partway through + for (const std::size_t depth : + { + nlohmann::detail::recursion_depth_limit() - 1, nlohmann::detail::recursion_depth_limit(), + nlohmann::detail::recursion_depth_limit() + 1, nlohmann::detail::recursion_depth_limit() + 2 + }) + { + CAPTURE(depth) + const json array = nested_array(depth, json(0)); + + std::vector expected_cbor(depth, 0x81); + expected_cbor.push_back(0x00); + CHECK(json::to_cbor(array) == expected_cbor); + + std::vector expected_msgpack(depth, 0x91); + expected_msgpack.push_back(0x00); + CHECK(json::to_msgpack(array) == expected_msgpack); + + std::string expected_ubjson(depth, '['); + expected_ubjson += "i"; + expected_ubjson += '\0'; + expected_ubjson.append(depth, ']'); + const auto packed_ubjson = json::to_ubjson(array); + CHECK(std::string(packed_ubjson.begin(), packed_ubjson.end()) == expected_ubjson); + + std::vector expected_bon8(depth, 0x81); + expected_bon8.push_back(0x90); + CHECK(json::to_bon8(array) == expected_bon8); + } + } + + SECTION("a deep object, and a BJData ndarray, past the recursion bound") + { + const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50; + + const json object = nested_object(depth, json(42)); + CHECK(json::from_cbor(json::to_cbor(object)) == object); + CHECK(json::from_msgpack(json::to_msgpack(object)) == object); + CHECK(json::from_ubjson(json::to_ubjson(object, true, true)) == object); + CHECK(json::from_bjdata(json::to_bjdata(object)) == object); + CHECK(json::from_bon8(json::to_bon8(object)) == object); + + const json ndarray = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}}); + const json deep_ndarray = nested_array(depth, ndarray); + CHECK(json::from_bjdata(json::to_bjdata(deep_ndarray)) == deep_ndarray); + } + + SECTION("a discarded value past the recursion bound still throws type_error.321") + { + const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50; + const json discarded_leaf(json::value_t::discarded); + const json deep_discarded = nested_array(depth, discarded_leaf); + + // with diagnostics, the message names the path to the discarded leaf +#if JSON_DIAGNOSTICS + std::string path; + for (std::size_t i = 0; i < depth; ++i) + { + path += "/0"; + } + const std::string prefix = "[json.exception.type_error.321] (" + path + ") "; +#else + const std::string prefix = "[json.exception.type_error.321] "; +#endif + + CHECK_THROWS_WITH_AS(json::to_cbor(deep_discarded), (prefix + "cannot serialize discarded value to CBOR").c_str(), json::type_error); + CHECK_THROWS_WITH_AS(json::to_msgpack(deep_discarded), (prefix + "cannot serialize discarded value to MessagePack").c_str(), json::type_error); + CHECK_THROWS_WITH_AS(json::to_ubjson(deep_discarded), (prefix + "cannot serialize discarded value to UBJSON").c_str(), json::type_error); + CHECK_THROWS_WITH_AS(json::to_bjdata(deep_discarded), (prefix + "cannot serialize discarded value to BJData").c_str(), json::type_error); + } + + SECTION("BON8 past the recursion bound: discarded values and errors") + { + const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50; + + // BON8 rejects a discarded value (type_error.321), deep or not + const json deep_discarded = nested_array(depth, json(json::value_t::discarded)); + CHECK_THROWS_AS(json::to_bon8(deep_discarded), json::type_error); + + // errors are thrown from below the bound as from above it + const json deep_invalid_string = nested_array(depth, json(std::string("\x80"))); + CHECK_THROWS_AS(json::to_bon8(deep_invalid_string), json::type_error); + + json deep_invalid_key = json::object(); + deep_invalid_key[std::string("\x80")] = 1; + deep_invalid_key = nested_object(depth, deep_invalid_key); + CHECK_THROWS_AS(json::to_bon8(deep_invalid_key), json::type_error); + + const json deep_too_large = nested_array(depth, json(9223372036854775808u)); + CHECK_THROWS_AS(json::to_bon8(deep_too_large), json::out_of_range); + } + + SECTION("does not overflow the C++ stack") + { + const std::size_t depth = 100000; + const json j = json::parse(std::string(depth, '[') + "0" + std::string(depth, ']')); + + std::vector packed; + CHECK_NOTHROW(packed = json::to_cbor(j)); + CHECK(json::from_cbor(packed) == j); + + CHECK_NOTHROW(packed = json::to_msgpack(j)); + CHECK(json::from_msgpack(packed) == j); + + CHECK_NOTHROW(packed = json::to_ubjson(j)); + CHECK(json::from_ubjson(packed) == j); + + CHECK_NOTHROW(packed = json::to_ubjson(j, true, false)); + CHECK(json::from_ubjson(packed) == j); + + CHECK_NOTHROW(packed = json::to_bjdata(j)); + CHECK(json::from_bjdata(packed) == j); + + CHECK_NOTHROW(packed = json::to_bon8(j)); + CHECK(json::from_bon8(packed) == j); + } + + SECTION("BON8 containers with more than four elements past the recursion bound") + { + // such containers are closed with 0xFE, which the iterative writer + // must emit when it leaves them; strings next to each other must + // still be separated with 0xFF + const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50; + json wide_array = json::array({1, "a", "b", json::array(), 5}); + json wide_object = json({{"k1", "v"}, {"k2", 2}, {"k3", ""}, {"k4", json::object()}, {"k5", nullptr}}); + for (std::size_t i = 0; i < depth; ++i) + { + wide_array = json::array({"s", wide_array, "t", wide_object, true}); + wide_object = json({{"a", wide_object}, {"b", "x"}, {"c", "y"}, {"d", wide_array.size()}, {"e", "z"}}); + } + + CHECK(json::from_bon8(json::to_bon8(wide_array)) == wide_array); + CHECK(json::from_bon8(json::to_bon8(wide_object)) == wide_object); + } + + SECTION("regression test for https://issues.oss-fuzz.com/issues/566583014") + { + // 200000 nested one-element CBOR arrays, the innermost holding null; + // round-tripping this used to recurse once per level on the way back + // out through to_cbor(), deep enough to overflow the stack + std::vector v(200000, 0x81); + v.push_back(0xf6); + const json j = json::from_cbor(v); + CHECK(json::to_cbor(j) == v); + + // the MessagePack analogue: fixarray of 1 nesting down to nil + std::vector v_msgpack(200000, 0x91); + v_msgpack.push_back(0xc0); + const json j_msgpack = json::from_msgpack(v_msgpack); + CHECK(json::to_msgpack(j_msgpack) == v_msgpack); + } + + SECTION("regression test for https://issues.oss-fuzz.com/issues/572238015") + { + // the BON8 analogue: 200000 nested one-element arrays down to null + std::vector v(200000, 0x81); + v.push_back(0xFA); + const json j = json::from_bon8(v); + CHECK(json::to_bon8(j) == v); + } +} + diff --git a/tests/src/unit-large_json_conversions.cpp b/tests/src/unit-large_json_conversions.cpp new file mode 100644 index 000000000..1fd30aa89 --- /dev/null +++ b/tests/src/unit-large_json_conversions.cpp @@ -0,0 +1,156 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include +#include +#include +#include + +// split from unit-large_json.cpp so that both run in parallel and stay below +// the ctest timeout under Valgrind + +namespace +{ + +// The text of a value nested depth levels deep around the number 0. Level i is +// an array if pattern[i % pattern.size()] is '[', and otherwise an object with +// the single member "a", which every object type enumerates in the same order. +std::string nested_text(std::size_t depth, const std::string& pattern) +{ + std::string text; + std::string closing; + for (std::size_t i = 0; i < depth; ++i) + { + const bool array = pattern[i % pattern.size()] == '['; + text += array ? "[" : "{\"a\":"; + closing += array ? ']' : '}'; + } + text += '0'; + text.append(closing.rbegin(), closing.rend()); + return text; +} + +} // namespace + +TEST_CASE("issue #5650 - stack overflow converting between specializations") +{ + // deep enough to exhaust the call stack, but small enough to stay cheap: + // parsing is iterative, so building the values below costs little + const std::size_t depth = 100000; + + const std::vector patterns = {"[", "{", "[{"}; + + SECTION("json to ordered_json") + { + for (const auto& pattern : patterns) + { + CAPTURE(pattern) + const std::string text = nested_text(depth, pattern); + const json j = json::parse(text); + + const nlohmann::ordered_json converted = j; + CHECK(converted.dump() == text); + } + } + + SECTION("ordered_json to json") + { + for (const auto& pattern : patterns) + { + CAPTURE(pattern) + const std::string text = nested_text(depth, pattern); + const nlohmann::ordered_json o = nlohmann::ordered_json::parse(text); + + const json converted = o; + CHECK(converted.dump() == text); + } + } + + SECTION("get()") + { + for (const auto& pattern : patterns) + { + CAPTURE(pattern) + const std::string text = nested_text(depth, pattern); + const json j = json::parse(text); + + CHECK(j.get().dump() == text); + } + } + + SECTION("depths around the bound of the recursive descent") + { + for (std::size_t d = 1; d <= 300; ++d) + { + CAPTURE(d) + for (const auto& pattern : patterns) + { + CAPTURE(pattern) + const std::string text = nested_text(d, pattern); + const json j = json::parse(text); + + const nlohmann::ordered_json converted = j; + CHECK(converted.dump() == text); + const json back = converted; + CHECK(back.dump() == text); + } + } + } + + SECTION("values below the bound are converted as values above it") + { + // Bury a value below the bound, where it is converted without the + // call stack, and compare it with the same value converted on its + // own by the containers' range constructors. Its objects have + // members that the two object types enumerate in different orders. + const auto bury = [](nlohmann::ordered_json value) + { + for (std::size_t i = 0; i < 200; ++i) + { + value = nlohmann::ordered_json::array({std::move(value)}); + } + return value; + }; + const auto dig = [](const json & value) + { + const json* current = &value; + for (std::size_t i = 0; i < 200; ++i) + { + current = ¤t->at(0); + } + return current; + }; + + nlohmann::ordered_json value = nlohmann::ordered_json::object(); + value["z"] = {1, -2, 3U, 4.5, true, nullptr, "six", nlohmann::ordered_json::binary({7, 8}, 9), + nlohmann::ordered_json::binary({10}), nlohmann::ordered_json::array(), nlohmann::ordered_json::object() + }; + value["y"] = {{"x", {{"w", 1}, {"v", 2}}}, {"u", {3, {{"t", 4}, {"s", 5}}}}}; + value["r"] = nlohmann::ordered_json::array({nlohmann::ordered_json(nlohmann::ordered_json::value_t::discarded)}); + + const json converted_above = value; + const json buried = bury(value); + const json& converted_below = *dig(buried); + + CHECK(converted_below.dump() == converted_above.dump()); + CHECK(converted_below.at("z").at(7).get_binary().subtype() == 9); + CHECK_FALSE(converted_below.at("z").at(8).get_binary().has_subtype()); + CHECK(converted_below.at("r").at(0).is_discarded()); + + // a discarded value is never equal to anything, so compare the rest + value.erase("r"); + const json without_discarded_above = value; + const json without_discarded_buried = bury(value); + CHECK(*dig(without_discarded_buried) == without_discarded_above); + } +}