From 5683f5e024549ee95cabf0298602ff142186408f Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Thu, 15 Oct 2026 22:45:56 +0200 Subject: [PATCH 1/3] Define less_general() only without three-way comparison (#5859) less_general() (#5840) calls operator<(value_t, value_t), which exists in C++20 only for GCC. Explicit instantiation of basic_json compiles every member, so clang-cl and MSVC failed on unit-explicit_instantiation and unit-regression2 in C++20 mode. Its only caller, operator<, is defined only without three-way comparison, as it was before #5840. Signed-off-by: Niels Lohmann --- include/nlohmann/json.hpp | 20 ++++++++++---------- single_include/nlohmann/json.hpp | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 366a8beed..37447262e 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -5518,16 +5518,6 @@ public: #endif } - JSON_HEDLEY_NEVER_INLINE - static bool less_general(const_reference lhs, const_reference rhs) noexcept - { - // default_result is used if we cannot compare values. In that case, - // we compare types. Note we have to call the operator explicitly, - // because MSVC has problems otherwise. - JSON_IMPLEMENT_OPERATOR( <, false, false, operator<(lhs_type, rhs_type), - compare_iteratively(lhs, rhs, true) == compare_result::less, false) - } - #if JSON_HAS_THREE_WAY_COMPARISON JSON_HEDLEY_NEVER_INLINE static std::partial_ordering three_way_general(const_reference lhs, const_reference rhs) noexcept // *NOPAD* @@ -5540,6 +5530,16 @@ public: lhs_type <=> rhs_type, // *NOPAD* to_partial_ordering(compare_iteratively(lhs, rhs, false)), true) } +#else + JSON_HEDLEY_NEVER_INLINE + static bool less_general(const_reference lhs, const_reference rhs) noexcept + { + // default_result is used if we cannot compare values. In that case, + // we compare types. Note we have to call the operator explicitly, + // because MSVC has problems otherwise. + JSON_IMPLEMENT_OPERATOR( <, false, false, operator<(lhs_type, rhs_type), + compare_iteratively(lhs, rhs, true) == compare_result::less, false) + } #endif /// @brief compare two leaves of the same type inside equal_structured() diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index f44422cd6..bcc8e5831 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -35552,16 +35552,6 @@ public: #endif } - JSON_HEDLEY_NEVER_INLINE - static bool less_general(const_reference lhs, const_reference rhs) noexcept - { - // default_result is used if we cannot compare values. In that case, - // we compare types. Note we have to call the operator explicitly, - // because MSVC has problems otherwise. - JSON_IMPLEMENT_OPERATOR( <, false, false, operator<(lhs_type, rhs_type), - compare_iteratively(lhs, rhs, true) == compare_result::less, false) - } - #if JSON_HAS_THREE_WAY_COMPARISON JSON_HEDLEY_NEVER_INLINE static std::partial_ordering three_way_general(const_reference lhs, const_reference rhs) noexcept // *NOPAD* @@ -35574,6 +35564,16 @@ public: lhs_type <=> rhs_type, // *NOPAD* to_partial_ordering(compare_iteratively(lhs, rhs, false)), true) } +#else + JSON_HEDLEY_NEVER_INLINE + static bool less_general(const_reference lhs, const_reference rhs) noexcept + { + // default_result is used if we cannot compare values. In that case, + // we compare types. Note we have to call the operator explicitly, + // because MSVC has problems otherwise. + JSON_IMPLEMENT_OPERATOR( <, false, false, operator<(lhs_type, rhs_type), + compare_iteratively(lhs, rhs, true) == compare_result::less, false) + } #endif /// @brief compare two leaves of the same type inside equal_structured() From cd6b428ef087fb0ce06a7daaa547f71d2907e948 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 16 Oct 2026 19:16:21 +0200 Subject: [PATCH 2/3] Fix CI failures on develop from the API performance stack (#5860) Fixes every CI failure develop has had since the API performance stack (#5831-#5842) was merged: compiler warnings and errors (Clang, GCC 4.9/11, MSVC 2015/2017/2022), clang-tidy, cpplint, Cppcheck, IWYU, the examples check, the no-exceptions, diagnostic-positions, implicit-conversions and strict-NUL test configurations, and the MinGW link (now with LLD). Signed-off-by: Niels Lohmann --- .github/workflows/windows.yml | 18 +- .../nlohmann/detail/conversions/to_json.hpp | 47 ++++- include/nlohmann/detail/conversions/zmij.hpp | 2 +- include/nlohmann/detail/exceptions.hpp | 3 +- .../nlohmann/detail/iterators/iter_impl.hpp | 6 +- include/nlohmann/detail/json_pointer.hpp | 7 +- include/nlohmann/json.hpp | 100 ++++++----- include/nlohmann/ordered_map.hpp | 1 + single_include/nlohmann/json.hpp | 167 +++++++++++------- tests/src/unit-constructor2.cpp | 14 +- tests/src/unit-diagnostics.cpp | 4 +- tests/src/unit-iterators1.cpp | 9 +- tests/src/unit-json_pointer.cpp | 4 +- tests/src/unit-serialization.cpp | 10 +- tests/src/unit-testsuites.cpp | 4 + tests/src/unit-throw_messages.cpp | 27 ++- 16 files changed, 255 insertions(+), 168 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index bbc1ef48b..6f1cf8cae 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -166,19 +166,21 @@ jobs: # link, but the binaries clang 11.0.1 and clang 18.1.8 then produce crash # before doctest prints its first line - 39 of 102 tests on clang 18. # Keep the objects small by splitting the test files instead. - # Objects with more than 65535 sections fail to link with "relocation - # truncated to fit: IMAGE_REL_AMD64_REL32 against `.rdata'": the MinGW - # linker stores the section an associative COMDAT section belongs to in - # 16 bits (x_associated in binutils' include/coff/internal.h), so it - # discards the jump tables of inline functions together with the wrong - # function. Each basic_json specialization a test file instantiates adds - # many sections, so test a second one in a file of its own. + # The tests link with LLD: GNU ld fails on objects with more than 32767 + # sections with "relocation truncated to fit: IMAGE_REL_AMD64_REL32 + # against `.rdata'", as it stores the section an associative COMDAT + # section belongs to in 16 bits (x_associated in binutils' + # include/coff/internal.h) and so discards the jump tables of inline + # functions together with the wrong function. Each basic_json + # specialization and input type a test file instantiates adds many + # sections, so unit-msgpack.cpp and unit-regression2.cpp kept crossing + # that limit as the library grew. - name: Run CMake run: cmake -S . -B build ^ -DCMAKE_CXX_COMPILER="C:/Program Files/LLVM/bin/clang++.exe" ^ -DCMAKE_CXX_FLAGS="--target=x86_64-w64-mingw32 -stdlib=libstdc++ -pthread" ^ -DCMAKE_CXX_FLAGS_DEBUG="-g0" ^ - -DCMAKE_EXE_LINKER_FLAGS="-lwinpthread" ^ + -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld -lwinpthread" ^ -G"MinGW Makefiles" ^ -DCMAKE_BUILD_TYPE=Debug ^ -DJSON_BuildTests=On diff --git a/include/nlohmann/detail/conversions/to_json.hpp b/include/nlohmann/detail/conversions/to_json.hpp index 5b217d7fe..05ae82370 100644 --- a/include/nlohmann/detail/conversions/to_json.hpp +++ b/include/nlohmann/detail/conversions/to_json.hpp @@ -13,7 +13,7 @@ #include // optional #endif -#include // begin, end +#include // begin, end, make_move_iterator, move_iterator #include // allocator_traits #include // basic_string, char_traits #include // tuple, get @@ -48,6 +48,32 @@ namespace detail * leave j with the type of the new value, but the pointer to the destroyed old one. */ +// a move iterator over the elements of an rvalue container, unless they are +// const (e.g., std::set): libstdc++ before GCC 5 defines the reference of a +// std::move_iterator as value_type&&, which cannot bind to a const element +template +std::move_iterator make_move_iterator_if_mutable_impl(const Iterator& it, std::true_type /*mutable*/) +{ + return std::make_move_iterator(it); +} + +template +Iterator make_move_iterator_if_mutable_impl(const Iterator& it, std::false_type /*mutable*/) +{ + return it; +} + +template +using is_mutable_iterator = std::integral_constant < bool, + !std::is_const())>::type>::value >; + +template +auto make_move_iterator_if_mutable(const Iterator& it) +-> decltype(make_move_iterator_if_mutable_impl(it, is_mutable_iterator {})) +{ + return make_move_iterator_if_mutable_impl(it, is_mutable_iterator {}); +} + template struct external_constructor; template<> @@ -174,7 +200,7 @@ struct external_constructor using std::begin; using std::end; - const typename BasicJsonType::json_value value(j.template create(std::make_move_iterator(begin(arr)), std::make_move_iterator(end(arr)))); + const typename BasicJsonType::json_value value(j.template create(make_move_iterator_if_mutable(begin(arr)), make_move_iterator_if_mutable(end(arr)))); j.replace_value(value_t::array, value); } @@ -235,7 +261,7 @@ struct external_constructor using std::begin; using std::end; - const typename BasicJsonType::json_value value(j.template create(std::make_move_iterator(begin(obj)), std::make_move_iterator(end(obj)))); + const typename BasicJsonType::json_value value(j.template create(make_move_iterator_if_mutable(begin(obj)), make_move_iterator_if_mutable(end(obj)))); j.replace_value(value_t::object, value); } @@ -413,25 +439,27 @@ using is_movable_container_rvalue = negation, has_dedicated_to_json, is_basic_json, - is_range_view_type, is_enum_keyed_map_type, std::is_same>>>; -// rvalue array-like containers +// rvalue array-like containers; object types are ruled out before views are +// checked, as std::ranges::view for the object_t of an ordered_json would +// need that ordered_json to be complete (GCC 11, see #3312) template using is_rvalue_array_to_json = conjunction, + negation>, + negation>, is_compatible_array_type, negation, std::is_same, - is_compatible_object_type, is_compatible_string_type, is_compatible_binary_type>>>; // rvalue containers: their elements are moved instead of copied template::value, int> = 0> -inline void to_json(BasicJsonType& j, CompatibleArrayType && arr) +inline void to_json(BasicJsonType& j, CompatibleArrayType && arr) // NOLINT(cppcoreguidelines-missing-std-forward) { external_constructor::construct_moving(j, arr); } @@ -509,7 +537,8 @@ inline void to_json(BasicJsonType& j, const EnumKeyedMap& map) } #endif -// rvalue object-like containers +// rvalue object-like containers (views have no key_type and mapped_type, so +// they are no compatible object type) template using is_rvalue_object_to_json = conjunction, negation>, @@ -517,7 +546,7 @@ using is_rvalue_object_to_json = conjunction, template::value, int> = 0> -inline void to_json(BasicJsonType& j, CompatibleObjectType && obj) +inline void to_json(BasicJsonType& j, CompatibleObjectType && obj) // NOLINT(cppcoreguidelines-missing-std-forward) { external_constructor::construct_moving(j, obj); } diff --git a/include/nlohmann/detail/conversions/zmij.hpp b/include/nlohmann/detail/conversions/zmij.hpp index 3820d891a..3b2e8e7a3 100644 --- a/include/nlohmann/detail/conversions/zmij.hpp +++ b/include/nlohmann/detail/conversions/zmij.hpp @@ -185,7 +185,7 @@ JSON_HEDLEY_ALWAYS_INLINE shortest_decimal to_shortest(std::uint64_t bits) noexc const uint128_parts p = umul192_hi128(p10.high, p10.low, bin_sig << static_cast(shift)); std::uint64_t integral = p.high >> static_cast(extra_shift); // (no casts in the shifts: MSVC 2015 warns C4554 on them despite the parentheses) - constexpr unsigned fractional_low_shift = static_cast(extra_shift); + constexpr auto fractional_low_shift = static_cast(extra_shift); constexpr unsigned fractional_high_shift = 64u - fractional_low_shift; const std::uint64_t fractional = (p.high << fractional_high_shift) | (p.low >> fractional_low_shift); std::uint64_t digit = 0; diff --git a/include/nlohmann/detail/exceptions.hpp b/include/nlohmann/detail/exceptions.hpp index b2b5f36dd..7d7f4e038 100644 --- a/include/nlohmann/detail/exceptions.hpp +++ b/include/nlohmann/detail/exceptions.hpp @@ -494,8 +494,7 @@ JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_missing_member(c static_cast(string_type); static_cast(val); // context-sensitive error message - const std::string error_msg = (op == "op") ? std::string("operation") : concat("operation '", op, '\''); - JSON_THROW(parse_error::create(exception_id::patch_invalid_operation, 0, concat(error_msg, string_type ? " must have string member '" : " must have member '", member, "'"), &val)); + JSON_THROW(parse_error::create(exception_id::patch_invalid_operation, 0, concat((op == "op") ? std::string("operation") : concat("operation '", op, '\''), string_type ? " must have string member '" : " must have member '", member, "'"), &val)); } template diff --git a/include/nlohmann/detail/iterators/iter_impl.hpp b/include/nlohmann/detail/iterators/iter_impl.hpp index b7c1df24a..120c1b893 100644 --- a/include/nlohmann/detail/iterators/iter_impl.hpp +++ b/include/nlohmann/detail/iterators/iter_impl.hpp @@ -680,12 +680,12 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - if (JSON_HEDLEY_LIKELY(m_object->is_object())) + if (JSON_HEDLEY_UNLIKELY(!m_object->is_object())) { - return m_it.object_iterator->first; + throw_literal(exception_id::iterator_key_not_object, "cannot use key() for non-object iterators", m_object); } - throw_literal(exception_id::iterator_key_not_object, "cannot use key() for non-object iterators", m_object); + return m_it.object_iterator->first; } /*! diff --git a/include/nlohmann/detail/json_pointer.hpp b/include/nlohmann/detail/json_pointer.hpp index e5fb3750b..63ab49f7d 100644 --- a/include/nlohmann/detail/json_pointer.hpp +++ b/include/nlohmann/detail/json_pointer.hpp @@ -11,6 +11,7 @@ #include // all_of #include // isdigit #include // errno, ERANGE +#include // size_t #include // strtoull #ifndef JSON_NO_IO #include // ostream @@ -334,7 +335,7 @@ class json_pointer { break; // not a plain number: let strtoull decide } - acc = static_cast(acc * 10 + static_cast(c - '0')); + acc = static_cast((acc * 10) + static_cast(c - '0')); if (JSON_HEDLEY_UNLIKELY(i == s.size() - 1)) { idx = acc; @@ -529,10 +530,10 @@ class json_pointer static std::size_t hash_of(std::size_t parent, const string_t& token) noexcept { - std::size_t h = parent * static_cast(0x9E3779B97F4A7C15ULL); + std::size_t h = parent * static_cast(0x9E3779B97F4A7C15ULL & (std::numeric_limits::max)()); for (std::size_t i = 0; i < token.size(); ++i) { - h = (h ^ static_cast(token[i])) * static_cast(0x100000001B3ULL); + h = (h ^ static_cast(token[i])) * static_cast(0x100000001B3ULL & (std::numeric_limits::max)()); } return h ^ (h >> 29u); } diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 37447262e..595fe11e5 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -29,6 +29,7 @@ #endif // JSON_NO_IO #include // make_move_iterator, random_access_iterator_tag #include // numeric_limits +#include // map #include // unique_ptr #include // swap, operator!= #include // length_error @@ -754,26 +755,20 @@ private: // one level deep); stops at the first structured child it meets static bool all_children_leaves(const basic_json& v) noexcept { + // array_t only needs forward iterators (see custom_array_type) if (v.m_data.m_type == value_t::array) { const auto& a = *v.m_data.m_value.array; - for (auto it = a.rbegin(); it != a.rend(); ++it) + return std::all_of(a.begin(), a.end(), [](const basic_json & el) { - if (!has_no_children(*it)) - { - return false; - } - } - return true; + return has_no_children(el); + }); } - for (const auto& el : *v.m_data.m_value.object) + const auto& o = *v.m_data.m_value.object; + return std::all_of(o.begin(), o.end(), [](const typename object_t::value_type & el) { - if (!has_no_children(el.second)) - { - return false; - } - } - return true; + return has_no_children(el.second); + }); } // The walk in destroy_container() may take the children of a @@ -1230,7 +1225,7 @@ public: } template < typename O = object_t, detail::enable_if_t < detail::has_stable_elements::value, int > = 0 > - static constexpr std::size_t object_capacity() noexcept + std::size_t object_capacity() const noexcept { return 0; } @@ -1242,7 +1237,7 @@ public: } template < typename O = object_t, detail::enable_if_t < !detail::has_stable_elements::value && !detail::is_nlohmann_ordered_map::value, int > = 0 > - static constexpr std::size_t object_capacity() noexcept + std::size_t object_capacity() const noexcept { return detail::unknown_size(); } @@ -3043,18 +3038,21 @@ public: JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE static void throw_iterator_range_of_null(const basic_json* obj) { + static_cast(obj); // unused when JSON_NOEXCEPTION is defined JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_of_null, detail::concat("cannot construct with iterators from ", obj->type_name()), obj)); } JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE static void throw_incompatible_reference_type(const basic_json& obj) { + static_cast(obj); // unused when JSON_NOEXCEPTION is defined JSON_THROW(type_error::create(detail::exception_id::incompatible_reference_type, detail::concat("incompatible ReferenceType for get_ref, actual type is ", obj.type_name()), &obj)); } JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE static void throw_array_index_exceeds_size_type(size_type idx) { + static_cast(idx); // unused when JSON_NOEXCEPTION is defined JSON_THROW(std::length_error(detail::concat("array index ", std::to_string(idx), " exceeds size_type"))); } @@ -3062,6 +3060,8 @@ public: JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE static void throw_array_index_out_of_range(size_type idx, const basic_json& j) { + static_cast(idx); // unused when JSON_NOEXCEPTION is defined + static_cast(j); JSON_THROW(out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat("array index ", std::to_string(idx), " is out of range"), &j)); } @@ -3070,6 +3070,8 @@ public: JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE static void throw_key_not_found(const KeyType& key, const basic_json& j) { + static_cast(key); // unused when JSON_NOEXCEPTION is defined + static_cast(j); JSON_THROW(out_of_range::create(detail::exception_id::key_not_found, detail::concat("key '", key, "' not found"), &j)); } @@ -4893,19 +4895,19 @@ public: iterator insert(const_iterator pos, const basic_json& val) // NOLINT(performance-unnecessary-value-param) { // insert only works for arrays - if (JSON_HEDLEY_LIKELY(is_array())) + if (JSON_HEDLEY_UNLIKELY(!is_array())) { - // check if iterator pos fits to this JSON value - if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) - { - detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); - } - - // insert to array and return iterator - return insert_iterator(pos, val); + detail::throw_cannot_use_with(detail::exception_id::insert_wrong_type, "insert()", *this); } - detail::throw_cannot_use_with(detail::exception_id::insert_wrong_type, "insert()", *this); + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); + } + + // insert to array and return iterator + return insert_iterator(pos, val); } /// @brief inserts element into array @@ -4913,21 +4915,21 @@ public: iterator insert(const_iterator pos, basic_json&& val) // NOLINT(performance-unnecessary-value-param) { // insert only works for arrays - if (JSON_HEDLEY_LIKELY(is_array())) + if (JSON_HEDLEY_UNLIKELY(!is_array())) { - // check if iterator pos fits to this JSON value - if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) - { - detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); - } - - // moving into a local first keeps this safe even if val aliases - // an element of this array - basic_json tmp(std::move(val)); - return insert_iterator(pos, std::move(tmp)); + detail::throw_cannot_use_with(detail::exception_id::insert_wrong_type, "insert()", *this); } - detail::throw_cannot_use_with(detail::exception_id::insert_wrong_type, "insert()", *this); + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); + } + + // moving into a local first keeps this safe even if val aliases + // an element of this array + basic_json tmp(std::move(val)); + return insert_iterator(pos, std::move(tmp)); } /// @brief inserts copies of element into array @@ -4935,19 +4937,19 @@ public: iterator insert(const_iterator pos, size_type cnt, const basic_json& val) // NOLINT(performance-unnecessary-value-param) { // insert only works for arrays - if (JSON_HEDLEY_LIKELY(is_array())) + if (JSON_HEDLEY_UNLIKELY(!is_array())) { - // check if iterator pos fits to this JSON value - if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) - { - detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); - } - - // insert to array and return iterator - return insert_iterator(pos, cnt, val); + detail::throw_cannot_use_with(detail::exception_id::insert_wrong_type, "insert()", *this); } - detail::throw_cannot_use_with(detail::exception_id::insert_wrong_type, "insert()", *this); + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); + } + + // insert to array and return iterator + return insert_iterator(pos, cnt, val); } /// @brief inserts range of elements into array @@ -5453,7 +5455,7 @@ public: // inline fast path for two values of the same non-owning scalar type; anything // else falls through to the out-of-line general comparison #define JSON_IMPLEMENT_SCALAR_FAST(op, null_result) \ - if (lhs.m_data.m_type == rhs.m_data.m_type) \ + if (lhs.m_data.m_type == rhs.m_data.m_type) /* NOLINT(readability/braces) */ \ { \ switch (lhs.m_data.m_type) \ { \ diff --git a/include/nlohmann/ordered_map.hpp b/include/nlohmann/ordered_map.hpp index 10fc764ea..e5f9164f6 100644 --- a/include/nlohmann/ordered_map.hpp +++ b/include/nlohmann/ordered_map.hpp @@ -23,6 +23,7 @@ #include #include #include +#include NLOHMANN_JSON_NAMESPACE_BEGIN diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index bcc8e5831..4debe551f 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -29,6 +29,7 @@ #endif // JSON_NO_IO #include // make_move_iterator, random_access_iterator_tag #include // numeric_limits +#include // map #include // unique_ptr #include // swap, operator!= #include // length_error @@ -5794,8 +5795,7 @@ JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_missing_member(c static_cast(string_type); static_cast(val); // context-sensitive error message - const std::string error_msg = (op == "op") ? std::string("operation") : concat("operation '", op, '\''); - JSON_THROW(parse_error::create(exception_id::patch_invalid_operation, 0, concat(error_msg, string_type ? " must have string member '" : " must have member '", member, "'"), &val)); + JSON_THROW(parse_error::create(exception_id::patch_invalid_operation, 0, concat((op == "op") ? std::string("operation") : concat("operation '", op, '\''), string_type ? " must have string member '" : " must have member '", member, "'"), &val)); } template @@ -6555,7 +6555,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // optional #endif -#include // begin, end +#include // begin, end, make_move_iterator, move_iterator #include // allocator_traits #include // basic_string, char_traits #include // tuple, get @@ -7204,6 +7204,32 @@ namespace detail * leave j with the type of the new value, but the pointer to the destroyed old one. */ +// a move iterator over the elements of an rvalue container, unless they are +// const (e.g., std::set): libstdc++ before GCC 5 defines the reference of a +// std::move_iterator as value_type&&, which cannot bind to a const element +template +std::move_iterator make_move_iterator_if_mutable_impl(const Iterator& it, std::true_type /*mutable*/) +{ + return std::make_move_iterator(it); +} + +template +Iterator make_move_iterator_if_mutable_impl(const Iterator& it, std::false_type /*mutable*/) +{ + return it; +} + +template +using is_mutable_iterator = std::integral_constant < bool, + !std::is_const())>::type>::value >; + +template +auto make_move_iterator_if_mutable(const Iterator& it) +-> decltype(make_move_iterator_if_mutable_impl(it, is_mutable_iterator {})) +{ + return make_move_iterator_if_mutable_impl(it, is_mutable_iterator {}); +} + template struct external_constructor; template<> @@ -7330,7 +7356,7 @@ struct external_constructor using std::begin; using std::end; - const typename BasicJsonType::json_value value(j.template create(std::make_move_iterator(begin(arr)), std::make_move_iterator(end(arr)))); + const typename BasicJsonType::json_value value(j.template create(make_move_iterator_if_mutable(begin(arr)), make_move_iterator_if_mutable(end(arr)))); j.replace_value(value_t::array, value); } @@ -7391,7 +7417,7 @@ struct external_constructor using std::begin; using std::end; - const typename BasicJsonType::json_value value(j.template create(std::make_move_iterator(begin(obj)), std::make_move_iterator(end(obj)))); + const typename BasicJsonType::json_value value(j.template create(make_move_iterator_if_mutable(begin(obj)), make_move_iterator_if_mutable(end(obj)))); j.replace_value(value_t::object, value); } @@ -7569,25 +7595,27 @@ using is_movable_container_rvalue = negation, has_dedicated_to_json, is_basic_json, - is_range_view_type, is_enum_keyed_map_type, std::is_same>>>; -// rvalue array-like containers +// rvalue array-like containers; object types are ruled out before views are +// checked, as std::ranges::view for the object_t of an ordered_json would +// need that ordered_json to be complete (GCC 11, see #3312) template using is_rvalue_array_to_json = conjunction, + negation>, + negation>, is_compatible_array_type, negation, std::is_same, - is_compatible_object_type, is_compatible_string_type, is_compatible_binary_type>>>; // rvalue containers: their elements are moved instead of copied template::value, int> = 0> -inline void to_json(BasicJsonType& j, CompatibleArrayType && arr) +inline void to_json(BasicJsonType& j, CompatibleArrayType && arr) // NOLINT(cppcoreguidelines-missing-std-forward) { external_constructor::construct_moving(j, arr); } @@ -7665,7 +7693,8 @@ inline void to_json(BasicJsonType& j, const EnumKeyedMap& map) } #endif -// rvalue object-like containers +// rvalue object-like containers (views have no key_type and mapped_type, so +// they are no compatible object type) template using is_rvalue_object_to_json = conjunction, negation>, @@ -7673,7 +7702,7 @@ using is_rvalue_object_to_json = conjunction, template::value, int> = 0> -inline void to_json(BasicJsonType& j, CompatibleObjectType && obj) +inline void to_json(BasicJsonType& j, CompatibleObjectType && obj) // NOLINT(cppcoreguidelines-missing-std-forward) { external_constructor::construct_moving(j, obj); } @@ -20715,12 +20744,12 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - if (JSON_HEDLEY_LIKELY(m_object->is_object())) + if (JSON_HEDLEY_UNLIKELY(!m_object->is_object())) { - return m_it.object_iterator->first; + throw_literal(exception_id::iterator_key_not_object, "cannot use key() for non-object iterators", m_object); } - throw_literal(exception_id::iterator_key_not_object, "cannot use key() for non-object iterators", m_object); + return m_it.object_iterator->first; } /*! @@ -20947,6 +20976,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // all_of #include // isdigit #include // errno, ERANGE +#include // size_t #include // strtoull #ifndef JSON_NO_IO #include // ostream @@ -21277,7 +21307,7 @@ class json_pointer { break; // not a plain number: let strtoull decide } - acc = static_cast(acc * 10 + static_cast(c - '0')); + acc = static_cast((acc * 10) + static_cast(c - '0')); if (JSON_HEDLEY_UNLIKELY(i == s.size() - 1)) { idx = acc; @@ -21472,10 +21502,10 @@ class json_pointer static std::size_t hash_of(std::size_t parent, const string_t& token) noexcept { - std::size_t h = parent * static_cast(0x9E3779B97F4A7C15ULL); + std::size_t h = parent * static_cast(0x9E3779B97F4A7C15ULL & (std::numeric_limits::max)()); for (std::size_t i = 0; i < token.size(); ++i) { - h = (h ^ static_cast(token[i])) * static_cast(0x100000001B3ULL); + h = (h ^ static_cast(token[i])) * static_cast(0x100000001B3ULL & (std::numeric_limits::max)()); } return h ^ (h >> 29u); } @@ -26454,7 +26484,7 @@ JSON_HEDLEY_ALWAYS_INLINE shortest_decimal to_shortest(std::uint64_t bits) noexc const uint128_parts p = umul192_hi128(p10.high, p10.low, bin_sig << static_cast(shift)); std::uint64_t integral = p.high >> static_cast(extra_shift); // (no casts in the shifts: MSVC 2015 warns C4554 on them despite the parentheses) - constexpr unsigned fractional_low_shift = static_cast(extra_shift); + constexpr auto fractional_low_shift = static_cast(extra_shift); constexpr unsigned fractional_high_shift = 64u - fractional_low_shift; const std::uint64_t fractional = (p.high << fractional_high_shift) | (p.low >> fractional_low_shift); std::uint64_t digit = 0; @@ -29704,6 +29734,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + NLOHMANN_JSON_NAMESPACE_BEGIN @@ -30788,26 +30820,20 @@ private: // one level deep); stops at the first structured child it meets static bool all_children_leaves(const basic_json& v) noexcept { + // array_t only needs forward iterators (see custom_array_type) if (v.m_data.m_type == value_t::array) { const auto& a = *v.m_data.m_value.array; - for (auto it = a.rbegin(); it != a.rend(); ++it) + return std::all_of(a.begin(), a.end(), [](const basic_json & el) { - if (!has_no_children(*it)) - { - return false; - } - } - return true; + return has_no_children(el); + }); } - for (const auto& el : *v.m_data.m_value.object) + const auto& o = *v.m_data.m_value.object; + return std::all_of(o.begin(), o.end(), [](const typename object_t::value_type & el) { - if (!has_no_children(el.second)) - { - return false; - } - } - return true; + return has_no_children(el.second); + }); } // The walk in destroy_container() may take the children of a @@ -31264,7 +31290,7 @@ public: } template < typename O = object_t, detail::enable_if_t < detail::has_stable_elements::value, int > = 0 > - static constexpr std::size_t object_capacity() noexcept + std::size_t object_capacity() const noexcept { return 0; } @@ -31276,7 +31302,7 @@ public: } template < typename O = object_t, detail::enable_if_t < !detail::has_stable_elements::value && !detail::is_nlohmann_ordered_map::value, int > = 0 > - static constexpr std::size_t object_capacity() noexcept + std::size_t object_capacity() const noexcept { return detail::unknown_size(); } @@ -33077,18 +33103,21 @@ public: JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE static void throw_iterator_range_of_null(const basic_json* obj) { + static_cast(obj); // unused when JSON_NOEXCEPTION is defined JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_of_null, detail::concat("cannot construct with iterators from ", obj->type_name()), obj)); } JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE static void throw_incompatible_reference_type(const basic_json& obj) { + static_cast(obj); // unused when JSON_NOEXCEPTION is defined JSON_THROW(type_error::create(detail::exception_id::incompatible_reference_type, detail::concat("incompatible ReferenceType for get_ref, actual type is ", obj.type_name()), &obj)); } JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE static void throw_array_index_exceeds_size_type(size_type idx) { + static_cast(idx); // unused when JSON_NOEXCEPTION is defined JSON_THROW(std::length_error(detail::concat("array index ", std::to_string(idx), " exceeds size_type"))); } @@ -33096,6 +33125,8 @@ public: JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE static void throw_array_index_out_of_range(size_type idx, const basic_json& j) { + static_cast(idx); // unused when JSON_NOEXCEPTION is defined + static_cast(j); JSON_THROW(out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat("array index ", std::to_string(idx), " is out of range"), &j)); } @@ -33104,6 +33135,8 @@ public: JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE static void throw_key_not_found(const KeyType& key, const basic_json& j) { + static_cast(key); // unused when JSON_NOEXCEPTION is defined + static_cast(j); JSON_THROW(out_of_range::create(detail::exception_id::key_not_found, detail::concat("key '", key, "' not found"), &j)); } @@ -34927,19 +34960,19 @@ public: iterator insert(const_iterator pos, const basic_json& val) // NOLINT(performance-unnecessary-value-param) { // insert only works for arrays - if (JSON_HEDLEY_LIKELY(is_array())) + if (JSON_HEDLEY_UNLIKELY(!is_array())) { - // check if iterator pos fits to this JSON value - if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) - { - detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); - } - - // insert to array and return iterator - return insert_iterator(pos, val); + detail::throw_cannot_use_with(detail::exception_id::insert_wrong_type, "insert()", *this); } - detail::throw_cannot_use_with(detail::exception_id::insert_wrong_type, "insert()", *this); + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); + } + + // insert to array and return iterator + return insert_iterator(pos, val); } /// @brief inserts element into array @@ -34947,21 +34980,21 @@ public: iterator insert(const_iterator pos, basic_json&& val) // NOLINT(performance-unnecessary-value-param) { // insert only works for arrays - if (JSON_HEDLEY_LIKELY(is_array())) + if (JSON_HEDLEY_UNLIKELY(!is_array())) { - // check if iterator pos fits to this JSON value - if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) - { - detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); - } - - // moving into a local first keeps this safe even if val aliases - // an element of this array - basic_json tmp(std::move(val)); - return insert_iterator(pos, std::move(tmp)); + detail::throw_cannot_use_with(detail::exception_id::insert_wrong_type, "insert()", *this); } - detail::throw_cannot_use_with(detail::exception_id::insert_wrong_type, "insert()", *this); + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); + } + + // moving into a local first keeps this safe even if val aliases + // an element of this array + basic_json tmp(std::move(val)); + return insert_iterator(pos, std::move(tmp)); } /// @brief inserts copies of element into array @@ -34969,19 +35002,19 @@ public: iterator insert(const_iterator pos, size_type cnt, const basic_json& val) // NOLINT(performance-unnecessary-value-param) { // insert only works for arrays - if (JSON_HEDLEY_LIKELY(is_array())) + if (JSON_HEDLEY_UNLIKELY(!is_array())) { - // check if iterator pos fits to this JSON value - if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) - { - detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); - } - - // insert to array and return iterator - return insert_iterator(pos, cnt, val); + detail::throw_cannot_use_with(detail::exception_id::insert_wrong_type, "insert()", *this); } - detail::throw_cannot_use_with(detail::exception_id::insert_wrong_type, "insert()", *this); + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); + } + + // insert to array and return iterator + return insert_iterator(pos, cnt, val); } /// @brief inserts range of elements into array @@ -35487,7 +35520,7 @@ public: // inline fast path for two values of the same non-owning scalar type; anything // else falls through to the out-of-line general comparison #define JSON_IMPLEMENT_SCALAR_FAST(op, null_result) \ - if (lhs.m_data.m_type == rhs.m_data.m_type) \ + if (lhs.m_data.m_type == rhs.m_data.m_type) /* NOLINT(readability/braces) */ \ { \ switch (lhs.m_data.m_type) \ { \ diff --git a/tests/src/unit-constructor2.cpp b/tests/src/unit-constructor2.cpp index 4896bb1f0..f17d0ad3a 100644 --- a/tests/src/unit-constructor2.cpp +++ b/tests/src/unit-constructor2.cpp @@ -198,11 +198,11 @@ TEST_CASE("copy constructor and destructor across value types") const json src_uint = 7u; const json src_float = 2.5; - json c_null(src_null); - json c_bool(src_bool); - json c_int(src_int); - json c_uint(src_uint); - json c_float(src_float); + json c_null(src_null); // NOLINT(performance-unnecessary-copy-initialization): tests the copy constructor + json c_bool(src_bool); // NOLINT(performance-unnecessary-copy-initialization): tests the copy constructor + json c_int(src_int); // NOLINT(performance-unnecessary-copy-initialization): tests the copy constructor + json c_uint(src_uint); // NOLINT(performance-unnecessary-copy-initialization): tests the copy constructor + json c_float(src_float); // NOLINT(performance-unnecessary-copy-initialization): tests the copy constructor CHECK(c_null.is_null()); CHECK(c_bool == true); @@ -300,12 +300,12 @@ TEST_CASE("copy constructor and destructor across value types") REQUIRE(v.size() == 70); std::vector copy(v); - CHECK(copy == v); + CHECK((copy == v)); // mutating the copy leaves the original alone copy[5] = "changed"; copy[6].push_back(1); - CHECK(copy != v); + CHECK((copy != v)); CHECK(v[5] == std::string(5, 'x')); CHECK(v[6].size() == 3); diff --git a/tests/src/unit-diagnostics.cpp b/tests/src/unit-diagnostics.cpp index 1d7b45a97..504a95322 100644 --- a/tests/src/unit-diagnostics.cpp +++ b/tests/src/unit-diagnostics.cpp @@ -817,8 +817,8 @@ template, struct relocating_map : nlohmann::ordered_map { using base_type = nlohmann::ordered_map; - using typename base_type::iterator; - using typename base_type::Container; + using iterator = typename base_type::iterator; + using Container = typename base_type::Container; using base_type::base_type; template diff --git a/tests/src/unit-iterators1.cpp b/tests/src/unit-iterators1.cpp index 629c6abd7..1f0a6ba4b 100644 --- a/tests/src/unit-iterators1.cpp +++ b/tests/src/unit-iterators1.cpp @@ -1637,6 +1637,7 @@ TEST_CASE_TEMPLATE("reverse iteration and member access", Json, nlohmann::json, { Json j = {1, 2, 3, 4}; + const std::vector reversed = {4, 3, 2, 1}; auto it = j.end(); std::vector seen; while (it != j.begin()) @@ -1644,7 +1645,7 @@ TEST_CASE_TEMPLATE("reverse iteration and member access", Json, nlohmann::json, --it; seen.push_back(it->template get()); } - CHECK(seen == std::vector {4, 3, 2, 1}); + CHECK(seen == reversed); auto it2 = j.end(); auto old = it2--; @@ -1656,14 +1657,14 @@ TEST_CASE_TEMPLATE("reverse iteration and member access", Json, nlohmann::json, { seen.push_back(r->template get()); } - CHECK(seen == std::vector {4, 3, 2, 1}); + CHECK(seen == reversed); seen.clear(); for (auto r = j.crbegin(); r != j.crend(); ++r) { - seen.push_back(*r); + seen.push_back(r->template get()); } - CHECK(seen == std::vector {4, 3, 2, 1}); + CHECK(seen == reversed); } SECTION("object: -- and rbegin/rend") diff --git a/tests/src/unit-json_pointer.cpp b/tests/src/unit-json_pointer.cpp index 39c4f790c..7748a375a 100644 --- a/tests/src/unit-json_pointer.cpp +++ b/tests/src/unit-json_pointer.cpp @@ -1230,11 +1230,11 @@ TEST_CASE("JSON pointer array index tokens match strtoull semantics") SECTION("random tokens") { - std::mt19937 gen(42); + std::mt19937 gen(42); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible const std::string alphabet = "0123456789012345678901234567890123456789-+ ex"; for (int i = 0; i < 5000; ++i) { - std::string token(static_cast(gen() % 23), '0'); + std::string token(gen() % 23, '0'); for (auto& c : token) { c = alphabet[gen() % alphabet.size()]; diff --git a/tests/src/unit-serialization.cpp b/tests/src/unit-serialization.cpp index 4913c30a6..b6608a49a 100644 --- a/tests/src/unit-serialization.cpp +++ b/tests/src/unit-serialization.cpp @@ -1374,7 +1374,13 @@ void check_dump_against_reference(const std::string& s, const bool with_strict = }; for (const auto error_handler : error_handlers) { +#if defined(JSON_NOEXCEPTION) + // a strict dump of invalid UTF-8 calls std::abort() instead of throwing + static_cast(with_strict); + if (error_handler == json::error_handler_t::strict) +#else if (!with_strict && error_handler == json::error_handler_t::strict) +#endif { continue; } @@ -1429,7 +1435,7 @@ TEST_CASE("invalid UTF-8 at every position is handled like the reference") "\xF4\x90\x80\x80", // sequences cut short by ASCII, a control character, a quote, a // backslash, DEL, or the lead byte of a well-formed character - "\xC3" "a", "\xE2\x82" "\n", "\xE2\x82\"", "\xF0\x9F\x98\\", "\xC3\x7F", "\xC3\xC3\xA4", "\xE2\xF0\x9F\x98\x80", + "\xC3" "a", "\xE2\x82" "\n", "\xE2\x82\"", "\xF0\x9F\x98\\", "\xC3\x7F", "\xC3\xC3\xA4", "\xE2\xF0\x9F\x98\x80", // NOLINT(bugprone-suspicious-missing-comma): "\xC3" "a" ends the hex escape // truncated sequences "\xC3", "\xE2\x82", "\xF0\x9F\x98" }; @@ -1489,7 +1495,7 @@ TEST_CASE("invalid UTF-8 at every position is handled like the reference") { for (const auto& pattern : patterns) { - check_dump_against_reference(make_string(filler, pos, pattern, length - pos), bytes <= (filler.size() == 1 ? 17 : 8)); + check_dump_against_reference(make_string(filler, pos, pattern, length - pos), bytes <= (filler.size() == 1 ? 17u : 8u)); } } } diff --git a/tests/src/unit-testsuites.cpp b/tests/src/unit-testsuites.cpp index 672981967..52d189457 100644 --- a/tests/src/unit-testsuites.cpp +++ b/tests/src/unit-testsuites.cpp @@ -218,7 +218,11 @@ TEST_CASE("compliance tests from nativejson-benchmark") n1e308[0] = '['; n1e308[1] = '1'; n1e308[310] = ']'; +#if JSON_STRICT_NUL_HANDLING + n1e308.pop_back(); // a NUL byte after the value is a syntax error in strict mode +#else n1e308[311] = '\0'; +#endif TEST_DOUBLE(n1e308, 1E308); } diff --git a/tests/src/unit-throw_messages.cpp b/tests/src/unit-throw_messages.cpp index 1c2289626..d7546cc93 100644 --- a/tests/src/unit-throw_messages.cpp +++ b/tests/src/unit-throw_messages.cpp @@ -54,8 +54,8 @@ TEST_CASE("exception messages of out-of-line throw sites") SECTION("at") { - const json carr = arr; - const json cobj = obj; + const json& carr = arr; + const json& cobj = obj; CHECK_MSG((void)arr.at(static_cast(10)), "[json.exception.out_of_range.401] array index 10 is out of range"); CHECK_MSG((void)carr.at(static_cast(3)), "[json.exception.out_of_range.401] array index 3 is out of range"); CHECK_MSG((void)num.at(static_cast(0)), "[json.exception.type_error.304] cannot use at() with number"); @@ -65,7 +65,7 @@ TEST_CASE("exception messages of out-of-line throw sites") CHECK_MSG((void)num.at("a"), "[json.exception.type_error.304] cannot use at() with number"); nlohmann::ordered_json oobj = {{"a", 1}}; - const nlohmann::ordered_json coobj = oobj; + const nlohmann::ordered_json& coobj = oobj; CHECK_MSG((void)oobj.at("z"), "[json.exception.out_of_range.403] key 'z' not found"); CHECK_MSG((void)coobj.at("z"), "[json.exception.out_of_range.403] key 'z' not found"); CHECK_MSG((void)coobj.at(std::string("zz")), "[json.exception.out_of_range.403] key 'zz' not found"); @@ -94,9 +94,10 @@ TEST_CASE("exception messages of out-of-line throw sites") CHECK_MSG(obj.update(arr.begin(), arr2.end()), "[json.exception.invalid_iterator.210] iterators do not fit"); } +#if !defined(JSON_NOEXCEPTION) // patch_msg() catches the exceptions, which JSON_NOEXCEPTION turns into std::abort() SECTION("patch") { - const auto patch_msg = [](const char* doc, const char* patch) + const auto patch_msg = [](const char* doc, const char* patch) -> std::string { try { @@ -105,16 +106,23 @@ TEST_CASE("exception messages of out-of-line throw sites") } catch (const json::exception& e) { - // with JSON_DIAGNOSTICS, what() names the offending value like "(/0) "; drop it + // with JSON_DIAGNOSTICS, what() names the offending value like "(/0) ", + // and with JSON_DIAGNOSTIC_POSITIONS its position like "(bytes 0-2) "; drop both std::string msg = e.what(); - const auto pos = msg.find(" (/"); - if (pos != std::string::npos) + for (const char* marker : + { + " (/", " (bytes " + }) { - msg.erase(pos + 1, msg.find(") ", pos) - pos + 1); + const auto pos = msg.find(marker); + if (pos != std::string::npos) + { + msg.erase(pos + 1, msg.find(") ", pos) - pos + 1); + } } return msg; } - return std::string(); + return {}; }; CHECK(patch_msg("{}", "{}") == "[json.exception.parse_error.104] parse error: JSON patch must be an array of objects"); @@ -132,4 +140,5 @@ TEST_CASE("exception messages of out-of-line throw sites") CHECK(patch_msg("{\"a\":1}", "[{\"op\":\"test\",\"path\":\"/a\",\"value\":2}]") == "[json.exception.other_error.501] unsuccessful: {\"op\":\"test\",\"path\":\"/a\",\"value\":2}"); CHECK(patch_msg("{\"a\":1}", "[{\"op\":\"move\",\"path\":\"/a\"}]") == "[json.exception.parse_error.105] parse error: operation 'move' must have member 'from'"); } +#endif } From 0e95fe8200adbf0e033d0c0818eea473e1d4d346 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 17 Oct 2026 08:28:28 +0200 Subject: [PATCH 3/3] Split the slowest Valgrind tests out of unit-large_json and unit-bjdata (#5862) Signed-off-by: Niels Lohmann --- tests/CMakeLists.txt | 2 +- tests/src/bjdata_round_trip.hpp | 103 +++++ tests/src/unit-bjdata.cpp | 65 --- tests/src/unit-bjdata_round_trip_draft2.cpp | 22 + tests/src/unit-bjdata_round_trip_draft3.cpp | 22 + tests/src/unit-large_json.cpp | 401 ------------------- tests/src/unit-large_json_binary_writers.cpp | 294 ++++++++++++++ tests/src/unit-large_json_conversions.cpp | 156 ++++++++ 8 files changed, 598 insertions(+), 467 deletions(-) create mode 100644 tests/src/bjdata_round_trip.hpp create mode 100644 tests/src/unit-bjdata_round_trip_draft2.cpp create mode 100644 tests/src/unit-bjdata_round_trip_draft3.cpp create mode 100644 tests/src/unit-large_json_binary_writers.cpp create mode 100644 tests/src/unit-large_json_conversions.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9318838bb..fd69a5f33 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -118,7 +118,7 @@ endif() if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") # avoid stack overflow, see https://github.com/nlohmann/json/issues/2955 - json_test_set_test_options("test-bon8;test-cbor;test-msgpack;test-ubjson;test-bjdata;test-binary_formats" LINK_OPTIONS /STACK:4000000) + json_test_set_test_options("test-bon8;test-cbor;test-msgpack;test-ubjson;test-bjdata;test-bjdata_round_trip_draft2;test-bjdata_round_trip_draft3;test-binary_formats" LINK_OPTIONS /STACK:4000000) endif() # disable exceptions for test-disabled_exceptions diff --git a/tests/src/bjdata_round_trip.hpp b/tests/src/bjdata_round_trip.hpp new file mode 100644 index 000000000..4fdde833b --- /dev/null +++ b/tests/src/bjdata_round_trip.hpp @@ -0,0 +1,103 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#pragma once + +#include // uint8_t +#include // vector + +#include "doctest_compatibility.h" + +#include + +#include "round_trip_corpus.hpp" + +// The BJData round-trip invariants, split by the BJData version that turns a +// corpus value into j1 (unit-bjdata_round_trip_draft2.cpp and +// unit-bjdata_round_trip_draft3.cpp). They were part of unit-bjdata.cpp, but +// took 80 % of its time; as separate tests, both halves run in parallel and +// stay below the ctest timeout under Valgrind. +namespace utils +{ + +// This checks what the parse_bjdata_fuzzer driver checks (see +// tests/src/fuzzer-parse_bjdata.cpp), so that a regression shows up in CI +// rather than as an OSS-Fuzz report: every value from_bjdata() returns +// (j1) can be serialized with any combination of options, the result can +// be parsed back (j2), and serializing j2 again with the same options +// yields a value-equal result. +// +// Beyond the driver, this also checks that j2 equals j1 and that +// serializing j2 reproduces the exact bytes, both except for values that +// contain a binary value: a binary value is only written as a binary +// value with Draft 3's optimized binary array, and otherwise read back as +// an array of integers, for which the writer may choose different (but +// equally valid) type markers when it is serialized again (see #5494). +// +// Values are compared with dump() rather than operator==, because a NaN +// never compares equal to itself. +// +// Only the corpus values written with @a initial_version become j1; each j1 +// is then serialized with every combination of options, both versions +// included. +inline void check_bjdata_round_trip(const nlohmann::json::bjdata_version_t initial_version) +{ + using nlohmann::json; + + struct options + { + bool use_size; + bool use_type; + json::bjdata_version_t version; + }; + const std::vector all_options = + { + {false, false, json::bjdata_version_t::draft2}, + {true, false, json::bjdata_version_t::draft2}, + {true, true, json::bjdata_version_t::draft2}, + {false, false, json::bjdata_version_t::draft3}, + {true, false, json::bjdata_version_t::draft3}, + {true, true, json::bjdata_version_t::draft3}, + }; + + for (const auto& j0 : utils::round_trip_corpus::values()) + { + // turn the corpus value into a value as from_bjdata() returns it + for (const auto& initial : all_options) + { + if (initial.version != initial_version) + { + continue; + } + + const json j1 = json::from_bjdata(json::to_bjdata(j0, initial.use_size, initial.use_type, initial.version)); + const bool has_binary = utils::round_trip_corpus::contains_binary(j1); + + for (const auto& o : all_options) + { + INFO("j1 = " << j1.dump() << ", use_size = " << o.use_size << ", use_type = " << o.use_type + << ", draft3 = " << (o.version == json::bjdata_version_t::draft3)); + + const std::vector vec = json::to_bjdata(j1, o.use_size, o.use_type, o.version); + json j2; + // anything the library writes must be parsable by the library + REQUIRE_NOTHROW(j2 = json::from_bjdata(vec)); + const std::vector vec2 = json::to_bjdata(j2, o.use_size, o.use_type, o.version); + CHECK(json::from_bjdata(vec2).dump() == j2.dump()); + + if (!has_binary) + { + CHECK(j2.dump() == j1.dump()); + CHECK(vec2 == vec); + } + } + } + } +} + +} // namespace utils diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index 2518f3386..2f91462da 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -28,7 +28,6 @@ using ordered_json = nlohmann::ordered_json; #include #include #include "make_test_data_available.hpp" -#include "round_trip_corpus.hpp" #include "test_utils.hpp" #include "sax_countdown.hpp" using utils::SaxCountdown; @@ -4441,70 +4440,6 @@ TEST_CASE("BJData use_type requires use_size") } } -TEST_CASE("BJData round-trip invariants") -{ - // This checks what the parse_bjdata_fuzzer driver checks (see - // tests/src/fuzzer-parse_bjdata.cpp), so that a regression shows up in CI - // rather than as an OSS-Fuzz report: every value from_bjdata() returns - // (j1) can be serialized with any combination of options, the result can - // be parsed back (j2), and serializing j2 again with the same options - // yields a value-equal result. - // - // Beyond the driver, this also checks that j2 equals j1 and that - // serializing j2 reproduces the exact bytes, both except for values that - // contain a binary value: a binary value is only written as a binary - // value with Draft 3's optimized binary array, and otherwise read back as - // an array of integers, for which the writer may choose different (but - // equally valid) type markers when it is serialized again (see #5494). - // - // Values are compared with dump() rather than operator==, because a NaN - // never compares equal to itself. - struct options - { - bool use_size; - bool use_type; - json::bjdata_version_t version; - }; - const std::vector all_options = - { - {false, false, json::bjdata_version_t::draft2}, - {true, false, json::bjdata_version_t::draft2}, - {true, true, json::bjdata_version_t::draft2}, - {false, false, json::bjdata_version_t::draft3}, - {true, false, json::bjdata_version_t::draft3}, - {true, true, json::bjdata_version_t::draft3}, - }; - - for (const auto& j0 : utils::round_trip_corpus::values()) - { - // turn the corpus value into a value as from_bjdata() returns it - for (const auto& initial : all_options) - { - const json j1 = json::from_bjdata(json::to_bjdata(j0, initial.use_size, initial.use_type, initial.version)); - const bool has_binary = utils::round_trip_corpus::contains_binary(j1); - - for (const auto& o : all_options) - { - INFO("j1 = " << j1.dump() << ", use_size = " << o.use_size << ", use_type = " << o.use_type - << ", draft3 = " << (o.version == json::bjdata_version_t::draft3)); - - const std::vector vec = json::to_bjdata(j1, o.use_size, o.use_type, o.version); - json j2; - // anything the library writes must be parsable by the library - REQUIRE_NOTHROW(j2 = json::from_bjdata(vec)); - const std::vector vec2 = json::to_bjdata(j2, o.use_size, o.use_type, o.version); - CHECK(json::from_bjdata(vec2).dump() == j2.dump()); - - if (!has_binary) - { - CHECK(j2.dump() == j1.dump()); - CHECK(vec2 == vec); - } - } - } - } -} - TEST_CASE("BJData round trip of a binary value is value-stable, not byte-stable") { // OSS-Fuzz issue 474480402: a Draft 3 optimized binary array is read as a diff --git a/tests/src/unit-bjdata_round_trip_draft2.cpp b/tests/src/unit-bjdata_round_trip_draft2.cpp new file mode 100644 index 000000000..6bbe6ea06 --- /dev/null +++ b/tests/src/unit-bjdata_round_trip_draft2.cpp @@ -0,0 +1,22 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include "bjdata_round_trip.hpp" + +// split from unit-bjdata.cpp, see bjdata_round_trip.hpp; the other half is in +// unit-bjdata_round_trip_draft3.cpp + +TEST_CASE("BJData round-trip invariants (j1 written with Draft 2)") +{ + utils::check_bjdata_round_trip(json::bjdata_version_t::draft2); +} diff --git a/tests/src/unit-bjdata_round_trip_draft3.cpp b/tests/src/unit-bjdata_round_trip_draft3.cpp new file mode 100644 index 000000000..97e7517b6 --- /dev/null +++ b/tests/src/unit-bjdata_round_trip_draft3.cpp @@ -0,0 +1,22 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include "bjdata_round_trip.hpp" + +// split from unit-bjdata.cpp, see bjdata_round_trip.hpp; the other half is in +// unit-bjdata_round_trip_draft2.cpp + +TEST_CASE("BJData round-trip invariants (j1 written with Draft 3)") +{ + utils::check_bjdata_round_trip(json::bjdata_version_t::draft3); +} diff --git a/tests/src/unit-large_json.cpp b/tests/src/unit-large_json.cpp index ecd6f9f81..ed988c242 100644 --- a/tests/src/unit-large_json.cpp +++ b/tests/src/unit-large_json.cpp @@ -13,8 +13,6 @@ using nlohmann::json; #include #include -#include -#include TEST_CASE("tests on very large JSONs") { @@ -55,24 +53,6 @@ const json* innermost_value(const json& j, std::size_t& depth) return current; } -// The text of a value nested depth levels deep around the number 0. Level i is -// an array if pattern[i % pattern.size()] is '[', and otherwise an object with -// the single member "a", which every object type enumerates in the same order. -std::string nested_text(std::size_t depth, const std::string& pattern) -{ - std::string text; - std::string closing; - for (std::size_t i = 0; i < depth; ++i) - { - const bool array = pattern[i % pattern.size()] == '['; - text += array ? "[" : "{\"a\":"; - closing += array ? ']' : '}'; - } - text += '0'; - text.append(closing.rbegin(), closing.rend()); - return text; -} - } // namespace TEST_CASE("tests on deeply nested JSONs") @@ -244,385 +224,4 @@ TEST_CASE("tests on deeply nested JSONs") CHECK(*innermost_value(j, unused) == 0); } } - - SECTION("issue #5650 - stack overflow converting between specializations") - { - const std::vector patterns = {"[", "{", "[{"}; - - SECTION("json to ordered_json") - { - for (const auto& pattern : patterns) - { - CAPTURE(pattern) - const std::string text = nested_text(depth, pattern); - const json j = json::parse(text); - - const nlohmann::ordered_json converted = j; - CHECK(converted.dump() == text); - } - } - - SECTION("ordered_json to json") - { - for (const auto& pattern : patterns) - { - CAPTURE(pattern) - const std::string text = nested_text(depth, pattern); - const nlohmann::ordered_json o = nlohmann::ordered_json::parse(text); - - const json converted = o; - CHECK(converted.dump() == text); - } - } - - SECTION("get()") - { - for (const auto& pattern : patterns) - { - CAPTURE(pattern) - const std::string text = nested_text(depth, pattern); - const json j = json::parse(text); - - CHECK(j.get().dump() == text); - } - } - - SECTION("depths around the bound of the recursive descent") - { - for (std::size_t d = 1; d <= 300; ++d) - { - CAPTURE(d) - for (const auto& pattern : patterns) - { - CAPTURE(pattern) - const std::string text = nested_text(d, pattern); - const json j = json::parse(text); - - const nlohmann::ordered_json converted = j; - CHECK(converted.dump() == text); - const json back = converted; - CHECK(back.dump() == text); - } - } - } - - SECTION("values below the bound are converted as values above it") - { - // Bury a value below the bound, where it is converted without the - // call stack, and compare it with the same value converted on its - // own by the containers' range constructors. Its objects have - // members that the two object types enumerate in different orders. - const auto bury = [](nlohmann::ordered_json value) - { - for (std::size_t i = 0; i < 200; ++i) - { - value = nlohmann::ordered_json::array({std::move(value)}); - } - return value; - }; - const auto dig = [](const json & value) - { - const json* current = &value; - for (std::size_t i = 0; i < 200; ++i) - { - current = ¤t->at(0); - } - return current; - }; - - nlohmann::ordered_json value = nlohmann::ordered_json::object(); - value["z"] = {1, -2, 3U, 4.5, true, nullptr, "six", nlohmann::ordered_json::binary({7, 8}, 9), - nlohmann::ordered_json::binary({10}), nlohmann::ordered_json::array(), nlohmann::ordered_json::object() - }; - value["y"] = {{"x", {{"w", 1}, {"v", 2}}}, {"u", {3, {{"t", 4}, {"s", 5}}}}}; - value["r"] = nlohmann::ordered_json::array({nlohmann::ordered_json(nlohmann::ordered_json::value_t::discarded)}); - - const json converted_above = value; - const json buried = bury(value); - const json& converted_below = *dig(buried); - - CHECK(converted_below.dump() == converted_above.dump()); - CHECK(converted_below.at("z").at(7).get_binary().subtype() == 9); - CHECK_FALSE(converted_below.at("z").at(8).get_binary().has_subtype()); - CHECK(converted_below.at("r").at(0).is_discarded()); - - // a discarded value is never equal to anything, so compare the rest - value.erase("r"); - const json without_discarded_above = value; - const json without_discarded_buried = bury(value); - CHECK(*dig(without_discarded_buried) == without_discarded_above); - } - } } - -namespace -{ -json nested_array(const std::size_t depth, json leaf) -{ - json j = std::move(leaf); - for (std::size_t i = 0; i < depth; ++i) - { - json a = json::array(); - a.push_back(std::move(j)); - j = std::move(a); - } - return j; -} - -json nested_object(const std::size_t depth, json leaf) -{ - json j = std::move(leaf); - for (std::size_t i = 0; i < depth; ++i) - { - json o = json::object(); - o["k"] = std::move(j); - j = std::move(o); - } - return j; -} -} // namespace - -TEST_CASE("issue #5392 - binary writers on deeply nested values") -{ - // 200 is past the point where the writers stop recursing, and still - // shallow enough that from_* and operator== (which still recurse) are fine. - const json deep_array = nested_array(200, json(0)); - const json deep_object = nested_object(200, json("x")); - const json empty_array = nested_array(200, json::array()); - const json empty_object = nested_object(200, json::object()); - const json mixed = nested_object(80, nested_array(80, json(true))); - - SECTION("roundtrip past the recursion bound") - { - CHECK(json::from_cbor(json::to_cbor(deep_array)) == deep_array); - CHECK(json::from_msgpack(json::to_msgpack(deep_array)) == deep_array); - CHECK(json::from_ubjson(json::to_ubjson(deep_array)) == deep_array); - CHECK(json::from_ubjson(json::to_ubjson(deep_array, true, false)) == deep_array); - CHECK(json::from_ubjson(json::to_ubjson(deep_array, true, true)) == deep_array); - CHECK(json::from_bjdata(json::to_bjdata(deep_array)) == deep_array); - CHECK(json::from_bon8(json::to_bon8(deep_array)) == deep_array); - - CHECK(json::from_cbor(json::to_cbor(deep_object)) == deep_object); - CHECK(json::from_msgpack(json::to_msgpack(deep_object)) == deep_object); - CHECK(json::from_ubjson(json::to_ubjson(deep_object)) == deep_object); - CHECK(json::from_ubjson(json::to_ubjson(deep_object, true, true)) == deep_object); - CHECK(json::from_bjdata(json::to_bjdata(deep_object)) == deep_object); - CHECK(json::from_bon8(json::to_bon8(deep_object)) == deep_object); - - CHECK(json::from_cbor(json::to_cbor(empty_array)) == empty_array); - CHECK(json::from_msgpack(json::to_msgpack(empty_array)) == empty_array); - CHECK(json::from_ubjson(json::to_ubjson(empty_array)) == empty_array); - CHECK(json::from_ubjson(json::to_ubjson(empty_array, true, true)) == empty_array); - CHECK(json::from_bon8(json::to_bon8(empty_array)) == empty_array); - - CHECK(json::from_cbor(json::to_cbor(empty_object)) == empty_object); - CHECK(json::from_msgpack(json::to_msgpack(empty_object)) == empty_object); - CHECK(json::from_ubjson(json::to_ubjson(empty_object)) == empty_object); - CHECK(json::from_bon8(json::to_bon8(empty_object)) == empty_object); - - CHECK(json::from_cbor(json::to_cbor(mixed)) == mixed); - CHECK(json::from_msgpack(json::to_msgpack(mixed)) == mixed); - CHECK(json::from_ubjson(json::to_ubjson(mixed)) == mixed); - CHECK(json::from_bjdata(json::to_bjdata(mixed)) == mixed); - CHECK(json::from_bon8(json::to_bon8(mixed)) == mixed); - } - - SECTION("the two ways of writing a value meet at the bound") - { - for (std::size_t depth = 120; depth <= 140; ++depth) - { - CAPTURE(depth) - - const json array = nested_array(depth, json(7)); - CHECK(json::from_cbor(json::to_cbor(array)) == array); - CHECK(json::from_msgpack(json::to_msgpack(array)) == array); - CHECK(json::from_ubjson(json::to_ubjson(array, true, true)) == array); - CHECK(json::from_bon8(json::to_bon8(array)) == array); - - const json object = nested_object(depth, json(7)); - CHECK(json::from_cbor(json::to_cbor(object)) == object); - CHECK(json::from_msgpack(json::to_msgpack(object)) == object); - CHECK(json::from_bjdata(json::to_bjdata(object)) == object); - CHECK(json::from_bon8(json::to_bon8(object)) == object); - } - } - - SECTION("a BJData ndarray below the bound is still an ndarray") - { - const json ndarray = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}}); - const json invalid = json({{"_ArrayType_", "nope"}, {"_ArraySize_", {1}}, {"_ArrayData_", {1}}}); - - const json deep_ndarray = nested_array(140, ndarray); - const json deep_invalid = nested_array(140, invalid); - - CHECK(json::from_bjdata(json::to_bjdata(deep_ndarray)) == deep_ndarray); - CHECK(json::from_bjdata(json::to_bjdata(deep_invalid)) == deep_invalid); - CHECK(json::from_bjdata(json::to_bjdata(ndarray)) == ndarray); - } - - SECTION("byte-exact across the switch-over") - { - // nested one-element arrays around the recursion bound: the exact - // bytes a writer produces do not depend on whether it stayed on the - // call stack or moved to the heap one partway through - for (const std::size_t depth : - { - nlohmann::detail::recursion_depth_limit() - 1, nlohmann::detail::recursion_depth_limit(), - nlohmann::detail::recursion_depth_limit() + 1, nlohmann::detail::recursion_depth_limit() + 2 - }) - { - CAPTURE(depth) - const json array = nested_array(depth, json(0)); - - std::vector expected_cbor(depth, 0x81); - expected_cbor.push_back(0x00); - CHECK(json::to_cbor(array) == expected_cbor); - - std::vector expected_msgpack(depth, 0x91); - expected_msgpack.push_back(0x00); - CHECK(json::to_msgpack(array) == expected_msgpack); - - std::string expected_ubjson(depth, '['); - expected_ubjson += "i"; - expected_ubjson += '\0'; - expected_ubjson.append(depth, ']'); - const auto packed_ubjson = json::to_ubjson(array); - CHECK(std::string(packed_ubjson.begin(), packed_ubjson.end()) == expected_ubjson); - - std::vector expected_bon8(depth, 0x81); - expected_bon8.push_back(0x90); - CHECK(json::to_bon8(array) == expected_bon8); - } - } - - SECTION("a deep object, and a BJData ndarray, past the recursion bound") - { - const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50; - - const json object = nested_object(depth, json(42)); - CHECK(json::from_cbor(json::to_cbor(object)) == object); - CHECK(json::from_msgpack(json::to_msgpack(object)) == object); - CHECK(json::from_ubjson(json::to_ubjson(object, true, true)) == object); - CHECK(json::from_bjdata(json::to_bjdata(object)) == object); - CHECK(json::from_bon8(json::to_bon8(object)) == object); - - const json ndarray = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}}); - const json deep_ndarray = nested_array(depth, ndarray); - CHECK(json::from_bjdata(json::to_bjdata(deep_ndarray)) == deep_ndarray); - } - - SECTION("a discarded value past the recursion bound still throws type_error.321") - { - const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50; - const json discarded_leaf(json::value_t::discarded); - const json deep_discarded = nested_array(depth, discarded_leaf); - - // with diagnostics, the message names the path to the discarded leaf -#if JSON_DIAGNOSTICS - std::string path; - for (std::size_t i = 0; i < depth; ++i) - { - path += "/0"; - } - const std::string prefix = "[json.exception.type_error.321] (" + path + ") "; -#else - const std::string prefix = "[json.exception.type_error.321] "; -#endif - - CHECK_THROWS_WITH_AS(json::to_cbor(deep_discarded), (prefix + "cannot serialize discarded value to CBOR").c_str(), json::type_error); - CHECK_THROWS_WITH_AS(json::to_msgpack(deep_discarded), (prefix + "cannot serialize discarded value to MessagePack").c_str(), json::type_error); - CHECK_THROWS_WITH_AS(json::to_ubjson(deep_discarded), (prefix + "cannot serialize discarded value to UBJSON").c_str(), json::type_error); - CHECK_THROWS_WITH_AS(json::to_bjdata(deep_discarded), (prefix + "cannot serialize discarded value to BJData").c_str(), json::type_error); - } - - SECTION("BON8 past the recursion bound: discarded values and errors") - { - const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50; - - // BON8 rejects a discarded value (type_error.321), deep or not - const json deep_discarded = nested_array(depth, json(json::value_t::discarded)); - CHECK_THROWS_AS(json::to_bon8(deep_discarded), json::type_error); - - // errors are thrown from below the bound as from above it - const json deep_invalid_string = nested_array(depth, json(std::string("\x80"))); - CHECK_THROWS_AS(json::to_bon8(deep_invalid_string), json::type_error); - - json deep_invalid_key = json::object(); - deep_invalid_key[std::string("\x80")] = 1; - deep_invalid_key = nested_object(depth, deep_invalid_key); - CHECK_THROWS_AS(json::to_bon8(deep_invalid_key), json::type_error); - - const json deep_too_large = nested_array(depth, json(9223372036854775808u)); - CHECK_THROWS_AS(json::to_bon8(deep_too_large), json::out_of_range); - } - - SECTION("does not overflow the C++ stack") - { - const std::size_t depth = 100000; - const json j = json::parse(std::string(depth, '[') + "0" + std::string(depth, ']')); - - std::vector packed; - CHECK_NOTHROW(packed = json::to_cbor(j)); - CHECK(json::from_cbor(packed) == j); - - CHECK_NOTHROW(packed = json::to_msgpack(j)); - CHECK(json::from_msgpack(packed) == j); - - CHECK_NOTHROW(packed = json::to_ubjson(j)); - CHECK(json::from_ubjson(packed) == j); - - CHECK_NOTHROW(packed = json::to_ubjson(j, true, false)); - CHECK(json::from_ubjson(packed) == j); - - CHECK_NOTHROW(packed = json::to_bjdata(j)); - CHECK(json::from_bjdata(packed) == j); - - CHECK_NOTHROW(packed = json::to_bon8(j)); - CHECK(json::from_bon8(packed) == j); - } - - SECTION("BON8 containers with more than four elements past the recursion bound") - { - // such containers are closed with 0xFE, which the iterative writer - // must emit when it leaves them; strings next to each other must - // still be separated with 0xFF - const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50; - json wide_array = json::array({1, "a", "b", json::array(), 5}); - json wide_object = json({{"k1", "v"}, {"k2", 2}, {"k3", ""}, {"k4", json::object()}, {"k5", nullptr}}); - for (std::size_t i = 0; i < depth; ++i) - { - wide_array = json::array({"s", wide_array, "t", wide_object, true}); - wide_object = json({{"a", wide_object}, {"b", "x"}, {"c", "y"}, {"d", wide_array.size()}, {"e", "z"}}); - } - - CHECK(json::from_bon8(json::to_bon8(wide_array)) == wide_array); - CHECK(json::from_bon8(json::to_bon8(wide_object)) == wide_object); - } - - SECTION("regression test for https://issues.oss-fuzz.com/issues/566583014") - { - // 200000 nested one-element CBOR arrays, the innermost holding null; - // round-tripping this used to recurse once per level on the way back - // out through to_cbor(), deep enough to overflow the stack - std::vector v(200000, 0x81); - v.push_back(0xf6); - const json j = json::from_cbor(v); - CHECK(json::to_cbor(j) == v); - - // the MessagePack analogue: fixarray of 1 nesting down to nil - std::vector v_msgpack(200000, 0x91); - v_msgpack.push_back(0xc0); - const json j_msgpack = json::from_msgpack(v_msgpack); - CHECK(json::to_msgpack(j_msgpack) == v_msgpack); - } - - SECTION("regression test for https://issues.oss-fuzz.com/issues/572238015") - { - // the BON8 analogue: 200000 nested one-element arrays down to null - std::vector v(200000, 0x81); - v.push_back(0xFA); - const json j = json::from_bon8(v); - CHECK(json::to_bon8(j) == v); - } -} - diff --git a/tests/src/unit-large_json_binary_writers.cpp b/tests/src/unit-large_json_binary_writers.cpp new file mode 100644 index 000000000..339c62135 --- /dev/null +++ b/tests/src/unit-large_json_binary_writers.cpp @@ -0,0 +1,294 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include +#include +#include +#include +#include + +// split from unit-large_json.cpp so that both run in parallel and stay below +// the ctest timeout under Valgrind + +namespace +{ +json nested_array(const std::size_t depth, json leaf) +{ + json j = std::move(leaf); + for (std::size_t i = 0; i < depth; ++i) + { + json a = json::array(); + a.push_back(std::move(j)); + j = std::move(a); + } + return j; +} + +json nested_object(const std::size_t depth, json leaf) +{ + json j = std::move(leaf); + for (std::size_t i = 0; i < depth; ++i) + { + json o = json::object(); + o["k"] = std::move(j); + j = std::move(o); + } + return j; +} +} // namespace + +TEST_CASE("issue #5392 - binary writers on deeply nested values") +{ + // 200 is past the point where the writers stop recursing, and still + // shallow enough that from_* and operator== (which still recurse) are fine. + const json deep_array = nested_array(200, json(0)); + const json deep_object = nested_object(200, json("x")); + const json empty_array = nested_array(200, json::array()); + const json empty_object = nested_object(200, json::object()); + const json mixed = nested_object(80, nested_array(80, json(true))); + + SECTION("roundtrip past the recursion bound") + { + CHECK(json::from_cbor(json::to_cbor(deep_array)) == deep_array); + CHECK(json::from_msgpack(json::to_msgpack(deep_array)) == deep_array); + CHECK(json::from_ubjson(json::to_ubjson(deep_array)) == deep_array); + CHECK(json::from_ubjson(json::to_ubjson(deep_array, true, false)) == deep_array); + CHECK(json::from_ubjson(json::to_ubjson(deep_array, true, true)) == deep_array); + CHECK(json::from_bjdata(json::to_bjdata(deep_array)) == deep_array); + CHECK(json::from_bon8(json::to_bon8(deep_array)) == deep_array); + + CHECK(json::from_cbor(json::to_cbor(deep_object)) == deep_object); + CHECK(json::from_msgpack(json::to_msgpack(deep_object)) == deep_object); + CHECK(json::from_ubjson(json::to_ubjson(deep_object)) == deep_object); + CHECK(json::from_ubjson(json::to_ubjson(deep_object, true, true)) == deep_object); + CHECK(json::from_bjdata(json::to_bjdata(deep_object)) == deep_object); + CHECK(json::from_bon8(json::to_bon8(deep_object)) == deep_object); + + CHECK(json::from_cbor(json::to_cbor(empty_array)) == empty_array); + CHECK(json::from_msgpack(json::to_msgpack(empty_array)) == empty_array); + CHECK(json::from_ubjson(json::to_ubjson(empty_array)) == empty_array); + CHECK(json::from_ubjson(json::to_ubjson(empty_array, true, true)) == empty_array); + CHECK(json::from_bon8(json::to_bon8(empty_array)) == empty_array); + + CHECK(json::from_cbor(json::to_cbor(empty_object)) == empty_object); + CHECK(json::from_msgpack(json::to_msgpack(empty_object)) == empty_object); + CHECK(json::from_ubjson(json::to_ubjson(empty_object)) == empty_object); + CHECK(json::from_bon8(json::to_bon8(empty_object)) == empty_object); + + CHECK(json::from_cbor(json::to_cbor(mixed)) == mixed); + CHECK(json::from_msgpack(json::to_msgpack(mixed)) == mixed); + CHECK(json::from_ubjson(json::to_ubjson(mixed)) == mixed); + CHECK(json::from_bjdata(json::to_bjdata(mixed)) == mixed); + CHECK(json::from_bon8(json::to_bon8(mixed)) == mixed); + } + + SECTION("the two ways of writing a value meet at the bound") + { + for (std::size_t depth = 120; depth <= 140; ++depth) + { + CAPTURE(depth) + + const json array = nested_array(depth, json(7)); + CHECK(json::from_cbor(json::to_cbor(array)) == array); + CHECK(json::from_msgpack(json::to_msgpack(array)) == array); + CHECK(json::from_ubjson(json::to_ubjson(array, true, true)) == array); + CHECK(json::from_bon8(json::to_bon8(array)) == array); + + const json object = nested_object(depth, json(7)); + CHECK(json::from_cbor(json::to_cbor(object)) == object); + CHECK(json::from_msgpack(json::to_msgpack(object)) == object); + CHECK(json::from_bjdata(json::to_bjdata(object)) == object); + CHECK(json::from_bon8(json::to_bon8(object)) == object); + } + } + + SECTION("a BJData ndarray below the bound is still an ndarray") + { + const json ndarray = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}}); + const json invalid = json({{"_ArrayType_", "nope"}, {"_ArraySize_", {1}}, {"_ArrayData_", {1}}}); + + const json deep_ndarray = nested_array(140, ndarray); + const json deep_invalid = nested_array(140, invalid); + + CHECK(json::from_bjdata(json::to_bjdata(deep_ndarray)) == deep_ndarray); + CHECK(json::from_bjdata(json::to_bjdata(deep_invalid)) == deep_invalid); + CHECK(json::from_bjdata(json::to_bjdata(ndarray)) == ndarray); + } + + SECTION("byte-exact across the switch-over") + { + // nested one-element arrays around the recursion bound: the exact + // bytes a writer produces do not depend on whether it stayed on the + // call stack or moved to the heap one partway through + for (const std::size_t depth : + { + nlohmann::detail::recursion_depth_limit() - 1, nlohmann::detail::recursion_depth_limit(), + nlohmann::detail::recursion_depth_limit() + 1, nlohmann::detail::recursion_depth_limit() + 2 + }) + { + CAPTURE(depth) + const json array = nested_array(depth, json(0)); + + std::vector expected_cbor(depth, 0x81); + expected_cbor.push_back(0x00); + CHECK(json::to_cbor(array) == expected_cbor); + + std::vector expected_msgpack(depth, 0x91); + expected_msgpack.push_back(0x00); + CHECK(json::to_msgpack(array) == expected_msgpack); + + std::string expected_ubjson(depth, '['); + expected_ubjson += "i"; + expected_ubjson += '\0'; + expected_ubjson.append(depth, ']'); + const auto packed_ubjson = json::to_ubjson(array); + CHECK(std::string(packed_ubjson.begin(), packed_ubjson.end()) == expected_ubjson); + + std::vector expected_bon8(depth, 0x81); + expected_bon8.push_back(0x90); + CHECK(json::to_bon8(array) == expected_bon8); + } + } + + SECTION("a deep object, and a BJData ndarray, past the recursion bound") + { + const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50; + + const json object = nested_object(depth, json(42)); + CHECK(json::from_cbor(json::to_cbor(object)) == object); + CHECK(json::from_msgpack(json::to_msgpack(object)) == object); + CHECK(json::from_ubjson(json::to_ubjson(object, true, true)) == object); + CHECK(json::from_bjdata(json::to_bjdata(object)) == object); + CHECK(json::from_bon8(json::to_bon8(object)) == object); + + const json ndarray = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}}); + const json deep_ndarray = nested_array(depth, ndarray); + CHECK(json::from_bjdata(json::to_bjdata(deep_ndarray)) == deep_ndarray); + } + + SECTION("a discarded value past the recursion bound still throws type_error.321") + { + const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50; + const json discarded_leaf(json::value_t::discarded); + const json deep_discarded = nested_array(depth, discarded_leaf); + + // with diagnostics, the message names the path to the discarded leaf +#if JSON_DIAGNOSTICS + std::string path; + for (std::size_t i = 0; i < depth; ++i) + { + path += "/0"; + } + const std::string prefix = "[json.exception.type_error.321] (" + path + ") "; +#else + const std::string prefix = "[json.exception.type_error.321] "; +#endif + + CHECK_THROWS_WITH_AS(json::to_cbor(deep_discarded), (prefix + "cannot serialize discarded value to CBOR").c_str(), json::type_error); + CHECK_THROWS_WITH_AS(json::to_msgpack(deep_discarded), (prefix + "cannot serialize discarded value to MessagePack").c_str(), json::type_error); + CHECK_THROWS_WITH_AS(json::to_ubjson(deep_discarded), (prefix + "cannot serialize discarded value to UBJSON").c_str(), json::type_error); + CHECK_THROWS_WITH_AS(json::to_bjdata(deep_discarded), (prefix + "cannot serialize discarded value to BJData").c_str(), json::type_error); + } + + SECTION("BON8 past the recursion bound: discarded values and errors") + { + const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50; + + // BON8 rejects a discarded value (type_error.321), deep or not + const json deep_discarded = nested_array(depth, json(json::value_t::discarded)); + CHECK_THROWS_AS(json::to_bon8(deep_discarded), json::type_error); + + // errors are thrown from below the bound as from above it + const json deep_invalid_string = nested_array(depth, json(std::string("\x80"))); + CHECK_THROWS_AS(json::to_bon8(deep_invalid_string), json::type_error); + + json deep_invalid_key = json::object(); + deep_invalid_key[std::string("\x80")] = 1; + deep_invalid_key = nested_object(depth, deep_invalid_key); + CHECK_THROWS_AS(json::to_bon8(deep_invalid_key), json::type_error); + + const json deep_too_large = nested_array(depth, json(9223372036854775808u)); + CHECK_THROWS_AS(json::to_bon8(deep_too_large), json::out_of_range); + } + + SECTION("does not overflow the C++ stack") + { + const std::size_t depth = 100000; + const json j = json::parse(std::string(depth, '[') + "0" + std::string(depth, ']')); + + std::vector packed; + CHECK_NOTHROW(packed = json::to_cbor(j)); + CHECK(json::from_cbor(packed) == j); + + CHECK_NOTHROW(packed = json::to_msgpack(j)); + CHECK(json::from_msgpack(packed) == j); + + CHECK_NOTHROW(packed = json::to_ubjson(j)); + CHECK(json::from_ubjson(packed) == j); + + CHECK_NOTHROW(packed = json::to_ubjson(j, true, false)); + CHECK(json::from_ubjson(packed) == j); + + CHECK_NOTHROW(packed = json::to_bjdata(j)); + CHECK(json::from_bjdata(packed) == j); + + CHECK_NOTHROW(packed = json::to_bon8(j)); + CHECK(json::from_bon8(packed) == j); + } + + SECTION("BON8 containers with more than four elements past the recursion bound") + { + // such containers are closed with 0xFE, which the iterative writer + // must emit when it leaves them; strings next to each other must + // still be separated with 0xFF + const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50; + json wide_array = json::array({1, "a", "b", json::array(), 5}); + json wide_object = json({{"k1", "v"}, {"k2", 2}, {"k3", ""}, {"k4", json::object()}, {"k5", nullptr}}); + for (std::size_t i = 0; i < depth; ++i) + { + wide_array = json::array({"s", wide_array, "t", wide_object, true}); + wide_object = json({{"a", wide_object}, {"b", "x"}, {"c", "y"}, {"d", wide_array.size()}, {"e", "z"}}); + } + + CHECK(json::from_bon8(json::to_bon8(wide_array)) == wide_array); + CHECK(json::from_bon8(json::to_bon8(wide_object)) == wide_object); + } + + SECTION("regression test for https://issues.oss-fuzz.com/issues/566583014") + { + // 200000 nested one-element CBOR arrays, the innermost holding null; + // round-tripping this used to recurse once per level on the way back + // out through to_cbor(), deep enough to overflow the stack + std::vector v(200000, 0x81); + v.push_back(0xf6); + const json j = json::from_cbor(v); + CHECK(json::to_cbor(j) == v); + + // the MessagePack analogue: fixarray of 1 nesting down to nil + std::vector v_msgpack(200000, 0x91); + v_msgpack.push_back(0xc0); + const json j_msgpack = json::from_msgpack(v_msgpack); + CHECK(json::to_msgpack(j_msgpack) == v_msgpack); + } + + SECTION("regression test for https://issues.oss-fuzz.com/issues/572238015") + { + // the BON8 analogue: 200000 nested one-element arrays down to null + std::vector v(200000, 0x81); + v.push_back(0xFA); + const json j = json::from_bon8(v); + CHECK(json::to_bon8(j) == v); + } +} + diff --git a/tests/src/unit-large_json_conversions.cpp b/tests/src/unit-large_json_conversions.cpp new file mode 100644 index 000000000..1fd30aa89 --- /dev/null +++ b/tests/src/unit-large_json_conversions.cpp @@ -0,0 +1,156 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include +#include +#include +#include + +// split from unit-large_json.cpp so that both run in parallel and stay below +// the ctest timeout under Valgrind + +namespace +{ + +// The text of a value nested depth levels deep around the number 0. Level i is +// an array if pattern[i % pattern.size()] is '[', and otherwise an object with +// the single member "a", which every object type enumerates in the same order. +std::string nested_text(std::size_t depth, const std::string& pattern) +{ + std::string text; + std::string closing; + for (std::size_t i = 0; i < depth; ++i) + { + const bool array = pattern[i % pattern.size()] == '['; + text += array ? "[" : "{\"a\":"; + closing += array ? ']' : '}'; + } + text += '0'; + text.append(closing.rbegin(), closing.rend()); + return text; +} + +} // namespace + +TEST_CASE("issue #5650 - stack overflow converting between specializations") +{ + // deep enough to exhaust the call stack, but small enough to stay cheap: + // parsing is iterative, so building the values below costs little + const std::size_t depth = 100000; + + const std::vector patterns = {"[", "{", "[{"}; + + SECTION("json to ordered_json") + { + for (const auto& pattern : patterns) + { + CAPTURE(pattern) + const std::string text = nested_text(depth, pattern); + const json j = json::parse(text); + + const nlohmann::ordered_json converted = j; + CHECK(converted.dump() == text); + } + } + + SECTION("ordered_json to json") + { + for (const auto& pattern : patterns) + { + CAPTURE(pattern) + const std::string text = nested_text(depth, pattern); + const nlohmann::ordered_json o = nlohmann::ordered_json::parse(text); + + const json converted = o; + CHECK(converted.dump() == text); + } + } + + SECTION("get()") + { + for (const auto& pattern : patterns) + { + CAPTURE(pattern) + const std::string text = nested_text(depth, pattern); + const json j = json::parse(text); + + CHECK(j.get().dump() == text); + } + } + + SECTION("depths around the bound of the recursive descent") + { + for (std::size_t d = 1; d <= 300; ++d) + { + CAPTURE(d) + for (const auto& pattern : patterns) + { + CAPTURE(pattern) + const std::string text = nested_text(d, pattern); + const json j = json::parse(text); + + const nlohmann::ordered_json converted = j; + CHECK(converted.dump() == text); + const json back = converted; + CHECK(back.dump() == text); + } + } + } + + SECTION("values below the bound are converted as values above it") + { + // Bury a value below the bound, where it is converted without the + // call stack, and compare it with the same value converted on its + // own by the containers' range constructors. Its objects have + // members that the two object types enumerate in different orders. + const auto bury = [](nlohmann::ordered_json value) + { + for (std::size_t i = 0; i < 200; ++i) + { + value = nlohmann::ordered_json::array({std::move(value)}); + } + return value; + }; + const auto dig = [](const json & value) + { + const json* current = &value; + for (std::size_t i = 0; i < 200; ++i) + { + current = ¤t->at(0); + } + return current; + }; + + nlohmann::ordered_json value = nlohmann::ordered_json::object(); + value["z"] = {1, -2, 3U, 4.5, true, nullptr, "six", nlohmann::ordered_json::binary({7, 8}, 9), + nlohmann::ordered_json::binary({10}), nlohmann::ordered_json::array(), nlohmann::ordered_json::object() + }; + value["y"] = {{"x", {{"w", 1}, {"v", 2}}}, {"u", {3, {{"t", 4}, {"s", 5}}}}}; + value["r"] = nlohmann::ordered_json::array({nlohmann::ordered_json(nlohmann::ordered_json::value_t::discarded)}); + + const json converted_above = value; + const json buried = bury(value); + const json& converted_below = *dig(buried); + + CHECK(converted_below.dump() == converted_above.dump()); + CHECK(converted_below.at("z").at(7).get_binary().subtype() == 9); + CHECK_FALSE(converted_below.at("z").at(8).get_binary().has_subtype()); + CHECK(converted_below.at("r").at(0).is_discarded()); + + // a discarded value is never equal to anything, so compare the rest + value.erase("r"); + const json without_discarded_above = value; + const json without_discarded_buried = bury(value); + CHECK(*dig(without_discarded_buried) == without_discarded_above); + } +}