Merge develop into fix/unflatten-linear and fix clang-tidy modernize-use-auto

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/conversions/from_json.hpp

@@ -785,6 +785,37 @@ void from_json(const BasicJsonType& j, std::unordered_map<Key, Value, Hash, KeyE
     from_json_pair_array_to_map(j, m);
 }
 
+#if defined(__cpp_lib_char8_t)
+// a std::u8string is read from a JSON string; the array of code units written by
+// versions before 3.13.0 is still accepted
+template<typename BasicJsonType, typename Tr, typename Allocator,
+         enable_if_t<std::is_same<typename json_type_of_t<BasicJsonType>::string_t::value_type, char>::value, int> = 0>
+inline void from_json(const BasicJsonType& j, std::basic_string<char8_t, Tr, Allocator>& s)
+{
+    if (j.is_string())
+    {
+        const auto& str = string_value(j);
+        const char* const data = str.data();
+        const std::size_t size = str.size();
+        std::basic_string<char8_t, Tr, Allocator> ret(s.get_allocator());
+        ret.resize(size);
+        for (std::size_t i = 0; i < size; ++i)
+        {
+            ret[i] = static_cast<char8_t>(static_cast<unsigned char>(data[i]));
+        }
+        s = std::move(ret);
+    }
+    else if (j.is_array())
+    {
+        from_json_array_impl(j, s, priority_tag<3> {});
+    }
+    else
+    {
+        throw_type_must_be("string or array", j);
+    }
+}
+#endif
+
 #if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM
 
 // Workaround for MSVC 19.51 (and possibly later): in large cpp files, the compiler may fail to resolve with generic has_from_json (issue #4996)

include/nlohmann/detail/conversions/to_json.hpp

@@ -625,16 +625,20 @@ inline void to_json(BasicJsonType& j, const T& t)
     to_json_tuple_impl(j, t, make_index_sequence<std::tuple_size<T>::value> {});
 }
 
-#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM
+// a std::u8string is a JSON string (it was an array of code units before 3.13.0);
+// defined outside the std::filesystem guard so that the result does not depend
+// on whether std::filesystem::path is supported
 #if defined(__cpp_lib_char8_t)
 template<typename BasicJsonType, typename Tr, typename Allocator>
 inline void to_json(BasicJsonType& j, const std::basic_string<char8_t, Tr, Allocator>& s)
 {
-    using OtherAllocator = typename std::allocator_traits<Allocator>::template rebind_alloc<char>;
-    j = std::basic_string<char, std::char_traits<char>, OtherAllocator>(s.begin(), s.end(), s.get_allocator());
+    using string_t = typename BasicJsonType::string_t;
+    // char may alias any object; the code units are copied unchanged
+    j = string_t(reinterpret_cast<const char*>(s.data()), s.size());
 }
 #endif
 
+#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM
 // Workaround for MSVC 19.51 (and possibly later): in large cpp files, the compiler may fail to resolve with generic has_to_json (issue #4996)
 template<typename BasicJsonType>
 struct has_to_json<BasicJsonType, std_fs::path, void> : std::true_type {};

include/nlohmann/detail/input/input_adapters.hpp

@@ -1312,15 +1312,23 @@ struct can_compare_ne_either_order : std::integral_constant < bool,
     can_compare_ne_impl<IteratorType, SentinelType>::value ||
     can_compare_ne_reversed<IteratorType, SentinelType>::value > {};
 
-// std::nullptr_t is excluded explicitly: a literal `nullptr` passed as a
-// trailing default argument (e.g. parse(s, nullptr, ...)) must never be
-// mistaken for a sentinel, and some compilers (e.g. GCC 4.8) unreliably
-// SFINAE the `operator!=` detection above for std::nullptr_t against
-// container/string types, which would otherwise make such calls ambiguous
-// with the compatible-input overload.
+// Two kinds of SentinelType are excluded explicitly:
+//
+// - std::nullptr_t: a literal `nullptr` passed as a trailing default argument
+//   (e.g. parse(s, nullptr, ...)) must never be mistaken for a sentinel, and
+//   some compilers (e.g. GCC 4.8) unreliably SFINAE the `operator!=` detection
+//   above for std::nullptr_t against container/string types, which would
+//   otherwise make such calls ambiguous with the compatible-input overload.
+// - function pointers and functions: a parser callback passed as plain function
+//   (e.g. parse(s, cb)) is never a sentinel. The callback's parameter types
+//   make basic_json an associated class, so ADL finds the friend
+//   operator!=(const_reference, ScalarType) of basic_json, which makes
+//   `s != cb` well-formed (both convert to basic_json) and would select the
+//   iterator/sentinel overload.
 template<typename IteratorType, typename SentinelType>
 struct can_compare_ne : std::integral_constant < bool,
     !std::is_same<SentinelType, std::nullptr_t>::value &&
+    !std::is_function<typename std::remove_pointer<SentinelType>::type>::value &&
     can_compare_ne_either_order<IteratorType, SentinelType>::value > {};
 
 template<typename T>

include/nlohmann/detail/json_pointer.hpp

@@ -1101,7 +1101,7 @@ class json_pointer
         /// @brief choose the seed of the hash function, before anything is added: it is derived from the address that is passed
         void reseed(const void* address) noexcept
         {
-            // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast): the address is only a source of entropy
+            // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast,hicpp-use-auto,modernize-use-auto): the address is only a source of entropy
             std::uint64_t x = reinterpret_cast<std::uintptr_t>(address);
             x ^= 0x9E3779B97F4A7C15ULL;
             x *= 0xBF58476D1CE4E5B9ULL;
@@ -1307,6 +1307,32 @@ class json_pointer
         }
     };
 
+    /*!
+    @brief the number of the deepest step of the previous path that is a prefix of the new one
+
+    Comparing every one of many steps with the beginning of the path would take
+    quadratic time. Instead, the length of the common beginning of the two paths
+    is computed once: a step whose prefix is not longer is a prefix of the new
+    path, too. For few steps, which is the usual case, @ref resolve_plain
+    compares them, which is cheaper.
+    */
+    template<typename Steps>
+    JSON_HEDLEY_NEVER_INLINE static std::size_t deepest_shared_step(const string_t& path, const std::size_t length, const string_t& previous, const Steps& steps) noexcept
+    {
+        const std::size_t same = common_prefix_length(path.data(), previous.data(), (std::min)(length, previous.size()));
+        std::size_t depth = steps.size();
+        for (; depth > 1; --depth)
+        {
+            // a step ends at a token boundary of the previous path
+            const std::size_t prefix = steps[depth - 1].first;
+            if (prefix <= same && (prefix == length || path[prefix] == '/'))
+            {
+                break;
+            }
+        }
+        return depth;
+    }
+
     /*!
     @brief resolve a plain JSON pointer without building its reference tokens
 
@@ -1335,13 +1361,20 @@ class json_pointer
         if (cache.path != nullptr && !steps.empty() && steps[0].second == root)
         {
             const string_t& previous = *cache.path;
-            for (depth = steps.size(); depth > 1; --depth)
+            if (JSON_HEDLEY_UNLIKELY(steps.size() > 16))
             {
-                // a step ends at a token boundary of the previous path
-                const std::size_t prefix = steps[depth - 1].first;
-                if (prefix <= length && (prefix == length || path[prefix] == '/') && std::memcmp(path.data(), previous.data(), prefix) == 0)
+                depth = deepest_shared_step(path, length, previous, steps);
+            }
+            else
+            {
+                for (depth = steps.size(); depth > 1; --depth)
                 {
-                    break;
+                    // a step ends at a token boundary of the previous path
+                    const std::size_t prefix = steps[depth - 1].first;
+                    if (prefix <= length && (prefix == length || path[prefix] == '/') && std::memcmp(path.data(), previous.data(), prefix) == 0)
+                    {
+                        break;
+                    }
                 }
             }
             steps.resize(depth);

include/nlohmann/detail/macro_scope.hpp

@@ -224,6 +224,12 @@
     #define JSON_NO_THREAD_LOCAL 1
 #endif
 
+// JSON_RECURSION_DEPTH_LIMIT (the number of nesting levels the walks over a
+// value recurse into before they continue with an explicit stack; 128 by
+// default, 0 to 254 allowed) is defined and checked where it is used, in
+// detail/recursion_depth_limit.hpp. Like JSON_NO_THREAD_LOCAL it is neither part
+// of the ABI tag nor undefined at the end of json.hpp.
+
 // allow disabling exceptions
 #if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION)
     #define JSON_THROW(exception) throw exception

include/nlohmann/detail/recursion_depth_limit.hpp

@@ -9,9 +9,22 @@
 #pragma once
 
 #include <cstddef> // size_t
+#include <type_traits> // false_type, is_integral, remove_cv, true_type
 
 #include <nlohmann/detail/abi_macros.hpp>
 
+// JSON_RECURSION_DEPTH_LIMIT: the number of nesting levels the operations that
+// walk a value recurse into on the call stack before they continue with an
+// explicit stack (see detail::recursion_depth_limit() below).
+//
+// May be defined by the user before the first include of the library; it has
+// to be the same in every translation unit of a program (it is not part of the
+// ABI tag, see https://json.nlohmann.me/api/macros/json_recursion_depth_limit/).
+// Allowed: an integer constant expression from 0 to 254. The default is 128.
+#ifndef JSON_RECURSION_DEPTH_LIMIT
+    #define JSON_RECURSION_DEPTH_LIMIT 128
+#endif
+
 NLOHMANN_JSON_NAMESPACE_BEGIN
 namespace detail
 {
@@ -20,21 +33,71 @@ namespace detail
 @brief the number of nesting levels an operation recurses into
 
 Operations that walk a value (copying, comparing, serializing, hashing, merging,
-...) recurse once
-per nesting level, which is fastest, but a value nested deeply enough would
-exhaust the call stack. So they recurse only this many levels deep and finish
-whatever lies below with an explicit stack. All of them share this limit.
+diffing, converting, destroying, and writing the binary formats CBOR,
+MessagePack, UBJSON, BJData, and BON8) recurse once per nesting level, which is
+fastest, but a value nested deeply enough would exhaust the call stack. So they
+recurse only this many levels deep and finish whatever lies below with an
+explicit stack that lives on the heap. All of them share this limit and none has
+a constant of its own. It is the value of the macro JSON_RECURSION_DEPTH_LIMIT,
+128 by default. (Parsing, the binary readers, flatten(), unflatten(), and
+to_bson() always use an explicit stack and do not read it, and the lockstep
+comparison of json_view documents has a fixed bound of its own, 48 levels.)
 
-Most of them pass the depth down as an argument. The copy constructor and the
-comparison operators cannot, as their signatures are fixed, so they count it
-in basic_json::nesting_depth() instead, a byte per thread; the limit must
-therefore stay below 255.
+Most of the operations pass the depth down as an argument. The copy constructor
+and the comparison operators cannot, as their signatures are fixed, so they
+count it in basic_json::nesting_depth() instead, a byte per thread. Only the
+levels that are descended into are counted, so the count never exceeds the limit.
+The limit must fit in the byte with some headroom, so that the count cannot wrap
+around: it must be below 255, and 254 is the largest value allowed. (More levels
+would hardly be useful: 254 levels need up to 576 KiB of stack in an unoptimized
+build already, twice the stack of the default.)
 
+The smallest value is 0: every operation then uses its explicit stack from the
+first level on and no operation recurses over the nesting of a value at all.
+That is slower, but needs a constant amount of call stack. It works because
+every use of the limit compares a depth with it (`depth >= limit`, or `<` for
+the opposite), which holds for 0 as for any other value; nothing computes
+`limit - 1` or `limit + 1`, so nothing wraps around. Each operation's explicit
+stack handles any nesting level, including the first, so none of them relies on
+having recursed first.
+
+The call stack that an operation needs is roughly a few KiB plus a cost per level
+times the limit. At the default of 128 levels, an unoptimized build needs up to
+about 290 KiB (`dump()` with Clang at -O0: about 2.3 KiB per level) and an
+optimized build up to about 48 KiB; with 254 levels it is up to 576 KiB and
+112 KiB, and with 0 it is 16 KiB or less. Lower the limit for threads with small
+stacks (RTOS tasks, fibers, musl threads), raise it to trade stack for speed on
+deeply nested values. All of this, with the measurements, is in the
+documentation of the macro.
+
+@sa https://json.nlohmann.me/api/macros/json_recursion_depth_limit/
 @sa https://github.com/nlohmann/json/issues/5387
 */
+/// the type of the value of JSON_RECURSION_DEPTH_LIMIT is bool or a character type
+template<typename T> struct recursion_depth_limit_is_char_or_bool : std::false_type {};
+template<> struct recursion_depth_limit_is_char_or_bool<bool> : std::true_type {};
+template<> struct recursion_depth_limit_is_char_or_bool<char> : std::true_type {};
+template<> struct recursion_depth_limit_is_char_or_bool<signed char> : std::true_type {};
+template<> struct recursion_depth_limit_is_char_or_bool<unsigned char> : std::true_type {};
+template<> struct recursion_depth_limit_is_char_or_bool<wchar_t> : std::true_type {};
+template<> struct recursion_depth_limit_is_char_or_bool<char16_t> : std::true_type {};
+template<> struct recursion_depth_limit_is_char_or_bool<char32_t> : std::true_type {};
+#if defined(__cpp_char8_t)
+template<> struct recursion_depth_limit_is_char_or_bool<char8_t> : std::true_type {};
+#endif
+
 constexpr std::size_t recursion_depth_limit() noexcept
 {
-    return 128;
+    // an integer: not a floating-point number, whose range check would pass for 1.5 or 254.0
+    // (bool and the character types are integer types, too, but true, false or 'a' are
+    // no way to say a limit; JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE refuses them as well)
+    static_assert(std::is_integral<decltype(JSON_RECURSION_DEPTH_LIMIT)>::value
+                  && !recursion_depth_limit_is_char_or_bool<typename std::remove_cv<decltype(JSON_RECURSION_DEPTH_LIMIT)>::type>::value,
+                  "JSON_RECURSION_DEPTH_LIMIT must be an integer constant (not bool or a character type)");
+    static_assert(JSON_RECURSION_DEPTH_LIMIT >= 0 && JSON_RECURSION_DEPTH_LIMIT <= 254,
+                  "JSON_RECURSION_DEPTH_LIMIT must be between 0 and 254: the nesting depth is counted in a byte per thread "
+                  "and may exceed the limit by one (see https://json.nlohmann.me/api/macros/json_recursion_depth_limit/)");
+    return static_cast<std::size_t>(JSON_RECURSION_DEPTH_LIMIT);
 }
 
 }  // namespace detail

include/nlohmann/detail/view/compare.hpp

@@ -79,6 +79,14 @@ NLOHMANN_VIEW_ALWAYS_INLINE bool bytes_equal(const char* x, const char* y, std::
     return true;
 }
 
+/// whether a float node holds the text nan, which an edit writes for a NaN (and records without a digit layout, so
+/// that a parsed float is told apart by one comparison of what the walk has loaded anyway): equal tokens are an
+/// equal value, except for this one
+NLOHMANN_VIEW_ALWAYS_INLINE bool float_token_is_nan(const node& n, const char* s) noexcept
+{
+    return NLOHMANN_VIEW_UNLIKELY(n.extra == 0xFFFFu) && n.len == 3 && s[0] == 'n';
+}
+
 /// whether the n1 bytes at x are before the n2 bytes at y (memcmp() order, then the shorter first);
 /// the first difference is found eight bytes at a time
 NLOHMANN_VIEW_ALWAYS_INLINE bool bytes_less(const char* x, std::size_t n1, const char* y, std::size_t n2) noexcept
@@ -688,7 +696,7 @@ class lockstep
             case value_t::string:
                 return a->len == b->len && bytes_equal(m_a.str(*a), m_b.str(*b), a->len);
             case value_t::number_float: // (the token, and the digit layout that the value is read with: they only differ in a damaged image)
-                return a->extra == b->extra && a->len == b->len && bytes_equal(m_a.str(*a), m_b.str(*b), a->len);
+                return same_float(a, b, std::integral_constant<bool, EA> {});
             case value_t::number_integer:
             case value_t::number_unsigned:
                 return integer_bits(*a) == integer_bits(*b);
@@ -783,6 +791,17 @@ class lockstep
         return true;
     }
 
+    /// (a read-only document holds no NaN)
+    bool same_float(const node* a, const node* b, std::false_type /*editable*/) const noexcept
+    {
+        return a->extra == b->extra && a->len == b->len && bytes_equal(m_a.str(*a), m_b.str(*b), a->len);
+    }
+
+    bool same_float(const node* a, const node* b, std::true_type /*editable*/) const noexcept
+    {
+        return a->extra == b->extra && a->len == b->len && bytes_equal(m_a.str(*a), m_b.str(*b), a->len) && !float_token_is_nan(*a, m_a.str(*a));
+    }
+
     bool elements(const node* a, const node* b, unsigned budget, lockstep_path::entry& stop) noexcept
     {
         const node* pa = nav_a::first(m_a, a);
@@ -839,9 +858,9 @@ bool certainly_equal(const view_side<BasicJsonType, ViewA>& a, const view_side<B
     {
         return false;
     }
-    if (na == nb)
+    if (na == nb && (!view_side<BasicJsonType, ViewA>::nav::editable() || a.document().edits == nullptr))
     {
-        return true; // (the same node of the same document: documents have distinct nodes)
+        return true; // (the same node of the same document: documents have distinct nodes; a document with edits is walked, as an edit can have written a NaN, which is not equal to itself)
     }
     path.resume = nullptr;
     if (path.next < path.length && path.nodes[path.next].a == na && path.nodes[path.next].b == nb)
@@ -886,7 +905,9 @@ void resume_array(const lockstep_path& /*path*/, const A& /*a*/, const B& /*b*/,
 template<typename BasicJsonType, typename ViewA, typename ViewB>
 bool floats_equal(const view_side<BasicJsonType, ViewA>& a, const view_side<BasicJsonType, ViewB>& b)
 {
-    return (a.value_node()->extra == b.value_node()->extra && a.float_token() == b.float_token()) || std::equal_to<typename BasicJsonType::number_float_t>()(a.floating(), b.floating());
+    // float_token_is_nan() reads at most the node length of the token, which is not null terminated
+    // NOLINTNEXTLINE(bugprone-suspicious-stringview-data-usage)
+    return (a.value_node()->extra == b.value_node()->extra && a.float_token() == b.float_token() && !(view_side<BasicJsonType, ViewA>::nav::editable() && float_token_is_nan(*a.value_node(), a.float_token().data()))) || std::equal_to<typename BasicJsonType::number_float_t>()(a.floating(), b.floating());
 }
 
 template<typename A, typename B>

include/nlohmann/detail/view/document_data.hpp

@@ -12,6 +12,7 @@
 #include <cstddef> // size_t
 #include <cstdint> // uint8_t, uint32_t
 #include <cstring> // memcmp, memcpy
+#include <functional> // less
 #include <limits> // numeric_limits
 #include <map> // map
 #include <memory> // unique_ptr
@@ -149,6 +150,24 @@ struct document_data
         node_capacity = inline_capacity;
     }
 
+    /// whether the n bytes at p overlap memory that the next parse releases or
+    /// reuses: the owned text, the decoded strings, an owned image, and the
+    /// current buffer of strings written by edits (earlier edit buffers are not
+    /// covered, as their sizes are not kept); for the debug check in read()
+    bool holds(const char* p, std::size_t n) const noexcept
+    {
+        const auto overlaps = [p, n](const char* b, std::size_t m) noexcept
+        {
+            // (std::less: a total order also for pointers into different objects; std::less<> needs C++14)
+            // NOLINTNEXTLINE(modernize-use-transparent-functors)
+            return n != 0 && m != 0 && std::less<const char*>()(p, b + m) && std::less<const char*>()(b, p + n);
+        };
+        return overlaps(owned.data(), owned.capacity())
+               || overlaps(arena.data(), arena.capacity())
+               || overlaps(reinterpret_cast<const char*>(owned_image.data()), owned_image.capacity()) // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
+               || (edits && !edits->texts.empty() && overlaps(edits->texts.back().get(), edits->text_cap));
+    }
+
     /// the largest node count whose size in bytes fits a std::size_t
     static constexpr std::size_t max_nodes() noexcept
     {

include/nlohmann/detail/view/image.hpp

@@ -416,6 +416,28 @@ inline bool float_token_finite(const unsigned char* s, std::size_t len, const nu
     return true;
 }
 
+/// whether an integer token (well-formed, no fraction or exponent) fits 64
+/// bits: the parser stores a number_integer or number_unsigned node for it,
+/// and a number_float node only for a token that does not fit
+inline bool integer_token_fits(const number_token& t)
+{
+    if (t.int_digits > 20)
+    {
+        return false;
+    }
+    if (t.int_digits == 20)
+    {
+        // (20 digits are compared with 2^64 - 1; a negative one is below -2^63)
+        // (only a leading 1 needs the comparison: the first digit of 20 is not 0)
+        return !t.negative && *t.int_start == '1' && std::memcmp(t.int_start, "18446744073709551615", 20) <= 0;
+    }
+    if (t.negative && t.int_digits == 19)
+    {
+        return *t.int_start != '9' || std::memcmp(t.int_start, "9223372036854775808", 19) <= 0;
+    }
+    return true;
+}
+
 /// whether an integer node matches its token the way the parser records it
 /// (after the bounds check; the token has at most 256 characters)
 inline bool check_integer(const node& n, const unsigned char* text)
@@ -535,7 +557,8 @@ inline bool check_float_ranges(std::vector<float_range>& ranges, const unsigned
         if (!plain_float_token(s, r.len, r.extra))
         {
             number_token t{};
-            if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t))
+            // (an integer token is a float node only if it does not fit 64 bits)
+            if (!scan_number_token(s, r.len, t) || (!t.is_float && integer_token_fits(t)) || !float_token_finite(s, r.len, t))
             {
                 return false;
             }

include/nlohmann/detail/view/macro_scope.hpp

@@ -71,6 +71,11 @@
     #define NLOHMANN_VIEW_THROW JSON_THROW_USER
 #endif
 
+// debug checks of preconditions (assert(), off with NDEBUG): JSON_ASSERT is
+// one of the macros that json.hpp undefines at its end
+#include <cassert> // assert
+#define NLOHMANN_VIEW_ASSERT(x) assert(x)
+
 // the parser stores a node's first word at once where the layout of `node` is
 // known to be little-endian (MSVC targets are); elsewhere field by field
 #if (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) || defined(_MSC_VER)

include/nlohmann/detail/view/macro_unscope.hpp

@@ -18,6 +18,7 @@
 #undef NLOHMANN_VIEW_NOCLONE
 #undef NLOHMANN_VIEW_NODISCARD
 #undef NLOHMANN_VIEW_THROW
+#undef NLOHMANN_VIEW_ASSERT
 #undef NLOHMANN_VIEW_LITTLE_ENDIAN
 #undef NLOHMANN_VIEW_REPEAT16
 #undef NLOHMANN_VIEW_REPEAT_SECOND_8