Hold the token carry buffer in a std::unique_ptr

clang-tidy rejected the raw new[]/delete[] of token_carry_buffer
(cppcoreguidelines-owning-memory) and the capacity computation
(readability-use-std-min-max). The storage is now owned by a
std::unique_ptr<char[]>, as in json_view's edit storage, and the new
capacity is a std::max.

istream parse instructions (Apple clang, twitter, citm, canada, floats)
are unchanged within noise.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/binary_reader.hpp

@@ -86,7 +86,7 @@ JSON_INLINE_VARIABLE constexpr std::size_t max_valueless_container_size = 1 << 2
 /*!
 @brief deserialization of BJData, BON8, BSON, CBOR, MessagePack, and UBJSON values
 */
-template<typename BasicJsonType, typename InputAdapterType, typename SAX = json_sax_dom_binary_parser<BasicJsonType, InputAdapterType>>
+template<typename BasicJsonType, typename InputAdapterType, typename SAX = json_sax_dom_parser<BasicJsonType, InputAdapterType>>
 class binary_reader
 {
     using number_integer_t = typename BasicJsonType::number_integer_t;

include/nlohmann/detail/input/input_adapters.hpp

@@ -8,17 +8,18 @@
 
 #pragma once
 
-#include <algorithm> // count, find, min
+#include <algorithm> // count, find, max, min
 #include <array> // array
 #include <cstddef> // size_t
 #include <cstdint> // uint32_t
 #include <cstring> // strlen
 #include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next, reverse_iterator
 #include <limits> // numeric_limits
+#include <memory> // unique_ptr
 #include <streambuf> // streambuf
 #include <string> // string, char_traits
 #include <type_traits> // enable_if, is_base_of, is_pointer, is_integral, remove_pointer
-#include <utility> // pair, declval
+#include <utility> // declval, move, pair
 #include <vector> // vector
 
 #ifndef JSON_NO_IO
@@ -72,6 +73,81 @@ enum class input_format_t { json, cbor, msgpack, ubjson, bson, bjdata, bon8 };
 
 #ifndef JSON_NO_IO
 /*!
+A growable array of characters for the part of a token that an input adapter
+copied out of its window: unlike std::vector<char>::push_back(), appending a
+character takes no reference to a local, and is a few instructions when there
+is room.
+*/
+class token_carry_buffer
+{
+  public:
+    token_carry_buffer() = default;
+    token_carry_buffer(const token_carry_buffer&) = delete;
+    token_carry_buffer& operator=(const token_carry_buffer&) = delete;
+    token_carry_buffer(token_carry_buffer&& rhs) noexcept
+        : m_data(std::move(rhs.m_data)), m_end(rhs.m_end), m_capacity_end(rhs.m_capacity_end)
+    {
+        rhs.m_end = rhs.m_capacity_end = nullptr;
+    }
+    token_carry_buffer& operator=(token_carry_buffer&&) = delete;
+    ~token_carry_buffer() = default;
+
+    void clear() noexcept
+    {
+        m_end = m_data.get();
+    }
+
+    void push_back(char c)
+    {
+        if (JSON_HEDLEY_UNLIKELY(m_end == m_capacity_end))
+        {
+            grow(1);
+        }
+        *m_end++ = c;
+    }
+
+    void append(const char* first, const char* last)
+    {
+        const auto n = static_cast<std::size_t>(last - first);
+        if (n > static_cast<std::size_t>(m_capacity_end - m_end))
+        {
+            grow(n);
+        }
+        if (n != 0)
+        {
+            std::memcpy(m_end, first, n);
+        }
+        m_end += n;
+    }
+
+    /// replace the contents of @a out
+    void copy_to(std::vector<char>& out) const
+    {
+        out.assign(m_data.get(), m_end);
+    }
+
+  private:
+    /// make room for @a extra more characters
+    JSON_HEDLEY_NEVER_INLINE void grow(std::size_t extra)
+    {
+        const auto size = static_cast<std::size_t>(m_end - m_data.get());
+        const auto capacity = static_cast<std::size_t>(m_capacity_end - m_data.get());
+        const std::size_t new_capacity = (std::max)(capacity < 32 ? static_cast<std::size_t>(32) : 2 * capacity, size + extra);
+        std::unique_ptr<char[]> data(new char[new_capacity]); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
+        if (size != 0)
+        {
+            std::memcpy(data.get(), m_data.get(), size);
+        }
+        m_data = std::move(data);
+        m_end = m_data.get() + size;
+        m_capacity_end = m_data.get() + new_capacity;
+    }
+
+    std::unique_ptr<char[]> m_data{}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,readability-redundant-member-init)
+    char* m_end = nullptr;
+    char* m_capacity_end = nullptr;
+};
+/*!
 Input adapter for stdio file access. This adapter read only 1 byte and do not use any
  buffer. This adapter is a very low level adapter.
 */
@@ -209,6 +285,7 @@ class input_stream_adapter
         , token_segment(rhs.token_segment)
         , current_in_get_area(rhs.current_in_get_area)
         , tracking_token(rhs.tracking_token)
+        , no_get_area(rhs.no_get_area)
 #endif
     {
         rhs.is = nullptr;
@@ -295,37 +372,20 @@ class input_stream_adapter
     std::char_traits<char>::int_type get_character()
     {
 #if !JSON_DISABLE_STREAM_BULK_SCAN
-        // sbumpc() refills the get area if it is exhausted
-        const bool refill = streambuf_access::get_ptr(sb) == streambuf_access::get_end(sb);
-        if (JSON_HEDLEY_UNLIKELY(refill))
-        {
-            save_token_segment();
-        }
-#endif
-
-        auto res = sb->sbumpc();
-        // set eof manually, as we don't use the istream interface.
-        if (JSON_HEDLEY_UNLIKELY(res == std::char_traits<char>::eof()))
-        {
-#if !JSON_DISABLE_STREAM_BULK_SCAN
-            current_in_get_area = false;
-#endif
-            is->clear(is->rdstate() | std::ios::eofbit);
-        }
-#if !JSON_DISABLE_STREAM_BULK_SCAN
-        else if (JSON_HEDLEY_UNLIKELY(refill))
-        {
-            // the consumed character is at gptr() - 1, unless the streambuf
-            // has no get area and returned it from uflow()
-            const char* const next = streambuf_access::get_ptr(sb);
-            after_refill(res, next != nullptr && next != streambuf_access::get_begin(sb));
-        }
-        else
+        // the character is in the get area: step over it, as sbumpc() does
+        const char* const next = streambuf_access::get_ptr(sb);
+        if (JSON_HEDLEY_LIKELY(next != streambuf_access::get_end(sb)))
         {
             current_in_get_area = true;
+            streambuf_access::bump(sb, 1);
+            return std::char_traits<char>::to_int_type(*next);
+        }
+        if (no_get_area)
+        {
+            return get_character_no_get_area();
         }
 #endif
-        return res;
+        return get_character_slow();
     }
 #endif
 
@@ -334,6 +394,18 @@ class input_stream_adapter
     {
 #if JSON_PRECISE_STREAM_POSITION
         commit_lookahead();
+#endif
+#if !JSON_DISABLE_STREAM_BULK_SCAN
+        // the elements are in the get area: copy them, as sgetn() does, but
+        // without the virtual call
+        const char* const first = streambuf_access::get_ptr(sb);
+        const auto available = static_cast<std::size_t>(streambuf_access::get_end(sb) - first);
+        if (JSON_HEDLEY_LIKELY(count * sizeof(T) <= available && count * sizeof(T) <= static_cast<std::size_t>((std::numeric_limits<int>::max)())))
+        {
+            std::memcpy(dest, first, count * sizeof(T));
+            streambuf_access::bump(sb, static_cast<int>(count * sizeof(T)));
+            return count * sizeof(T);
+        }
 #endif
         auto res = static_cast<std::size_t>(sb->sgetn(reinterpret_cast<char*>(dest), static_cast<std::streamsize>(count * sizeof(T))));
         if (JSON_HEDLEY_UNLIKELY(res < count * sizeof(T)))
@@ -421,7 +493,7 @@ class input_stream_adapter
     // last one if @a drop_last.
     void copy_token(std::vector<char>& out, bool drop_last) const
     {
-        out = token_carry;
+        token_carry.copy_to(out);
         if (token_segment != nullptr)
         {
             out.insert(out.end(), token_segment, current_end());
@@ -434,6 +506,47 @@ class input_stream_adapter
 #endif
 
   private:
+#if !JSON_PRECISE_STREAM_POSITION
+    // the get area is exhausted, or there is none
+#if !JSON_DISABLE_STREAM_BULK_SCAN
+    JSON_HEDLEY_NEVER_INLINE
+#endif
+    std::char_traits<char>::int_type get_character_slow()
+    {
+#if !JSON_DISABLE_STREAM_BULK_SCAN
+        // sbumpc() refills the get area if it is exhausted
+        const bool refill = streambuf_access::get_ptr(sb) == streambuf_access::get_end(sb);
+        if (JSON_HEDLEY_UNLIKELY(refill))
+        {
+            save_token_segment();
+        }
+#endif
+
+        auto res = sb->sbumpc();
+        // set eof manually, as we don't use the istream interface.
+        if (JSON_HEDLEY_UNLIKELY(res == std::char_traits<char>::eof()))
+        {
+#if !JSON_DISABLE_STREAM_BULK_SCAN
+            current_in_get_area = false;
+#endif
+            is->clear(is->rdstate() | std::ios::eofbit);
+        }
+#if !JSON_DISABLE_STREAM_BULK_SCAN
+        else if (JSON_HEDLEY_UNLIKELY(refill))
+        {
+            // the consumed character is at gptr() - 1, unless the streambuf
+            // has no get area and returned it from uflow()
+            const char* const next = streambuf_access::get_ptr(sb);
+            after_refill(res, next != nullptr && next != streambuf_access::get_begin(sb));
+        }
+        else
+        {
+            current_in_get_area = true;
+        }
+#endif
+        return res;
+    }
+#endif
 #if !JSON_DISABLE_STREAM_BULK_SCAN
     // Reads the protected get-area pointers of any std::streambuf. Naming the
     // members through a derived class makes forming the member pointers
@@ -457,6 +570,11 @@ class input_stream_adapter
         {
             (s->*(&streambuf_access::gbump))(n);
         }
+        // sbumpc() for an exhausted get area
+        static std::streambuf::int_type next_character(std::streambuf* s)
+        {
+            return (s->*(&streambuf_access::uflow))();
+        }
     };
 
     // one past the character last returned by get_character(), if that is in
@@ -477,16 +595,53 @@ class input_stream_adapter
     {
         if (token_segment != nullptr)
         {
-            token_carry.insert(token_carry.end(), token_segment, streambuf_access::get_end(sb));
+            token_carry.append(token_segment, streambuf_access::get_end(sb));
             token_segment = nullptr;
         }
     }
 
+#if !JSON_PRECISE_STREAM_POSITION
+    // The streambuf has no get area (e.g., std::cin synchronized with stdio):
+    // every character comes from uflow(), and the token is kept by value.
+    JSON_HEDLEY_NEVER_INLINE std::char_traits<char>::int_type get_character_no_get_area()
+    {
+        // the get area is exhausted, so this is what sbumpc() does
+        const auto res = streambuf_access::next_character(sb);
+        if (JSON_HEDLEY_UNLIKELY(res == std::char_traits<char>::eof() || streambuf_access::get_end(sb) != nullptr))
+        {
+            return get_character_no_get_area_other(res);
+        }
+        // current_in_get_area is false: only the first character of a get
+        // area, or the end of the input, is read here
+        if (tracking_token)
+        {
+            token_carry.push_back(std::char_traits<char>::to_char_type(res));
+        }
+        return res;
+    }
+
+    // @a res came from uflow() of a streambuf without a get area: it is the
+    // end of the input, or the streambuf has a get area now
+    JSON_HEDLEY_NEVER_INLINE std::char_traits<char>::int_type get_character_no_get_area_other(std::char_traits<char>::int_type res)
+    {
+        if (res == std::char_traits<char>::eof())
+        {
+            current_in_get_area = false;
+            is->clear(is->rdstate() | std::ios::eofbit);
+            return res;
+        }
+        // the consumed character is at gptr() - 1
+        after_refill(res, streambuf_access::get_ptr(sb) != streambuf_access::get_begin(sb));
+        return res;
+    }
+#endif
+
     // character @a c was just read into a new get area (@a in_get_area) or
     // returned without one
     void after_refill(std::char_traits<char>::int_type c, bool in_get_area)
     {
         current_in_get_area = in_get_area;
+        no_get_area = !in_get_area;
         if (!tracking_token)
         {
             return;
@@ -525,13 +680,16 @@ class input_stream_adapter
 #endif
 #if !JSON_DISABLE_STREAM_BULK_SCAN
     /// the part of the current token that was in earlier get areas
-    std::vector<char> token_carry{}; // NOLINT(readability-redundant-member-init)
+    token_carry_buffer token_carry{}; // NOLINT(readability-redundant-member-init)
     /// start of the current token's part in the get area, or nullptr
     const char* token_segment = nullptr;
     /// whether the character last returned by get_character() is in the get area
     bool current_in_get_area = false;
     /// whether a token is tracked at all; the binary readers never start one
     bool tracking_token = false;
+    /// whether the streambuf had no get area when it was last asked for a
+    /// character
+    bool no_get_area = false;
 #endif
 };
 #endif  // JSON_NO_IO

include/nlohmann/detail/input/lexer.hpp

@@ -14,7 +14,6 @@
 #include <cstdio> // snprintf
 #include <cstring> // memcmp
 #include <initializer_list> // initializer_list
-#include <limits> // numeric_limits
 #include <string> // char_traits, string
 #include <utility> // move
 #include <vector> // vector
@@ -1817,15 +1816,6 @@ scan_number_done:
     /// significand (see scan_number_bulk_contiguous())
     bool convert_accumulated_float(const float_significand& s, std::true_type /*native*/) noexcept
     {
-        // accept() only needs to know that the value is finite (see the parser):
-        // w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a
-        // value that underflows is a finite number as well. Only a token that
-        // may overflow has to be converted to find out.
-        if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
-        {
-            value_float = 0;
-            return true;
-        }
         value_float = decimal_to_float<number_float_t>(s);
         return true;
     }

include/nlohmann/detail/input/parser.hpp

@@ -250,13 +250,6 @@ class parser
         // value to avoid a goto (see comment where set to true)
         bool skip_to_state_evaluation = false;
 
-        // a document that starts with a container pushes onto the stack at
-        // once: allocate it in one step instead of growing it from 1 byte
-        if (last_token == token_type::begin_object || last_token == token_type::begin_array)
-        {
-            states.reserve(32);
-        }
-
         while (true)
         {
             if (!skip_to_state_evaluation)

include/nlohmann/json.hpp

@@ -161,8 +161,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     template<typename BasicJsonType, typename InputAdapterType>
     friend class ::nlohmann::detail::json_sax_dom_parser;
     template<typename BasicJsonType, typename InputAdapterType>
-    friend class ::nlohmann::detail::json_sax_dom_binary_parser;
-    template<typename BasicJsonType, typename InputAdapterType>
     friend class ::nlohmann::detail::json_sax_dom_callback_parser;
 #if JSON_DIAGNOSTIC_POSITIONS
     friend struct ::nlohmann::detail::diagnostic_positions;
@@ -6289,7 +6287,7 @@ public:
            from_ubjson/from_bjdata/from_bon8/from_bson overloads
 
     Building the @ref binary_reader as a named local, rather than as a
-    temporary that @ref detail::json_sax_dom_binary_parser::parse is called on in
+    temporary that @ref detail::json_sax_dom_parser::parse is called on in
     the same expression, avoids a false-positive cppcheck accessMoved
     warning in each of the 16 callers.
     */
@@ -6300,7 +6298,7 @@ public:
                                        const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error)
     {
         basic_json result;
-        detail::json_sax_dom_binary_parser<basic_json, InputAdapterType> sdp(result, allow_exceptions);
+        detail::json_sax_dom_parser<basic_json, InputAdapterType> sdp(result, allow_exceptions);
         binary_reader<InputAdapterType> reader(std::move(ia), format, error_handler);
         if (!reader.sax_parse(&sdp, strict, tag_handler))
         {

tests/src/unit-class_parser.cpp

@@ -2952,32 +2952,3 @@ TEST_CASE("diagnostic positions: value lifetime, input adapters, and SAX")
     }
 }
 #endif
-
-TEST_CASE("accept() agrees with parse() on floats around the overflow limit")
-{
-    // accept() does not convert a float that cannot overflow; the tokens
-    // around the limit (and beyond it) must still be rejected exactly as
-    // parse() rejects them
-    const char* const numbers[] =
-    {
-        "1e0", "123.456e-5", "1e-400", "-1e-400", "1e307", "1e308", "1e309", "-1e309",
-        "10e307", "100e307", "0.1e309", "0.1e310", "1E+308", "1E+309",
-        "1.7976931348623157e308", "1.7976931348623159e308", "1.8e308", "-1.7976931348623157e308",
-        "12345678901234567e291", "12345678901234567e292", "123456789012345678e290", "123456789012345678e291",
-        "1234567890123456789e289", "1234567890123456789e290", "9999999999999999999e289", "9999999999999999999e290",
-        "0.00000000000000000001e328", "0.00000000000000000001e329"
-    };
-
-    for (const char* number : numbers)
-    {
-        for (const std::string text :
-                {
-                    std::string(number), "[" + std::string(number) + "]", "{\"a\":" + std::string(number) + "}"
-                })
-        {
-            INFO(text);
-            const bool parses = !json::parse(text, nullptr, false).is_discarded();
-            CHECK(json::accept(text) == parses);
-        }
-    }
-}

tests/src/unit-ordered_json.cpp

@@ -237,274 +237,3 @@ TEST_CASE("regression test for issue #5673 - ordered_json::emplace with a non-rv
         CHECK(oj.dump() == "{\"a\":1}");
     }
 }
-
-namespace
-{
-// the sizes around which the DOM parser changes how it stores the elements of
-// an array (16) and the members of an object in a vector (32, and 64 for the
-// key index)
-const std::size_t container_sizes[] = {0, 1, 2, 3, 15, 16, 17, 18, 31, 32, 33, 34, 63, 64, 65, 66, 200};
-
-template<typename Json>
-std::string array_text(const std::size_t n, const std::size_t depth)
-{
-    std::string text = "[";
-    for (std::size_t i = 0; i < n; ++i)
-    {
-        text += (i == 0 ? "" : ",");
-        // alternate scalars, empty containers and nested containers
-        switch (i % 4)
-        {
-            case 0:
-                text += std::to_string(i);
-                break;
-            case 1:
-                text += "\"s" + std::to_string(i) + "\"";
-                break;
-            case 2:
-                text += depth == 0 ? "[]" : array_text<Json>(n / 2, depth - 1);
-                break;
-            default:
-                text += depth == 0 ? "{}" : "{\"a\":" + array_text<Json>(n / 3, depth - 1) + ",\"b\":null}";
-                break;
-        }
-    }
-    return text + "]";
-}
-
-template<typename Json>
-std::string object_text(const std::size_t n, const std::size_t depth, const bool duplicates)
-{
-    std::string text = "{";
-    for (std::size_t i = 0; i < n; ++i)
-    {
-        text += (i == 0 ? "" : ",");
-        text += "\"key" + std::to_string(i) + "\":";
-        switch (i % 4)
-        {
-            case 0:
-                text += std::to_string(i);
-                break;
-            case 1:
-                text += depth == 0 ? "[]" : array_text<Json>(n / 2, depth - 1);
-                break;
-            case 2:
-                text += depth == 0 ? "{}" : object_text<Json>(n / 2, depth - 1, duplicates);
-                break;
-            default:
-                text += "\"value\"";
-                break;
-        }
-    }
-    if (duplicates)
-    {
-        // keys that are there already, the first and the last one, with other values
-        text += n == 0 ? "" : ",";
-        text += "\"key0\":[1,2],\"key" + std::to_string(n - 1) + "\":{\"again\":true}";
-    }
-    return text + "}";
-}
-
-template<typename Json>
-void check_same_dump(const std::string& text)
-{
-    // the text is what dump() writes for the parsed value, and the value
-    // survives a second round trip: arrays keep their elements in order, an
-    // ordered_json keeps its members in order of appearance
-    const Json parsed = Json::parse(text);
-    const Json again = Json::parse(parsed.dump());
-    CHECK(parsed == again);
-    CHECK(parsed.dump() == again.dump());
-    CHECK(Json::accept(text));
-}
-} // namespace
-
-TEST_CASE("parsing containers of sizes around the limits of the DOM parser")
-{
-    SECTION("arrays")
-    {
-        for (const std::size_t n : container_sizes)
-        {
-            const std::string text = array_text<json>(n, 2);
-            INFO("size ", n);
-
-            const json parsed = json::parse(text);
-            CHECK(parsed.is_array());
-            CHECK(parsed.size() == n);
-            for (std::size_t i = 0; i < n; i += 4)
-            {
-                CHECK(parsed[i] == i);
-            }
-            check_same_dump<json>(text);
-            check_same_dump<ordered_json>(text);
-            CHECK(ordered_json::parse(text).dump() == parsed.dump());
-        }
-    }
-
-    SECTION("objects")
-    {
-        for (const bool duplicates :
-                {
-                    false, true
-                })
-        {
-            for (const std::size_t n : container_sizes)
-            {
-                const std::string text = object_text<json>(n, 2, duplicates);
-                INFO("size ", n, " duplicates ", duplicates);
-
-                const json parsed = json::parse(text);
-                const ordered_json ordered = ordered_json::parse(text);
-                CHECK(parsed.is_object());
-                CHECK(ordered.is_object());
-                CHECK(parsed.size() == ordered.size());
-                // the same members, in the order of their first appearance
-                std::size_t i = 0;
-                for (auto it = ordered.begin(); it != ordered.end(); ++it, ++i)
-                {
-                    if (i < n)
-                    {
-                        CHECK(it.key() == "key" + std::to_string(i));
-                    }
-                    CHECK(parsed.at(it.key()) == json(it.value()));
-                }
-                if (duplicates && n > 1)
-                {
-                    // the later value replaced the earlier one in its place
-                    CHECK(ordered.at("key0") == ordered_json::parse("[1,2]"));
-                    CHECK(ordered.at("key" + std::to_string(n - 1)) == ordered_json::parse("{\"again\":true}"));
-                    CHECK(ordered.begin().key() == "key0");
-                    CHECK(ordered.size() == n);
-                }
-                check_same_dump<json>(text);
-                check_same_dump<ordered_json>(text);
-            }
-        }
-    }
-
-    SECTION("objects with many different and repeated keys")
-    {
-        // 300 members of which a third repeat an earlier key
-        std::string text = "{";
-        for (std::size_t i = 0; i < 300; ++i)
-        {
-            text += (i == 0 ? "" : ",");
-            text += "\"k" + std::to_string(i % 200) + "\":" + std::to_string(i);
-        }
-        text += "}";
-
-        const ordered_json ordered = ordered_json::parse(text);
-        const json parsed = json::parse(text);
-        CHECK(ordered.size() == 200);
-        CHECK(parsed.size() == 200);
-        std::size_t i = 0;
-        for (auto it = ordered.begin(); it != ordered.end(); ++it, ++i)
-        {
-            CHECK(it.key() == "k" + std::to_string(i));
-            // the last value written
-            CHECK(it.value() == (i < 100 ? i + 200 : i));
-            CHECK(parsed.at(it.key()) == json(it.value()));
-        }
-    }
-}
-
-namespace
-{
-// a deterministic source of structure for parsing tests: builds a JSON text
-// and, at the same time, the value it has to parse to
-class tree_generator
-{
-  public:
-    explicit tree_generator(std::uint32_t seed) : state(seed) {}
-
-    template<typename Json>
-    Json make(std::string& text, const std::size_t depth)
-    {
-        switch (next(depth == 0 ? 5 : 9))
-        {
-            case 0:
-                text += "null";
-                return Json(nullptr);
-            case 1:
-                text += "true";
-                return Json(true);
-            case 2:
-            {
-                const int n = static_cast<int>(next(2000)) - 1000;
-                text += std::to_string(n);
-                return Json(n);
-            }
-            case 3:
-                text += "\"text\"";
-                return Json("text");
-            case 4:
-                text += "2.5";
-                return Json(2.5);
-            case 5:
-            case 6:
-            case 7:
-            {
-                Json array = Json::array();
-                text += "[";
-                const std::size_t n = size();
-                for (std::size_t i = 0; i < n; ++i)
-                {
-                    text += (i == 0 ? "" : ",");
-                    array.push_back(make<Json>(text, depth - 1));
-                }
-                text += "]";
-                return array;
-            }
-            default:
-            {
-                Json object = Json::object();
-                text += "{";
-                const std::size_t n = size();
-                for (std::size_t i = 0; i < n; ++i)
-                {
-                    const std::string key = "k" + std::to_string(i);
-                    text += (i == 0 ? "\"" : ",\"") + key + "\":";
-                    object[key] = make<Json>(text, depth - 1);
-                }
-                text += "}";
-                return object;
-            }
-        }
-    }
-
-  private:
-    std::uint32_t next(const std::uint32_t bound)
-    {
-        state = state * 1664525U + 1013904223U;
-        return (state >> 8) % bound;
-    }
-
-    std::size_t size()
-    {
-        return container_sizes[next(sizeof(container_sizes) / sizeof(*container_sizes))];
-    }
-
-    std::uint32_t state;
-};
-} // namespace
-
-TEST_CASE("parsing random trees of arrays and objects")
-{
-    for (std::uint32_t seed = 1; seed <= 150; ++seed)
-    {
-        INFO("seed ", seed);
-
-        std::string text;
-        std::string ordered_text;
-        const json expected = tree_generator(seed).make<json>(text, 3);
-        const ordered_json ordered_expected = tree_generator(seed).make<ordered_json>(ordered_text, 3);
-        CHECK(text == ordered_text);
-
-        CHECK(json::parse(text) == expected);
-        CHECK(json::accept(text));
-        const ordered_json ordered = ordered_json::parse(text);
-        CHECK(ordered == ordered_expected);
-        CHECK(ordered.dump() == ordered_expected.dump());
-        CHECK(json::parse(text).dump() == expected.dump());
-    }
-}

tests/src/unit-stream-bulk-scan.cpp

@@ -48,14 +48,16 @@ namespace
 class chunked_streambuf : public std::streambuf
 {
   public:
-    chunked_streambuf(std::string s, std::size_t chunk_size)
-        : m_data(std::move(s)), m_chunk_size(chunk_size)
+    // The first @a unbuffered characters are served without a get area, the
+    // rest in get areas of @a chunk_size.
+    chunked_streambuf(std::string s, std::size_t chunk_size, std::size_t unbuffered = 0)
+        : m_data(std::move(s)), m_chunk_size(chunk_size), m_unbuffered(unbuffered)
     {}
 
   protected:
     int_type underflow() override
     {
-        if (m_chunk_size == 0)
+        if (m_chunk_size == 0 || m_pos < m_unbuffered)
         {
             return m_pos < m_data.size() ? traits_type::to_int_type(m_data[m_pos]) : traits_type::eof();
         }
@@ -77,7 +79,7 @@ class chunked_streambuf : public std::streambuf
 
     int_type uflow() override
     {
-        if (m_chunk_size == 0)
+        if (m_chunk_size == 0 || m_pos < m_unbuffered)
         {
             return m_pos < m_data.size() ? traits_type::to_int_type(m_data[m_pos++]) : traits_type::eof();
         }
@@ -88,6 +90,7 @@ class chunked_streambuf : public std::streambuf
     std::string m_data;
     std::size_t m_pos = 0;
     std::size_t m_chunk_size;
+    std::size_t m_unbuffered;
 };
 
 // how a test reads from the stream
@@ -101,9 +104,9 @@ enum class read_mode
 // Read @a input from a chunked_streambuf and describe everything observable:
 // the values or the exception, the stream state after each value, and the
 // characters left in the stream.
-std::string observe(const std::string& input, std::size_t chunk_size, read_mode mode)
+std::string observe(const std::string& input, std::size_t chunk_size, read_mode mode, std::size_t unbuffered = 0)
 {
-    chunked_streambuf buf(input, chunk_size);
+    chunked_streambuf buf(input, chunk_size, unbuffered);
     std::istream is(&buf);
     std::string result;
     for (int i = 0; i < 4; ++i)
@@ -212,6 +215,31 @@ TEST_CASE("parsing from a std::istream scans the get area in bulk")
         }
     }
 
+    SECTION("results do not depend on a get area that appears late")
+    {
+        // the first characters come without a get area, then the streambuf
+        // starts to use one: the token that is read at that point is complete
+        for (const auto& input : corpus())
+        {
+            const std::string expected = observe(input, 0, read_mode::extract);
+            for (const std::size_t unbuffered :
+                    {
+                        1u, 2u, 3u, 5u, 8u, 13u, 21u
+                    })
+            {
+                for (const std::size_t chunk_size :
+                        {
+                            1u, 2u, 3u, 7u, 64u
+                        })
+                {
+                    CAPTURE(input)
+                    CAPTURE(unbuffered)
+                    CAPTURE(chunk_size)
+                    CHECK(observe(input, chunk_size, read_mode::extract, unbuffered) == expected);
+                }
+            }
+        }
+    }
 #endif // !defined(JSON_NOEXCEPTION)
 
     SECTION("values are the same as when parsing from a string")