diff --git a/docs/mkdocs/docs/api/basic_json_document/shrink_to_fit.md b/docs/mkdocs/docs/api/basic_json_document/shrink_to_fit.md index f8f8a4312..128e7f996 100644 --- a/docs/mkdocs/docs/api/basic_json_document/shrink_to_fit.md +++ b/docs/mkdocs/docs/api/basic_json_document/shrink_to_fit.md @@ -23,6 +23,10 @@ Linear in [`node_count()`](node_count.md) plus the length of the decoded strings ## Notes +The node index of a small document (an estimate of up to 1024 nodes, which is up to about 4 KiB of minified text) is +allocated together with the document's header, so that parsing needs one allocation. That part cannot be released +separately: `shrink_to_fit()` only trims a node index that has its own allocation, and the decoded strings. + !!! warning "Invalidates views" Unlike moving the document, `shrink_to_fit()` **invalidates every view taken from this document before the diff --git a/include/nlohmann/detail/view/builder.hpp b/include/nlohmann/detail/view/builder.hpp index f7b560f50..a0446a1e1 100644 --- a/include/nlohmann/detail/view/builder.hpp +++ b/include/nlohmann/detail/view/builder.hpp @@ -543,7 +543,15 @@ run_found: NLOHMANN_VIEW_NOINLINE void arena_grow(std::size_t n) { const std::size_t used = arena_used(); - m_doc.arena.resize((std::max)(m_doc.arena.size() * 2, used + n + 256)); + std::size_t wanted = used + n + 256; + if (m_doc.arena.empty()) + { + // the first growth: a decoded string is never longer than its source, + // so a fraction of the input (capped, as the buffer is zero-filled) + // covers the escapes of a typical document without regrowing it + wanted = (std::max)(wanted, (std::min)((static_cast(m_end - m_begin) / 4) + 16, std::size_t{4096})); + } + m_doc.arena.resize((std::max)(m_doc.arena.size() * 2, wanted)); m_arena_write = m_doc.arena.data() + used; m_arena_end = m_doc.arena.data() + m_doc.arena.size(); } diff --git a/include/nlohmann/detail/view/document_data.hpp b/include/nlohmann/detail/view/document_data.hpp index afef5a7a2..2c5f97767 100644 --- a/include/nlohmann/detail/view/document_data.hpp +++ b/include/nlohmann/detail/view/document_data.hpp @@ -82,14 +82,17 @@ struct document_data }; std::unique_ptr edits; ///< created by the first edit + /// the most nodes that are allocated together with the header (16 KiB) + static constexpr std::size_t max_inline_nodes = 1024; + /// One allocation for the header and room for `count` nodes, so that a /// small document needs one allocation instead of two. Large documents get /// a separate node array instead (so it can be trimmed). When a document - /// outgrows the inline array, that array stays unused: at most 256 nodes - /// (4 KiB), and only for documents whose estimate was too small. + /// outgrows the inline array, that array stays unused: at most 1024 nodes + /// (16 KiB), and only for documents whose estimate was too small. static document_data* create(std::size_t count) { - count = count <= 256 ? count : 0; + count = count <= max_inline_nodes ? count : 0; void* mem = ::operator new (sizeof(document_data) + (count * sizeof(node))); auto* d = new (mem) document_data(); // NOLINT(cppcoreguidelines-owning-memory): owned by the returned pointer, freed by deleter // (aligned: sizeof is a multiple of the alignment; through void*, as GCC's -Wcast-align wants) diff --git a/include/nlohmann/json_view.hpp b/include/nlohmann/json_view.hpp index 071c14904..d413a55b9 100644 --- a/include/nlohmann/json_view.hpp +++ b/include/nlohmann/json_view.hpp @@ -54,9 +54,11 @@ // IWYU pragma: no_include // IWYU pragma: no_include "nlohmann/detail/abi_config.hpp" // IWYU pragma: no_include "nlohmann/detail/abi_macros.hpp" +// IWYU pragma: no_include "nlohmann/detail/conversions/from_json.hpp" // IWYU pragma: no_include "nlohmann/detail/input/input_adapters.hpp" // IWYU pragma: no_include "nlohmann/detail/json_pointer.hpp" // IWYU pragma: no_include "nlohmann/detail/meta/cpp_future.hpp" +// IWYU pragma: no_include "nlohmann/detail/meta/type_traits.hpp" // IWYU pragma: no_include "nlohmann/detail/string_concat.hpp" // IWYU pragma: no_include "nlohmann/detail/value_t.hpp" // IWYU pragma: no_include "nlohmann/json.hpp" @@ -1313,7 +1315,7 @@ class basic_json_document /// member is set (added if missing), an array element assigned, and "-" /// or the size of the array appends. A null parent becomes what /// basic_json's operator[](json_pointer) makes of it: an array for "-" - /// and for digits (padded with nulls up to the index), an object otherwise. + /// and for array indices (padded with nulls up to the index), an object otherwise. template view_type set(const json_pointer& ptr, V&& value) { @@ -1329,7 +1331,11 @@ class basic_json_document { return push_back(parent, std::forward(value)); } - const bool digits = std::all_of(token.begin(), token.end(), [](const char c) + // (a token that cannot be an array index, such as "01" or "", names an + // object member, as in basic_json) + const bool digits = !token.empty() + && (token.size() == 1 || token[0] != '0') + && std::all_of(token.begin(), token.end(), [](const char c) { return c >= '0' && c <= '9'; }); diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 28e024f3e..c8c3f6e33 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -54,9 +54,11 @@ // IWYU pragma: no_include // IWYU pragma: no_include "nlohmann/detail/abi_config.hpp" // IWYU pragma: no_include "nlohmann/detail/abi_macros.hpp" +// IWYU pragma: no_include "nlohmann/detail/conversions/from_json.hpp" // IWYU pragma: no_include "nlohmann/detail/input/input_adapters.hpp" // IWYU pragma: no_include "nlohmann/detail/json_pointer.hpp" // IWYU pragma: no_include "nlohmann/detail/meta/cpp_future.hpp" +// IWYU pragma: no_include "nlohmann/detail/meta/type_traits.hpp" // IWYU pragma: no_include "nlohmann/detail/string_concat.hpp" // IWYU pragma: no_include "nlohmann/detail/value_t.hpp" // IWYU pragma: no_include "nlohmann/json.hpp" @@ -410,14 +412,17 @@ struct document_data }; std::unique_ptr edits; ///< created by the first edit + /// the most nodes that are allocated together with the header (16 KiB) + static constexpr std::size_t max_inline_nodes = 1024; + /// One allocation for the header and room for `count` nodes, so that a /// small document needs one allocation instead of two. Large documents get /// a separate node array instead (so it can be trimmed). When a document - /// outgrows the inline array, that array stays unused: at most 256 nodes - /// (4 KiB), and only for documents whose estimate was too small. + /// outgrows the inline array, that array stays unused: at most 1024 nodes + /// (16 KiB), and only for documents whose estimate was too small. static document_data* create(std::size_t count) { - count = count <= 256 ? count : 0; + count = count <= max_inline_nodes ? count : 0; void* mem = ::operator new (sizeof(document_data) + (count * sizeof(node))); auto* d = new (mem) document_data(); // NOLINT(cppcoreguidelines-owning-memory): owned by the returned pointer, freed by deleter // (aligned: sizeof is a multiple of the alignment; through void*, as GCC's -Wcast-align wants) @@ -1747,7 +1752,15 @@ run_found: NLOHMANN_VIEW_NOINLINE void arena_grow(std::size_t n) { const std::size_t used = arena_used(); - m_doc.arena.resize((std::max)(m_doc.arena.size() * 2, used + n + 256)); + std::size_t wanted = used + n + 256; + if (m_doc.arena.empty()) + { + // the first growth: a decoded string is never longer than its source, + // so a fraction of the input (capped, as the buffer is zero-filled) + // covers the escapes of a typical document without regrowing it + wanted = (std::max)(wanted, (std::min)((static_cast(m_end - m_begin) / 4) + 16, std::size_t{4096})); + } + m_doc.arena.resize((std::max)(m_doc.arena.size() * 2, wanted)); m_arena_write = m_doc.arena.data() + used; m_arena_end = m_doc.arena.data() + m_doc.arena.size(); } @@ -9105,7 +9118,7 @@ class basic_json_document /// member is set (added if missing), an array element assigned, and "-" /// or the size of the array appends. A null parent becomes what /// basic_json's operator[](json_pointer) makes of it: an array for "-" - /// and for digits (padded with nulls up to the index), an object otherwise. + /// and for array indices (padded with nulls up to the index), an object otherwise. template view_type set(const json_pointer& ptr, V&& value) { @@ -9121,7 +9134,11 @@ class basic_json_document { return push_back(parent, std::forward(value)); } - const bool digits = std::all_of(token.begin(), token.end(), [](const char c) + // (a token that cannot be an array index, such as "01" or "", names an + // object member, as in basic_json) + const bool digits = !token.empty() + && (token.size() == 1 || token[0] != '0') + && std::all_of(token.begin(), token.end(), [](const char c) { return c >= '0' && c <= '9'; }); diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index b4756c9b5..73b277d56 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -1311,7 +1311,7 @@ TEST_CASE("BSON declared document size bounds the content") // records the events that a SAX parser receives struct recorder : nlohmann::json_sax { - std::string events; + std::string events{}; bool null() override { diff --git a/tests/src/unit-json_pointer.cpp b/tests/src/unit-json_pointer.cpp index 1a4e726ed..b28ac4023 100644 --- a/tests/src/unit-json_pointer.cpp +++ b/tests/src/unit-json_pointer.cpp @@ -1122,17 +1122,17 @@ TEST_CASE("flatten of structured values") // the same for other keys that are not valid array indices CHECK_THROWS_WITH_AS(json({{"/a/0", 1}, {"/a/1a", 2}}).unflatten(), - "[json.exception.out_of_range.404] unresolved reference token '1a'", json::out_of_range&); + "[json.exception.out_of_range.404] unresolved reference token '1a'", json::out_of_range&); CHECK_THROWS_WITH_AS(json({{"/a/0", 1}, {"/a/-", 2}}).unflatten(), - "[json.exception.out_of_range.404] unresolved reference token '-'", json::out_of_range&); + "[json.exception.out_of_range.404] unresolved reference token '-'", json::out_of_range&); } SECTION("unflatten with the empty token among array indices") { CHECK_THROWS_WITH_AS(json({{"/a/0", 1}, {"/a/", 2}}).unflatten(), - "[json.exception.out_of_range.404] unresolved reference token ''", json::out_of_range&); + "[json.exception.out_of_range.404] unresolved reference token ''", json::out_of_range&); CHECK_THROWS_WITH_AS(json({{"/a/0", 1}, {"/a/1", 2}, {"/a/", 3}}).unflatten(), - "[json.exception.out_of_range.404] unresolved reference token ''", json::out_of_range&); + "[json.exception.out_of_range.404] unresolved reference token ''", json::out_of_range&); // without an index "0", the empty token is an ordinary object key CHECK(json({{"/a/", 2}}).unflatten() == json({{"a", {{"", 2}}}})); diff --git a/tests/src/unit-json_view.cpp b/tests/src/unit-json_view.cpp index e87162d98..ee2300e3e 100644 --- a/tests/src/unit-json_view.cpp +++ b/tests/src/unit-json_view.cpp @@ -666,6 +666,61 @@ TEST_CASE("json_view") CHECK(small.root().materialize() == json::parse("[1,[2,3],{\"a\":\"b\\n\"}]")); } + SECTION("block sizes") + { + // documents whose node estimate lies around the size that is allocated + // with the header, and whose decoded strings need the first growth of + // the buffer, in all sizes around the points where the policies change + for (const int count : + { + 1, 2, 10, 50, 100, 255, 256, 257, 500, 1000, 1023, 1024, 1025, 2000, 5000 + }) + { + // strings with escapes (decoded text, minified) + std::string escaped = "["; + for (int i = 0; i < count; ++i) + { + escaped += (i != 0 ? ",\"a\\n" : "\"a\\n") + std::to_string(i) + "\\u00e4\""; + } + escaped += ']'; + json_document e = json_document::parse(escaped); + CHECK(e.root().materialize() == json::parse(escaped)); + e.shrink_to_fit(); + CHECK(e.root().materialize() == json::parse(escaped)); + + // dense numbers (more nodes per byte than the estimate expects) + std::string dense = "["; + for (int i = 0; i < count; ++i) + { + dense += (i != 0 ? ",1" : "1"); + } + dense += ']'; + json_document n = json_document::parse(dense); + CHECK(n.node_count() == static_cast(count) + 1); + CHECK(n.root().materialize() == json::parse(dense)); + n.shrink_to_fit(); + CHECK(n.root().size() == static_cast(count)); + + // the same documents, pretty-printed + const std::string pretty = json::parse(escaped).dump(2); + json_document p = json_document::parse(pretty); + CHECK(p.root().materialize() == json::parse(escaped)); + } + + // a large document with a single escape does not need a buffer of the + // size of its input + std::string large = R"(["a\n")"; + for (int i = 0; i < 100000; ++i) + { + large += ",1"; + } + large += ']'; + const json_document l = json_document::parse(large); + CHECK(l.root().size() == 100001); + CHECK(l.root()[0].get_string() == "a\n"); + CHECK(l.root()[100000].get() == 1); + } + SECTION("source offsets") { const std::string text = R"( {"key": "value", "escaped": "a\nb", "n": 42})"; diff --git a/tests/src/unit-json_view_from_json.cpp b/tests/src/unit-json_view_from_json.cpp index ec296e5db..4a776dd52 100644 --- a/tests/src/unit-json_view_from_json.cpp +++ b/tests/src/unit-json_view_from_json.cpp @@ -41,6 +41,13 @@ using nlohmann::ordered_json_document; #include #endif +// NLOHMANN_JSON_SERIALIZE_ENUM uses a static std::pair; same() compares floating-point values on purpose +DOCTEST_CLANG_SUPPRESS_WARNING_PUSH +DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors") +DOCTEST_CLANG_SUPPRESS_WARNING("-Wfloat-equal") +DOCTEST_GCC_SUPPRESS_WARNING_PUSH +DOCTEST_GCC_SUPPRESS_WARNING("-Wfloat-equal") + // The from_json() functions of the NLOHMANN_DEFINE_TYPE_* and // NLOHMANN_JSON_SERIALIZE_ENUM macros read views directly; other from_json() // functions get the value of materialize(). @@ -61,10 +68,11 @@ struct point struct shape { - std::string name; - std::vector points; - point center; - std::map named; + // initialized for GCC -Weffc++ + std::string name{}; + std::vector points{}; + point center{}; + std::map named{}; NLOHMANN_DEFINE_TYPE_INTRUSIVE(shape, name, points, center, named) }; @@ -79,7 +87,7 @@ NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(with_defaults, a, b, c) struct with_names { int value = 0; - std::string text; + std::string text{}; // initialized for GCC -Weffc++ }; NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_NAMES(with_names, "Value", value, "Text", text) @@ -447,3 +455,6 @@ TEST_CASE("json_view from_json") } #endif } + +DOCTEST_CLANG_SUPPRESS_WARNING_POP +DOCTEST_GCC_SUPPRESS_WARNING_POP diff --git a/tests/src/unit-pointer_access.cpp b/tests/src/unit-pointer_access.cpp index 33a4e2047..2b84ceede 100644 --- a/tests/src/unit-pointer_access.cpp +++ b/tests/src/unit-pointer_access.cpp @@ -141,6 +141,7 @@ TEST_CASE("pointer access") // check if pointers are returned correctly const test_type* p1 = value.get_ptr(); CHECK(p1 == value.get_ptr()); + // @infer-ignore NULLPTR_DEREFERENCE CHECK(*p1 == value.get()); const test_type* p2 = value.get_ptr(); @@ -170,6 +171,7 @@ TEST_CASE("pointer access") // check if pointers are returned correctly test_type* p1 = value.get_ptr(); CHECK(p1 == value.get_ptr()); + // @infer-ignore NULLPTR_DEREFERENCE CHECK(*p1 == value.get()); const test_type* p2 = value.get_ptr(); @@ -199,6 +201,7 @@ TEST_CASE("pointer access") // check if pointers are returned correctly const test_type* p1 = value.get_ptr(); CHECK(p1 == value.get_ptr()); + // @infer-ignore NULLPTR_DEREFERENCE CHECK(*p1 == value.get()); const test_type* p2 = value.get_ptr(); @@ -232,6 +235,7 @@ TEST_CASE("pointer access") const test_type* p2 = value.get_ptr(); CHECK(p2 == value.get_ptr()); + // @infer-ignore NULLPTR_DEREFERENCE CHECK(*p2 == value.get()); const test_type* const p3 = value.get_ptr(); @@ -257,6 +261,7 @@ TEST_CASE("pointer access") // check if pointers are returned correctly const test_type* p1 = value.get_ptr(); CHECK(p1 == value.get_ptr()); + // @infer-ignore NULLPTR_DEREFERENCE CHECK(*p1 == value.get()); const test_type* p2 = value.get_ptr();