From 4971200b5476a30e2bd8d90d1d5a7f399cb877c6 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 21 Oct 2026 10:21:29 +0200 Subject: [PATCH] Add images of json_documents: save() and load() (#5632) * Add images of json_documents: save() and load() An image is a document stored so that loading it needs no parsing: save() writes the node index, the text and the decoded strings; the static load() reads an image written by save(). load() takes a pointer and size, a borrowed vector, or an owned rvalue vector; the nodes are copied so they are aligned and can be edited, while the text and decoded strings stay in the image. image_check controls how much load() trusts the input: full checks structure, bounds, strings and numbers, the parser's own guarantees; bounds checks structure and bounds only; none skips all checks, for images from a trusted source. Layout is little-endian only ("NJVI" header, nodes, text, decoded strings), following the idea of zero-copy formats such as FlatBuffers and YaFF; the check follows FlatBuffers' Verifier. New errors: parse_error.116 for a malformed image or a failed check, type_error.320 for a discarded document or a big-endian target. A dedicated fuzzer and 6,000 seeded corruptions, checked under ASan/UBSan, found and fixed two gaps: unchecked reserved header fields, and unbounded null/boolean offsets that could make dump() throw std::length_error. Signed-off-by: Niels Lohmann * Fix MSVC errors in image test and add image_check doc anchor Move the raw string literal out of the CHECK macro (MSVC preprocessor), cast 64-bit header fields to std::size_t (C4244 on 32-bit), and give the image_check section in load.md a real anchor for the mkdocs strict build. Signed-off-by: Niels Lohmann * Check each distinct string and float range once in the full image check The full check scanned the contents of every string node and every float token over its own range, so many nodes pointing to one large range made load() quadratic. The structural walk now only collects the ranges; after it, each kind is sorted and checked once per distinct range. Ranges of one kind that overlap without being identical are rejected, as save() never writes them (nodes that share a value share the whole range). The cost is linear in the size of the image plus sorting the ranges. Signed-off-by: Niels Lohmann * Adapt the image tests to named documents and last-wins lookups Signed-off-by: Niels Lohmann * Test that the targets of links never reach an image Signed-off-by: Niels Lohmann * Test lookups in loaded images with duplicate and colliding keys Signed-off-by: Niels Lohmann * Release the list of large objects once load() has built the indexes Signed-off-by: Niels Lohmann * Test that invalid UTF-8 of a loaded image is not copied into an editable document Signed-off-by: Niels Lohmann * Document that editable documents refuse invalid UTF-8 copied from a loaded image Signed-off-by: Niels Lohmann * Expect first-wins lookups in the image tests Signed-off-by: Niels Lohmann * Avoid useless casts of header fields in the image tests Signed-off-by: Niels Lohmann * Fix clang-tidy 22 findings in unit-json_view_image.cpp and image.hpp Signed-off-by: Niels Lohmann * Fix MSVC C4244 and C2017 in the image test Signed-off-by: Niels Lohmann * Fix the indentation of a loop in the image test Signed-off-by: Niels Lohmann * Make json_view.hpp pass include-what-you-use ci_single_binaries runs IWYU with --error on every header. For json_view.hpp it suggested adding (std::array is used), the headers that json.hpp already provides (abi_config, abi_macros, input_adapters, json_pointer, cpp_future, string_concat, value_t, json_fwd), and for std::nullptr_t, and removing . - include - keep (nullptr_t, size_t; IWYU attributes them to and ) - tell IWYU not to suggest the headers that json.hpp provides: the amalgamated json_view.hpp only includes json.hpp, so including them here would duplicate their definitions - drop edit_storage.hpp, which edit.hpp includes Signed-off-by: Niels Lohmann * Silence a clang-tidy C++17 finding in the image test bugprone-suspicious-stringview-data-usage flags data() of the source view under C++17, although the length is passed and checked by the REQUIRE before it. Signed-off-by: Niels Lohmann * Build the image link test without exceptions load_result() exists only with exceptions; with JSON_NOEXCEPTION the test loads the image with the check directly (a failed check aborts). Signed-off-by: Niels Lohmann * Regenerate the amalgamated headers Signed-off-by: Niels Lohmann * Follow the renames from the review of the view builder The review of #5620 renamed document_data's tape/tape_size/tape_cap to nodes/node_count/node_capacity and made the decode arena a std::vector; adapt the code of this change to it. Signed-off-by: Niels Lohmann * Rename the emit lambda in the image writer for Qt compatibility Qt defines emit as an empty macro, so a Qt header included before json_view.hpp broke the lambda's declaration (see #5865). The slots variables in the image tests are renamed as well. Signed-off-by: Niels Lohmann * Address the review of json_document images Signed-off-by: Niels Lohmann * Regenerate the amalgamated headers Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .github/labeler.yml | 2 +- BUILD.bazel | 1 + .../docs/api/basic_json_document/index.md | 15 + .../docs/api/basic_json_document/load.md | 179 +++ .../docs/api/basic_json_document/parse.md | 1 + .../docs/api/basic_json_document/read.md | 1 + .../docs/api/basic_json_document/save.md | 104 ++ .../examples/basic_json_document__load.cpp | 52 + .../examples/basic_json_document__load.output | 5 + .../examples/basic_json_document__save.cpp | 30 + .../examples/basic_json_document__save.output | 3 + docs/mkdocs/docs/features/index.md | 3 +- docs/mkdocs/docs/features/json_view.md | 52 + docs/mkdocs/docs/home/architecture.md | 8 + docs/mkdocs/docs/home/exceptions.md | 44 +- docs/mkdocs/mkdocs.yml | 2 + .../nlohmann/detail/view/document_data.hpp | 5 +- include/nlohmann/detail/view/image.hpp | 744 ++++++++++ include/nlohmann/detail/view/number.hpp | 100 +- include/nlohmann/detail/view/serializer.hpp | 20 +- include/nlohmann/json_view.hpp | 69 +- single_include/nlohmann/json_view.hpp | 1258 ++++++++++++++--- tests/Makefile | 5 +- tests/fuzzing.md | 6 + tests/src/fuzzer-json_view_image.cpp | 95 ++ tests/src/unit-json_view_image.cpp | 1237 ++++++++++++++++ 26 files changed, 3797 insertions(+), 244 deletions(-) create mode 100644 docs/mkdocs/docs/api/basic_json_document/load.md create mode 100644 docs/mkdocs/docs/api/basic_json_document/save.md create mode 100644 docs/mkdocs/docs/examples/basic_json_document__load.cpp create mode 100644 docs/mkdocs/docs/examples/basic_json_document__load.output create mode 100644 docs/mkdocs/docs/examples/basic_json_document__save.cpp create mode 100644 docs/mkdocs/docs/examples/basic_json_document__save.output create mode 100644 include/nlohmann/detail/view/image.hpp create mode 100644 tests/src/fuzzer-json_view_image.cpp create mode 100644 tests/src/unit-json_view_image.cpp diff --git a/.github/labeler.yml b/.github/labeler.yml index 014d9f9bb..076cad183 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -63,7 +63,7 @@ labels: - "single_include/nlohmann/json_view\\.hpp" - "tests/src/unit-json_view.*" - "tests/src/json_view_test_helpers\\.hpp" - - "tests/src/fuzzer-parse_json_view\\.cpp" + - "tests/src/fuzzer-(parse_json_view|json_view_image)\\.cpp" - "tools/amalgamate/config_json_view\\.json" - "docs/mkdocs/docs/features/json_view\\.md" - "docs/mkdocs/docs/api/basic_json_(document|view)/.*" diff --git a/BUILD.bazel b/BUILD.bazel index 4c218fa57..78505d796 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -73,6 +73,7 @@ cc_library( "include/nlohmann/detail/view/edit.hpp", "include/nlohmann/detail/view/edit_storage.hpp", "include/nlohmann/detail/view/errors.hpp", + "include/nlohmann/detail/view/image.hpp", "include/nlohmann/detail/view/input.hpp", "include/nlohmann/detail/view/iterator.hpp", "include/nlohmann/detail/view/lookup.hpp", diff --git a/docs/mkdocs/docs/api/basic_json_document/index.md b/docs/mkdocs/docs/api/basic_json_document/index.md index 66e83b50e..d1321959a 100644 --- a/docs/mkdocs/docs/api/basic_json_document/index.md +++ b/docs/mkdocs/docs/api/basic_json_document/index.md @@ -49,14 +49,21 @@ bookkeeping edits need, and calling any of them on one fails to compile (`#!cpp - **view_type** - the type of view returned by [`root()`](root.md) (`#!cpp basic_json_view`) - **value_t** - the JSON type enumeration, see [`basic_json::value_t`](../basic_json/value_t.md) +- **image_check** - how [`load()`](load.md) validates an image (`full`, `bounds`, `none`), see [`image_check`](load.md#image_check) ## Member functions - [(constructor)](basic_json_document.md) + +### Parsing + - [**parse**](parse.md) (_static_) - deserialize from a compatible input, borrowing or owning it as appropriate - [**parse_copy**](parse_copy.md) (_static_) - deserialize a copy of a compatible input - [**accept**](accept.md) (_static_) - check whether the input is valid JSON - [**read**](read.md) - (re-)parse into this document, reusing its memory + +### Access + - [**root**](root.md) - the view of the root value - [**is_discarded**](is_discarded.md) - return whether the last parse failed - [**source**](source.md) - the parsed text @@ -64,6 +71,14 @@ bookkeeping edits need, and calling any of them on one fails to compile (`#!cpp - [**node_count**](node_count.md) - the number of index entries (values plus object keys) - [**memory_usage**](memory_usage.md) - the number of bytes held by the document - [**shrink_to_fit**](shrink_to_fit.md) - release unused index capacity + +### Images + +- [**save**](save.md) - the document as an image that `load()` reads without parsing +- [**load**](load.md) (_static_) - read an image written by `save()` + +### Edits + - [**set**](set.md) - replace a value, or set an object member, an array element, or the value a JSON pointer refers to (`#!cpp Editable` documents only) - [**push_back**](push_back.md) - append to an array (`#!cpp Editable` documents only) diff --git a/docs/mkdocs/docs/api/basic_json_document/load.md b/docs/mkdocs/docs/api/basic_json_document/load.md new file mode 100644 index 000000000..36223f63c --- /dev/null +++ b/docs/mkdocs/docs/api/basic_json_document/load.md @@ -0,0 +1,179 @@ +# nlohmann::basic_json_document::load + +```cpp +// (1) +static basic_json_document load(const std::uint8_t* image, std::size_t size, + const image_check check = image_check::full); + +// (2) +static basic_json_document load(const std::vector& image, + const image_check check = image_check::full); + +// (3) +static basic_json_document load(std::vector&& image, + const image_check check = image_check::full); +``` + +1. Reads an image [`save()`](save.md) wrote, from a pointer and a byte count. The image is **borrowed**: `image` + must stay alive and unchanged for as long as the returned document, and any view taken from it, is used. +2. Reads an image from a `#!cpp std::vector`. Also **borrowed** -- equivalent to overload 1 called with + `#!cpp image.data()` and `#!cpp image.size()`. +3. Reads an image, keeping the vector instead of copying it: `image` is moved into the document (no copy), which + then owns it for as long as it needs the text and the decoded strings. [`owns_source()`](owns_source.md) is + `#!cpp true` afterward. + +In every overload, the node index is copied into storage the document itself owns -- so that it is properly aligned, +and, for an [editable](index.md#edits) document, can be edited -- while the text and the decoded strings stay in +`image`. The hash indexes [large objects](../../features/json_view.md) use for lookup are rebuilt, exactly as after +parsing. + +## Parameters + +`image` (in) +: the image [`save()`](save.md) wrote (overloads 1 and 2), or one to take ownership of (overload 3) + +`size` (in) +: the number of bytes at `image` (overload 1) + +`check` (in) +: how thoroughly to validate `image` before trusting it; see [`image_check`](#image_check) below (optional, + `#!cpp image_check::full` by default) + +## Return value + +The document read from the image. + +## Exception safety + +Overloads 1 and 2 give the strong guarantee: `image` is only read, never written, so a thrown exception leaves the +caller's buffer untouched. + +Overload 3 moves `image` into the document *before* validating it, so that a good image is kept without a copy. If +loading then fails, the partially built document -- and the vector now inside it -- is discarded along with the +exception, and `image` itself is left **empty**, not restored to what was passed in. Move a copy in instead, or +validate with overload 2 first, if the original vector must survive a failed load. + +## Exceptions + +On a big-endian target, throws [`type_error.320`](../../home/exceptions.md#jsonexceptiontype_error320) -- the same +exception [`save()`](save.md#exceptions) throws there, since the image format is little-endian only. + +Otherwise throws [`parse_error.116`](../../home/exceptions.md#jsonexceptionparse_error116) if `image` is not one +`save()` could have written, or fails the requested `check`: + +| message | when | +|------------------------|------------------------------------------------------------------------------------------------------| +| `too short` | `image` is `#!cpp nullptr`, or `size` is smaller than the 64-byte header | +| `unknown format` | the header's magic bytes or version do not match, or a reserved header field is not zero | +| `sizes out of range` | the node count, text size, or decoded-string size the header describes does not fit `size`, or the `#!cpp '\0'` after the text or after the decoded strings is missing | +| `the check failed` | `check` is not `#!cpp image_check::none`, and the image fails it -- see [`image_check`](#image_check) | + +!!! failure "Example messages" + + ``` + [json.exception.parse_error.116] parse error: invalid json_document image: too short + ``` + ``` + [json.exception.parse_error.116] parse error: invalid json_document image: unknown format + ``` + ``` + [json.exception.parse_error.116] parse error: invalid json_document image: sizes out of range + ``` + ``` + [json.exception.parse_error.116] parse error: invalid json_document image: the check failed + ``` + +## Complexity + +Linear in the number of nodes, which are always copied into the document. `#!cpp image_check::none` does not read +the text or the decoded strings. `#!cpp image_check::bounds` reads one byte of the text for each float token (its sign, +to check the recorded digits against the token's length), and nothing else of the text or the decoded strings. + +With `#!cpp image_check::full`, linear in the size of the image (the nodes, the text, and the decoded strings), plus +sorting the ranges of the strings and of the float tokens (at most one per node): the contents of each distinct range +are checked once, however many nodes refer to it. + +## Notes + +### `image_check` { #image_check } + +```cpp +using image_check = detail::view::image_check; + +enum class image_check +{ + full, + bounds, + none +}; +``` + +How thoroughly `load()` validates `image` before trusting it. + +`full` (the default) +: Checks everything the parser itself guarantees: structure and bounds; that every string is valid UTF-8 (and, for a + string still in the source text, that it contains no quote, backslash, or control character); and that every number + token is well-formed and matches the value stored for it. Strings and float tokens may share a range only if the + ranges are identical (as nodes that share a value do), and never overlap otherwise. Reading and serializing a + checked image is safe and always produces valid JSON, exactly as for a parsed document. This is the right choice for + an image from anything you do not fully control -- a file, a cache shared with other processes, a peer on the + network. + +`bounds` +: Checks structure and bounds only: that every offset and count in the node index stays inside the image. So + `load()`, [`root()`](root.md), element access, and [`materialize()`](../basic_json_view/materialize.md) can never + read outside the image, even on a damaged one. It does *not* guarantee that the image describes valid JSON: a + string that a `full` check would have rejected can make [`dump()`](../basic_json_view/dump.md) write invalid UTF-8 + or invalid JSON, or throw [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316), and a number can + read back with a value that does not match how it is spelled. An [editable document](../json_editable_document.md) + does not take such a string over: [`set`](set.md), [`push_back`](push_back.md) and [`insert`](insert.md) throw + `type_error.316` when they copy it from a view of the loaded document, and leave the editable document unchanged. + `bounds` skips scanning the text and the decoded strings, so it fits a cache your own process just wrote and reads + straight back, where damage would mean a bug or a hardware fault rather than adversarial input. + +`none` +: Checks nothing. Reading a damaged image is undefined behavior, so use it only for an image you trust as much as + your own memory. + +**Lifetime.** Overloads 1 and 2 borrow `image`: it must stay alive and byte-for-byte unchanged for as long as the +returned document, and any [view](../basic_json_view/index.md) taken from it, is used -- exactly like a document +[`parse()`](parse.md) borrowed its input for. Overload 3 avoids this by keeping the vector itself; see +[`owns_source`](owns_source.md). + +!!! warning "Experimental" + + The image format is versioned but not yet stable, and may change in an incompatible way before it is declared + stable; `load()` already rejects an image written by a different format version with `parse_error.116` + ("unknown format"). Use images to cache a document within one build of the library, or to hand one to another + process running the *same* build on the *same* (little-endian) machine -- not as a long-term storage format. + +## Examples + +??? example "Caching a document, ownership, and a rejected image" + + The example below saves a parsed document as an image, checks that `load()` reproduces the original + [`dump()`](../basic_json_view/dump.md) without parsing, and shows the difference between + `load(std::move(image))` (owned) and `load(image)` (borrowed). It then damages one byte of the image and shows + `image_check::full` rejecting it with `parse_error.116`, while `image_check::bounds` -- meant for a cache the + process already trusts -- still reads it without going out of bounds. + + ```cpp + --8<-- "examples/basic_json_document__load.cpp" + ``` + + Output: + + ```json + --8<-- "examples/basic_json_document__load.output" + ``` + +## See also + +- [save](save.md) - write the document as an image +- [owns_source](owns_source.md) - return whether the document holds its own copy of the text +- [parse](parse.md) - deserialize from JSON text instead of an image +- [Images](../../features/json_view.md#images) - why and when to use images + +## Version history + +- Added in version 3.13.0. diff --git a/docs/mkdocs/docs/api/basic_json_document/parse.md b/docs/mkdocs/docs/api/basic_json_document/parse.md index 3e2506264..7bc00a9f1 100644 --- a/docs/mkdocs/docs/api/basic_json_document/parse.md +++ b/docs/mkdocs/docs/api/basic_json_document/parse.md @@ -149,6 +149,7 @@ buffer of a given length, pass a pair of pointers: `#!cpp parse(ptr, ptr + len)` - [accept](accept.md) - check whether the input is valid JSON - [read](read.md) - (re-)parse into this document, reusing its memory - [owns_source](owns_source.md) - return whether the document holds its own copy of the text +- [load](load.md) - read a document from an image instead of parsing JSON text - [`BasicJsonType::parse`](../basic_json/parse.md) - the corresponding function of `basic_json` ## Version history diff --git a/docs/mkdocs/docs/api/basic_json_document/read.md b/docs/mkdocs/docs/api/basic_json_document/read.md index 979bec869..174fcc477 100644 --- a/docs/mkdocs/docs/api/basic_json_document/read.md +++ b/docs/mkdocs/docs/api/basic_json_document/read.md @@ -80,6 +80,7 @@ compile; see [`parse`](parse.md#notes). - [parse](parse.md) - deserialize from a compatible input - [root](root.md) - the view of the root value +- [load](load.md) - read a document from an image instead of parsing JSON text ## Version history diff --git a/docs/mkdocs/docs/api/basic_json_document/save.md b/docs/mkdocs/docs/api/basic_json_document/save.md new file mode 100644 index 000000000..f69d5c9f8 --- /dev/null +++ b/docs/mkdocs/docs/api/basic_json_document/save.md @@ -0,0 +1,104 @@ +# nlohmann::basic_json_document::save + +```cpp +std::vector save() const; +``` + +Writes the document as an *image*: a byte buffer that [`load`](load.md) reads back without parsing. The image holds +the node index, the source text (plus, for an edited document, the number tokens edits wrote), and the decoded +strings (plus the strings edits wrote) -- everything [`root()`](root.md) needs, with nothing left to parse. + +An edited document is written in its *current* state, with its values in document order, the way the library's own +parser would have produced them for that JSON text: a member [`set`](set.md) added goes at the end, an +[`erase`](erase.md)d member leaves no trace, and a float that is not finite (NaN or positive/negative infinity) +becomes null, the same substitution [`dump()`](../basic_json_view/dump.md) makes. The same document always saves to +the same bytes -- also across `BasicJsonType` and `#!cpp Editable`, since the image reflects document order and +values only, not which specialization produced them. + +## Return value + +The image, as a `#!cpp std::vector`. Pass it, or a pointer to its data together with its size, to +[`load`](load.md) to read the document back. + +## Exception safety + +Strong guarantee: `save()` does not modify `#!cpp *this` (it is `#!cpp const`), so if it throws, the document is left +exactly as it was, and the partially built image is discarded with the exception. + +## Exceptions + +Throws [`type_error.320`](../../home/exceptions.md#jsonexceptiontype_error320) if the document is +[discarded](is_discarded.md) -- a default-constructed document, or one a failed [`parse()`](parse.md)/ +[`read()`](read.md) with `allow_exceptions == false` left discarded. + +On a big-endian target, throws `type_error.320` with a different message instead: the image format is little-endian +only (see [Notes](#notes)). + +Throws [`out_of_range.416`](../../home/exceptions.md#jsonexceptionout_of_range416) if the node count, the text, or +the decoded strings of the image would individually reach 4 GiB -- the same 32-bit offsets +[`parse()`](parse.md#exceptions) and, for edits, [`set`](set.md)/[`push_back`](push_back.md) are already limited to. + +!!! failure "Example messages" + + ``` + [json.exception.type_error.320] cannot save a discarded json_document + ``` + ``` + [json.exception.type_error.320] json_document images need a little-endian target + ``` + ``` + [json.exception.out_of_range.416] images of 4 GiB or more are not supported by json_document + ``` + +## Complexity + +Linear in the size of the document: the number of nodes, plus the length of the text and the decoded strings that end +up in the image. + +## Notes + +**Format.** The image begins with a 64-byte header (the magic bytes `#!cpp "NJVI"`, a version number, the node count, +and the sizes of the text and the decoded strings, all little-endian), followed by the nodes +([16 bytes each](../../home/architecture.md#node-index-of-json-views)), the text and a `#!cpp '\0'`, and the decoded +strings and a `#!cpp '\0'`. [`load`](load.md) checks the header, and the sizes it describes, before reading anything +else -- see [`load`'s Exceptions](load.md#exceptions). + +!!! warning "Experimental" + + The image format is versioned but not yet stable: it may change in an incompatible way before it is declared + stable. Use images to cache a document within one build of the library, or to hand one to another process running + the *same* build on the *same* (little-endian) machine -- not as a long-term storage format. Keep the original + JSON text if you need to read a saved document back with a future library version. + +**Little-endian only.** The image is written as raw little-endian bytes, with no byte-swapping. `save()` (and +[`load`](load.md)) throw `type_error.320` on a big-endian target rather than silently produce bytes a big-endian +reader could not interpret correctly. + +## Examples + +??? example "Caching a parsed document as an image" + + The example below saves a parsed configuration as an image -- the way a service might cache one to answer later + requests without parsing the text again -- and confirms that loading it back gives exactly the same result as + parsing did, and that saving is deterministic. + + ```cpp + --8<-- "examples/basic_json_document__save.cpp" + ``` + + Output: + + ```json + --8<-- "examples/basic_json_document__save.output" + ``` + +## See also + +- [load](load.md) - read an image written by `save()` +- [owns_source](owns_source.md) - return whether the document holds its own copy of the text +- [`basic_json_view::dump`](../basic_json_view/dump.md) - serialize the document to JSON text instead of an image +- [Images](../../features/json_view.md#images) - why and when to use images + +## Version history + +- Added in version 3.13.0. diff --git a/docs/mkdocs/docs/examples/basic_json_document__load.cpp b/docs/mkdocs/docs/examples/basic_json_document__load.cpp new file mode 100644 index 000000000..52a085a10 --- /dev/null +++ b/docs/mkdocs/docs/examples/basic_json_document__load.cpp @@ -0,0 +1,52 @@ +#include +#include +#include +#include +#include + +using json = nlohmann::json; +using json_document = nlohmann::json_document; +using image_check = json_document::image_check; + +int main() +{ + std::cout << std::boolalpha; + + // the image of a parsed document -- as if read back from a cache file or + // received from another process running the same build of the library + const std::string text = R"({"name": "cache", "note": "caf\u00e9", "replicas": ["db2", "db3"]})"; + const json_document parsed = json_document::parse(text); + const std::vector image = parsed.save(); + + // (1)/(2) load() needs no parsing, yet dumps exactly what parsing did + const json_document borrowed = json_document::load(image); + std::cout << (borrowed.root().dump() == parsed.root().dump()) << '\n'; + std::cout << borrowed.owns_source() << '\n'; // borrowed: still points into `image` + + // (3) load(std::move(image)) keeps the vector instead of copying it + std::vector to_move = image; + const json_document owned = json_document::load(std::move(to_move)); + std::cout << owned.owns_source() << '\n'; + + // a damaged image -- the last byte of the decoded string "note" holds + // (an escape sequence, so it was unescaped into the document's own + // buffer), flipped, as storage or transport corruption might do + std::vector damaged = image; + damaged[damaged.size() - 2] = 0xFF; + + // image_check::full inspects strings and numbers, so it catches the damage + try + { + static_cast(json_document::load(damaged, image_check::full)); + } + catch (const json::parse_error& e) + { + std::cout << e.id << '\n'; + } + + // image_check::bounds only checks structure and bounds, so a cache the + // process already trusts loads without the extra scan -- reading a value + // the damage did not touch is still safe + const json_document trusted = json_document::load(damaged, image_check::bounds); + std::cout << trusted.root()["name"].get() << '\n'; +} diff --git a/docs/mkdocs/docs/examples/basic_json_document__load.output b/docs/mkdocs/docs/examples/basic_json_document__load.output new file mode 100644 index 000000000..4dc66a29b --- /dev/null +++ b/docs/mkdocs/docs/examples/basic_json_document__load.output @@ -0,0 +1,5 @@ +true +false +true +116 +cache diff --git a/docs/mkdocs/docs/examples/basic_json_document__save.cpp b/docs/mkdocs/docs/examples/basic_json_document__save.cpp new file mode 100644 index 000000000..6a3cc978b --- /dev/null +++ b/docs/mkdocs/docs/examples/basic_json_document__save.cpp @@ -0,0 +1,30 @@ +#include +#include +#include +#include +#include + +using json_document = nlohmann::json_document; + +int main() +{ + std::cout << std::boolalpha; + + // a configuration a service parses once and then caches as an image, so + // that later requests can load() it instead of parsing the text again + const std::string text = R"({"name": "cache", "host": "db1", "port": 6379, "replicas": ["db2", "db3"]})"; + const json_document config = json_document::parse(text); + + // save() turns the parsed document into a byte buffer: a 64-byte header, + // the node index, the source text, and the decoded strings + const std::vector image = config.save(); + std::cout << image.size() << '\n'; + + // the same document always saves to the same bytes + std::cout << (image == json_document::parse(text).save()) << '\n'; + + // loading the image back needs no parsing, yet dumps exactly what + // parsing the text produced + const json_document reloaded = json_document::load(image); + std::cout << (reloaded.root().dump() == config.root().dump()) << '\n'; +} diff --git a/docs/mkdocs/docs/examples/basic_json_document__save.output b/docs/mkdocs/docs/examples/basic_json_document__save.output new file mode 100644 index 000000000..5ce0ade9a --- /dev/null +++ b/docs/mkdocs/docs/examples/basic_json_document__save.output @@ -0,0 +1,3 @@ +316 +true +true diff --git a/docs/mkdocs/docs/features/index.md b/docs/mkdocs/docs/features/index.md index b1173b662..4ffe7e585 100644 --- a/docs/mkdocs/docs/features/index.md +++ b/docs/mkdocs/docs/features/index.md @@ -14,7 +14,8 @@ C++ types, and finally serialize it again. [parsing untrusted input](parsing/untrusted_input.md). - [Zero-copy JSON views](json_view.md) — read a JSON text through a flat index instead of building a `json` tree; strings and numbers stay in the input and are only decoded when needed. - [Editable documents](json_view.md#editing-a-document) can also be modified. +- [Editable documents](json_view.md#editing-a-document) — modify a parsed document without building a `json` tree. +- [Images](json_view.md#images) — save a parsed document and load it again without parsing it. - [Comments](comments.md) and [trailing commas](trailing_commas.md) — opt-in relaxations of the JSON grammar. ## Accessing and modifying values diff --git a/docs/mkdocs/docs/features/json_view.md b/docs/mkdocs/docs/features/json_view.md index 83b53de57..4f0938273 100644 --- a/docs/mkdocs/docs/features/json_view.md +++ b/docs/mkdocs/docs/features/json_view.md @@ -312,6 +312,57 @@ edits. See [`basic_json_document`'s Edits](../api/basic_json_document/index.md#e throws (the *basic* guarantee, not the strong one `dump()` and the read-only functions provide). How edits are kept in the index is described in the [architecture overview](../home/architecture.md#node-index-of-json-views). +## Images + +[`save()`](../api/basic_json_document/save.md) writes a document as an *image*: a byte buffer that +[`load()`](../api/basic_json_document/load.md) reads back into a document without parsing -- no lexing, no building +the node index, nothing but copying the nodes and pointing the text and the decoded strings at the image. Where +[`parse_copy()`](../api/basic_json_document/parse_copy.md) still has to scan the whole input, +[`load()`](../api/basic_json_document/load.md) turns that scan into a copy of the node index alone. + +**Why.** A document that is parsed once and then read many times -- a configuration loaded at startup, a template +rendered on every request, a large reference dataset a worker process needs in memory -- pays for parsing once but +can amortize [`save()`](../api/basic_json_document/save.md)'s cost across every later load. That makes images useful +for a cache: save a document the first time it is parsed (to a file, a shared-memory segment, an in-process cache), +and [`load()`](../api/basic_json_document/load.md) it on every later use instead of parsing the source text again. +They are just as useful for handing a parsed document to another process (or a forked worker) running the same build +of the library, since [`load()`](../api/basic_json_document/load.md) turns the transfer into a copy of the node index +plus pointers into the received bytes, not a re-parse. + +**Choosing a check.** [`load()`](../api/basic_json_document/load.md) takes an +[`image_check`](../api/basic_json_document/load.md#image_check) that trades validation against speed: +`image_check::full` (the default) checks everything the parser itself guarantees, so a checked image is exactly as +safe to read and serialize as a freshly parsed document -- the right choice whenever the image did not come straight +from this process's own [`save()`](../api/basic_json_document/save.md), such as a file or a network peer. +`image_check::bounds` only checks structure and bounds -- cheaper, since it skips scanning the text and the decoded +strings -- and fits a cache the process trusts, one it wrote and reads back itself. `image_check::none` skips +validation entirely, for an image trusted as much as the process's own memory. See +[`load()`'s Notes](../api/basic_json_document/load.md#notes) for exactly what each level does and does not guarantee. + +??? example "Example: cache a parsed configuration as an image" + + ```cpp + --8<-- "examples/basic_json_document__save.cpp" + ``` + + Output: + + ```json + --8<-- "examples/basic_json_document__save.output" + ``` + +!!! warning "Experimental" + + The image format is versioned but not yet stable, and may change in an incompatible way before it is declared + stable. It is little-endian only, and tied to the library build that wrote it -- use it to cache a document or to + hand one to another process running the *same* build, not as a long-term storage format; keep the original JSON + text if a saved document needs to be readable by a future library version. + +The idea of a document you can read without parsing comes from zero-copy formats such as +[FlatBuffers](https://github.com/google/flatbuffers) and [YaFF](https://github.com/yandex/yaff); the check +[`load()`](../api/basic_json_document/load.md) runs follows the idea of FlatBuffers' Verifier. No code is taken from +either. + ## Choosing between `json`, `ordered_json`, the SAX interface, and `json_view` | | [`json`](../api/json.md) / [`ordered_json`](../api/ordered_json.md) | [SAX interface](parsing/sax_interface.md) | [`json_document`](../api/json_document.md) / [`json_view`](../api/json_view.md) | [`json_editable_document`](../api/json_editable_document.md) / [`json_editable_view`](../api/json_editable_view.md) | @@ -320,6 +371,7 @@ the index is described in the [architecture overview](../home/architecture.md#no | **Mutability** | freely mutable | not applicable (a one-shot event stream) | read-only | [`set`](../api/basic_json_document/set.md)/[`push_back`](../api/basic_json_document/push_back.md)/[`insert`](../api/basic_json_document/insert.md)/[`erase`](../api/basic_json_document/erase.md) edit in place; the source text is never rewritten | | **What you get** | a full tree you can read, write, and keep as long as you like | a sequence of callbacks; whatever your handler builds from them | a flat index plus, on demand, [`materialize()`](../api/basic_json_view/materialize.md)d `json`/`ordered_json` values for the parts you actually use | the same, plus [`dump()`](../api/basic_json_view/dump.md) of an edited document that keeps the member order and, with [`number_format::source`](../api/basic_json_view/number_format.md), the spelling of every untouched number | | **Typical use** | general-purpose JSON handling: config, request/response bodies you build or modify, anything you hold onto | validating or projecting a text into your own data structure without ever holding the whole thing as JSON | large or high-volume input where you only need part of it, or need it repeatedly, and can keep the source text (or a copy) alive for as long as the document lives | a document you read, patch a few fields of, and write back -- a configuration file, for instance -- where the rest of it should come back exactly as it was | +| **Caching/reload** | not applicable -- re-parse, or roll your own serialization | not applicable | [`save()`](../api/basic_json_document/save.md)/[`load()`](../api/basic_json_document/load.md): cache the parsed index as an image and reload it without parsing | same, saving the document's current -- possibly edited -- state | ## Version history diff --git a/docs/mkdocs/docs/home/architecture.md b/docs/mkdocs/docs/home/architecture.md index 5ee7fcc5d..ff7db49cc 100644 --- a/docs/mkdocs/docs/home/architecture.md +++ b/docs/mkdocs/docs/home/architecture.md @@ -262,6 +262,14 @@ edited: - Views of read-only documents compile without any of this: how views walk the index is a template parameter (`navigation`). +Images ([`save`](../api/basic_json_document/save.md) and [`load`](../api/basic_json_document/load.md), +[`detail/view/image.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/view/image.hpp)) store +the nodes as they are: a 64-byte header (the magic bytes `NJVI`, a format version, the sizes, and reserved bytes that +must be zero), the nodes, the text, and the decoded strings. An edited document is first written in document order, as +the parser would have written it (without links), and the numbers of the hash indexes are cleared, since `load` +rebuilds the indexes. So a change of the node layout is a change of the image format: it must raise `image_version`, +and `load` then rejects images of other versions (`parse_error.116`) instead of misreading them. + ## Input adapters Input is read via **input adapters** that abstract a source. Every input adapter provides this interface: diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index 4fa8eb062..8f1016214 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -395,6 +395,23 @@ A UBJSON high-precision number could not be parsed. [json.exception.parse_error.115] parse error at byte 5: syntax error while parsing UBJSON high-precision number: invalid number text: 1A ``` +### json.exception.parse_error.116 + +[`basic_json_document::load()`](../api/basic_json_document/load.md) rejected an +[image](../features/json_view.md#images): either the bytes are not one [`save()`](../api/basic_json_document/save.md) +could have written (too short, an unknown magic number or format version, or sizes that do not fit the buffer), or +they are, but fail the requested [`image_check`](../api/basic_json_document/load.md#image_check). + +!!! failure "Example message" + + ``` + [json.exception.parse_error.116] parse error: invalid json_document image: the check failed + ``` + +!!! note + + This exception was added in version 3.13.0, together with [images](../features/json_view.md#images). + ## Iterator errors This exception is thrown if iterators passed to a library function do not match @@ -831,6 +848,26 @@ from JSON text. This exception was added in version 3.13.0, together with editable [`json_document`s](../features/json_view.md). +### json.exception.type_error.320 + +[`basic_json_document::save()`](../api/basic_json_document/save.md) cannot write an +[image](../features/json_view.md#images) of a [discarded](../api/basic_json_document/is_discarded.md) document. +[`save()`](../api/basic_json_document/save.md) and [`load()`](../api/basic_json_document/load.md) also throw this +exception on a big-endian target, since the image format is little-endian only. + +!!! failure "Example messages" + + ``` + [json.exception.type_error.320] cannot save a discarded json_document + ``` + ``` + [json.exception.type_error.320] json_document images need a little-endian target + ``` + +!!! note + + This exception was added in version 3.13.0, together with [images](../features/json_view.md#images). + ### json.exception.type_error.321 A discarded value (one created by [`parse()`](../api/basic_json/parse.md) with a callback that returns `false` for the @@ -1088,7 +1125,9 @@ MessagePack's ext type and BSON's binary subtype are each stored in a single byt so they do not support an input of 4294967280 bytes (4 GiB minus 16 bytes) or more. The same 32-bit limit applies to an **editable** document's own storage: [`set`](../api/basic_json_document/set.md) and [`push_back`](../api/basic_json_document/push_back.md) throw this exception once the strings and number tokens written by edits reach 4 GiB in total, or once more than -4294967295 arrays/objects have had an element set or appended to them. +4294967295 arrays/objects have had an element set or appended to them. The same limit applies to an +[image](../features/json_view.md#images): [`save()`](../api/basic_json_document/save.md) throws it if the node +count, the text, or the decoded strings it would write would individually reach 4 GiB. !!! failure "Example messages" @@ -1098,6 +1137,9 @@ this exception once the strings and number tokens written by edits reach 4 GiB i ``` [json.exception.out_of_range.416] edits of 4 GiB or more are not supported by json_document ``` + ``` + [json.exception.out_of_range.416] images of 4 GiB or more are not supported by json_document + ``` !!! note diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 301250f2d..35b5be3e0 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -242,6 +242,7 @@ nav: - 'erase': api/basic_json_document/erase.md - 'insert': api/basic_json_document/insert.md - 'is_discarded': api/basic_json_document/is_discarded.md + - 'load': api/basic_json_document/load.md - 'memory_usage': api/basic_json_document/memory_usage.md - 'node_count': api/basic_json_document/node_count.md - 'owns_source': api/basic_json_document/owns_source.md @@ -250,6 +251,7 @@ nav: - 'push_back': api/basic_json_document/push_back.md - 'read': api/basic_json_document/read.md - 'root': api/basic_json_document/root.md + - 'save': api/basic_json_document/save.md - 'set': api/basic_json_document/set.md - 'shrink_to_fit': api/basic_json_document/shrink_to_fit.md - 'source': api/basic_json_document/source.md diff --git a/include/nlohmann/detail/view/document_data.hpp b/include/nlohmann/detail/view/document_data.hpp index e31400a1e..55f9d8c76 100644 --- a/include/nlohmann/detail/view/document_data.hpp +++ b/include/nlohmann/detail/view/document_data.hpp @@ -10,7 +10,7 @@ #include // array #include // size_t -#include // uint32_t +#include // uint8_t, uint32_t #include // memcmp, memcpy #include // numeric_limits #include // map @@ -42,7 +42,9 @@ struct document_data std::size_t inline_capacity = 0; std::size_t max_depth = 0; ///< deepest nesting of arrays and objects (the root container is depth 1) std::vector arena; ///< decoded strings that contained escapes (no terminator: they are handed out with their length) + std::size_t arena_size = 0; ///< bytes of decoded strings at base[1] (the arena, or those of a loaded image) std::string owned; ///< owned copy of the input, if any (a std::string, so that its NUL serves as the parser's sentinel) + std::vector owned_image; ///< a loaded image the document owns (the text and the decoded strings point into it) // hash indexes of the keys of large objects (see object_index.hpp) static constexpr std::uint32_t index_min_members = 128; @@ -119,6 +121,7 @@ struct document_data document_data() noexcept : arena() // NOLINT(readability-redundant-member-init) , owned() // NOLINT(readability-redundant-member-init) + , owned_image() // NOLINT(readability-redundant-member-init) , indexes() // NOLINT(readability-redundant-member-init) , index_slots() // NOLINT(readability-redundant-member-init) , large_objects() // NOLINT(readability-redundant-member-init) diff --git a/include/nlohmann/detail/view/image.hpp b/include/nlohmann/detail/view/image.hpp new file mode 100644 index 000000000..702a55530 --- /dev/null +++ b/include/nlohmann/detail/view/image.hpp @@ -0,0 +1,744 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#pragma once + +#include // min, sort +#include // array +#include // offsetof, size_t +#include // int64_t, uint8_t, uint16_t, uint32_t, uint64_t +#include // memcmp, memcpy, memset +#include // numeric_limits +#include // string +#include // vector + +#include +#include +#include +#include +#include +#include +#include +#include + +// Images: a document stored so that loading it needs no parsing. +// +// Layout (little-endian): a 64-byte header, the nodes, the text (the source, +// followed by the number tokens written by edits), a NUL, the decoded strings +// (followed by the strings written by edits), a NUL. The idea is that of +// zero-copy formats such as FlatBuffers (https://github.com/google/flatbuffers) +// and YaFF (https://github.com/yandex/yaff); no code is taken from them. +// check_image follows the idea of FlatBuffers' Verifier (bounds and +// structure) and also checks what the parser guarantees about strings and +// numbers, so that reading and serializing a checked image is safe and yields +// valid JSON. + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ +namespace view +{ + +/// how load() checks an image +enum class image_check +{ + /// everything the parser guarantees: structure and bounds, strings (valid + /// UTF-8; source strings without quotes, backslashes, and control + /// characters), and numbers (well-formed, matching the stored values) + full, + /// structure and bounds only: reading and serializing are safe, but a + /// crafted image can yield invalid UTF-8, strings that serialize to + /// invalid JSON, or numbers that differ from their text + bounds, + /// none: for images from a trusted source only (a damaged image is + /// undefined behavior) + none, +}; + +struct image_header +{ + std::array magic; ///< "NJVI" + std::uint32_t version; ///< 1 + std::uint64_t node_count; + std::uint64_t text_size; + std::uint64_t arena_size; + std::array reserved; ///< zero (for later versions) +}; +static_assert(sizeof(image_header) == 64, "the image header must be 64 bytes"); + +constexpr std::uint32_t image_version = 1; + +/// the largest node count and text or string size of an image (as for parsed +/// documents, offsets and counts must fit 32 bits) +constexpr std::uint64_t image_limit = 0xFFFFFFF0u; + +/// Copy the current structure of an edited document into nodes in document +/// order, as the parser would have written them. Text written by edits is +/// appended to text_tail (number tokens) and arena_tail (strings); floats that +/// are not finite become null, as dump() writes them. +inline void compact_nodes(const document_data& d, std::size_t arena_size, std::vector& out, std::string& text_tail, std::string& arena_tail) +{ + struct frame + { + const node* cur; + const node* end; + std::size_t index; ///< the container's node in out + std::uint32_t count; + bool object; + }; + std::vector stack; + const auto string_node = [&](const node & s) + { + node r = s; + r.extra = 0; + r.flags = static_cast(s.flags & node_flags::storage); + if (r.flags == node_flags::edited) + { + r.off = static_cast(arena_size + arena_tail.size()); + arena_tail.append(d.str(s), s.len); + r.flags = node_flags::escaped; + } + return r; + }; + const auto copy_node = [&](const node * v) + { + node r = *v; + switch (static_cast(v->kind)) + { + case value_t::object: + case value_t::array: + r.flags = 0; + r.extra = 0; + r.off = (v->flags & (node_flags::moved | node_flags::is_new)) != 0 ? 0 : v->off; + r.len = 0; // counted below + r.next = 0; // set when the container is complete + stack.push_back(frame{d.first_child_edited(v), d.child_end_edited(v), out.size(), 0, v->kind == static_cast(value_t::object)}); + break; + case value_t::string: + r = string_node(*v); + break; + case value_t::number_integer: + case value_t::number_unsigned: + if ((v->flags & node_flags::storage) == node_flags::edited) + { + r.off = static_cast(d.size + text_tail.size()); + text_tail.append(d.str(*v), number_length(*v)); + } + r.flags = 0; + break; + case value_t::number_float: + if ((v->flags & node_flags::storage) == node_flags::edited) + { + const char* const t = d.str(*v); + if (t[0] == 'n' || t[0] == 'i' || (v->len > 1 && t[1] == 'i')) + { + r = node{}; // nan and infinity: null, as dump() writes them + r.kind = static_cast(value_t::null); + break; + } + r.off = static_cast(d.size + text_tail.size()); + text_tail.append(t, v->len); + r.extra = 0xFFFFu; // the digit layout is not recorded + } + r.flags = 0; + break; + case value_t::boolean: + r.flags = static_cast(v->flags & node_flags::is_true); + break; + case value_t::null: + case value_t::binary: + case value_t::discarded: + default: + r.flags = 0; + break; + } + out.push_back(r); + }; + copy_node(d.nodes); + while (!stack.empty()) + { + const frame top = stack.back(); + if (top.cur == top.end) + { + node& c = out[top.index]; + c.len = top.count; + c.next = static_cast(out.size() - top.index); + stack.pop_back(); + continue; + } + if (top.object) + { + out.push_back(string_node(*top.cur)); + } + const node* const value = top.cur + (top.object ? 1 : 0); + stack.back().cur = document_data::after(value); + ++stack.back().count; + copy_node(document_data::deref(value)); // may grow the stack + } +} + +/// the document as an image +inline std::vector save_image(const document_data& d) +{ +#if !NLOHMANN_VIEW_LITTLE_ENDIAN + throw_type_error(320, "json_document images need a little-endian target"); // LCOV_EXCL_LINE +#endif + const std::size_t arena_size = d.arena_size; + std::vector compacted; + std::string text_tail; + std::string arena_tail; + if (d.edits) + { + compact_nodes(d, arena_size, compacted, text_tail, arena_tail); + } + const node* const nodes = d.edits ? compacted.data() : d.nodes; + const std::size_t count = d.edits ? compacted.size() : d.node_count; + const std::size_t text_size = d.size + text_tail.size(); + const std::size_t total_arena = arena_size + arena_tail.size(); + if (NLOHMANN_VIEW_UNLIKELY(text_size >= image_limit || total_arena >= image_limit || count >= image_limit)) + { + // LCOV_EXCL_START (4 GiB) + throw_out_of_range(416, "images of 4 GiB or more are not supported by json_document"); + // LCOV_EXCL_STOP + } + image_header h{}; + h.magic = {{'N', 'J', 'V', 'I'}}; + h.version = image_version; + h.node_count = count; + h.text_size = text_size; + h.arena_size = total_arena; + std::vector image(sizeof(h) + (count * sizeof(node)) + text_size + 1 + total_arena + 1); + std::uint8_t* o = image.data(); + std::memcpy(o, &h, sizeof(h)); + o += sizeof(h); + std::memcpy(o, nodes, count * sizeof(node)); + // the hash indexes are rebuilt by load(), whose check rejects any other + // extra of an object, and the same document saves to the same bytes + for (std::size_t i = 0; i < count; ++i) + { + if (nodes[i].kind == static_cast(value_t::object) && nodes[i].extra != 0) + { + std::memset(o + (i * sizeof(node)) + offsetof(node, extra), 0, sizeof(nodes[i].extra)); + } + } + o += count * sizeof(node); + const auto append = [&o](const char* s, std::size_t n) + { + if (n != 0) + { + std::memcpy(o, s, n); + o += n; + } + }; + append(d.src, d.size); + append(text_tail.data(), text_tail.size()); + *o++ = 0; + append(d.base[1], arena_size); + append(arena_tail.data(), arena_tail.size()); + *o = 0; + return image; +} + +/// the parts of a number token that the checks need +struct number_token +{ + const unsigned char* int_start; + std::size_t int_digits; + std::size_t frac_digits; + std::int64_t exponent; + bool negative; + bool is_float; ///< a fraction or an exponent +}; + +/// whether [s, s + len) is a JSON number (the grammar the parser accepts) +inline bool scan_number_token(const unsigned char* s, std::size_t len, number_token& t) +{ + const unsigned char* const e = s + len; + const unsigned char* p = s; + t.negative = p != e && *p == '-'; + p += t.negative ? 1 : 0; + if (p == e) + { + return false; + } + t.int_start = p; + if (*p == '0') + { + ++p; + } + else if (*p >= '1' && *p <= '9') + { + do + { + ++p; + } + while (p != e && is_digit(*p)); + } + else + { + return false; + } + t.int_digits = static_cast(p - t.int_start); + t.frac_digits = 0; + t.is_float = false; + if (p != e && *p == '.') + { + const unsigned char* const f0 = ++p; + while (p != e && is_digit(*p)) + { + ++p; + } + if (p == f0) + { + return false; + } + t.frac_digits = static_cast(p - f0); + t.is_float = true; + } + t.exponent = 0; + if (p != e && (*p | 0x20u) == 'e') + { + ++p; + const bool exp_negative = p != e && *p == '-'; + p += (p != e && (*p == '+' || *p == '-')) ? 1 : 0; + if (p == e || !is_digit(*p)) + { + return false; + } + do + { + t.exponent = t.exponent < 100000 ? (t.exponent * 10) + (*p - '0') : t.exponent; + ++p; + } + while (p != e && is_digit(*p)); + t.exponent = exp_negative ? -t.exponent : t.exponent; + t.is_float = true; + } + return p == e; +} + +/// the digit layout the parser records for a float token (compaction +/// writes "many" instead; the caller accepts both) +inline std::uint16_t float_layout(const number_token& t) +{ + return static_cast((std::min)(t.int_digits, std::size_t{255}) | ((std::min)(t.frac_digits, std::size_t{255}) << 8u)); +} + +/// whether a float token (well-formed, per scan_number_token) is finite as +/// double; parse() rejects floats that overflow, and as there, only a number +/// whose magnitude could reach 1e308 needs the conversion +inline bool float_token_finite(const unsigned char* s, std::size_t len, const number_token& t) +{ + if (static_cast(t.int_digits) + t.exponent > 300) + { + node n{}; + n.kind = static_cast(value_t::number_float); + n.len = static_cast(len); + const auto v = float_value(reinterpret_cast(s), n); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + return v <= (std::numeric_limits::max)() && v >= -(std::numeric_limits::max)(); + } + return true; +} + +/// whether an integer node matches its token the way the parser records it +/// (after the bounds check; the token has at most 256 characters) +inline bool check_integer(const node& n, const unsigned char* text) +{ + const std::size_t len = number_length(n); + const unsigned char* const s = text + n.off; + number_token t{}; + if (!scan_number_token(s, len, t)) + { + return false; + } + // the token's value is the stored one; number_integer nodes of edits can + // be non-negative (as basic_json keeps the type of a value) + const bool integer = n.kind == static_cast(value_t::number_integer); + if (t.is_float || t.int_digits > 20 || (t.negative && !integer)) + { + return false; + } + // (at most 19 digits cannot overflow; 20 digits are compared with 2^64 - 1) + if (t.int_digits == 20 && std::memcmp(t.int_start, "18446744073709551615", 20) > 0) + { + return false; + } + std::uint64_t m = 0; + for (const unsigned char* d = t.int_start; d != t.int_start + t.int_digits; ++d) + { + m = (m * 10) + static_cast(*d - '0'); + } + if (integer && m > (t.negative ? std::uint64_t{1} << 63u : (std::uint64_t{1} << 63u) - 1)) + { + return false; + } + return integer_bits(n) == (t.negative ? 0 - m : m); +} + +/// a byte range of the text or of the decoded strings +struct byte_range +{ + std::uint32_t off; + std::uint32_t len; +}; + +/// a float token and the digit layout its node records +struct float_range +{ + std::uint32_t off; + std::uint32_t len; + std::uint16_t extra; +}; + +/// Run check once for each distinct range of ranges (which are within bounds +/// and not empty). Ranges that are not identical must not overlap: save() +/// writes every string and token to a place of its own, and nodes that share +/// a value (copies within a document) share the whole range. This bounds the +/// work by the size of the text, however many nodes point to the same bytes. +template +bool check_distinct_ranges(std::vector& ranges, Check check) +{ + std::sort(ranges.begin(), ranges.end(), [](const byte_range & a, const byte_range & b) + { + return a.off != b.off ? a.off < b.off : a.len < b.len; + }); + std::size_t end = 0; + for (std::size_t i = 0; i < ranges.size(); ++i) + { + const byte_range r = ranges[i]; + if (i != 0 && r.off == ranges[i - 1].off && r.len == ranges[i - 1].len) + { + continue; + } + if (r.off < end || !check(r)) + { + return false; + } + end = static_cast(r.off) + r.len; + } + return true; +} + +/// Check the float nodes: each token once (nodes of the same token must +/// record layouts that match it), tokens must not overlap. +inline bool check_float_ranges(std::vector& ranges, const unsigned char* text) +{ + std::sort(ranges.begin(), ranges.end(), [](const float_range & a, const float_range & b) + { + if (a.off != b.off) + { + return a.off < b.off; + } + if (a.len != b.len) + { + return a.len < b.len; + } + return a.extra < b.extra; + }); + std::size_t end = 0; + std::size_t i = 0; + while (i < ranges.size()) + { + const float_range r = ranges[i]; + if (r.off < end) + { + return false; + } + number_token t{}; + const unsigned char* const s = text + r.off; + if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t)) + { + return false; + } + // the digit layout the parser records (or "many", as compaction writes it) + const std::uint16_t layout = float_layout(t); + for (; i < ranges.size() && ranges[i].off == r.off && ranges[i].len == r.len; ++i) + { + if (ranges[i].extra != layout && ranges[i].extra != 0xFFFFu) + { + return false; + } + } + end = static_cast(r.off) + r.len; + } + return true; +} + +/// the content checks of check_image: source strings as the parser leaves +/// them (no quotes, backslashes, or control characters), decoded strings +/// (valid UTF-8), and float tokens +inline bool check_contents(const unsigned char* text, const unsigned char* arena, + std::vector& source_strings, std::vector& decoded_strings, + std::vector& floats) +{ + return check_distinct_ranges(source_strings, [text](const byte_range & r) + { + const unsigned char* const b = text + r.off; + return scan_string_run(b, b + r.len) == b + r.len; + }) + && check_distinct_ranges(decoded_strings, [arena](const byte_range & r) + { + return valid_utf8_prefix(arena + r.off, r.len) == r.len; + }) + && check_float_ranges(floats, text); +} + +/// Check the nodes of a loaded image against its text and decoded strings: +/// kinds, flags, and `extra`; extents and element counts of arrays and +/// objects; keys; bounds; string contents (source strings as the parser +/// leaves them: no quotes, backslashes, or control characters; all strings +/// valid UTF-8); and number tokens. +/// +/// The structure and the bounds are checked node by node. The contents of +/// strings and of float tokens are checked afterwards, once for each distinct +/// range (see check_distinct_ranges), so that the full check is linear in the +/// size of the image plus the sorting of the ranges, and not in the number of +/// nodes times the size of the text. +inline bool check_image(const node* nodes, std::size_t count, const unsigned char* text, std::size_t text_size, + const unsigned char* arena, std::size_t arena_size, bool full) +{ + struct frame + { + std::size_t end; + std::uint32_t len; + std::uint32_t seen; + bool object; + bool expect_key; + }; + std::vector stack; + std::vector source_strings; + std::vector decoded_strings; + std::vector floats; + const auto check_string = [&](const node & n) -> bool + { + if ((n.flags & ~node_flags::escaped) != 0 || n.extra != 0) + { + return false; + } + const bool decoded = (n.flags & node_flags::escaped) != 0; + const std::size_t limit = decoded ? arena_size : text_size; + if (n.off > limit || n.len > limit - n.off) + { + return false; + } + if (full && n.len != 0) + { + (decoded ? decoded_strings : source_strings).push_back(byte_range{n.off, n.len}); + } + return true; + }; + // bounds of a number token; the recorded digit layout must lie within it + const auto number_in_bounds = [&](const node & n) -> bool + { + const std::size_t len = number_length(n); + if (len == 0 || n.off > text_size || len > text_size - n.off) + { + return false; + } + if (n.kind != static_cast(value_t::number_float)) + { + return (n.extra >> 8u) == 0; + } + // float_value() reads the sign, the integer digits, and the point and + // fraction digits the layout records (a layout of more than 19 digits + // means the general conversion, which stays within the token) + const std::size_t int_digits = n.extra & 0xFFu; + const std::size_t frac_digits = n.extra >> 8u; + const std::size_t need = (text[n.off] == '-' ? 1u : 0u) + int_digits + (frac_digits != 0 ? frac_digits + 1 : 0); + return int_digits + frac_digits > 19 || need <= len; + }; + std::size_t i = 0; + for (;;) + { + // close finished arrays and objects + while (!stack.empty() && i == stack.back().end) + { + const frame f = stack.back(); + if (f.seen != f.len || (f.object && !f.expect_key)) + { + return false; + } + stack.pop_back(); + if (!stack.empty()) + { + ++stack.back().seen; + stack.back().expect_key = true; + } + } + if (i == count) + { + return stack.empty() && (!full || check_contents(text, arena, source_strings, decoded_strings, floats)); + } + if (i != 0 && stack.empty()) + { + return false; // nodes after the root + } + const node& n = nodes[i]; + if (!stack.empty() && stack.back().object && stack.back().expect_key) + { + if (n.kind != static_cast(value_t::string) || !check_string(n)) + { + return false; + } + stack.back().expect_key = false; + ++i; + continue; + } + bool complete = true; + switch (static_cast(n.kind)) + { + case value_t::null: + // (the offset of a literal is read to size the output of dump()) + if (n.flags != 0 || n.extra != 0 || n.off > text_size) + { + return false; + } + break; + case value_t::boolean: + if ((n.flags & ~node_flags::is_true) != 0 || n.extra != 0 || n.off > text_size) + { + return false; + } + break; + case value_t::string: + if (!check_string(n)) + { + return false; + } + break; + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + if (n.flags != 0 || !number_in_bounds(n)) + { + return false; + } + if (full) + { + // (the contents of float tokens are checked later; the token of an + // integer has at most 256 characters) + if (n.kind == static_cast(value_t::number_float)) + { + floats.push_back(float_range{n.off, n.len, n.extra}); + } + else if (!check_integer(n, text)) + { + return false; + } + } + break; + case value_t::array: + case value_t::object: + { + const std::size_t limit = stack.empty() ? count : stack.back().end; + if (n.flags != 0 || n.extra != 0 || n.next == 0 || n.next > limit - i || n.off > text_size) + { + return false; + } + stack.push_back(frame{i + n.next, n.len, 0, n.kind == static_cast(value_t::object), true}); + complete = false; + break; + } + case value_t::binary: + case value_t::discarded: + default: + return false; + } + ++i; + if (complete && !stack.empty()) + { + ++stack.back().seen; + stack.back().expect_key = true; + } + } +} + +[[noreturn]] NLOHMANN_VIEW_NOINLINE inline void throw_invalid_image(const char* what) +{ + throw_parse_error(116, concat("invalid json_document image: ", what)); +} + +/// Read an image into d. The text and the decoded strings stay in the image; +/// the nodes are copied (so that they are aligned, and edits can change them). +inline void load_image(document_data& d, const std::uint8_t* image, std::size_t size, image_check check) +{ +#if !NLOHMANN_VIEW_LITTLE_ENDIAN + throw_type_error(320, "json_document images need a little-endian target"); // LCOV_EXCL_LINE +#endif + if (image == nullptr || size < sizeof(image_header)) + { + throw_invalid_image("too short"); + } + image_header h{}; + std::memcpy(&h, image, sizeof(h)); + // (the reserved fields are for later versions) + if (std::memcmp(h.magic.data(), "NJVI", 4) != 0 || h.version != image_version + || (h.reserved[0] | h.reserved[1] | h.reserved[2] | h.reserved[3]) != 0) + { + throw_invalid_image("unknown format"); + } + const std::size_t room = size - sizeof(h); + if (h.node_count == 0 || h.node_count > room / sizeof(node) || h.node_count >= image_limit || h.text_size >= image_limit || h.arena_size >= image_limit) + { + throw_invalid_image("sizes out of range"); + } + const auto count = static_cast(h.node_count); + const auto text_size = static_cast(h.text_size); + const auto arena_size = static_cast(h.arena_size); + const std::size_t text_at = sizeof(h) + (count * sizeof(node)); + // the text, a NUL, the decoded strings, a NUL, and nothing after them + if (size - text_at < 2 || text_size > size - text_at - 2 || arena_size != size - text_at - text_size - 2 + || image[text_at + text_size] != 0 || image[size - 1] != 0) + { + throw_invalid_image("sizes out of range"); + } + + d.discarded = true; + d.edits.reset(); + d.base[2] = nullptr; + d.owned.clear(); + if (d.owned_image.empty() || image != d.owned_image.data()) + { + d.owned_image.clear(); + } + d.arena.clear(); + d.indexes.clear(); + d.index_slots.clear(); + d.large_objects.clear(); + d.node_count = 0; + d.reserve(count); + std::memcpy(d.nodes, image + sizeof(h), count * sizeof(node)); + d.node_count = count; + const std::uint8_t* const text = image + text_at; + const std::uint8_t* const arena = text + text_size + 1; + d.src = reinterpret_cast(text); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + d.size = text_size; + d.base[0] = d.src; + d.base[1] = reinterpret_cast(arena); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + d.arena_size = arena_size; + if (check != image_check::none && !check_image(d.nodes, count, text, text_size, arena, arena_size, check == image_check::full)) + { + throw_invalid_image("the check failed"); + } + // the hash indexes of large objects, as after parsing + for (std::size_t i = 0; i < count; ++i) + { + const node& n = d.nodes[i]; + if (n.kind == static_cast(value_t::object) && n.len >= document_data::index_min_members) + { + d.large_objects.push_back(static_cast(i)); + } + } + build_object_indexes(d); + std::vector().swap(d.large_objects); // (only needed while building) + d.discarded = false; +} + +} // namespace view +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/view/number.hpp b/include/nlohmann/detail/view/number.hpp index fce98e9ba..10e2aa85d 100644 --- a/include/nlohmann/detail/view/number.hpp +++ b/include/nlohmann/detail/view/number.hpp @@ -26,43 +26,79 @@ namespace detail namespace view { +/*! +@brief locate the decimal point and the end of the mantissa of a float token + +Also checks that the token is a JSON number. Tokens of the parser and of edits +always are; an image loaded with image_check::bounds can hold any bytes, which +must not reach the conversion (it expects a well-formed token). +*/ +inline bool float_token_layout(const char* first, const char* last, std::size_t& dot, std::size_t& mantissa_end) noexcept +{ + const auto digit = [last](const char* q) + { + return q != last && is_digit(static_cast(*q)); + }; + const char* p = first; + p += (p != last && *p == '-') ? 1 : 0; + if (!digit(p) || (*p == '0' && digit(p + 1))) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + dot = std::string::npos; + if (p != last && *p == '.') + { + dot = static_cast(p - first); + if (!digit(++p)) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + } + mantissa_end = static_cast(p - first); + if (p != last && (*p == 'e' || *p == 'E')) + { + ++p; + p += (p != last && (*p == '+' || *p == '-')) ? 1 : 0; + if (!digit(p)) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + } + return p == last; +} + /*! @brief the value of the float token of a node, as parse() converts it Uses the lexer's conversion (detail::convert_float), so that the values are bit-identical to parse(): float and double are converted without allocation -and independent of the locale. The digit layout recorded while parsing locates -the decimal point and the exponent without scanning the token. +and independent of the locale. A token that is not a JSON number (only in a +damaged image loaded with image_check::bounds) yields 0. */ template NLOHMANN_VIEW_NOINLINE FloatType float_value(const char* first, const node& n) { const char* const last = first + n.len; - const std::size_t neg = first[0] == '-' ? 1 : 0; - const std::size_t int_digits = n.extra & 0xFFu; - const std::size_t frac_digits = n.extra >> 8u; - std::size_t dot = std::string::npos; - std::size_t mantissa_end = n.len; - if (int_digits != 255 && frac_digits != 255) + std::size_t dot = 0; + std::size_t mantissa_end = 0; + if (NLOHMANN_VIEW_UNLIKELY(!float_token_layout(first, last, dot, mantissa_end))) { - dot = frac_digits != 0 ? neg + int_digits : std::string::npos; - mantissa_end = neg + int_digits + (frac_digits != 0 ? 1 + frac_digits : 0); - } - else - { - // more digits than the layout records: locate them - for (std::size_t i = 0; i < n.len; ++i) - { - if (first[i] == '.') - { - dot = i; - } - else if (first[i] == 'e' || first[i] == 'E') - { - mantissa_end = i; - break; - } - } + return FloatType{}; } return convert_float(first, last, dot, mantissa_end); } @@ -93,16 +129,20 @@ NLOHMANN_VIEW_ALWAYS_INLINE float_significand layout_decimal(const unsigned char } if (p != e) { - // [eE][+-]digits; huge exponents saturate (the parser rejected overflow) + // [eE][+-]digits; huge exponents saturate (the parser rejected + // overflow). The token is not read beyond e, and the digits are taken + // as unsigned, so that a token that is not well-formed (a damaged + // image loaded with image_check::bounds) yields a wrong value, but no + // overflow. ++p; - const bool exp_negative = *p == '-'; - p += (*p == '-' || *p == '+') ? 1 : 0; + const bool exp_negative = p != e && *p == '-'; + p += (p != e && (*p == '-' || *p == '+')) ? 1 : 0; std::int64_t exp_value = 0; for (; p != e; ++p) { if (exp_value < 0x10000000) { - exp_value = (exp_value * 10) + (*p - '0'); + exp_value = (exp_value * 10) + static_cast(*p - '0'); } } q += exp_negative ? -exp_value : exp_value; diff --git a/include/nlohmann/detail/view/serializer.hpp b/include/nlohmann/detail/view/serializer.hpp index 5409b477b..6a6719a65 100644 --- a/include/nlohmann/detail/view/serializer.hpp +++ b/include/nlohmann/detail/view/serializer.hpp @@ -327,7 +327,9 @@ class view_serializer m_out.put('"'); } - /// as serializer::dump_escaped() for valid UTF-8 (the view has no other) + /// as serializer::dump_escaped(); strings of a document are valid UTF-8, + /// except in a damaged image loaded with image_check::bounds, for which + /// this throws what basic_json::dump() throws for the string template void write_escaped(const unsigned char* s, std::size_t n) { @@ -351,12 +353,13 @@ class view_serializer } std::uint32_t codepoint = s[i]; std::size_t len = 1; - if (codepoint >= 0xC0) + if (codepoint >= 0x80) { - len = 2; - if (codepoint >= 0xE0) + len = validate_one_utf8(s + i, n - i); + if (NLOHMANN_VIEW_UNLIKELY(len == 0)) { - len = codepoint >= 0xF0 ? 4 : 3; + invalid_utf8(s, n); + return; } codepoint &= 0xFFu >> (len + 1); for (std::size_t k = 1; k < len; ++k) @@ -369,6 +372,13 @@ class view_serializer } } + /// throw what basic_json::dump() throws for a string that is not valid UTF-8 + NLOHMANN_VIEW_NOINLINE static void invalid_utf8(const unsigned char* s, std::size_t n) + { + const string_t dumped = BasicJsonType(string_t(reinterpret_cast(s), n)).dump(); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + static_cast(dumped); + } + template void write_codepoint(std::uint32_t codepoint, const unsigned char* bytes, std::size_t len) { diff --git a/include/nlohmann/json_view.hpp b/include/nlohmann/json_view.hpp index da18cd698..fd43d3990 100644 --- a/include/nlohmann/json_view.hpp +++ b/include/nlohmann/json_view.hpp @@ -29,7 +29,7 @@ #include // all_of, min #include // array #include // nullptr_t, size_t // IWYU pragma: keep -#include // uint32_t +#include // uint8_t, uint32_t #include // memcpy, strlen #include // istream // IWYU pragma: keep #include // distance, input_iterator_tag, iterator_traits @@ -71,6 +71,7 @@ #include #include #include +#include #include #include #include @@ -1028,7 +1029,7 @@ class basic_json_document /// whether the document holds its own copy of the text bool owns_source() const noexcept { - return m_data && !m_data->owned.empty() && m_data->src == m_data->owned.data(); + return m_data && ((!m_data->owned.empty() && m_data->src == m_data->owned.data()) || !m_data->owned_image.empty()); } /// number of index nodes (values plus object keys) @@ -1037,7 +1038,7 @@ class basic_json_document return m_data ? m_data->node_count : 0; } - /// bytes held by the document (index, decoded strings, owned text) + /// bytes held by the document (index, decoded strings, and owned text or image) std::size_t memory_usage() const noexcept { if (!m_data) @@ -1046,7 +1047,7 @@ class basic_json_document } return sizeof(document_data) + (m_data->inline_capacity * sizeof(detail::view::node)) + (m_data->nodes != m_data->inline_nodes ? m_data->node_capacity * sizeof(detail::view::node) : 0) - + m_data->arena.capacity() + m_data->owned.capacity() + + m_data->arena.capacity() + m_data->owned.capacity() + m_data->owned_image.capacity() + (m_data->indexes.capacity() * sizeof(document_data::object_index)) + (m_data->index_slots.capacity() * sizeof(std::uint32_t)) + (m_data->large_objects.capacity() * sizeof(std::uint32_t)) + (m_data->edits != nullptr ? m_data->edits->bytes : 0); @@ -1066,8 +1067,10 @@ class basic_json_document // allocate everything first, so that an exception leaves the document // unchanged + // (the decoded strings of a loaded image stay in the image) + const bool arena_in_use = d.base[1] == d.arena.data(); const bool shrink_arena = d.arena.capacity() > d.arena.size(); - std::vector arena(shrink_arena ? d.arena : std::vector()); + std::vector arena(shrink_arena && arena_in_use ? d.arena : std::vector()); // (edits link to the nodes of the index, which then stays in place) const bool shrink_nodes = d.nodes != d.inline_nodes && d.node_count != d.node_capacity && d.edits == nullptr; const bool into_header = d.node_count <= d.inline_capacity; @@ -1088,7 +1091,10 @@ class basic_json_document if (shrink_arena) { d.arena.swap(arena); - d.base[1] = d.arena.data(); + if (arena_in_use) + { + d.base[1] = d.arena.data(); + } } if (d.indexes.capacity() > d.indexes.size()) { @@ -1100,6 +1106,55 @@ class basic_json_document } } + //////////// + // images // + //////////// + + /// how load() checks an image (full, bounds, or none) + using image_check = detail::view::image_check; + + /// The document as an image that load() reads without parsing: the node + /// index, the text, and the decoded strings. An edited document is + /// written in its current state (floats that are not finite become null, + /// as in dump()). + std::vector save() const + { + if (NLOHMANN_VIEW_UNLIKELY(!m_data || m_data->discarded)) + { + detail::view::throw_type_error(320, "cannot save a discarded json_document"); + } + return detail::view::save_image(*m_data); + } + + /// Read an image written by save(). The image is borrowed: it must stay + /// alive and unchanged while the document is used. + NLOHMANN_VIEW_NODISCARD + static basic_json_document load(const std::uint8_t* image, std::size_t size, const image_check check = image_check::full) + { + basic_json_document d; + d.ensure_data(nullptr, 0); + detail::view::load_image(*d.m_data, image, size, check); + return d; + } + + /// read an image (borrowed) + NLOHMANN_VIEW_NODISCARD + static basic_json_document load(const std::vector& image, const image_check check = image_check::full) + { + return load(image.data(), image.size(), check); + } + + /// read an image and keep it (no copy) + NLOHMANN_VIEW_NODISCARD + static basic_json_document load(std::vector&& image, const image_check check = image_check::full) + { + basic_json_document d; + d.ensure_data(nullptr, 0); + d.m_data->owned_image = std::move(image); + detail::view::load_image(*d.m_data, d.m_data->owned_image.data(), d.m_data->owned_image.size(), check); + return d; + } + /////////// // edits // /////////// @@ -1334,6 +1389,7 @@ class basic_json_document { d.owned.clear(); } + d.owned_image.clear(); d.src = src; d.size = size; d.node_count = 0; @@ -1358,6 +1414,7 @@ class basic_json_document { d.base[0] = d.src; d.base[1] = d.arena.data(); + d.arena_size = d.arena.size(); detail::view::build_object_indexes(d); std::vector().swap(d.large_objects); // (only needed while parsing) d.discarded = false; diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 7453e82d3..0682c08be 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -29,7 +29,7 @@ #include // all_of, min #include // array #include // nullptr_t, size_t // IWYU pragma: keep -#include // uint32_t +#include // uint8_t, uint32_t #include // memcpy, strlen #include // istream // IWYU pragma: keep #include // distance, input_iterator_tag, iterator_traits @@ -101,7 +101,7 @@ #include // array #include // size_t -#include // uint32_t +#include // uint8_t, uint32_t #include // memcmp, memcpy #include // numeric_limits #include // map @@ -356,7 +356,9 @@ struct document_data std::size_t inline_capacity = 0; std::size_t max_depth = 0; ///< deepest nesting of arrays and objects (the root container is depth 1) std::vector arena; ///< decoded strings that contained escapes (no terminator: they are handed out with their length) + std::size_t arena_size = 0; ///< bytes of decoded strings at base[1] (the arena, or those of a loaded image) std::string owned; ///< owned copy of the input, if any (a std::string, so that its NUL serves as the parser's sentinel) + std::vector owned_image; ///< a loaded image the document owns (the text and the decoded strings point into it) // hash indexes of the keys of large objects (see object_index.hpp) static constexpr std::uint32_t index_min_members = 128; @@ -433,6 +435,7 @@ struct document_data document_data() noexcept : arena() // NOLINT(readability-redundant-member-init) , owned() // NOLINT(readability-redundant-member-init) + , owned_image() // NOLINT(readability-redundant-member-init) , indexes() // NOLINT(readability-redundant-member-init) , index_slots() // NOLINT(readability-redundant-member-init) , large_objects() // NOLINT(readability-redundant-member-init) @@ -4326,6 +4329,985 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // min, sort +#include // array +#include // offsetof, size_t +#include // int64_t, uint8_t, uint16_t, uint32_t, uint64_t +#include // memcmp, memcpy, memset +#include // numeric_limits +#include // string +#include // vector + +// #include +// #include + +// #include + +// #include + +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // size_t +#include // int64_t, uint64_t +#include // numeric_limits +#include // string +#include // integral_constant + +// #include +// #include + +// #include + +// #include + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ +namespace view +{ + +/*! +@brief locate the decimal point and the end of the mantissa of a float token + +Also checks that the token is a JSON number. Tokens of the parser and of edits +always are; an image loaded with image_check::bounds can hold any bytes, which +must not reach the conversion (it expects a well-formed token). +*/ +inline bool float_token_layout(const char* first, const char* last, std::size_t& dot, std::size_t& mantissa_end) noexcept +{ + const auto digit = [last](const char* q) + { + return q != last && is_digit(static_cast(*q)); + }; + const char* p = first; + p += (p != last && *p == '-') ? 1 : 0; + if (!digit(p) || (*p == '0' && digit(p + 1))) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + dot = std::string::npos; + if (p != last && *p == '.') + { + dot = static_cast(p - first); + if (!digit(++p)) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + } + mantissa_end = static_cast(p - first); + if (p != last && (*p == 'e' || *p == 'E')) + { + ++p; + p += (p != last && (*p == '+' || *p == '-')) ? 1 : 0; + if (!digit(p)) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + } + return p == last; +} + +/*! +@brief the value of the float token of a node, as parse() converts it + +Uses the lexer's conversion (detail::convert_float), so that the values are +bit-identical to parse(): float and double are converted without allocation +and independent of the locale. A token that is not a JSON number (only in a +damaged image loaded with image_check::bounds) yields 0. +*/ +template +NLOHMANN_VIEW_NOINLINE FloatType float_value(const char* first, const node& n) +{ + const char* const last = first + n.len; + std::size_t dot = 0; + std::size_t mantissa_end = 0; + if (NLOHMANN_VIEW_UNLIKELY(!float_token_layout(first, last, dot, mantissa_end))) + { + return FloatType{}; + } + return convert_float(first, last, dot, mantissa_end); +} + +/*! +@brief the digits of a float token with at most 19 digits, from its layout + +The digit layout recorded while parsing says where the integer digits, the +fraction digits, and the exponent are, so the digits are read eight at a +time without scanning. + +@param[in] p first character of the token +@param[in] e end of the token +@param[in] limit end of the readable memory (the source text) +*/ +NLOHMANN_VIEW_ALWAYS_INLINE float_significand layout_decimal(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept +{ + const bool negative = *p == '-'; + p += negative ? 1 : 0; + std::uint64_t w = parse_upto19(p, int_digits, limit); + p += int_digits; + std::int64_t q = 0; + if (frac_digits != 0) + { + w = (w * int_pow10(frac_digits)) + parse_upto19(p + 1, frac_digits, limit); + p += 1 + frac_digits; + q = -static_cast(frac_digits); + } + if (p != e) + { + // [eE][+-]digits; huge exponents saturate (the parser rejected + // overflow). The token is not read beyond e, and the digits are taken + // as unsigned, so that a token that is not well-formed (a damaged + // image loaded with image_check::bounds) yields a wrong value, but no + // overflow. + ++p; + const bool exp_negative = p != e && *p == '-'; + p += (p != e && (*p == '-' || *p == '+')) ? 1 : 0; + std::int64_t exp_value = 0; + for (; p != e; ++p) + { + if (exp_value < 0x10000000) + { + exp_value = (exp_value * 10) + static_cast(*p - '0'); + } + } + q += exp_negative ? -exp_value : exp_value; + } + + float_significand d; + d.w = w; + d.exponent = q; + d.negative = negative; + return d; +} + +/*! +@brief the value of a float token with at most 19 digits, from its layout + +The result is correctly rounded by the lexer's conversion +(detail::decimal_to_float(): Clinger's fast path where both operands are +exact, else the Eisel-Lemire algorithm, which needs no fallback for up to 19 +digits), so it is the value parse() produces. +*/ +template +NLOHMANN_VIEW_ALWAYS_INLINE FloatType layout_float(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept +{ + return decimal_to_float(layout_decimal(p, e, int_digits, frac_digits, limit)); +} + +/// the value of a float set by an edit: its token (the shortest round-trip +/// text, or "nan", "inf", "-inf") in the edit arena +template +NLOHMANN_VIEW_NOINLINE FloatType edited_float(const char* token, const node& n) +{ + if (token[0] == 'n') + { + return std::numeric_limits::quiet_NaN(); + } + if (token[0] == 'i' || (token[0] == '-' && token[1] == 'i')) + { + return token[0] == 'i' ? std::numeric_limits::infinity() : -std::numeric_limits::infinity(); + } + return float_value(token, n); +} + +/// the value of the float token of a node, as parse() converts it; floats and +/// doubles with at most 19 digits are converted from the digit layout +template +FloatType float_value(const document_data& d, const node& n) +{ + if (NLOHMANN_VIEW_UNLIKELY((n.flags & node_flags::storage) == node_flags::edited)) + { + return edited_float(d.str(n), n); + } + return float_value(d, n, std::integral_constant::value> {}); +} + +template +FloatType float_value(const document_data& d, const node& n, std::true_type /*binary32 or binary64*/) +{ + const unsigned int_digits = n.extra & 0xFFu; + const unsigned frac_digits = n.extra >> 8u; + if (NLOHMANN_VIEW_LIKELY(int_digits + frac_digits <= 19)) // (255 marks "many") + { + // (a float token not written by an edit is in the text) + const auto* const first = reinterpret_cast(d.src + n.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + return layout_float(first, first + n.len, int_digits, frac_digits, reinterpret_cast(d.src + d.size)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + } + return float_value(d.str(n), n); +} + +template +FloatType float_value(const document_data& d, const node& n, std::false_type /*other*/) +{ + return float_value(d.str(n), n); +} + +} // namespace view +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include + +// #include + + +// Images: a document stored so that loading it needs no parsing. +// +// Layout (little-endian): a 64-byte header, the nodes, the text (the source, +// followed by the number tokens written by edits), a NUL, the decoded strings +// (followed by the strings written by edits), a NUL. The idea is that of +// zero-copy formats such as FlatBuffers (https://github.com/google/flatbuffers) +// and YaFF (https://github.com/yandex/yaff); no code is taken from them. +// check_image follows the idea of FlatBuffers' Verifier (bounds and +// structure) and also checks what the parser guarantees about strings and +// numbers, so that reading and serializing a checked image is safe and yields +// valid JSON. + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ +namespace view +{ + +/// how load() checks an image +enum class image_check +{ + /// everything the parser guarantees: structure and bounds, strings (valid + /// UTF-8; source strings without quotes, backslashes, and control + /// characters), and numbers (well-formed, matching the stored values) + full, + /// structure and bounds only: reading and serializing are safe, but a + /// crafted image can yield invalid UTF-8, strings that serialize to + /// invalid JSON, or numbers that differ from their text + bounds, + /// none: for images from a trusted source only (a damaged image is + /// undefined behavior) + none, +}; + +struct image_header +{ + std::array magic; ///< "NJVI" + std::uint32_t version; ///< 1 + std::uint64_t node_count; + std::uint64_t text_size; + std::uint64_t arena_size; + std::array reserved; ///< zero (for later versions) +}; +static_assert(sizeof(image_header) == 64, "the image header must be 64 bytes"); + +constexpr std::uint32_t image_version = 1; + +/// the largest node count and text or string size of an image (as for parsed +/// documents, offsets and counts must fit 32 bits) +constexpr std::uint64_t image_limit = 0xFFFFFFF0u; + +/// Copy the current structure of an edited document into nodes in document +/// order, as the parser would have written them. Text written by edits is +/// appended to text_tail (number tokens) and arena_tail (strings); floats that +/// are not finite become null, as dump() writes them. +inline void compact_nodes(const document_data& d, std::size_t arena_size, std::vector& out, std::string& text_tail, std::string& arena_tail) +{ + struct frame + { + const node* cur; + const node* end; + std::size_t index; ///< the container's node in out + std::uint32_t count; + bool object; + }; + std::vector stack; + const auto string_node = [&](const node & s) + { + node r = s; + r.extra = 0; + r.flags = static_cast(s.flags & node_flags::storage); + if (r.flags == node_flags::edited) + { + r.off = static_cast(arena_size + arena_tail.size()); + arena_tail.append(d.str(s), s.len); + r.flags = node_flags::escaped; + } + return r; + }; + const auto copy_node = [&](const node * v) + { + node r = *v; + switch (static_cast(v->kind)) + { + case value_t::object: + case value_t::array: + r.flags = 0; + r.extra = 0; + r.off = (v->flags & (node_flags::moved | node_flags::is_new)) != 0 ? 0 : v->off; + r.len = 0; // counted below + r.next = 0; // set when the container is complete + stack.push_back(frame{d.first_child_edited(v), d.child_end_edited(v), out.size(), 0, v->kind == static_cast(value_t::object)}); + break; + case value_t::string: + r = string_node(*v); + break; + case value_t::number_integer: + case value_t::number_unsigned: + if ((v->flags & node_flags::storage) == node_flags::edited) + { + r.off = static_cast(d.size + text_tail.size()); + text_tail.append(d.str(*v), number_length(*v)); + } + r.flags = 0; + break; + case value_t::number_float: + if ((v->flags & node_flags::storage) == node_flags::edited) + { + const char* const t = d.str(*v); + if (t[0] == 'n' || t[0] == 'i' || (v->len > 1 && t[1] == 'i')) + { + r = node{}; // nan and infinity: null, as dump() writes them + r.kind = static_cast(value_t::null); + break; + } + r.off = static_cast(d.size + text_tail.size()); + text_tail.append(t, v->len); + r.extra = 0xFFFFu; // the digit layout is not recorded + } + r.flags = 0; + break; + case value_t::boolean: + r.flags = static_cast(v->flags & node_flags::is_true); + break; + case value_t::null: + case value_t::binary: + case value_t::discarded: + default: + r.flags = 0; + break; + } + out.push_back(r); + }; + copy_node(d.nodes); + while (!stack.empty()) + { + const frame top = stack.back(); + if (top.cur == top.end) + { + node& c = out[top.index]; + c.len = top.count; + c.next = static_cast(out.size() - top.index); + stack.pop_back(); + continue; + } + if (top.object) + { + out.push_back(string_node(*top.cur)); + } + const node* const value = top.cur + (top.object ? 1 : 0); + stack.back().cur = document_data::after(value); + ++stack.back().count; + copy_node(document_data::deref(value)); // may grow the stack + } +} + +/// the document as an image +inline std::vector save_image(const document_data& d) +{ +#if !NLOHMANN_VIEW_LITTLE_ENDIAN + throw_type_error(320, "json_document images need a little-endian target"); // LCOV_EXCL_LINE +#endif + const std::size_t arena_size = d.arena_size; + std::vector compacted; + std::string text_tail; + std::string arena_tail; + if (d.edits) + { + compact_nodes(d, arena_size, compacted, text_tail, arena_tail); + } + const node* const nodes = d.edits ? compacted.data() : d.nodes; + const std::size_t count = d.edits ? compacted.size() : d.node_count; + const std::size_t text_size = d.size + text_tail.size(); + const std::size_t total_arena = arena_size + arena_tail.size(); + if (NLOHMANN_VIEW_UNLIKELY(text_size >= image_limit || total_arena >= image_limit || count >= image_limit)) + { + // LCOV_EXCL_START (4 GiB) + throw_out_of_range(416, "images of 4 GiB or more are not supported by json_document"); + // LCOV_EXCL_STOP + } + image_header h{}; + h.magic = {{'N', 'J', 'V', 'I'}}; + h.version = image_version; + h.node_count = count; + h.text_size = text_size; + h.arena_size = total_arena; + std::vector image(sizeof(h) + (count * sizeof(node)) + text_size + 1 + total_arena + 1); + std::uint8_t* o = image.data(); + std::memcpy(o, &h, sizeof(h)); + o += sizeof(h); + std::memcpy(o, nodes, count * sizeof(node)); + // the hash indexes are rebuilt by load(), whose check rejects any other + // extra of an object, and the same document saves to the same bytes + for (std::size_t i = 0; i < count; ++i) + { + if (nodes[i].kind == static_cast(value_t::object) && nodes[i].extra != 0) + { + std::memset(o + (i * sizeof(node)) + offsetof(node, extra), 0, sizeof(nodes[i].extra)); + } + } + o += count * sizeof(node); + const auto append = [&o](const char* s, std::size_t n) + { + if (n != 0) + { + std::memcpy(o, s, n); + o += n; + } + }; + append(d.src, d.size); + append(text_tail.data(), text_tail.size()); + *o++ = 0; + append(d.base[1], arena_size); + append(arena_tail.data(), arena_tail.size()); + *o = 0; + return image; +} + +/// the parts of a number token that the checks need +struct number_token +{ + const unsigned char* int_start; + std::size_t int_digits; + std::size_t frac_digits; + std::int64_t exponent; + bool negative; + bool is_float; ///< a fraction or an exponent +}; + +/// whether [s, s + len) is a JSON number (the grammar the parser accepts) +inline bool scan_number_token(const unsigned char* s, std::size_t len, number_token& t) +{ + const unsigned char* const e = s + len; + const unsigned char* p = s; + t.negative = p != e && *p == '-'; + p += t.negative ? 1 : 0; + if (p == e) + { + return false; + } + t.int_start = p; + if (*p == '0') + { + ++p; + } + else if (*p >= '1' && *p <= '9') + { + do + { + ++p; + } + while (p != e && is_digit(*p)); + } + else + { + return false; + } + t.int_digits = static_cast(p - t.int_start); + t.frac_digits = 0; + t.is_float = false; + if (p != e && *p == '.') + { + const unsigned char* const f0 = ++p; + while (p != e && is_digit(*p)) + { + ++p; + } + if (p == f0) + { + return false; + } + t.frac_digits = static_cast(p - f0); + t.is_float = true; + } + t.exponent = 0; + if (p != e && (*p | 0x20u) == 'e') + { + ++p; + const bool exp_negative = p != e && *p == '-'; + p += (p != e && (*p == '+' || *p == '-')) ? 1 : 0; + if (p == e || !is_digit(*p)) + { + return false; + } + do + { + t.exponent = t.exponent < 100000 ? (t.exponent * 10) + (*p - '0') : t.exponent; + ++p; + } + while (p != e && is_digit(*p)); + t.exponent = exp_negative ? -t.exponent : t.exponent; + t.is_float = true; + } + return p == e; +} + +/// the digit layout the parser records for a float token (compaction +/// writes "many" instead; the caller accepts both) +inline std::uint16_t float_layout(const number_token& t) +{ + return static_cast((std::min)(t.int_digits, std::size_t{255}) | ((std::min)(t.frac_digits, std::size_t{255}) << 8u)); +} + +/// whether a float token (well-formed, per scan_number_token) is finite as +/// double; parse() rejects floats that overflow, and as there, only a number +/// whose magnitude could reach 1e308 needs the conversion +inline bool float_token_finite(const unsigned char* s, std::size_t len, const number_token& t) +{ + if (static_cast(t.int_digits) + t.exponent > 300) + { + node n{}; + n.kind = static_cast(value_t::number_float); + n.len = static_cast(len); + const auto v = float_value(reinterpret_cast(s), n); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + return v <= (std::numeric_limits::max)() && v >= -(std::numeric_limits::max)(); + } + return true; +} + +/// whether an integer node matches its token the way the parser records it +/// (after the bounds check; the token has at most 256 characters) +inline bool check_integer(const node& n, const unsigned char* text) +{ + const std::size_t len = number_length(n); + const unsigned char* const s = text + n.off; + number_token t{}; + if (!scan_number_token(s, len, t)) + { + return false; + } + // the token's value is the stored one; number_integer nodes of edits can + // be non-negative (as basic_json keeps the type of a value) + const bool integer = n.kind == static_cast(value_t::number_integer); + if (t.is_float || t.int_digits > 20 || (t.negative && !integer)) + { + return false; + } + // (at most 19 digits cannot overflow; 20 digits are compared with 2^64 - 1) + if (t.int_digits == 20 && std::memcmp(t.int_start, "18446744073709551615", 20) > 0) + { + return false; + } + std::uint64_t m = 0; + for (const unsigned char* d = t.int_start; d != t.int_start + t.int_digits; ++d) + { + m = (m * 10) + static_cast(*d - '0'); + } + if (integer && m > (t.negative ? std::uint64_t{1} << 63u : (std::uint64_t{1} << 63u) - 1)) + { + return false; + } + return integer_bits(n) == (t.negative ? 0 - m : m); +} + +/// a byte range of the text or of the decoded strings +struct byte_range +{ + std::uint32_t off; + std::uint32_t len; +}; + +/// a float token and the digit layout its node records +struct float_range +{ + std::uint32_t off; + std::uint32_t len; + std::uint16_t extra; +}; + +/// Run check once for each distinct range of ranges (which are within bounds +/// and not empty). Ranges that are not identical must not overlap: save() +/// writes every string and token to a place of its own, and nodes that share +/// a value (copies within a document) share the whole range. This bounds the +/// work by the size of the text, however many nodes point to the same bytes. +template +bool check_distinct_ranges(std::vector& ranges, Check check) +{ + std::sort(ranges.begin(), ranges.end(), [](const byte_range & a, const byte_range & b) + { + return a.off != b.off ? a.off < b.off : a.len < b.len; + }); + std::size_t end = 0; + for (std::size_t i = 0; i < ranges.size(); ++i) + { + const byte_range r = ranges[i]; + if (i != 0 && r.off == ranges[i - 1].off && r.len == ranges[i - 1].len) + { + continue; + } + if (r.off < end || !check(r)) + { + return false; + } + end = static_cast(r.off) + r.len; + } + return true; +} + +/// Check the float nodes: each token once (nodes of the same token must +/// record layouts that match it), tokens must not overlap. +inline bool check_float_ranges(std::vector& ranges, const unsigned char* text) +{ + std::sort(ranges.begin(), ranges.end(), [](const float_range & a, const float_range & b) + { + if (a.off != b.off) + { + return a.off < b.off; + } + if (a.len != b.len) + { + return a.len < b.len; + } + return a.extra < b.extra; + }); + std::size_t end = 0; + std::size_t i = 0; + while (i < ranges.size()) + { + const float_range r = ranges[i]; + if (r.off < end) + { + return false; + } + number_token t{}; + const unsigned char* const s = text + r.off; + if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t)) + { + return false; + } + // the digit layout the parser records (or "many", as compaction writes it) + const std::uint16_t layout = float_layout(t); + for (; i < ranges.size() && ranges[i].off == r.off && ranges[i].len == r.len; ++i) + { + if (ranges[i].extra != layout && ranges[i].extra != 0xFFFFu) + { + return false; + } + } + end = static_cast(r.off) + r.len; + } + return true; +} + +/// the content checks of check_image: source strings as the parser leaves +/// them (no quotes, backslashes, or control characters), decoded strings +/// (valid UTF-8), and float tokens +inline bool check_contents(const unsigned char* text, const unsigned char* arena, + std::vector& source_strings, std::vector& decoded_strings, + std::vector& floats) +{ + return check_distinct_ranges(source_strings, [text](const byte_range & r) + { + const unsigned char* const b = text + r.off; + return scan_string_run(b, b + r.len) == b + r.len; + }) + && check_distinct_ranges(decoded_strings, [arena](const byte_range & r) + { + return valid_utf8_prefix(arena + r.off, r.len) == r.len; + }) + && check_float_ranges(floats, text); +} + +/// Check the nodes of a loaded image against its text and decoded strings: +/// kinds, flags, and `extra`; extents and element counts of arrays and +/// objects; keys; bounds; string contents (source strings as the parser +/// leaves them: no quotes, backslashes, or control characters; all strings +/// valid UTF-8); and number tokens. +/// +/// The structure and the bounds are checked node by node. The contents of +/// strings and of float tokens are checked afterwards, once for each distinct +/// range (see check_distinct_ranges), so that the full check is linear in the +/// size of the image plus the sorting of the ranges, and not in the number of +/// nodes times the size of the text. +inline bool check_image(const node* nodes, std::size_t count, const unsigned char* text, std::size_t text_size, + const unsigned char* arena, std::size_t arena_size, bool full) +{ + struct frame + { + std::size_t end; + std::uint32_t len; + std::uint32_t seen; + bool object; + bool expect_key; + }; + std::vector stack; + std::vector source_strings; + std::vector decoded_strings; + std::vector floats; + const auto check_string = [&](const node & n) -> bool + { + if ((n.flags & ~node_flags::escaped) != 0 || n.extra != 0) + { + return false; + } + const bool decoded = (n.flags & node_flags::escaped) != 0; + const std::size_t limit = decoded ? arena_size : text_size; + if (n.off > limit || n.len > limit - n.off) + { + return false; + } + if (full && n.len != 0) + { + (decoded ? decoded_strings : source_strings).push_back(byte_range{n.off, n.len}); + } + return true; + }; + // bounds of a number token; the recorded digit layout must lie within it + const auto number_in_bounds = [&](const node & n) -> bool + { + const std::size_t len = number_length(n); + if (len == 0 || n.off > text_size || len > text_size - n.off) + { + return false; + } + if (n.kind != static_cast(value_t::number_float)) + { + return (n.extra >> 8u) == 0; + } + // float_value() reads the sign, the integer digits, and the point and + // fraction digits the layout records (a layout of more than 19 digits + // means the general conversion, which stays within the token) + const std::size_t int_digits = n.extra & 0xFFu; + const std::size_t frac_digits = n.extra >> 8u; + const std::size_t need = (text[n.off] == '-' ? 1u : 0u) + int_digits + (frac_digits != 0 ? frac_digits + 1 : 0); + return int_digits + frac_digits > 19 || need <= len; + }; + std::size_t i = 0; + for (;;) + { + // close finished arrays and objects + while (!stack.empty() && i == stack.back().end) + { + const frame f = stack.back(); + if (f.seen != f.len || (f.object && !f.expect_key)) + { + return false; + } + stack.pop_back(); + if (!stack.empty()) + { + ++stack.back().seen; + stack.back().expect_key = true; + } + } + if (i == count) + { + return stack.empty() && (!full || check_contents(text, arena, source_strings, decoded_strings, floats)); + } + if (i != 0 && stack.empty()) + { + return false; // nodes after the root + } + const node& n = nodes[i]; + if (!stack.empty() && stack.back().object && stack.back().expect_key) + { + if (n.kind != static_cast(value_t::string) || !check_string(n)) + { + return false; + } + stack.back().expect_key = false; + ++i; + continue; + } + bool complete = true; + switch (static_cast(n.kind)) + { + case value_t::null: + // (the offset of a literal is read to size the output of dump()) + if (n.flags != 0 || n.extra != 0 || n.off > text_size) + { + return false; + } + break; + case value_t::boolean: + if ((n.flags & ~node_flags::is_true) != 0 || n.extra != 0 || n.off > text_size) + { + return false; + } + break; + case value_t::string: + if (!check_string(n)) + { + return false; + } + break; + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + if (n.flags != 0 || !number_in_bounds(n)) + { + return false; + } + if (full) + { + // (the contents of float tokens are checked later; the token of an + // integer has at most 256 characters) + if (n.kind == static_cast(value_t::number_float)) + { + floats.push_back(float_range{n.off, n.len, n.extra}); + } + else if (!check_integer(n, text)) + { + return false; + } + } + break; + case value_t::array: + case value_t::object: + { + const std::size_t limit = stack.empty() ? count : stack.back().end; + if (n.flags != 0 || n.extra != 0 || n.next == 0 || n.next > limit - i || n.off > text_size) + { + return false; + } + stack.push_back(frame{i + n.next, n.len, 0, n.kind == static_cast(value_t::object), true}); + complete = false; + break; + } + case value_t::binary: + case value_t::discarded: + default: + return false; + } + ++i; + if (complete && !stack.empty()) + { + ++stack.back().seen; + stack.back().expect_key = true; + } + } +} + +[[noreturn]] NLOHMANN_VIEW_NOINLINE inline void throw_invalid_image(const char* what) +{ + throw_parse_error(116, concat("invalid json_document image: ", what)); +} + +/// Read an image into d. The text and the decoded strings stay in the image; +/// the nodes are copied (so that they are aligned, and edits can change them). +inline void load_image(document_data& d, const std::uint8_t* image, std::size_t size, image_check check) +{ +#if !NLOHMANN_VIEW_LITTLE_ENDIAN + throw_type_error(320, "json_document images need a little-endian target"); // LCOV_EXCL_LINE +#endif + if (image == nullptr || size < sizeof(image_header)) + { + throw_invalid_image("too short"); + } + image_header h{}; + std::memcpy(&h, image, sizeof(h)); + // (the reserved fields are for later versions) + if (std::memcmp(h.magic.data(), "NJVI", 4) != 0 || h.version != image_version + || (h.reserved[0] | h.reserved[1] | h.reserved[2] | h.reserved[3]) != 0) + { + throw_invalid_image("unknown format"); + } + const std::size_t room = size - sizeof(h); + if (h.node_count == 0 || h.node_count > room / sizeof(node) || h.node_count >= image_limit || h.text_size >= image_limit || h.arena_size >= image_limit) + { + throw_invalid_image("sizes out of range"); + } + const auto count = static_cast(h.node_count); + const auto text_size = static_cast(h.text_size); + const auto arena_size = static_cast(h.arena_size); + const std::size_t text_at = sizeof(h) + (count * sizeof(node)); + // the text, a NUL, the decoded strings, a NUL, and nothing after them + if (size - text_at < 2 || text_size > size - text_at - 2 || arena_size != size - text_at - text_size - 2 + || image[text_at + text_size] != 0 || image[size - 1] != 0) + { + throw_invalid_image("sizes out of range"); + } + + d.discarded = true; + d.edits.reset(); + d.base[2] = nullptr; + d.owned.clear(); + if (d.owned_image.empty() || image != d.owned_image.data()) + { + d.owned_image.clear(); + } + d.arena.clear(); + d.indexes.clear(); + d.index_slots.clear(); + d.large_objects.clear(); + d.node_count = 0; + d.reserve(count); + std::memcpy(d.nodes, image + sizeof(h), count * sizeof(node)); + d.node_count = count; + const std::uint8_t* const text = image + text_at; + const std::uint8_t* const arena = text + text_size + 1; + d.src = reinterpret_cast(text); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + d.size = text_size; + d.base[0] = d.src; + d.base[1] = reinterpret_cast(arena); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + d.arena_size = arena_size; + if (check != image_check::none && !check_image(d.nodes, count, text, text_size, arena, arena_size, check == image_check::full)) + { + throw_invalid_image("the check failed"); + } + // the hash indexes of large objects, as after parsing + for (std::size_t i = 0; i < count; ++i) + { + const node& n = d.nodes[i]; + if (n.kind == static_cast(value_t::object) && n.len >= document_data::index_min_members) + { + d.large_objects.push_back(static_cast(i)); + } + } + build_object_indexes(d); + std::vector().swap(d.large_objects); // (only needed while building) + d.discarded = false; +} + +} // namespace view +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -4786,192 +5768,6 @@ NLOHMANN_JSON_NAMESPACE_END // #include // #include -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - - - -#include // size_t -#include // int64_t, uint64_t -#include // numeric_limits -#include // string -#include // integral_constant - -// #include -// #include - -// #include - -// #include - -// #include - - -NLOHMANN_JSON_NAMESPACE_BEGIN -namespace detail -{ -namespace view -{ - -/*! -@brief the value of the float token of a node, as parse() converts it - -Uses the lexer's conversion (detail::convert_float), so that the values are -bit-identical to parse(): float and double are converted without allocation -and independent of the locale. The digit layout recorded while parsing locates -the decimal point and the exponent without scanning the token. -*/ -template -NLOHMANN_VIEW_NOINLINE FloatType float_value(const char* first, const node& n) -{ - const char* const last = first + n.len; - const std::size_t neg = first[0] == '-' ? 1 : 0; - const std::size_t int_digits = n.extra & 0xFFu; - const std::size_t frac_digits = n.extra >> 8u; - std::size_t dot = std::string::npos; - std::size_t mantissa_end = n.len; - if (int_digits != 255 && frac_digits != 255) - { - dot = frac_digits != 0 ? neg + int_digits : std::string::npos; - mantissa_end = neg + int_digits + (frac_digits != 0 ? 1 + frac_digits : 0); - } - else - { - // more digits than the layout records: locate them - for (std::size_t i = 0; i < n.len; ++i) - { - if (first[i] == '.') - { - dot = i; - } - else if (first[i] == 'e' || first[i] == 'E') - { - mantissa_end = i; - break; - } - } - } - return convert_float(first, last, dot, mantissa_end); -} - -/*! -@brief the digits of a float token with at most 19 digits, from its layout - -The digit layout recorded while parsing says where the integer digits, the -fraction digits, and the exponent are, so the digits are read eight at a -time without scanning. - -@param[in] p first character of the token -@param[in] e end of the token -@param[in] limit end of the readable memory (the source text) -*/ -NLOHMANN_VIEW_ALWAYS_INLINE float_significand layout_decimal(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept -{ - const bool negative = *p == '-'; - p += negative ? 1 : 0; - std::uint64_t w = parse_upto19(p, int_digits, limit); - p += int_digits; - std::int64_t q = 0; - if (frac_digits != 0) - { - w = (w * int_pow10(frac_digits)) + parse_upto19(p + 1, frac_digits, limit); - p += 1 + frac_digits; - q = -static_cast(frac_digits); - } - if (p != e) - { - // [eE][+-]digits; huge exponents saturate (the parser rejected overflow) - ++p; - const bool exp_negative = *p == '-'; - p += (*p == '-' || *p == '+') ? 1 : 0; - std::int64_t exp_value = 0; - for (; p != e; ++p) - { - if (exp_value < 0x10000000) - { - exp_value = (exp_value * 10) + (*p - '0'); - } - } - q += exp_negative ? -exp_value : exp_value; - } - - float_significand d; - d.w = w; - d.exponent = q; - d.negative = negative; - return d; -} - -/*! -@brief the value of a float token with at most 19 digits, from its layout - -The result is correctly rounded by the lexer's conversion -(detail::decimal_to_float(): Clinger's fast path where both operands are -exact, else the Eisel-Lemire algorithm, which needs no fallback for up to 19 -digits), so it is the value parse() produces. -*/ -template -NLOHMANN_VIEW_ALWAYS_INLINE FloatType layout_float(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept -{ - return decimal_to_float(layout_decimal(p, e, int_digits, frac_digits, limit)); -} - -/// the value of a float set by an edit: its token (the shortest round-trip -/// text, or "nan", "inf", "-inf") in the edit arena -template -NLOHMANN_VIEW_NOINLINE FloatType edited_float(const char* token, const node& n) -{ - if (token[0] == 'n') - { - return std::numeric_limits::quiet_NaN(); - } - if (token[0] == 'i' || (token[0] == '-' && token[1] == 'i')) - { - return token[0] == 'i' ? std::numeric_limits::infinity() : -std::numeric_limits::infinity(); - } - return float_value(token, n); -} - -/// the value of the float token of a node, as parse() converts it; floats and -/// doubles with at most 19 digits are converted from the digit layout -template -FloatType float_value(const document_data& d, const node& n) -{ - if (NLOHMANN_VIEW_UNLIKELY((n.flags & node_flags::storage) == node_flags::edited)) - { - return edited_float(d.str(n), n); - } - return float_value(d, n, std::integral_constant::value> {}); -} - -template -FloatType float_value(const document_data& d, const node& n, std::true_type /*binary32 or binary64*/) -{ - const unsigned int_digits = n.extra & 0xFFu; - const unsigned frac_digits = n.extra >> 8u; - if (NLOHMANN_VIEW_LIKELY(int_digits + frac_digits <= 19)) // (255 marks "many") - { - // (a float token not written by an edit is in the text) - const auto* const first = reinterpret_cast(d.src + n.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - return layout_float(first, first + n.len, int_digits, frac_digits, reinterpret_cast(d.src + d.size)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - } - return float_value(d.str(n), n); -} - -template -FloatType float_value(const document_data& d, const node& n, std::false_type /*other*/) -{ - return float_value(d.str(n), n); -} - -} // namespace view -} // namespace detail -NLOHMANN_JSON_NAMESPACE_END NLOHMANN_JSON_NAMESPACE_BEGIN @@ -5617,7 +6413,9 @@ class view_serializer m_out.put('"'); } - /// as serializer::dump_escaped() for valid UTF-8 (the view has no other) + /// as serializer::dump_escaped(); strings of a document are valid UTF-8, + /// except in a damaged image loaded with image_check::bounds, for which + /// this throws what basic_json::dump() throws for the string template void write_escaped(const unsigned char* s, std::size_t n) { @@ -5641,12 +6439,13 @@ class view_serializer } std::uint32_t codepoint = s[i]; std::size_t len = 1; - if (codepoint >= 0xC0) + if (codepoint >= 0x80) { - len = 2; - if (codepoint >= 0xE0) + len = validate_one_utf8(s + i, n - i); + if (NLOHMANN_VIEW_UNLIKELY(len == 0)) { - len = codepoint >= 0xF0 ? 4 : 3; + invalid_utf8(s, n); + return; } codepoint &= 0xFFu >> (len + 1); for (std::size_t k = 1; k < len; ++k) @@ -5659,6 +6458,13 @@ class view_serializer } } + /// throw what basic_json::dump() throws for a string that is not valid UTF-8 + NLOHMANN_VIEW_NOINLINE static void invalid_utf8(const unsigned char* s, std::size_t n) + { + const string_t dumped = BasicJsonType(string_t(reinterpret_cast(s), n)).dump(); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + static_cast(dumped); + } + template void write_codepoint(std::uint32_t codepoint, const unsigned char* bytes, std::size_t len) { @@ -6933,7 +7739,7 @@ class basic_json_document /// whether the document holds its own copy of the text bool owns_source() const noexcept { - return m_data && !m_data->owned.empty() && m_data->src == m_data->owned.data(); + return m_data && ((!m_data->owned.empty() && m_data->src == m_data->owned.data()) || !m_data->owned_image.empty()); } /// number of index nodes (values plus object keys) @@ -6942,7 +7748,7 @@ class basic_json_document return m_data ? m_data->node_count : 0; } - /// bytes held by the document (index, decoded strings, owned text) + /// bytes held by the document (index, decoded strings, and owned text or image) std::size_t memory_usage() const noexcept { if (!m_data) @@ -6951,7 +7757,7 @@ class basic_json_document } return sizeof(document_data) + (m_data->inline_capacity * sizeof(detail::view::node)) + (m_data->nodes != m_data->inline_nodes ? m_data->node_capacity * sizeof(detail::view::node) : 0) - + m_data->arena.capacity() + m_data->owned.capacity() + + m_data->arena.capacity() + m_data->owned.capacity() + m_data->owned_image.capacity() + (m_data->indexes.capacity() * sizeof(document_data::object_index)) + (m_data->index_slots.capacity() * sizeof(std::uint32_t)) + (m_data->large_objects.capacity() * sizeof(std::uint32_t)) + (m_data->edits != nullptr ? m_data->edits->bytes : 0); @@ -6971,8 +7777,10 @@ class basic_json_document // allocate everything first, so that an exception leaves the document // unchanged + // (the decoded strings of a loaded image stay in the image) + const bool arena_in_use = d.base[1] == d.arena.data(); const bool shrink_arena = d.arena.capacity() > d.arena.size(); - std::vector arena(shrink_arena ? d.arena : std::vector()); + std::vector arena(shrink_arena && arena_in_use ? d.arena : std::vector()); // (edits link to the nodes of the index, which then stays in place) const bool shrink_nodes = d.nodes != d.inline_nodes && d.node_count != d.node_capacity && d.edits == nullptr; const bool into_header = d.node_count <= d.inline_capacity; @@ -6993,7 +7801,10 @@ class basic_json_document if (shrink_arena) { d.arena.swap(arena); - d.base[1] = d.arena.data(); + if (arena_in_use) + { + d.base[1] = d.arena.data(); + } } if (d.indexes.capacity() > d.indexes.size()) { @@ -7005,6 +7816,55 @@ class basic_json_document } } + //////////// + // images // + //////////// + + /// how load() checks an image (full, bounds, or none) + using image_check = detail::view::image_check; + + /// The document as an image that load() reads without parsing: the node + /// index, the text, and the decoded strings. An edited document is + /// written in its current state (floats that are not finite become null, + /// as in dump()). + std::vector save() const + { + if (NLOHMANN_VIEW_UNLIKELY(!m_data || m_data->discarded)) + { + detail::view::throw_type_error(320, "cannot save a discarded json_document"); + } + return detail::view::save_image(*m_data); + } + + /// Read an image written by save(). The image is borrowed: it must stay + /// alive and unchanged while the document is used. + NLOHMANN_VIEW_NODISCARD + static basic_json_document load(const std::uint8_t* image, std::size_t size, const image_check check = image_check::full) + { + basic_json_document d; + d.ensure_data(nullptr, 0); + detail::view::load_image(*d.m_data, image, size, check); + return d; + } + + /// read an image (borrowed) + NLOHMANN_VIEW_NODISCARD + static basic_json_document load(const std::vector& image, const image_check check = image_check::full) + { + return load(image.data(), image.size(), check); + } + + /// read an image and keep it (no copy) + NLOHMANN_VIEW_NODISCARD + static basic_json_document load(std::vector&& image, const image_check check = image_check::full) + { + basic_json_document d; + d.ensure_data(nullptr, 0); + d.m_data->owned_image = std::move(image); + detail::view::load_image(*d.m_data, d.m_data->owned_image.data(), d.m_data->owned_image.size(), check); + return d; + } + /////////// // edits // /////////// @@ -7239,6 +8099,7 @@ class basic_json_document { d.owned.clear(); } + d.owned_image.clear(); d.src = src; d.size = size; d.node_count = 0; @@ -7263,6 +8124,7 @@ class basic_json_document { d.base[0] = d.src; d.base[1] = d.arena.data(); + d.arena_size = d.arena.size(); detail::view::build_object_indexes(d); std::vector().swap(d.large_objects); // (only needed while parsing) d.discarded = false; diff --git a/tests/Makefile b/tests/Makefile index e8e61e603..e03b9dc7d 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -10,7 +10,7 @@ CXXFLAGS += -std=c++11 CPPFLAGS += -I ../single_include FUZZER_ENGINE = src/fuzzer-driver_afl.cpp -FUZZERS = parse_afl_fuzzer parse_bson_fuzzer parse_cbor_fuzzer parse_msgpack_fuzzer parse_ubjson_fuzzer parse_bjdata_fuzzer parse_bon8_fuzzer parse_json_view_fuzzer +FUZZERS = parse_afl_fuzzer parse_bson_fuzzer parse_cbor_fuzzer parse_msgpack_fuzzer parse_ubjson_fuzzer parse_bjdata_fuzzer parse_bon8_fuzzer parse_json_view_fuzzer json_view_image_fuzzer fuzzers: $(FUZZERS) parse_afl_fuzzer: @@ -19,6 +19,9 @@ parse_afl_fuzzer: parse_json_view_fuzzer: $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(FUZZER_ENGINE) src/fuzzer-parse_json_view.cpp -o $@ +json_view_image_fuzzer: + $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(FUZZER_ENGINE) src/fuzzer-json_view_image.cpp -o $@ + parse_bson_fuzzer: $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(FUZZER_ENGINE) src/fuzzer-parse_bson.cpp -o $@ diff --git a/tests/fuzzing.md b/tests/fuzzing.md index feafbfe76..2399bdfd2 100644 --- a/tests/fuzzing.md +++ b/tests/fuzzing.md @@ -12,6 +12,12 @@ produces, and that a rejected input makes both parsers throw with an identical ` are taken from the low bits of the first input byte (the byte stays part of the text). It takes plain JSON text, so it reuses the `corpus_json` corpus rather than a format of its own. +`json_view_image_fuzzer` (`tests/src/fuzzer-json_view_image.cpp`) tests the images of `json_document` (`save()` and +`load()`). It uses each input twice: as an image, which `load()` must either reject with `parse_error.116` or read +safely (with `image_check::full`, the document must also serialize to the JSON it reads as), and as a JSON text, whose +image must load and serialize to the same text. A corpus of images can be made from JSON files with a small program +that calls `json_document::parse(text).save()`; plain JSON files work as well. + ## What the fuzzers check Each fuzzer driver (`tests/src/fuzzer-parse_*.cpp`) parses its input twice: once with `allow_exceptions = false` and diff --git a/tests/src/fuzzer-json_view_image.cpp b/tests/src/fuzzer-json_view_image.cpp new file mode 100644 index 000000000..8495468a4 --- /dev/null +++ b/tests/src/fuzzer-json_view_image.cpp @@ -0,0 +1,95 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +/* +This file implements a test of json_document images suitable for fuzz +testing. The input is used twice: + +- as an image: json_document::load() with image_check::full must either throw + a parse_error or yield a document that serializes to the JSON text it reads + as; with image_check::bounds, reading and serializing must be safe (checked + by the sanitizers), and serializing may only throw type_error.316 +- as a JSON text: if json_document::parse() accepts it, the image of the + document must load (with every check) and serialize to the same text + +The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer +drivers. +*/ + +#include +#include +#include +#include +#include +#include + +// the checks below are assertions; NDEBUG would compile them away +#ifdef NDEBUG + #error "the fuzzer drivers must be built without NDEBUG" +#endif + +using json = nlohmann::json; +using json_document = nlohmann::json_document; +using image_check = json_document::image_check; + +// see http://llvm.org/docs/LibFuzzer.html +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) +{ + // the input as an image + for (const image_check check : + { + image_check::full, image_check::bounds + }) + { + json_document d; + try + { + d = json_document::load(data, size, check); + } + catch (const json::parse_error& e) + { + assert(e.id == 116); + continue; + } + std::string dumped; + try + { + dumped = d.root().dump(); + } + catch (const json::type_error& e) + { + // invalid UTF-8 can only pass the bounds check + assert(check == image_check::bounds && e.id == 316); + continue; + } + const json j = d.root().materialize(); + if (check == image_check::full) + { + assert(json::parse(dumped) == j); + // an image of the loaded document is the input + assert(d.save() == std::vector(data, data + size)); + } + } + + // the input as a JSON text + const std::string text(reinterpret_cast(data), size); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const json_document parsed = json_document::parse(text, false); + if (!parsed.is_discarded()) + { + const std::vector image = parsed.save(); + for (const image_check check : + { + image_check::full, image_check::bounds, image_check::none + }) + { + const json_document loaded = json_document::load(image, check); + assert(loaded.root().dump() == parsed.root().dump()); + } + } + return 0; +} diff --git a/tests/src/unit-json_view_image.cpp b/tests/src/unit-json_view_image.cpp new file mode 100644 index 000000000..994bd211c --- /dev/null +++ b/tests/src/unit-json_view_image.cpp @@ -0,0 +1,1237 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; +using nlohmann::ordered_json; +using nlohmann::json_document; +using nlohmann::json_editable_document; +using nlohmann::ordered_json_document; +using nlohmann::ordered_json_editable_document; +using image_check = json_document::image_check; +using nlohmann::detail::view::node; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if !(defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + +namespace +{ +#if !defined(JSON_NOEXCEPTION) +std::string exception_of(const std::function& f) +{ + try + { + f(); + } + catch (const json::exception& e) + { + return e.what(); + } + return ""; +} + +const char* const check_failed = "[json.exception.parse_error.116] parse error: invalid json_document image: the check failed"; +#endif + +std::string read_file(const std::string& name) +{ + const std::ifstream f(std::string(TEST_DATA_DIRECTORY) + name, std::ios::binary); + std::stringstream ss; + ss << f.rdbuf(); + return ss.str(); +} + +// the dump of a loaded image, through a named document (root() of a temporary +// document does not compile, and its views would dangle) +std::string loaded_dump(const std::vector& image, image_check check = image_check::full) +{ + const json_document d = json_document::load(image, check); + return d.root().dump(); +} + +// the offsets of the parts of an image +constexpr std::size_t header_size = 64; + +std::uint64_t header_field(const std::vector& image, std::size_t offset) +{ + std::uint64_t v = 0; + std::memcpy(&v, image.data() + offset, sizeof(v)); + return v; +} + +// a header field as a size; the cast is from a variable, which GCC's +// -Wuseless-cast does not flag where std::uint64_t and std::size_t coincide +std::size_t header_size_field(const std::vector& image, std::size_t offset) +{ + const std::uint64_t v = header_field(image, offset); + return static_cast(v); +} + +void set_header_field(std::vector& image, std::size_t offset, std::uint64_t v) +{ + std::memcpy(image.data() + offset, &v, sizeof(v)); +} + +std::size_t node_count(const std::vector& image) +{ + return header_size_field(image, 8); +} + +std::size_t text_at(const std::vector& image) +{ + return header_size + (node_count(image) * sizeof(node)); +} + +node node_at(const std::vector& image, std::size_t i) +{ + node n{}; + std::memcpy(&n, image.data() + header_size + (i * sizeof(node)), sizeof(node)); + return n; +} + +void set_node(std::vector& image, std::size_t i, const node& n) +{ + std::memcpy(image.data() + header_size + (i * sizeof(node)), &n, sizeof(node)); +} + +#if !defined(JSON_NOEXCEPTION) +/// the result of loading an image with a check: "" or the exception message +std::string load_result(const std::vector& image, image_check check) +{ + return exception_of([&] + { + const json_document d = json_document::load(image, check); + static_cast(d); + }); +} + +/// a copy of the image with node i changed by f +template +std::vector corrupted(const std::vector& image, std::size_t i, F f) +{ + std::vector b = image; + node n = node_at(b, i); + f(n); + set_node(b, i, n); + return b; +} +#endif + +/// a document and the documents loaded from its image must be equal +template +void check_round_trip(const Document& d) +{ + const std::vector image = d.save(); + for (const image_check check : + { + image_check::full, image_check::bounds, image_check::none + }) + { + const json_document l = json_document::load(image, check); + CHECK(l.root().dump() == d.root().dump()); + CHECK(l.root().dump(2) == d.root().dump(2)); + CHECK(l.root().materialize() == json(d.root().materialize())); + // an image of a loaded document is the same image + CHECK(l.save() == image); + } + // an editable document can be loaded, too + const ordered_json_editable_document e = ordered_json_editable_document::load(image); + CHECK(e.root().dump() == d.root().dump()); +} + +std::uint32_t rng() +{ + static std::mt19937 generator(5295); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible + // result_type is std::uint_fast32_t, which may be wider than 32 bits + const std::mt19937::result_type value = generator(); + return static_cast(value); +} +} // namespace + +TEST_CASE("json_view images: round trips") +{ + SECTION("small documents") + { + for (const char* text : + { + "null", "true", "false", "0", "-0", "42", "-42", "18446744073709551615", "-9223372036854775808", + "123456789012345678901234567890", "1.5", "-1.25e-300", "1E308", "0.1000000000000000000000000001", + "\"\"", "\"text\"", R"("esc\"aped\n\u00e9\ud83d\ude00")", "\"\xc3\xa9\xe3\x81\x82\"", + "[]", "{}", "[[]]", "[{}]", "{\"\":{}}", + R"({"a": [1, 2.5, "x\ty", true, null, {"b": []}], "c": {"d": -3, "eA": "f"}})", + R"({"k": 1, "k": 2, "l": [], "k": 3})", + " [1 , 2 ] " + }) + { + CAPTURE(text) + // false positive: parse() returns a document with a root + // @infer-ignore NULLPTR_DEREFERENCE + check_round_trip(json_document::parse(text)); + // false positive: parse() returns a document with a root + // @infer-ignore NULLPTR_DEREFERENCE + check_round_trip(ordered_json_document::parse(text)); + } + } + + SECTION("files") + { + for (const char* name : + { + "/json_testsuite/sample.json", "/nativejson-benchmark/canada.json", "/nativejson-benchmark/citm_catalog.json", + "/nativejson-benchmark/twitter.json", "/json_tests/pass1.json", "/json_tests/pass2.json", "/json_tests/pass3.json" + }) + { + CAPTURE(name) + const std::string text = read_file(name); + const json_document d = json_document::parse(text); + check_round_trip(d); + // what a loaded document reads is what parse() produces + const json_document l = json_document::load(d.save()); + CHECK(l.root().materialize() == json::parse(text)); + } + } + + SECTION("images are deterministic") + { + const std::string text = R"({"b": [1, 2, {"c": "\u00e9"}], "a": 1.5})"; + const json_document d = json_document::parse(text); + CHECK(d.save() == json_document::parse(text).save()); + CHECK(d.save() == json_editable_document::parse(text).save()); + CHECK(d.save() == ordered_json_document::parse(text).save()); + const json_document copy = json_document::parse_copy(text); + CHECK(copy.save() == d.save()); + } + + SECTION("large objects get their hash index again") + { + std::string text = "{"; + for (int i = 0; i < 1000; ++i) + { + text += (i != 0 ? ",\"k" : "\"k") + std::to_string(i) + "\":" + std::to_string(i); + } + text += R"(,"k7":"a duplicate","inner":{)"; + for (int i = 0; i < 200; ++i) + { + text += (i != 0 ? ",\"m" : "\"m") + std::to_string(i) + "\":" + std::to_string(-i); + } + text += "}}"; + const json_document d = json_document::parse(text); + const std::vector image = d.save(); + for (const image_check check : + { + image_check::full, image_check::none + }) + { + const json_document l = json_document::load(image, check); + for (int i = 0; i < 1000; ++i) + { + CHECK(l.root()["k" + std::to_string(i)] == d.root()["k" + std::to_string(i)]); + } + CHECK(l.root()["k7"].get() == 7); // the first of duplicate keys + CHECK(l.root()["inner"]["m199"].get() == -199); + CHECK(!l.root().contains("k1000")); + // the index is not part of the image + CHECK(l.save() == image); + } + // the nodes of objects in the image do not carry the number of an index + CHECK(node_at(image, 0).extra == 0); + } + + SECTION("duplicate keys of a large object: lookups return the first member") + { + std::string text = "{"; + for (int i = 0; i < 200; ++i) + { + text += (i != 0 ? ",\"k" : "\"k") + std::to_string(i) + "\":" + std::to_string(i); + } + // three members for k5 (the first is a number), and duplicates of k0 and k199 + text += R"(,"k5":"two","k0":null,"k199":[],"k5":"three"})"; + const json_document d = json_document::parse(text); + REQUIRE(d.root().size() == 204); + CHECK(d.root()["k5"] == 5); + const std::vector image = d.save(); + for (const image_check check : + { + image_check::full, image_check::bounds, image_check::none + }) + { + const json_document l = json_document::load(image, check); + CHECK(l.root().size() == 204); + CHECK(l.root()["k5"] == 5); + CHECK(l.root().at("k5") == 5); + CHECK(l.root().find("k5").value() == 5); + CHECK(l.root()["k0"] == 0); + CHECK(l.root()["k199"] == 199); + CHECK(l.root()["k100"] == 100); + // materialize() keeps the semantics of parse(): the last value + CHECK(l.root().materialize() == d.root().materialize()); + CHECK(l.root().materialize()["k5"] == "three"); + } + } + + SECTION("objects with colliding keys") + { + // the hash is not seeded: keys can be found that land in one slot of + // a table (see json_view: "colliding keys") + const auto keys_for = [](std::size_t table_size, std::size_t colliding_count, std::size_t spread_count) + { + std::vector colliding; + std::vector spread; + for (std::uint64_t counter = 0; colliding.size() < colliding_count || spread.size() < spread_count; ++counter) + { + std::string key(8, 'a'); + std::uint64_t x = counter; + for (std::size_t i = 0; i < 8; ++i, x /= 26) + { + key[i] = static_cast('a' + (x % 26)); + } + const bool lands_in_slot_zero = (nlohmann::detail::view::key_hash(key.data(), key.size()) & (table_size - 1)) == 0; + if (lands_in_slot_zero && colliding.size() < colliding_count) + { + colliding.push_back(key); + } + else if (!lands_in_slot_zero && spread.size() < spread_count) + { + spread.push_back(key); + } + } + colliding.insert(colliding.end(), spread.begin(), spread.end()); + return colliding; + }; + const auto make_text = [](const std::vector& keys) + { + std::string text = "{"; + for (std::size_t i = 0; i < keys.size(); ++i) + { + text += (i != 0 ? ",\"" : "\"") + keys[i] + "\":" + std::to_string(i); + } + return text + "}"; + }; + + // 300 keys in one slot (the table is 1024 slots): the table would + // exceed the probe limit, so the object has none and is searched + // linearly; 40 of 200 keys in one slot (512 slots): a table with a + // long chain + const struct // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays): an unnamed type + { + std::size_t table_size; + std::size_t colliding; + std::size_t spread; + } cases[] = {{1024, 300, 0}, {512, 40, 160}}; + for (const auto& c : cases) + { + CAPTURE(c.table_size) + std::vector keys = keys_for(c.table_size, c.colliding, c.spread); + const std::size_t members = keys.size(); + // a duplicate of a colliding key, of the first and of a spread key + const std::vector duplicated = {keys[c.colliding - 1], keys[0], keys[members - 1]}; + std::string text = make_text(keys); + text.pop_back(); + for (const std::string& key : duplicated) + { + text += ",\"" + key + R"(":"last")"; + } + text += "}"; + const json_document d = json_document::parse(text); + const std::vector image = d.save(); + for (const image_check check : + { + image_check::full, image_check::bounds, image_check::none + }) + { + const json_document l = json_document::load(image, check); + REQUIRE(l.root().size() == members + duplicated.size()); + for (std::size_t i = 0; i < members; ++i) + { + CAPTURE(i) + // lookups find the first member, also of the duplicated keys + CHECK(l.root()[keys[i]] == i); + CHECK(l.root().at(keys[i]) == l.root()[keys[i]]); + CHECK(l.root().find(keys[i]).key() == keys[i]); + CHECK(!l.root().contains(keys[i] + "x")); + } + CHECK(!l.root().contains("missing")); + CHECK(l.root().materialize() == json::parse(text)); + } + } + } + + SECTION("loading releases the list of large objects") + { + std::string text = "["; + for (int object = 0; object < 400; ++object) + { + text += object != 0 ? ",{" : "{"; + for (int i = 0; i < 128; ++i) + { + text += (i != 0 ? ",\"" : "\"") + std::to_string(i) + "\":" + std::to_string(i); + } + text += '}'; + } + text += "]"; + json_document parsed = json_document::parse(text); + const std::vector image = parsed.save(); + json_document loaded = json_document::load(image); + CHECK(loaded.root()[399]["127"] == 127); + // Parsing and loading build the same tables, and keep nothing else: + // not the positions of the objects to index (2 KiB here). The slack + // covers the nodes in the header of the document, which are sized + // differently. + parsed.shrink_to_fit(); + loaded.shrink_to_fit(); + CHECK(loaded.memory_usage() <= parsed.memory_usage() + 512); + } +} + +TEST_CASE("json_view images: edited documents") +{ + const std::string text = R"({"name": "x", "n": 1, "f": 2.5, "list": [1, 2, 3], "obj": {"a": "\u00e9", "b": [true]}, "s": "a\"b"})"; + + SECTION("every kind of edit") + { + ordered_json_editable_document d = ordered_json_editable_document::parse(text); + d.set(d.root()["name"], "a new \"name\""); // string in the edit arena + d.set(d.root()["n"], -17); // negative integer + d.set(d.root(), "p", 5); // non-negative number_integer + d.set(d.root(), "u", 18446744073709551615u); // unsigned + d.set(d.root()["f"], 0.1); // float token + d.set(d.root(), "nan", std::numeric_limits::quiet_NaN()); + d.set(d.root(), "inf", -std::numeric_limits::infinity()); + d.push_back(d.root()["list"], "pushed"); // moved array + d.insert(d.root()["list"], 0, ordered_json::object({{"new", {1, 2}}})); + d.erase(d.root()["list"], 2); + d.erase(d.root(), "s"); + d.set(d.root()["obj"], "c", ordered_json::array({1, "two", 3.5, nullptr, false})); // new object member with a new array + d.set(d.root(), "copy", d.root()["obj"]); // a copy of a subtree + d.set(d.root(), "key \xc3\xa9", true); // a key in the edit arena + + const std::vector image = d.save(); + const ordered_json expected = ordered_json::parse(d.root().dump()); + for (const image_check check : + { + image_check::full, image_check::bounds, image_check::none + }) + { + const ordered_json_document l = ordered_json_document::load(image, check); + CHECK(l.root().dump() == d.root().dump()); + CHECK(l.root().materialize() == expected); + CHECK(l.root()["nan"].is_null()); + CHECK(l.root()["inf"].is_null()); + CHECK(l.root()["p"].is_number_integer()); + CHECK(l.root()["p"].get() == 5); + CHECK(l.root()["f"].get() == 0.1); + CHECK(l.root()["u"].get() == 18446744073709551615u); + } + // the node index is in document order again: an image of the loaded + // document is the same image + CHECK(ordered_json_document::load(image).save() == image); + // number tokens of edits follow the source; the text is the source's + // prefix + const ordered_json_document l = ordered_json_document::load(image); + REQUIRE(l.source().size() > text.size()); + CHECK(std::string(l.source().data(), text.size()) == text); // NOLINT(bugprone-suspicious-stringview-data-usage): the size is given and checked above + } + + SECTION("a loaded document can be edited and saved again") + { + const std::vector first = json_editable_document::parse(text).save(); + json_editable_document d = json_editable_document::load(first); + d.set(d.root()["obj"]["a"], "changed"); + d.push_back(d.root()["list"], 4); + d.set(d.root(), "z", json::array({json::object()})); + const std::vector second = d.save(); + const json_document l = json_document::load(second); + CHECK(l.root().dump() == d.root().dump()); + CHECK(l.root()["obj"]["a"] == "changed"); + CHECK(l.root()["list"].size() == 4); + } + + SECTION("the targets of links do not reach the image") + { + // Edits that make an entry of a moved sequence link to a value mark the + // value as linked (node_flags::linked). An image has no links and no + // such flag: the full and the bounds check reject it. + const std::string nested = R"({"a": [1, 2, {"x": [3]}, "s", 1.5, true, null, [4, 5]], "b": {"c": 1, "d": [1, 2, 3], "e": {"f": "g"}}, "h": "str"})"; + const auto check_image = [](const json_editable_document & d) + { + const std::vector image = d.save(); + for (std::size_t i = 0; i < node_count(image); ++i) + { + CAPTURE(i) + CHECK((node_at(image, i).flags & nlohmann::detail::view::node_flags::linked) == 0); + CHECK(node_at(image, i).kind != nlohmann::detail::view::kind_link); + } + for (const image_check check : + { + image_check::full, image_check::bounds + }) + { +#if !defined(JSON_NOEXCEPTION) + CHECK(load_result(image, check).empty()); +#else + const json_document loaded = json_document::load(image, check); + CHECK(loaded.root().dump() == d.root().dump()); +#endif + } + check_round_trip(d); + // a loaded document is edited and saved again + json_editable_document e = json_editable_document::load(image); + CHECK(e.root().dump() == d.root().dump()); + e.set(e.root(), "after", 1); + CHECK(loaded_dump(e.save()) == e.root().dump()); + }; + + SECTION("a container replaced by a scalar after an earlier edit") + { + json_editable_document d = json_editable_document::parse(nested); + d.set(d.root()["a"][0], 9); + d.set(d.root()["a"][2], 5); + check_image(d); + d.set(d.root()["a"][2], "now a string"); + check_image(d); + d.set(d.root()["a"][7], 0.25); + check_image(d); + d.set(d.root()["b"]["e"], true); + d.set(d.root()["b"]["d"], 7); + check_image(d); + } + + SECTION("insert and push_back into parsed arrays") + { + json_editable_document d = json_editable_document::parse(nested); + d.push_back(d.root()["a"], 6); + check_image(d); + d.insert(d.root()["a"], 0, "first"); + check_image(d); + d.insert(d.root()["b"]["d"], 1, json::array({1, 2})); + d.push_back(d.root()["b"]["d"], json::object({{"k", nullptr}})); + check_image(d); + // links to values that are replaced afterwards + d.set(d.root()["a"][3], json::object()); + d.set(d.root()["a"][4], false); + d.set(d.root()["a"][5], "replaced"); + d.set(d.root()["a"][9], 1e300); + check_image(d); + } + + SECTION("members set in parsed objects") + { + json_editable_document d = json_editable_document::parse(nested); + d.set(d.root(), "new", 1); + d.set(d.root()["b"], "c", "changed"); + d.set(d.root()["b"], "e", json::array({1, {{"z", 2}}})); + d.set(d.root()["b"], "more", d.root()["a"]); + check_image(d); + d.set(d.root()["a"][2], 3); // a link target in a copied subtree + d.erase(d.root()["b"], "c"); + d.set(d.root(), "h", json::object()); + check_image(d); + } + + SECTION("erase") + { + json_editable_document d = json_editable_document::parse(nested); + d.push_back(d.root()["a"], 6); + d.erase(d.root()["a"], 0); + d.erase(d.root()["a"], 1); + d.set(d.root()["a"][0], 1); + d.erase(d.root()["b"], "d"); + check_image(d); + } + } + +#if !defined(JSON_NOEXCEPTION) + SECTION("invalid UTF-8 of a loaded image is not copied into an editable document") + { + // loading with the bounds check (or none) does not look at the encoding + // of strings: dump() of such a view throws, and an editable document + // must not take the string over + const auto patched = [](const std::string & json_text) + { + std::vector image = json_document::parse(json_text).save(); + bool found = false; + for (std::size_t i = 0; i + 1 < image.size(); ++i) + { + if (image[i] == 'Q' && image[i + 1] == 'Z') + { + image[i] = 0xC3; + image[i + 1] = 0x28; // an invalid sequence + found = true; + } + } + REQUIRE(found); + return image; + }; + + for (const image_check check : + { + image_check::bounds, image_check::none + }) + { + // as a value + { + const json_document loaded = json_document::load(patched(R"(["abQZ"])"), check); + json_editable_document e = json_editable_document::parse("[]"); + CHECK_THROWS_WITH_AS(e.push_back(e.root(), loaded.root()[0]), "[json.exception.type_error.316] invalid UTF-8 byte at index 3: 0x28", json::type_error&); + CHECK_THROWS_WITH_AS(e.insert(e.root(), 0, loaded.root()[0]), "[json.exception.type_error.316] invalid UTF-8 byte at index 3: 0x28", json::type_error&); + CHECK_THROWS_WITH_AS(e.set(e.root(), loaded.root()[0]), "[json.exception.type_error.316] invalid UTF-8 byte at index 3: 0x28", json::type_error&); + CHECK_THROWS_WITH_AS(e.push_back(e.root(), loaded.root()), "[json.exception.type_error.316] invalid UTF-8 byte at index 3: 0x28", json::type_error&); + // nothing changed + CHECK(e.root().dump() == "[]"); + CHECK(e.root().size() == 0); + CHECK(loaded_dump(e.save()) == "[]"); + // the valid part of the same document can be copied + e.push_back(e.root(), loaded.root().size()); + CHECK(e.root().dump() == "[1]"); + } + // as a key, and in a nested value + { + const json_document loaded = json_document::load(patched(R"([{"abQZ": 1}, [["x", "QZ"]]])"), check); + json_editable_document e = json_editable_document::parse(R"({"keep": [1]})"); + CHECK_THROWS_WITH_AS(e.set(e.root(), "k", loaded.root()[0]), "[json.exception.type_error.316] invalid UTF-8 byte at index 3: 0x28", json::type_error&); + CHECK_THROWS_WITH_AS(e.set(e.root(), "k", loaded.root()[1]), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0x28", json::type_error&); + CHECK_THROWS_WITH_AS(e.push_back(e.root()["keep"], loaded.root()[1]), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0x28", json::type_error&); + CHECK(e.root().dump() == R"({"keep":[1]})"); + CHECK(loaded_dump(e.save()) == R"({"keep":[1]})"); + } + } + } +#endif + + SECTION("the root replaced") + { + json_editable_document d = json_editable_document::parse(text); + d.set(d.root(), json::array({1, "x"})); + check_round_trip(d); + d.set(d.root(), 3.5); + check_round_trip(d); + d.set(d.root(), "text"); + check_round_trip(d); + } +} + +TEST_CASE("json_view images: ownership") +{ + const std::string text = R"({"a": "esc\u00e9aped", "b": [1, 2]})"; + const std::vector image = json_document::parse(text).save(); + + SECTION("borrowed") + { + const json_document d = json_document::load(image); + CHECK(!d.owns_source()); + CHECK(d.root()["a"] == "esc\xc3\xa9" "aped"); + const json_document p = json_document::load(image.data(), image.size()); + CHECK(!p.owns_source()); + CHECK(p.root() == d.root()); + // the text is the image's + CHECK(d.source().data() == reinterpret_cast(image.data() + text_at(image))); + } + + SECTION("owned") + { + std::vector copy = image; + const std::uint8_t* const data = copy.data(); + json_document d = json_document::load(std::move(copy)); + CHECK(d.owns_source()); + CHECK(d.source().data() == reinterpret_cast(data + text_at(image))); + CHECK(d.memory_usage() >= image.size()); + CHECK(d.root()["b"][1] == 2); + // read() replaces the image + d.read(std::string("[1]")); + CHECK(d.owns_source()); + CHECK(d.root().dump() == "[1]"); + const std::string borrowed = "[2]"; + d.read(borrowed); + CHECK(!d.owns_source()); + } + + SECTION("shrink_to_fit keeps the decoded strings of the image") + { + json_document d = json_document::parse(R"(["\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9"])"); + d.shrink_to_fit(); + const std::vector img = d.save(); + json_document l = json_document::load(img); + l.shrink_to_fit(); + CHECK(l.root().dump() == d.root().dump()); + CHECK(l.save() == img); + } +} + +// the remaining tests are about the exceptions of load() and save() +#if !defined(JSON_NOEXCEPTION) +TEST_CASE("json_view images: errors") +{ + SECTION("a literal as the root: dump() after loading") + { + for (const char* text : + { + "null", "true", "false" + }) + { + std::vector image = json_document::parse(text).save(); + node n = node_at(image, 0); + n.off = static_cast(image.size()); + set_node(image, 0, n); + CHECK(load_result(image, image_check::full) == check_failed); + CHECK(load_result(image, image_check::bounds) == check_failed); + } + } + + SECTION("saving a discarded document") + { + const json_document empty{}; + CHECK(exception_of([&] { static_cast(empty.save()); }) == "[json.exception.type_error.320] cannot save a discarded json_document"); + const json_document failed = json_document::parse("[1,", false); + CHECK(exception_of([&] { static_cast(failed.save()); }) == "[json.exception.type_error.320] cannot save a discarded json_document"); + } + + const std::vector image = json_document::parse(R"({"a": [1, "\u00e9"]})").save(); + const std::string prefix = "[json.exception.parse_error.116] parse error: invalid json_document image: "; + + SECTION("header and sizes") + { + CHECK(exception_of([] + { + const json_document d = json_document::load(nullptr, 0); + static_cast(d); + }) == prefix + "too short"); + CHECK(exception_of([&] + { + const json_document d = json_document::load(image.data(), 63); + static_cast(d); + }) == prefix + "too short"); + + std::vector bad = image; + bad[0] = 'X'; + CHECK(load_result(bad, image_check::full) == prefix + "unknown format"); + bad = image; + bad[4] = 2; // version + CHECK(load_result(bad, image_check::full) == prefix + "unknown format"); + for (std::size_t reserved = 32; reserved < 64; reserved += 8) + { + bad = image; + bad[reserved + 3] = 1; + CHECK(load_result(bad, image_check::none) == prefix + "unknown format"); + } + + const auto sizes = [&](std::size_t offset, std::uint64_t v) + { + std::vector b = image; + set_header_field(b, offset, v); + return load_result(b, image_check::none); + }; + CHECK(sizes(8, 0) == prefix + "sizes out of range"); // no nodes + CHECK(sizes(8, 1000) == prefix + "sizes out of range"); // more nodes than bytes + CHECK(sizes(8, 0xFFFFFFF0u) == prefix + "sizes out of range"); + CHECK(sizes(16, 0xFFFFFFF0u) == prefix + "sizes out of range"); // text size + CHECK(sizes(16, header_field(image, 16) + 1) == prefix + "sizes out of range"); + CHECK(sizes(16, image.size()) == prefix + "sizes out of range"); + CHECK(sizes(24, 0xFFFFFFF0u) == prefix + "sizes out of range"); // decoded string size + CHECK(sizes(24, header_field(image, 24) - 1) == prefix + "sizes out of range"); + + // the NULs after the text and the decoded strings + bad = image; + bad[text_at(image) + header_size_field(image, 16)] = 'x'; + CHECK(load_result(bad, image_check::none) == prefix + "sizes out of range"); + bad = image; + bad.back() = 'x'; + CHECK(load_result(bad, image_check::none) == prefix + "sizes out of range"); + // nothing after the image + bad = image; + bad.push_back(0); + CHECK(load_result(bad, image_check::none) == prefix + "sizes out of range"); + // nodes, but not even room for the NULs + bad.assign(image.begin(), image.begin() + static_cast(text_at(image))); + CHECK(load_result(bad, image_check::none) == prefix + "sizes out of range"); + } +} + +TEST_CASE("json_view images: check") +{ + // nodes: 0 { 1 "s" 2 "x\"y" (escaped) 3 "i" 4 -12 5 "u" 6 7 7 "f" 8 1.5e300 9 "b" 10 true 11 "n" 12 null + // 13 "a" 14 [ 15 "t" 16 {} ] + const std::string text = R"({"s":"x\"y","i":-12,"u":7,"f":1.5e300,"b":true,"n":null,"a":["t",{}]})"; + const std::vector image = json_document::parse(text).save(); + REQUIRE(load_result(image, image_check::full).empty()); + REQUIRE(node_count(image) == 17); + + // bounds: rejected by both checks; content: only by the full one + const auto rejected = [&](const std::vector& b, bool bounds) + { + CHECK(load_result(b, image_check::full) == check_failed); + CHECK(load_result(b, image_check::bounds) == (bounds ? check_failed : "")); + }; + + SECTION("kinds") + { + const std::array kinds = {{8, 9, 10, 200}}; // binary, discarded, link, unknown + for (const std::uint8_t kind : kinds) + { + rejected(corrupted(image, 12, [&](node & n) + { + n.kind = kind; + }), true); + } + // a key that is not a string + rejected(corrupted(image, 1, [](node & n) + { + n.kind = 0; + n.len = 0; + n.off = 0; + }), true); + } + + SECTION("flags and extra") + { + rejected(corrupted(image, 12, [](node & n) + { + n.flags = 4; + }), true); + rejected(corrupted(image, 12, [](node & n) + { + n.extra = 1; + }), true); + rejected(corrupted(image, 10, [](node & n) + { + n.flags = 5; + }), true); + rejected(corrupted(image, 10, [](node & n) + { + n.extra = 1; + }), true); + rejected(corrupted(image, 1, [](node & n) + { + n.flags = 2; // a string in the edit arena + }), true); + rejected(corrupted(image, 1, [](node & n) + { + n.extra = 3; + }), true); + rejected(corrupted(image, 4, [](node & n) + { + n.flags = 2; + }), true); + rejected(corrupted(image, 4, [](node & n) + { + n.extra = static_cast(n.extra | 0x100u); // an integer with fraction digits + }), true); + rejected(corrupted(image, 0, [](node & n) + { + n.flags = 8; // moved + }), true); + rejected(corrupted(image, 0, [](node & n) + { + n.extra = 1; // a hash index + }), true); + // the flag of the targets of links (editable documents) is not part of an image + for (std::size_t i = 0; i < node_count(image); ++i) + { + CAPTURE(i) + rejected(corrupted(image, i, [](node & n) + { + n.flags = static_cast(n.flags | nlohmann::detail::view::node_flags::linked); + }), true); + } + } + + SECTION("bounds") + { + const std::size_t text_size = header_size_field(image, 16); + const std::size_t arena_size = header_size_field(image, 24); + rejected(corrupted(image, 1, [&](node & n) + { + n.off = static_cast(text_size + 1); + }), true); + rejected(corrupted(image, 1, [&](node & n) + { + n.len = static_cast(text_size); + }), true); + rejected(corrupted(image, 2, [&](node & n) + { + n.len = static_cast(arena_size + 1); + }), true); + rejected(corrupted(image, 6, [&](node & n) + { + n.off = static_cast(text_size); + }), true); + rejected(corrupted(image, 6, [&](node & n) + { + n.off = static_cast(text_size + 5); + }), true); + rejected(corrupted(image, 6, [](node & n) + { + n.extra = 0; // no digits + }), true); + rejected(corrupted(image, 8, [&](node & n) + { + n.len = static_cast(text_size); + }), true); + rejected(corrupted(image, 8, [](node & n) + { + n.len = 2; // shorter than the recorded digits + }), true); + rejected(corrupted(image, 14, [&](node & n) + { + n.off = static_cast(text_size + 1); + }), true); + // literals: their offset sizes the output of dump() + rejected(corrupted(image, 10, [&](node & n) + { + n.off = static_cast(text_size + 1); + }), true); + rejected(corrupted(image, 12, [&](node & n) + { + n.off = 0xFFFFFFFFu; + }), true); + } + + SECTION("structure") + { + rejected(corrupted(image, 0, [](node & n) + { + n.next = 0; + }), true); + rejected(corrupted(image, 0, [](node & n) + { + n.next = 18; // beyond the image + }), true); + rejected(corrupted(image, 14, [](node & n) + { + n.next = 4; // beyond the enclosing object + }), true); + rejected(corrupted(image, 0, [](node & n) + { + n.len = 6; // member count + }), true); + rejected(corrupted(image, 14, [](node & n) + { + n.len = 3; // element count + }), true); + rejected(corrupted(image, 0, [](node & n) + { + n.next = 14; // the object ends after the key "a" + n.len = 7; + }), true); + rejected(corrupted(image, 0, [](node & n) + { + n.next = 13; // nodes after the root + n.len = 6; + }), true); + rejected(corrupted(image, 0, [](node & n) + { + n.kind = 2; // an array: the "keys" are values, and the counts do not match + }), true); + const std::vector as_array = corrupted(image, 16, [](node & n) + { + n.kind = 2; // {} as []: fine + }); + CHECK(load_result(as_array, image_check::full).empty()); + const std::string expected_dump = R"({"s":"x\"y","i":-12,"u":7,"f":1.5e+300,"b":true,"n":null,"a":["t",[]]})"; + CHECK(loaded_dump(as_array) == expected_dump); + } + + SECTION("strings") + { + // a quote in a source string (the full check only) + std::vector b = image; + const std::size_t t = text_at(image); + const node t15 = node_at(image, 15); + b[t + t15.off] = '"'; + rejected(b, false); + // a control character + b[t + t15.off] = '\n'; + rejected(b, false); + // invalid UTF-8 in a decoded string + b = image; + const node s2 = node_at(image, 2); + b[t + header_size_field(image, 16) + 1 + s2.off] = 0xFF; + rejected(b, false); + } + + SECTION("numbers") + { + const std::size_t t = text_at(image); + const node i4 = node_at(image, 4); + const node u6 = node_at(image, 6); + const node f8 = node_at(image, 8); + const auto at_token = [&](const node & n, std::size_t k, std::uint8_t c) + { + std::vector b = image; + b[t + n.off + k] = c; + return b; + }; + rejected(at_token(i4, 1, 'x'), false); // -x2 + rejected(at_token(i4, 1, '0'), false); // -02 + rejected(at_token(i4, 0, '1'), false); // 112 != -12 + rejected(at_token(f8, 1, 'x'), false); // 1x5e300 + rejected(at_token(f8, 2, 'e'), false); // 1.ee300 + rejected(at_token(f8, 4, 'x'), false); // 1.5ex00 + rejected(at_token(f8, 3, '0'), false); // 1.50300: another layout + rejected(at_token(f8, 4, '9'), false); // 1.5e900: overflow + rejected(at_token(f8, 0, 'x'), false); + rejected(at_token(u6, 0, '8'), false); // 8 != 7 + rejected(corrupted(image, 4, [](node & n) + { + n.kind = 6; // "-12" as unsigned: a sign + n.extra = 3; + }), false); + // a non-negative number_integer (as edits write it): fine + const std::vector positive = corrupted(image, 6, [](node & n) + { + n.kind = 5; + n.extra = 0; + }); + CHECK(load_result(positive, image_check::full).empty()); + const json_document pos = json_document::load(positive); + CHECK(pos.root()["u"].is_number_integer()); + rejected(corrupted(image, 8, [](node & n) + { + n.kind = 6; // a float token as integer + n.extra = 7; + }), false); + } + + SECTION("float tokens of an image checked for bounds only") + { + // A float node whose layout records "many" digits is converted from + // its token alone; a token that is not a JSON number reads as 0. + const std::vector img = json_document::parse("[1.5e300,2]").save(); + const std::size_t t = text_at(img); + for (const char* token : + { + "x.5e300", "01.5e30", "1.xe300", "1.5ex00", "1.5e+x0", "1.5e30x", "-.5e300", "1.5E300" + }) + { + CAPTURE(token) + std::vector b = img; + node n = node_at(b, 1); + n.extra = 0xFFFFu; + set_node(b, 1, n); + std::memcpy(b.data() + t + n.off, token, n.len); + const json_document d = json_document::load(b, image_check::bounds); + const auto v = d.root()[0].get(); + CHECK(v == (std::string(token) == "1.5E300" ? 1.5e300 : 0.0)); + CHECK(load_result(b, image_check::full) == (std::string(token) == "1.5E300" ? "" : check_failed)); + } + } + + SECTION("nodes that share a range") + { + // [big string, then n strings made to point to the big string]: every + // node shares the one range (the check reads it once) + const std::size_t n = 200; + std::string long_text = "[\"" + std::string(5000, 'a') + "\""; + for (std::size_t k = 0; k < n; ++k) + { + long_text += ",\"x\""; + } + long_text += "]"; + const std::vector img = json_document::parse(long_text).save(); + const node big = node_at(img, 1); + std::vector b = img; + for (std::size_t k = 0; k < n; ++k) + { + set_node(b, 2 + k, big); + } + CHECK(load_result(b, image_check::full).empty()); + const json_document d = json_document::load(b); + CHECK(d.root().size() == n + 1); + CHECK(d.root()[n].get() == std::string(5000, 'a')); + CHECK(d.root().dump() == loaded_dump(b, image_check::none)); + + // the same for decoded strings and float tokens, with a node that + // records another digit layout than its token + const std::vector img2 = json_document::parse(R"(["a\"b", "a\"b", 1.25, 1.25, 1.25e3])").save(); + CHECK(load_result(img2, image_check::full).empty()); + std::vector same = img2; + set_node(same, 2, node_at(img2, 1)); + set_node(same, 4, node_at(img2, 3)); + CHECK(load_result(same, image_check::full).empty()); + // (a raw string with a backslash must not be a macro argument: MSVC C2017) + const std::string expected_same = R"(["a\"b","a\"b",1.25,1.25,1250.0])"; + CHECK(loaded_dump(same) == expected_same); + std::vector layout = same; + node f4 = node_at(layout, 4); + f4.extra = 0x0100u; // the layout of "1.", and not that of "1.25" + set_node(layout, 4, f4); + rejected(layout, false); + f4.extra = 0xFFFFu; // "many" digits: fine, as compaction writes it + set_node(layout, 4, f4); + CHECK(load_result(layout, image_check::full).empty()); + } + + SECTION("ranges that overlap") + { + // save() writes every string and every token to a place of its own + // (nodes that share a value share the whole range): ranges that + // overlap without being identical are a damaged image, though each + // range is a valid string or token + // nodes: 0 [ 1 "abcdef" 2 "ghijkl" 3 1.2525 4 9.9 5 "a\"bcd" (decoded) 6 "e\"fgh" (decoded) + const std::vector img = json_document::parse(R"(["abcdef", "ghijkl", 1.2525, 9.9, "a\"bcd", "e\"fgh"])").save(); + REQUIRE(load_result(img, image_check::full).empty()); + // node i with the range (off of node of + shift, length), and extra + const auto aliased = [&](std::size_t i, std::size_t of, std::uint32_t shift, std::uint32_t length, std::uint16_t extra) + { + std::vector b = img; + node n = node_at(b, of); + n.off += shift; + n.len = length; + n.extra = extra; + set_node(b, i, n); + return b; + }; + // source strings + rejected(aliased(2, 1, 0, 4, 0), false); // "abcd": the start of another string + rejected(aliased(2, 1, 1, 4, 0), false); // "bcde": inside + rejected(aliased(2, 1, 2, 4, 0), false); // "cdef": the end + CHECK(load_result(aliased(2, 1, 0, 6, 0), image_check::full).empty()); // the whole range: identical + // decoded strings: inside, and partially overlapping (the arena holds a"bcde"fgh) + rejected(aliased(6, 5, 1, 3, 0), false); + rejected(aliased(6, 5, 3, 5, 0), false); + CHECK(load_result(aliased(6, 5, 0, 5, 0), image_check::full).empty()); + // float tokens: "1.25" and "525" (an integer token as a float) inside "1.2525" + rejected(aliased(4, 3, 0, 4, 0x0201u), false); + rejected(aliased(4, 3, 3, 3, 0x0003u), false); + CHECK(load_result(aliased(4, 3, 0, 6, 0x0401u), image_check::full).empty()); + // a string and a float token may use the same bytes (each is checked by its own kind) + std::vector shared = img; + node as_string = node_at(img, 2); + as_string.off = node_at(img, 3).off; + as_string.len = 6; + set_node(shared, 2, as_string); + CHECK(load_result(shared, image_check::full).empty()); + // empty strings inside others are not ranges + CHECK(load_result(aliased(2, 1, 2, 0, 0), image_check::full).empty()); + } + + SECTION("copies within an edited document") + { + // copies within a document share the value: identical ranges + json_editable_document d = json_editable_document::parse(R"({"s": "ab", "e": "x\"y", "f": 1.5, "g": 1.5e300, "a": [1.5, "ab"]})"); + d.set(d.root(), "c1", d.root()["a"]); + d.set(d.root(), "c2", d.root()["a"]); + d.set(d.root(), "c3", d.root()); + d.push_back(d.root()["a"], d.root()["e"]); + d.push_back(d.root()["a"], d.root()["g"]); + d.push_back(d.root()["a"], d.root()["g"]); + d.set(d.root()["s"], "changed"); + d.set(d.root()["f"], 7.25); + const std::vector saved = d.save(); + CHECK(load_result(saved, image_check::full).empty()); + CHECK(loaded_dump(saved) == d.root().dump()); + check_round_trip(d); + } + + SECTION("integer ranges") + { + // tokens of many digits, which the parser stores as floats + const std::string big = R"([123456789012345678901234, 99999999999999999999, 9223372036854775808])"; + const std::vector img = json_document::parse(big).save(); + const auto as_integer = [&](std::size_t i, std::uint8_t kind, std::uint16_t extra) + { + std::vector b = img; + node n = node_at(b, i); + n.kind = kind; + n.extra = extra; + set_node(b, i, n); + return load_result(b, image_check::full); + }; + CHECK(as_integer(1, 6, 24) == check_failed); // more than 20 digits + CHECK(as_integer(2, 6, 20) == check_failed); // more than 2^64 - 1 + CHECK(as_integer(3, 5, 18) == check_failed); // more than 2^63 - 1 as number_integer + } +} + +TEST_CASE("json_view images: damaged images") +{ + // A damaged image must be rejected, or read safely; with the full check, + // it also serializes to the JSON it reads as. + const std::vector texts = + { + R"({"a": [1, -2, 3.25, "x\u00e9y", true, null], "b": {"c": "\"q\"", "d": 1e10}, "e": ""})", + R"([[[[]]], {"k": {"k": {"k": 12345678901234567890}}}, "\ud83d\ude00", -0.0, 0])", + }; + for (const std::string& text : texts) + { + const std::vector image = json_document::parse(text).save(); + for (int round = 0; round < 3000; ++round) + { + std::vector b = image; + const std::uint32_t flips = 1 + (rng() % 3); + for (std::uint32_t k = 0; k < flips; ++k) + { + // mostly the nodes, where the damage matters most + const std::size_t at = rng() % 4 != 0 ? header_size + (rng() % (b.size() - header_size)) : rng() % b.size(); + b[at] = static_cast(rng() % 3 == 0 ? rng() : b[at] ^ (1u << (rng() % 8))); + } + for (const image_check check : + { + image_check::full, image_check::bounds + }) + { + json_document d; + try + { + d = json_document::load(b, check); + } + catch (const json::parse_error& e) + { + CHECK(e.id == 116); + continue; + } + std::string dumped; + std::string dumped_ascii; + try + { + dumped = d.root().dump(); + dumped_ascii = d.root().dump(-1, ' ', true); + } + catch (const json::type_error& e) + { + // invalid UTF-8 (the bounds check only) + CHECK(check == image_check::bounds); + CHECK(e.id == 316); + continue; + } + const json j = d.root().materialize(); + if (check == image_check::full) + { + CHECK(json::parse(dumped) == j); + CHECK(json::parse(dumped_ascii) == j); + } + } + } + } +} +#endif + +#else + +TEST_CASE("json_view images: big-endian targets") +{ + const json_document d = json_document::parse("[1]"); + CHECK_THROWS_WITH_AS(d.save(), "[json.exception.type_error.320] json_document images need a little-endian target", json::type_error&); +} + +#endif