diff --git a/.github/labeler.yml b/.github/labeler.yml index 014d9f9bb..076cad183 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -63,7 +63,7 @@ labels: - "single_include/nlohmann/json_view\\.hpp" - "tests/src/unit-json_view.*" - "tests/src/json_view_test_helpers\\.hpp" - - "tests/src/fuzzer-parse_json_view\\.cpp" + - "tests/src/fuzzer-(parse_json_view|json_view_image)\\.cpp" - "tools/amalgamate/config_json_view\\.json" - "docs/mkdocs/docs/features/json_view\\.md" - "docs/mkdocs/docs/api/basic_json_(document|view)/.*" diff --git a/BUILD.bazel b/BUILD.bazel index 4c218fa57..78505d796 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -73,6 +73,7 @@ cc_library( "include/nlohmann/detail/view/edit.hpp", "include/nlohmann/detail/view/edit_storage.hpp", "include/nlohmann/detail/view/errors.hpp", + "include/nlohmann/detail/view/image.hpp", "include/nlohmann/detail/view/input.hpp", "include/nlohmann/detail/view/iterator.hpp", "include/nlohmann/detail/view/lookup.hpp", diff --git a/docs/mkdocs/docs/api/basic_json_document/index.md b/docs/mkdocs/docs/api/basic_json_document/index.md index 66e83b50e..d1321959a 100644 --- a/docs/mkdocs/docs/api/basic_json_document/index.md +++ b/docs/mkdocs/docs/api/basic_json_document/index.md @@ -49,14 +49,21 @@ bookkeeping edits need, and calling any of them on one fails to compile (`#!cpp - **view_type** - the type of view returned by [`root()`](root.md) (`#!cpp basic_json_view`) - **value_t** - the JSON type enumeration, see [`basic_json::value_t`](../basic_json/value_t.md) +- **image_check** - how [`load()`](load.md) validates an image (`full`, `bounds`, `none`), see [`image_check`](load.md#image_check) ## Member functions - [(constructor)](basic_json_document.md) + +### Parsing + - [**parse**](parse.md) (_static_) - deserialize from a compatible input, borrowing or owning it as appropriate - [**parse_copy**](parse_copy.md) (_static_) - deserialize a copy of a compatible input - [**accept**](accept.md) (_static_) - check whether the input is valid JSON - [**read**](read.md) - (re-)parse into this document, reusing its memory + +### Access + - [**root**](root.md) - the view of the root value - [**is_discarded**](is_discarded.md) - return whether the last parse failed - [**source**](source.md) - the parsed text @@ -64,6 +71,14 @@ bookkeeping edits need, and calling any of them on one fails to compile (`#!cpp - [**node_count**](node_count.md) - the number of index entries (values plus object keys) - [**memory_usage**](memory_usage.md) - the number of bytes held by the document - [**shrink_to_fit**](shrink_to_fit.md) - release unused index capacity + +### Images + +- [**save**](save.md) - the document as an image that `load()` reads without parsing +- [**load**](load.md) (_static_) - read an image written by `save()` + +### Edits + - [**set**](set.md) - replace a value, or set an object member, an array element, or the value a JSON pointer refers to (`#!cpp Editable` documents only) - [**push_back**](push_back.md) - append to an array (`#!cpp Editable` documents only) diff --git a/docs/mkdocs/docs/api/basic_json_document/load.md b/docs/mkdocs/docs/api/basic_json_document/load.md new file mode 100644 index 000000000..36223f63c --- /dev/null +++ b/docs/mkdocs/docs/api/basic_json_document/load.md @@ -0,0 +1,179 @@ +# nlohmann::basic_json_document::load + +```cpp +// (1) +static basic_json_document load(const std::uint8_t* image, std::size_t size, + const image_check check = image_check::full); + +// (2) +static basic_json_document load(const std::vector& image, + const image_check check = image_check::full); + +// (3) +static basic_json_document load(std::vector&& image, + const image_check check = image_check::full); +``` + +1. Reads an image [`save()`](save.md) wrote, from a pointer and a byte count. The image is **borrowed**: `image` + must stay alive and unchanged for as long as the returned document, and any view taken from it, is used. +2. Reads an image from a `#!cpp std::vector`. Also **borrowed** -- equivalent to overload 1 called with + `#!cpp image.data()` and `#!cpp image.size()`. +3. Reads an image, keeping the vector instead of copying it: `image` is moved into the document (no copy), which + then owns it for as long as it needs the text and the decoded strings. [`owns_source()`](owns_source.md) is + `#!cpp true` afterward. + +In every overload, the node index is copied into storage the document itself owns -- so that it is properly aligned, +and, for an [editable](index.md#edits) document, can be edited -- while the text and the decoded strings stay in +`image`. The hash indexes [large objects](../../features/json_view.md) use for lookup are rebuilt, exactly as after +parsing. + +## Parameters + +`image` (in) +: the image [`save()`](save.md) wrote (overloads 1 and 2), or one to take ownership of (overload 3) + +`size` (in) +: the number of bytes at `image` (overload 1) + +`check` (in) +: how thoroughly to validate `image` before trusting it; see [`image_check`](#image_check) below (optional, + `#!cpp image_check::full` by default) + +## Return value + +The document read from the image. + +## Exception safety + +Overloads 1 and 2 give the strong guarantee: `image` is only read, never written, so a thrown exception leaves the +caller's buffer untouched. + +Overload 3 moves `image` into the document *before* validating it, so that a good image is kept without a copy. If +loading then fails, the partially built document -- and the vector now inside it -- is discarded along with the +exception, and `image` itself is left **empty**, not restored to what was passed in. Move a copy in instead, or +validate with overload 2 first, if the original vector must survive a failed load. + +## Exceptions + +On a big-endian target, throws [`type_error.320`](../../home/exceptions.md#jsonexceptiontype_error320) -- the same +exception [`save()`](save.md#exceptions) throws there, since the image format is little-endian only. + +Otherwise throws [`parse_error.116`](../../home/exceptions.md#jsonexceptionparse_error116) if `image` is not one +`save()` could have written, or fails the requested `check`: + +| message | when | +|------------------------|------------------------------------------------------------------------------------------------------| +| `too short` | `image` is `#!cpp nullptr`, or `size` is smaller than the 64-byte header | +| `unknown format` | the header's magic bytes or version do not match, or a reserved header field is not zero | +| `sizes out of range` | the node count, text size, or decoded-string size the header describes does not fit `size`, or the `#!cpp '\0'` after the text or after the decoded strings is missing | +| `the check failed` | `check` is not `#!cpp image_check::none`, and the image fails it -- see [`image_check`](#image_check) | + +!!! failure "Example messages" + + ``` + [json.exception.parse_error.116] parse error: invalid json_document image: too short + ``` + ``` + [json.exception.parse_error.116] parse error: invalid json_document image: unknown format + ``` + ``` + [json.exception.parse_error.116] parse error: invalid json_document image: sizes out of range + ``` + ``` + [json.exception.parse_error.116] parse error: invalid json_document image: the check failed + ``` + +## Complexity + +Linear in the number of nodes, which are always copied into the document. `#!cpp image_check::none` does not read +the text or the decoded strings. `#!cpp image_check::bounds` reads one byte of the text for each float token (its sign, +to check the recorded digits against the token's length), and nothing else of the text or the decoded strings. + +With `#!cpp image_check::full`, linear in the size of the image (the nodes, the text, and the decoded strings), plus +sorting the ranges of the strings and of the float tokens (at most one per node): the contents of each distinct range +are checked once, however many nodes refer to it. + +## Notes + +### `image_check` { #image_check } + +```cpp +using image_check = detail::view::image_check; + +enum class image_check +{ + full, + bounds, + none +}; +``` + +How thoroughly `load()` validates `image` before trusting it. + +`full` (the default) +: Checks everything the parser itself guarantees: structure and bounds; that every string is valid UTF-8 (and, for a + string still in the source text, that it contains no quote, backslash, or control character); and that every number + token is well-formed and matches the value stored for it. Strings and float tokens may share a range only if the + ranges are identical (as nodes that share a value do), and never overlap otherwise. Reading and serializing a + checked image is safe and always produces valid JSON, exactly as for a parsed document. This is the right choice for + an image from anything you do not fully control -- a file, a cache shared with other processes, a peer on the + network. + +`bounds` +: Checks structure and bounds only: that every offset and count in the node index stays inside the image. So + `load()`, [`root()`](root.md), element access, and [`materialize()`](../basic_json_view/materialize.md) can never + read outside the image, even on a damaged one. It does *not* guarantee that the image describes valid JSON: a + string that a `full` check would have rejected can make [`dump()`](../basic_json_view/dump.md) write invalid UTF-8 + or invalid JSON, or throw [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316), and a number can + read back with a value that does not match how it is spelled. An [editable document](../json_editable_document.md) + does not take such a string over: [`set`](set.md), [`push_back`](push_back.md) and [`insert`](insert.md) throw + `type_error.316` when they copy it from a view of the loaded document, and leave the editable document unchanged. + `bounds` skips scanning the text and the decoded strings, so it fits a cache your own process just wrote and reads + straight back, where damage would mean a bug or a hardware fault rather than adversarial input. + +`none` +: Checks nothing. Reading a damaged image is undefined behavior, so use it only for an image you trust as much as + your own memory. + +**Lifetime.** Overloads 1 and 2 borrow `image`: it must stay alive and byte-for-byte unchanged for as long as the +returned document, and any [view](../basic_json_view/index.md) taken from it, is used -- exactly like a document +[`parse()`](parse.md) borrowed its input for. Overload 3 avoids this by keeping the vector itself; see +[`owns_source`](owns_source.md). + +!!! warning "Experimental" + + The image format is versioned but not yet stable, and may change in an incompatible way before it is declared + stable; `load()` already rejects an image written by a different format version with `parse_error.116` + ("unknown format"). Use images to cache a document within one build of the library, or to hand one to another + process running the *same* build on the *same* (little-endian) machine -- not as a long-term storage format. + +## Examples + +??? example "Caching a document, ownership, and a rejected image" + + The example below saves a parsed document as an image, checks that `load()` reproduces the original + [`dump()`](../basic_json_view/dump.md) without parsing, and shows the difference between + `load(std::move(image))` (owned) and `load(image)` (borrowed). It then damages one byte of the image and shows + `image_check::full` rejecting it with `parse_error.116`, while `image_check::bounds` -- meant for a cache the + process already trusts -- still reads it without going out of bounds. + + ```cpp + --8<-- "examples/basic_json_document__load.cpp" + ``` + + Output: + + ```json + --8<-- "examples/basic_json_document__load.output" + ``` + +## See also + +- [save](save.md) - write the document as an image +- [owns_source](owns_source.md) - return whether the document holds its own copy of the text +- [parse](parse.md) - deserialize from JSON text instead of an image +- [Images](../../features/json_view.md#images) - why and when to use images + +## Version history + +- Added in version 3.13.0. diff --git a/docs/mkdocs/docs/api/basic_json_document/parse.md b/docs/mkdocs/docs/api/basic_json_document/parse.md index 3e2506264..7bc00a9f1 100644 --- a/docs/mkdocs/docs/api/basic_json_document/parse.md +++ b/docs/mkdocs/docs/api/basic_json_document/parse.md @@ -149,6 +149,7 @@ buffer of a given length, pass a pair of pointers: `#!cpp parse(ptr, ptr + len)` - [accept](accept.md) - check whether the input is valid JSON - [read](read.md) - (re-)parse into this document, reusing its memory - [owns_source](owns_source.md) - return whether the document holds its own copy of the text +- [load](load.md) - read a document from an image instead of parsing JSON text - [`BasicJsonType::parse`](../basic_json/parse.md) - the corresponding function of `basic_json` ## Version history diff --git a/docs/mkdocs/docs/api/basic_json_document/read.md b/docs/mkdocs/docs/api/basic_json_document/read.md index 979bec869..174fcc477 100644 --- a/docs/mkdocs/docs/api/basic_json_document/read.md +++ b/docs/mkdocs/docs/api/basic_json_document/read.md @@ -80,6 +80,7 @@ compile; see [`parse`](parse.md#notes). - [parse](parse.md) - deserialize from a compatible input - [root](root.md) - the view of the root value +- [load](load.md) - read a document from an image instead of parsing JSON text ## Version history diff --git a/docs/mkdocs/docs/api/basic_json_document/save.md b/docs/mkdocs/docs/api/basic_json_document/save.md new file mode 100644 index 000000000..f69d5c9f8 --- /dev/null +++ b/docs/mkdocs/docs/api/basic_json_document/save.md @@ -0,0 +1,104 @@ +# nlohmann::basic_json_document::save + +```cpp +std::vector save() const; +``` + +Writes the document as an *image*: a byte buffer that [`load`](load.md) reads back without parsing. The image holds +the node index, the source text (plus, for an edited document, the number tokens edits wrote), and the decoded +strings (plus the strings edits wrote) -- everything [`root()`](root.md) needs, with nothing left to parse. + +An edited document is written in its *current* state, with its values in document order, the way the library's own +parser would have produced them for that JSON text: a member [`set`](set.md) added goes at the end, an +[`erase`](erase.md)d member leaves no trace, and a float that is not finite (NaN or positive/negative infinity) +becomes null, the same substitution [`dump()`](../basic_json_view/dump.md) makes. The same document always saves to +the same bytes -- also across `BasicJsonType` and `#!cpp Editable`, since the image reflects document order and +values only, not which specialization produced them. + +## Return value + +The image, as a `#!cpp std::vector`. Pass it, or a pointer to its data together with its size, to +[`load`](load.md) to read the document back. + +## Exception safety + +Strong guarantee: `save()` does not modify `#!cpp *this` (it is `#!cpp const`), so if it throws, the document is left +exactly as it was, and the partially built image is discarded with the exception. + +## Exceptions + +Throws [`type_error.320`](../../home/exceptions.md#jsonexceptiontype_error320) if the document is +[discarded](is_discarded.md) -- a default-constructed document, or one a failed [`parse()`](parse.md)/ +[`read()`](read.md) with `allow_exceptions == false` left discarded. + +On a big-endian target, throws `type_error.320` with a different message instead: the image format is little-endian +only (see [Notes](#notes)). + +Throws [`out_of_range.416`](../../home/exceptions.md#jsonexceptionout_of_range416) if the node count, the text, or +the decoded strings of the image would individually reach 4 GiB -- the same 32-bit offsets +[`parse()`](parse.md#exceptions) and, for edits, [`set`](set.md)/[`push_back`](push_back.md) are already limited to. + +!!! failure "Example messages" + + ``` + [json.exception.type_error.320] cannot save a discarded json_document + ``` + ``` + [json.exception.type_error.320] json_document images need a little-endian target + ``` + ``` + [json.exception.out_of_range.416] images of 4 GiB or more are not supported by json_document + ``` + +## Complexity + +Linear in the size of the document: the number of nodes, plus the length of the text and the decoded strings that end +up in the image. + +## Notes + +**Format.** The image begins with a 64-byte header (the magic bytes `#!cpp "NJVI"`, a version number, the node count, +and the sizes of the text and the decoded strings, all little-endian), followed by the nodes +([16 bytes each](../../home/architecture.md#node-index-of-json-views)), the text and a `#!cpp '\0'`, and the decoded +strings and a `#!cpp '\0'`. [`load`](load.md) checks the header, and the sizes it describes, before reading anything +else -- see [`load`'s Exceptions](load.md#exceptions). + +!!! warning "Experimental" + + The image format is versioned but not yet stable: it may change in an incompatible way before it is declared + stable. Use images to cache a document within one build of the library, or to hand one to another process running + the *same* build on the *same* (little-endian) machine -- not as a long-term storage format. Keep the original + JSON text if you need to read a saved document back with a future library version. + +**Little-endian only.** The image is written as raw little-endian bytes, with no byte-swapping. `save()` (and +[`load`](load.md)) throw `type_error.320` on a big-endian target rather than silently produce bytes a big-endian +reader could not interpret correctly. + +## Examples + +??? example "Caching a parsed document as an image" + + The example below saves a parsed configuration as an image -- the way a service might cache one to answer later + requests without parsing the text again -- and confirms that loading it back gives exactly the same result as + parsing did, and that saving is deterministic. + + ```cpp + --8<-- "examples/basic_json_document__save.cpp" + ``` + + Output: + + ```json + --8<-- "examples/basic_json_document__save.output" + ``` + +## See also + +- [load](load.md) - read an image written by `save()` +- [owns_source](owns_source.md) - return whether the document holds its own copy of the text +- [`basic_json_view::dump`](../basic_json_view/dump.md) - serialize the document to JSON text instead of an image +- [Images](../../features/json_view.md#images) - why and when to use images + +## Version history + +- Added in version 3.13.0. diff --git a/docs/mkdocs/docs/examples/basic_json_document__load.cpp b/docs/mkdocs/docs/examples/basic_json_document__load.cpp new file mode 100644 index 000000000..52a085a10 --- /dev/null +++ b/docs/mkdocs/docs/examples/basic_json_document__load.cpp @@ -0,0 +1,52 @@ +#include +#include +#include +#include +#include + +using json = nlohmann::json; +using json_document = nlohmann::json_document; +using image_check = json_document::image_check; + +int main() +{ + std::cout << std::boolalpha; + + // the image of a parsed document -- as if read back from a cache file or + // received from another process running the same build of the library + const std::string text = R"({"name": "cache", "note": "caf\u00e9", "replicas": ["db2", "db3"]})"; + const json_document parsed = json_document::parse(text); + const std::vector image = parsed.save(); + + // (1)/(2) load() needs no parsing, yet dumps exactly what parsing did + const json_document borrowed = json_document::load(image); + std::cout << (borrowed.root().dump() == parsed.root().dump()) << '\n'; + std::cout << borrowed.owns_source() << '\n'; // borrowed: still points into `image` + + // (3) load(std::move(image)) keeps the vector instead of copying it + std::vector to_move = image; + const json_document owned = json_document::load(std::move(to_move)); + std::cout << owned.owns_source() << '\n'; + + // a damaged image -- the last byte of the decoded string "note" holds + // (an escape sequence, so it was unescaped into the document's own + // buffer), flipped, as storage or transport corruption might do + std::vector damaged = image; + damaged[damaged.size() - 2] = 0xFF; + + // image_check::full inspects strings and numbers, so it catches the damage + try + { + static_cast(json_document::load(damaged, image_check::full)); + } + catch (const json::parse_error& e) + { + std::cout << e.id << '\n'; + } + + // image_check::bounds only checks structure and bounds, so a cache the + // process already trusts loads without the extra scan -- reading a value + // the damage did not touch is still safe + const json_document trusted = json_document::load(damaged, image_check::bounds); + std::cout << trusted.root()["name"].get() << '\n'; +} diff --git a/docs/mkdocs/docs/examples/basic_json_document__load.output b/docs/mkdocs/docs/examples/basic_json_document__load.output new file mode 100644 index 000000000..4dc66a29b --- /dev/null +++ b/docs/mkdocs/docs/examples/basic_json_document__load.output @@ -0,0 +1,5 @@ +true +false +true +116 +cache diff --git a/docs/mkdocs/docs/examples/basic_json_document__save.cpp b/docs/mkdocs/docs/examples/basic_json_document__save.cpp new file mode 100644 index 000000000..6a3cc978b --- /dev/null +++ b/docs/mkdocs/docs/examples/basic_json_document__save.cpp @@ -0,0 +1,30 @@ +#include +#include +#include +#include +#include + +using json_document = nlohmann::json_document; + +int main() +{ + std::cout << std::boolalpha; + + // a configuration a service parses once and then caches as an image, so + // that later requests can load() it instead of parsing the text again + const std::string text = R"({"name": "cache", "host": "db1", "port": 6379, "replicas": ["db2", "db3"]})"; + const json_document config = json_document::parse(text); + + // save() turns the parsed document into a byte buffer: a 64-byte header, + // the node index, the source text, and the decoded strings + const std::vector image = config.save(); + std::cout << image.size() << '\n'; + + // the same document always saves to the same bytes + std::cout << (image == json_document::parse(text).save()) << '\n'; + + // loading the image back needs no parsing, yet dumps exactly what + // parsing the text produced + const json_document reloaded = json_document::load(image); + std::cout << (reloaded.root().dump() == config.root().dump()) << '\n'; +} diff --git a/docs/mkdocs/docs/examples/basic_json_document__save.output b/docs/mkdocs/docs/examples/basic_json_document__save.output new file mode 100644 index 000000000..5ce0ade9a --- /dev/null +++ b/docs/mkdocs/docs/examples/basic_json_document__save.output @@ -0,0 +1,3 @@ +316 +true +true diff --git a/docs/mkdocs/docs/features/index.md b/docs/mkdocs/docs/features/index.md index b1173b662..4ffe7e585 100644 --- a/docs/mkdocs/docs/features/index.md +++ b/docs/mkdocs/docs/features/index.md @@ -14,7 +14,8 @@ C++ types, and finally serialize it again. [parsing untrusted input](parsing/untrusted_input.md). - [Zero-copy JSON views](json_view.md) — read a JSON text through a flat index instead of building a `json` tree; strings and numbers stay in the input and are only decoded when needed. - [Editable documents](json_view.md#editing-a-document) can also be modified. +- [Editable documents](json_view.md#editing-a-document) — modify a parsed document without building a `json` tree. +- [Images](json_view.md#images) — save a parsed document and load it again without parsing it. - [Comments](comments.md) and [trailing commas](trailing_commas.md) — opt-in relaxations of the JSON grammar. ## Accessing and modifying values diff --git a/docs/mkdocs/docs/features/json_view.md b/docs/mkdocs/docs/features/json_view.md index 83b53de57..4f0938273 100644 --- a/docs/mkdocs/docs/features/json_view.md +++ b/docs/mkdocs/docs/features/json_view.md @@ -312,6 +312,57 @@ edits. See [`basic_json_document`'s Edits](../api/basic_json_document/index.md#e throws (the *basic* guarantee, not the strong one `dump()` and the read-only functions provide). How edits are kept in the index is described in the [architecture overview](../home/architecture.md#node-index-of-json-views). +## Images + +[`save()`](../api/basic_json_document/save.md) writes a document as an *image*: a byte buffer that +[`load()`](../api/basic_json_document/load.md) reads back into a document without parsing -- no lexing, no building +the node index, nothing but copying the nodes and pointing the text and the decoded strings at the image. Where +[`parse_copy()`](../api/basic_json_document/parse_copy.md) still has to scan the whole input, +[`load()`](../api/basic_json_document/load.md) turns that scan into a copy of the node index alone. + +**Why.** A document that is parsed once and then read many times -- a configuration loaded at startup, a template +rendered on every request, a large reference dataset a worker process needs in memory -- pays for parsing once but +can amortize [`save()`](../api/basic_json_document/save.md)'s cost across every later load. That makes images useful +for a cache: save a document the first time it is parsed (to a file, a shared-memory segment, an in-process cache), +and [`load()`](../api/basic_json_document/load.md) it on every later use instead of parsing the source text again. +They are just as useful for handing a parsed document to another process (or a forked worker) running the same build +of the library, since [`load()`](../api/basic_json_document/load.md) turns the transfer into a copy of the node index +plus pointers into the received bytes, not a re-parse. + +**Choosing a check.** [`load()`](../api/basic_json_document/load.md) takes an +[`image_check`](../api/basic_json_document/load.md#image_check) that trades validation against speed: +`image_check::full` (the default) checks everything the parser itself guarantees, so a checked image is exactly as +safe to read and serialize as a freshly parsed document -- the right choice whenever the image did not come straight +from this process's own [`save()`](../api/basic_json_document/save.md), such as a file or a network peer. +`image_check::bounds` only checks structure and bounds -- cheaper, since it skips scanning the text and the decoded +strings -- and fits a cache the process trusts, one it wrote and reads back itself. `image_check::none` skips +validation entirely, for an image trusted as much as the process's own memory. See +[`load()`'s Notes](../api/basic_json_document/load.md#notes) for exactly what each level does and does not guarantee. + +??? example "Example: cache a parsed configuration as an image" + + ```cpp + --8<-- "examples/basic_json_document__save.cpp" + ``` + + Output: + + ```json + --8<-- "examples/basic_json_document__save.output" + ``` + +!!! warning "Experimental" + + The image format is versioned but not yet stable, and may change in an incompatible way before it is declared + stable. It is little-endian only, and tied to the library build that wrote it -- use it to cache a document or to + hand one to another process running the *same* build, not as a long-term storage format; keep the original JSON + text if a saved document needs to be readable by a future library version. + +The idea of a document you can read without parsing comes from zero-copy formats such as +[FlatBuffers](https://github.com/google/flatbuffers) and [YaFF](https://github.com/yandex/yaff); the check +[`load()`](../api/basic_json_document/load.md) runs follows the idea of FlatBuffers' Verifier. No code is taken from +either. + ## Choosing between `json`, `ordered_json`, the SAX interface, and `json_view` | | [`json`](../api/json.md) / [`ordered_json`](../api/ordered_json.md) | [SAX interface](parsing/sax_interface.md) | [`json_document`](../api/json_document.md) / [`json_view`](../api/json_view.md) | [`json_editable_document`](../api/json_editable_document.md) / [`json_editable_view`](../api/json_editable_view.md) | @@ -320,6 +371,7 @@ the index is described in the [architecture overview](../home/architecture.md#no | **Mutability** | freely mutable | not applicable (a one-shot event stream) | read-only | [`set`](../api/basic_json_document/set.md)/[`push_back`](../api/basic_json_document/push_back.md)/[`insert`](../api/basic_json_document/insert.md)/[`erase`](../api/basic_json_document/erase.md) edit in place; the source text is never rewritten | | **What you get** | a full tree you can read, write, and keep as long as you like | a sequence of callbacks; whatever your handler builds from them | a flat index plus, on demand, [`materialize()`](../api/basic_json_view/materialize.md)d `json`/`ordered_json` values for the parts you actually use | the same, plus [`dump()`](../api/basic_json_view/dump.md) of an edited document that keeps the member order and, with [`number_format::source`](../api/basic_json_view/number_format.md), the spelling of every untouched number | | **Typical use** | general-purpose JSON handling: config, request/response bodies you build or modify, anything you hold onto | validating or projecting a text into your own data structure without ever holding the whole thing as JSON | large or high-volume input where you only need part of it, or need it repeatedly, and can keep the source text (or a copy) alive for as long as the document lives | a document you read, patch a few fields of, and write back -- a configuration file, for instance -- where the rest of it should come back exactly as it was | +| **Caching/reload** | not applicable -- re-parse, or roll your own serialization | not applicable | [`save()`](../api/basic_json_document/save.md)/[`load()`](../api/basic_json_document/load.md): cache the parsed index as an image and reload it without parsing | same, saving the document's current -- possibly edited -- state | ## Version history diff --git a/docs/mkdocs/docs/home/architecture.md b/docs/mkdocs/docs/home/architecture.md index 5ee7fcc5d..ff7db49cc 100644 --- a/docs/mkdocs/docs/home/architecture.md +++ b/docs/mkdocs/docs/home/architecture.md @@ -262,6 +262,14 @@ edited: - Views of read-only documents compile without any of this: how views walk the index is a template parameter (`navigation`). +Images ([`save`](../api/basic_json_document/save.md) and [`load`](../api/basic_json_document/load.md), +[`detail/view/image.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/view/image.hpp)) store +the nodes as they are: a 64-byte header (the magic bytes `NJVI`, a format version, the sizes, and reserved bytes that +must be zero), the nodes, the text, and the decoded strings. An edited document is first written in document order, as +the parser would have written it (without links), and the numbers of the hash indexes are cleared, since `load` +rebuilds the indexes. So a change of the node layout is a change of the image format: it must raise `image_version`, +and `load` then rejects images of other versions (`parse_error.116`) instead of misreading them. + ## Input adapters Input is read via **input adapters** that abstract a source. Every input adapter provides this interface: diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index 4fa8eb062..8f1016214 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -395,6 +395,23 @@ A UBJSON high-precision number could not be parsed. [json.exception.parse_error.115] parse error at byte 5: syntax error while parsing UBJSON high-precision number: invalid number text: 1A ``` +### json.exception.parse_error.116 + +[`basic_json_document::load()`](../api/basic_json_document/load.md) rejected an +[image](../features/json_view.md#images): either the bytes are not one [`save()`](../api/basic_json_document/save.md) +could have written (too short, an unknown magic number or format version, or sizes that do not fit the buffer), or +they are, but fail the requested [`image_check`](../api/basic_json_document/load.md#image_check). + +!!! failure "Example message" + + ``` + [json.exception.parse_error.116] parse error: invalid json_document image: the check failed + ``` + +!!! note + + This exception was added in version 3.13.0, together with [images](../features/json_view.md#images). + ## Iterator errors This exception is thrown if iterators passed to a library function do not match @@ -831,6 +848,26 @@ from JSON text. This exception was added in version 3.13.0, together with editable [`json_document`s](../features/json_view.md). +### json.exception.type_error.320 + +[`basic_json_document::save()`](../api/basic_json_document/save.md) cannot write an +[image](../features/json_view.md#images) of a [discarded](../api/basic_json_document/is_discarded.md) document. +[`save()`](../api/basic_json_document/save.md) and [`load()`](../api/basic_json_document/load.md) also throw this +exception on a big-endian target, since the image format is little-endian only. + +!!! failure "Example messages" + + ``` + [json.exception.type_error.320] cannot save a discarded json_document + ``` + ``` + [json.exception.type_error.320] json_document images need a little-endian target + ``` + +!!! note + + This exception was added in version 3.13.0, together with [images](../features/json_view.md#images). + ### json.exception.type_error.321 A discarded value (one created by [`parse()`](../api/basic_json/parse.md) with a callback that returns `false` for the @@ -1088,7 +1125,9 @@ MessagePack's ext type and BSON's binary subtype are each stored in a single byt so they do not support an input of 4294967280 bytes (4 GiB minus 16 bytes) or more. The same 32-bit limit applies to an **editable** document's own storage: [`set`](../api/basic_json_document/set.md) and [`push_back`](../api/basic_json_document/push_back.md) throw this exception once the strings and number tokens written by edits reach 4 GiB in total, or once more than -4294967295 arrays/objects have had an element set or appended to them. +4294967295 arrays/objects have had an element set or appended to them. The same limit applies to an +[image](../features/json_view.md#images): [`save()`](../api/basic_json_document/save.md) throws it if the node +count, the text, or the decoded strings it would write would individually reach 4 GiB. !!! failure "Example messages" @@ -1098,6 +1137,9 @@ this exception once the strings and number tokens written by edits reach 4 GiB i ``` [json.exception.out_of_range.416] edits of 4 GiB or more are not supported by json_document ``` + ``` + [json.exception.out_of_range.416] images of 4 GiB or more are not supported by json_document + ``` !!! note diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 301250f2d..35b5be3e0 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -242,6 +242,7 @@ nav: - 'erase': api/basic_json_document/erase.md - 'insert': api/basic_json_document/insert.md - 'is_discarded': api/basic_json_document/is_discarded.md + - 'load': api/basic_json_document/load.md - 'memory_usage': api/basic_json_document/memory_usage.md - 'node_count': api/basic_json_document/node_count.md - 'owns_source': api/basic_json_document/owns_source.md @@ -250,6 +251,7 @@ nav: - 'push_back': api/basic_json_document/push_back.md - 'read': api/basic_json_document/read.md - 'root': api/basic_json_document/root.md + - 'save': api/basic_json_document/save.md - 'set': api/basic_json_document/set.md - 'shrink_to_fit': api/basic_json_document/shrink_to_fit.md - 'source': api/basic_json_document/source.md diff --git a/include/nlohmann/detail/view/document_data.hpp b/include/nlohmann/detail/view/document_data.hpp index e31400a1e..55f9d8c76 100644 --- a/include/nlohmann/detail/view/document_data.hpp +++ b/include/nlohmann/detail/view/document_data.hpp @@ -10,7 +10,7 @@ #include // array #include // size_t -#include // uint32_t +#include // uint8_t, uint32_t #include // memcmp, memcpy #include // numeric_limits #include // map @@ -42,7 +42,9 @@ struct document_data std::size_t inline_capacity = 0; std::size_t max_depth = 0; ///< deepest nesting of arrays and objects (the root container is depth 1) std::vector arena; ///< decoded strings that contained escapes (no terminator: they are handed out with their length) + std::size_t arena_size = 0; ///< bytes of decoded strings at base[1] (the arena, or those of a loaded image) std::string owned; ///< owned copy of the input, if any (a std::string, so that its NUL serves as the parser's sentinel) + std::vector owned_image; ///< a loaded image the document owns (the text and the decoded strings point into it) // hash indexes of the keys of large objects (see object_index.hpp) static constexpr std::uint32_t index_min_members = 128; @@ -119,6 +121,7 @@ struct document_data document_data() noexcept : arena() // NOLINT(readability-redundant-member-init) , owned() // NOLINT(readability-redundant-member-init) + , owned_image() // NOLINT(readability-redundant-member-init) , indexes() // NOLINT(readability-redundant-member-init) , index_slots() // NOLINT(readability-redundant-member-init) , large_objects() // NOLINT(readability-redundant-member-init) diff --git a/include/nlohmann/detail/view/image.hpp b/include/nlohmann/detail/view/image.hpp new file mode 100644 index 000000000..702a55530 --- /dev/null +++ b/include/nlohmann/detail/view/image.hpp @@ -0,0 +1,744 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#pragma once + +#include // min, sort +#include // array +#include // offsetof, size_t +#include // int64_t, uint8_t, uint16_t, uint32_t, uint64_t +#include // memcmp, memcpy, memset +#include // numeric_limits +#include // string +#include // vector + +#include +#include +#include +#include +#include +#include +#include +#include + +// Images: a document stored so that loading it needs no parsing. +// +// Layout (little-endian): a 64-byte header, the nodes, the text (the source, +// followed by the number tokens written by edits), a NUL, the decoded strings +// (followed by the strings written by edits), a NUL. The idea is that of +// zero-copy formats such as FlatBuffers (https://github.com/google/flatbuffers) +// and YaFF (https://github.com/yandex/yaff); no code is taken from them. +// check_image follows the idea of FlatBuffers' Verifier (bounds and +// structure) and also checks what the parser guarantees about strings and +// numbers, so that reading and serializing a checked image is safe and yields +// valid JSON. + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ +namespace view +{ + +/// how load() checks an image +enum class image_check +{ + /// everything the parser guarantees: structure and bounds, strings (valid + /// UTF-8; source strings without quotes, backslashes, and control + /// characters), and numbers (well-formed, matching the stored values) + full, + /// structure and bounds only: reading and serializing are safe, but a + /// crafted image can yield invalid UTF-8, strings that serialize to + /// invalid JSON, or numbers that differ from their text + bounds, + /// none: for images from a trusted source only (a damaged image is + /// undefined behavior) + none, +}; + +struct image_header +{ + std::array magic; ///< "NJVI" + std::uint32_t version; ///< 1 + std::uint64_t node_count; + std::uint64_t text_size; + std::uint64_t arena_size; + std::array reserved; ///< zero (for later versions) +}; +static_assert(sizeof(image_header) == 64, "the image header must be 64 bytes"); + +constexpr std::uint32_t image_version = 1; + +/// the largest node count and text or string size of an image (as for parsed +/// documents, offsets and counts must fit 32 bits) +constexpr std::uint64_t image_limit = 0xFFFFFFF0u; + +/// Copy the current structure of an edited document into nodes in document +/// order, as the parser would have written them. Text written by edits is +/// appended to text_tail (number tokens) and arena_tail (strings); floats that +/// are not finite become null, as dump() writes them. +inline void compact_nodes(const document_data& d, std::size_t arena_size, std::vector& out, std::string& text_tail, std::string& arena_tail) +{ + struct frame + { + const node* cur; + const node* end; + std::size_t index; ///< the container's node in out + std::uint32_t count; + bool object; + }; + std::vector stack; + const auto string_node = [&](const node & s) + { + node r = s; + r.extra = 0; + r.flags = static_cast(s.flags & node_flags::storage); + if (r.flags == node_flags::edited) + { + r.off = static_cast(arena_size + arena_tail.size()); + arena_tail.append(d.str(s), s.len); + r.flags = node_flags::escaped; + } + return r; + }; + const auto copy_node = [&](const node * v) + { + node r = *v; + switch (static_cast(v->kind)) + { + case value_t::object: + case value_t::array: + r.flags = 0; + r.extra = 0; + r.off = (v->flags & (node_flags::moved | node_flags::is_new)) != 0 ? 0 : v->off; + r.len = 0; // counted below + r.next = 0; // set when the container is complete + stack.push_back(frame{d.first_child_edited(v), d.child_end_edited(v), out.size(), 0, v->kind == static_cast(value_t::object)}); + break; + case value_t::string: + r = string_node(*v); + break; + case value_t::number_integer: + case value_t::number_unsigned: + if ((v->flags & node_flags::storage) == node_flags::edited) + { + r.off = static_cast(d.size + text_tail.size()); + text_tail.append(d.str(*v), number_length(*v)); + } + r.flags = 0; + break; + case value_t::number_float: + if ((v->flags & node_flags::storage) == node_flags::edited) + { + const char* const t = d.str(*v); + if (t[0] == 'n' || t[0] == 'i' || (v->len > 1 && t[1] == 'i')) + { + r = node{}; // nan and infinity: null, as dump() writes them + r.kind = static_cast(value_t::null); + break; + } + r.off = static_cast(d.size + text_tail.size()); + text_tail.append(t, v->len); + r.extra = 0xFFFFu; // the digit layout is not recorded + } + r.flags = 0; + break; + case value_t::boolean: + r.flags = static_cast(v->flags & node_flags::is_true); + break; + case value_t::null: + case value_t::binary: + case value_t::discarded: + default: + r.flags = 0; + break; + } + out.push_back(r); + }; + copy_node(d.nodes); + while (!stack.empty()) + { + const frame top = stack.back(); + if (top.cur == top.end) + { + node& c = out[top.index]; + c.len = top.count; + c.next = static_cast(out.size() - top.index); + stack.pop_back(); + continue; + } + if (top.object) + { + out.push_back(string_node(*top.cur)); + } + const node* const value = top.cur + (top.object ? 1 : 0); + stack.back().cur = document_data::after(value); + ++stack.back().count; + copy_node(document_data::deref(value)); // may grow the stack + } +} + +/// the document as an image +inline std::vector save_image(const document_data& d) +{ +#if !NLOHMANN_VIEW_LITTLE_ENDIAN + throw_type_error(320, "json_document images need a little-endian target"); // LCOV_EXCL_LINE +#endif + const std::size_t arena_size = d.arena_size; + std::vector compacted; + std::string text_tail; + std::string arena_tail; + if (d.edits) + { + compact_nodes(d, arena_size, compacted, text_tail, arena_tail); + } + const node* const nodes = d.edits ? compacted.data() : d.nodes; + const std::size_t count = d.edits ? compacted.size() : d.node_count; + const std::size_t text_size = d.size + text_tail.size(); + const std::size_t total_arena = arena_size + arena_tail.size(); + if (NLOHMANN_VIEW_UNLIKELY(text_size >= image_limit || total_arena >= image_limit || count >= image_limit)) + { + // LCOV_EXCL_START (4 GiB) + throw_out_of_range(416, "images of 4 GiB or more are not supported by json_document"); + // LCOV_EXCL_STOP + } + image_header h{}; + h.magic = {{'N', 'J', 'V', 'I'}}; + h.version = image_version; + h.node_count = count; + h.text_size = text_size; + h.arena_size = total_arena; + std::vector image(sizeof(h) + (count * sizeof(node)) + text_size + 1 + total_arena + 1); + std::uint8_t* o = image.data(); + std::memcpy(o, &h, sizeof(h)); + o += sizeof(h); + std::memcpy(o, nodes, count * sizeof(node)); + // the hash indexes are rebuilt by load(), whose check rejects any other + // extra of an object, and the same document saves to the same bytes + for (std::size_t i = 0; i < count; ++i) + { + if (nodes[i].kind == static_cast(value_t::object) && nodes[i].extra != 0) + { + std::memset(o + (i * sizeof(node)) + offsetof(node, extra), 0, sizeof(nodes[i].extra)); + } + } + o += count * sizeof(node); + const auto append = [&o](const char* s, std::size_t n) + { + if (n != 0) + { + std::memcpy(o, s, n); + o += n; + } + }; + append(d.src, d.size); + append(text_tail.data(), text_tail.size()); + *o++ = 0; + append(d.base[1], arena_size); + append(arena_tail.data(), arena_tail.size()); + *o = 0; + return image; +} + +/// the parts of a number token that the checks need +struct number_token +{ + const unsigned char* int_start; + std::size_t int_digits; + std::size_t frac_digits; + std::int64_t exponent; + bool negative; + bool is_float; ///< a fraction or an exponent +}; + +/// whether [s, s + len) is a JSON number (the grammar the parser accepts) +inline bool scan_number_token(const unsigned char* s, std::size_t len, number_token& t) +{ + const unsigned char* const e = s + len; + const unsigned char* p = s; + t.negative = p != e && *p == '-'; + p += t.negative ? 1 : 0; + if (p == e) + { + return false; + } + t.int_start = p; + if (*p == '0') + { + ++p; + } + else if (*p >= '1' && *p <= '9') + { + do + { + ++p; + } + while (p != e && is_digit(*p)); + } + else + { + return false; + } + t.int_digits = static_cast(p - t.int_start); + t.frac_digits = 0; + t.is_float = false; + if (p != e && *p == '.') + { + const unsigned char* const f0 = ++p; + while (p != e && is_digit(*p)) + { + ++p; + } + if (p == f0) + { + return false; + } + t.frac_digits = static_cast(p - f0); + t.is_float = true; + } + t.exponent = 0; + if (p != e && (*p | 0x20u) == 'e') + { + ++p; + const bool exp_negative = p != e && *p == '-'; + p += (p != e && (*p == '+' || *p == '-')) ? 1 : 0; + if (p == e || !is_digit(*p)) + { + return false; + } + do + { + t.exponent = t.exponent < 100000 ? (t.exponent * 10) + (*p - '0') : t.exponent; + ++p; + } + while (p != e && is_digit(*p)); + t.exponent = exp_negative ? -t.exponent : t.exponent; + t.is_float = true; + } + return p == e; +} + +/// the digit layout the parser records for a float token (compaction +/// writes "many" instead; the caller accepts both) +inline std::uint16_t float_layout(const number_token& t) +{ + return static_cast((std::min)(t.int_digits, std::size_t{255}) | ((std::min)(t.frac_digits, std::size_t{255}) << 8u)); +} + +/// whether a float token (well-formed, per scan_number_token) is finite as +/// double; parse() rejects floats that overflow, and as there, only a number +/// whose magnitude could reach 1e308 needs the conversion +inline bool float_token_finite(const unsigned char* s, std::size_t len, const number_token& t) +{ + if (static_cast(t.int_digits) + t.exponent > 300) + { + node n{}; + n.kind = static_cast(value_t::number_float); + n.len = static_cast(len); + const auto v = float_value(reinterpret_cast(s), n); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + return v <= (std::numeric_limits::max)() && v >= -(std::numeric_limits::max)(); + } + return true; +} + +/// whether an integer node matches its token the way the parser records it +/// (after the bounds check; the token has at most 256 characters) +inline bool check_integer(const node& n, const unsigned char* text) +{ + const std::size_t len = number_length(n); + const unsigned char* const s = text + n.off; + number_token t{}; + if (!scan_number_token(s, len, t)) + { + return false; + } + // the token's value is the stored one; number_integer nodes of edits can + // be non-negative (as basic_json keeps the type of a value) + const bool integer = n.kind == static_cast(value_t::number_integer); + if (t.is_float || t.int_digits > 20 || (t.negative && !integer)) + { + return false; + } + // (at most 19 digits cannot overflow; 20 digits are compared with 2^64 - 1) + if (t.int_digits == 20 && std::memcmp(t.int_start, "18446744073709551615", 20) > 0) + { + return false; + } + std::uint64_t m = 0; + for (const unsigned char* d = t.int_start; d != t.int_start + t.int_digits; ++d) + { + m = (m * 10) + static_cast(*d - '0'); + } + if (integer && m > (t.negative ? std::uint64_t{1} << 63u : (std::uint64_t{1} << 63u) - 1)) + { + return false; + } + return integer_bits(n) == (t.negative ? 0 - m : m); +} + +/// a byte range of the text or of the decoded strings +struct byte_range +{ + std::uint32_t off; + std::uint32_t len; +}; + +/// a float token and the digit layout its node records +struct float_range +{ + std::uint32_t off; + std::uint32_t len; + std::uint16_t extra; +}; + +/// Run check once for each distinct range of ranges (which are within bounds +/// and not empty). Ranges that are not identical must not overlap: save() +/// writes every string and token to a place of its own, and nodes that share +/// a value (copies within a document) share the whole range. This bounds the +/// work by the size of the text, however many nodes point to the same bytes. +template +bool check_distinct_ranges(std::vector& ranges, Check check) +{ + std::sort(ranges.begin(), ranges.end(), [](const byte_range & a, const byte_range & b) + { + return a.off != b.off ? a.off < b.off : a.len < b.len; + }); + std::size_t end = 0; + for (std::size_t i = 0; i < ranges.size(); ++i) + { + const byte_range r = ranges[i]; + if (i != 0 && r.off == ranges[i - 1].off && r.len == ranges[i - 1].len) + { + continue; + } + if (r.off < end || !check(r)) + { + return false; + } + end = static_cast(r.off) + r.len; + } + return true; +} + +/// Check the float nodes: each token once (nodes of the same token must +/// record layouts that match it), tokens must not overlap. +inline bool check_float_ranges(std::vector& ranges, const unsigned char* text) +{ + std::sort(ranges.begin(), ranges.end(), [](const float_range & a, const float_range & b) + { + if (a.off != b.off) + { + return a.off < b.off; + } + if (a.len != b.len) + { + return a.len < b.len; + } + return a.extra < b.extra; + }); + std::size_t end = 0; + std::size_t i = 0; + while (i < ranges.size()) + { + const float_range r = ranges[i]; + if (r.off < end) + { + return false; + } + number_token t{}; + const unsigned char* const s = text + r.off; + if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t)) + { + return false; + } + // the digit layout the parser records (or "many", as compaction writes it) + const std::uint16_t layout = float_layout(t); + for (; i < ranges.size() && ranges[i].off == r.off && ranges[i].len == r.len; ++i) + { + if (ranges[i].extra != layout && ranges[i].extra != 0xFFFFu) + { + return false; + } + } + end = static_cast(r.off) + r.len; + } + return true; +} + +/// the content checks of check_image: source strings as the parser leaves +/// them (no quotes, backslashes, or control characters), decoded strings +/// (valid UTF-8), and float tokens +inline bool check_contents(const unsigned char* text, const unsigned char* arena, + std::vector& source_strings, std::vector& decoded_strings, + std::vector& floats) +{ + return check_distinct_ranges(source_strings, [text](const byte_range & r) + { + const unsigned char* const b = text + r.off; + return scan_string_run(b, b + r.len) == b + r.len; + }) + && check_distinct_ranges(decoded_strings, [arena](const byte_range & r) + { + return valid_utf8_prefix(arena + r.off, r.len) == r.len; + }) + && check_float_ranges(floats, text); +} + +/// Check the nodes of a loaded image against its text and decoded strings: +/// kinds, flags, and `extra`; extents and element counts of arrays and +/// objects; keys; bounds; string contents (source strings as the parser +/// leaves them: no quotes, backslashes, or control characters; all strings +/// valid UTF-8); and number tokens. +/// +/// The structure and the bounds are checked node by node. The contents of +/// strings and of float tokens are checked afterwards, once for each distinct +/// range (see check_distinct_ranges), so that the full check is linear in the +/// size of the image plus the sorting of the ranges, and not in the number of +/// nodes times the size of the text. +inline bool check_image(const node* nodes, std::size_t count, const unsigned char* text, std::size_t text_size, + const unsigned char* arena, std::size_t arena_size, bool full) +{ + struct frame + { + std::size_t end; + std::uint32_t len; + std::uint32_t seen; + bool object; + bool expect_key; + }; + std::vector stack; + std::vector source_strings; + std::vector decoded_strings; + std::vector floats; + const auto check_string = [&](const node & n) -> bool + { + if ((n.flags & ~node_flags::escaped) != 0 || n.extra != 0) + { + return false; + } + const bool decoded = (n.flags & node_flags::escaped) != 0; + const std::size_t limit = decoded ? arena_size : text_size; + if (n.off > limit || n.len > limit - n.off) + { + return false; + } + if (full && n.len != 0) + { + (decoded ? decoded_strings : source_strings).push_back(byte_range{n.off, n.len}); + } + return true; + }; + // bounds of a number token; the recorded digit layout must lie within it + const auto number_in_bounds = [&](const node & n) -> bool + { + const std::size_t len = number_length(n); + if (len == 0 || n.off > text_size || len > text_size - n.off) + { + return false; + } + if (n.kind != static_cast(value_t::number_float)) + { + return (n.extra >> 8u) == 0; + } + // float_value() reads the sign, the integer digits, and the point and + // fraction digits the layout records (a layout of more than 19 digits + // means the general conversion, which stays within the token) + const std::size_t int_digits = n.extra & 0xFFu; + const std::size_t frac_digits = n.extra >> 8u; + const std::size_t need = (text[n.off] == '-' ? 1u : 0u) + int_digits + (frac_digits != 0 ? frac_digits + 1 : 0); + return int_digits + frac_digits > 19 || need <= len; + }; + std::size_t i = 0; + for (;;) + { + // close finished arrays and objects + while (!stack.empty() && i == stack.back().end) + { + const frame f = stack.back(); + if (f.seen != f.len || (f.object && !f.expect_key)) + { + return false; + } + stack.pop_back(); + if (!stack.empty()) + { + ++stack.back().seen; + stack.back().expect_key = true; + } + } + if (i == count) + { + return stack.empty() && (!full || check_contents(text, arena, source_strings, decoded_strings, floats)); + } + if (i != 0 && stack.empty()) + { + return false; // nodes after the root + } + const node& n = nodes[i]; + if (!stack.empty() && stack.back().object && stack.back().expect_key) + { + if (n.kind != static_cast(value_t::string) || !check_string(n)) + { + return false; + } + stack.back().expect_key = false; + ++i; + continue; + } + bool complete = true; + switch (static_cast(n.kind)) + { + case value_t::null: + // (the offset of a literal is read to size the output of dump()) + if (n.flags != 0 || n.extra != 0 || n.off > text_size) + { + return false; + } + break; + case value_t::boolean: + if ((n.flags & ~node_flags::is_true) != 0 || n.extra != 0 || n.off > text_size) + { + return false; + } + break; + case value_t::string: + if (!check_string(n)) + { + return false; + } + break; + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + if (n.flags != 0 || !number_in_bounds(n)) + { + return false; + } + if (full) + { + // (the contents of float tokens are checked later; the token of an + // integer has at most 256 characters) + if (n.kind == static_cast(value_t::number_float)) + { + floats.push_back(float_range{n.off, n.len, n.extra}); + } + else if (!check_integer(n, text)) + { + return false; + } + } + break; + case value_t::array: + case value_t::object: + { + const std::size_t limit = stack.empty() ? count : stack.back().end; + if (n.flags != 0 || n.extra != 0 || n.next == 0 || n.next > limit - i || n.off > text_size) + { + return false; + } + stack.push_back(frame{i + n.next, n.len, 0, n.kind == static_cast(value_t::object), true}); + complete = false; + break; + } + case value_t::binary: + case value_t::discarded: + default: + return false; + } + ++i; + if (complete && !stack.empty()) + { + ++stack.back().seen; + stack.back().expect_key = true; + } + } +} + +[[noreturn]] NLOHMANN_VIEW_NOINLINE inline void throw_invalid_image(const char* what) +{ + throw_parse_error(116, concat("invalid json_document image: ", what)); +} + +/// Read an image into d. The text and the decoded strings stay in the image; +/// the nodes are copied (so that they are aligned, and edits can change them). +inline void load_image(document_data& d, const std::uint8_t* image, std::size_t size, image_check check) +{ +#if !NLOHMANN_VIEW_LITTLE_ENDIAN + throw_type_error(320, "json_document images need a little-endian target"); // LCOV_EXCL_LINE +#endif + if (image == nullptr || size < sizeof(image_header)) + { + throw_invalid_image("too short"); + } + image_header h{}; + std::memcpy(&h, image, sizeof(h)); + // (the reserved fields are for later versions) + if (std::memcmp(h.magic.data(), "NJVI", 4) != 0 || h.version != image_version + || (h.reserved[0] | h.reserved[1] | h.reserved[2] | h.reserved[3]) != 0) + { + throw_invalid_image("unknown format"); + } + const std::size_t room = size - sizeof(h); + if (h.node_count == 0 || h.node_count > room / sizeof(node) || h.node_count >= image_limit || h.text_size >= image_limit || h.arena_size >= image_limit) + { + throw_invalid_image("sizes out of range"); + } + const auto count = static_cast(h.node_count); + const auto text_size = static_cast(h.text_size); + const auto arena_size = static_cast(h.arena_size); + const std::size_t text_at = sizeof(h) + (count * sizeof(node)); + // the text, a NUL, the decoded strings, a NUL, and nothing after them + if (size - text_at < 2 || text_size > size - text_at - 2 || arena_size != size - text_at - text_size - 2 + || image[text_at + text_size] != 0 || image[size - 1] != 0) + { + throw_invalid_image("sizes out of range"); + } + + d.discarded = true; + d.edits.reset(); + d.base[2] = nullptr; + d.owned.clear(); + if (d.owned_image.empty() || image != d.owned_image.data()) + { + d.owned_image.clear(); + } + d.arena.clear(); + d.indexes.clear(); + d.index_slots.clear(); + d.large_objects.clear(); + d.node_count = 0; + d.reserve(count); + std::memcpy(d.nodes, image + sizeof(h), count * sizeof(node)); + d.node_count = count; + const std::uint8_t* const text = image + text_at; + const std::uint8_t* const arena = text + text_size + 1; + d.src = reinterpret_cast(text); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + d.size = text_size; + d.base[0] = d.src; + d.base[1] = reinterpret_cast(arena); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + d.arena_size = arena_size; + if (check != image_check::none && !check_image(d.nodes, count, text, text_size, arena, arena_size, check == image_check::full)) + { + throw_invalid_image("the check failed"); + } + // the hash indexes of large objects, as after parsing + for (std::size_t i = 0; i < count; ++i) + { + const node& n = d.nodes[i]; + if (n.kind == static_cast(value_t::object) && n.len >= document_data::index_min_members) + { + d.large_objects.push_back(static_cast(i)); + } + } + build_object_indexes(d); + std::vector().swap(d.large_objects); // (only needed while building) + d.discarded = false; +} + +} // namespace view +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/view/number.hpp b/include/nlohmann/detail/view/number.hpp index fce98e9ba..10e2aa85d 100644 --- a/include/nlohmann/detail/view/number.hpp +++ b/include/nlohmann/detail/view/number.hpp @@ -26,43 +26,79 @@ namespace detail namespace view { +/*! +@brief locate the decimal point and the end of the mantissa of a float token + +Also checks that the token is a JSON number. Tokens of the parser and of edits +always are; an image loaded with image_check::bounds can hold any bytes, which +must not reach the conversion (it expects a well-formed token). +*/ +inline bool float_token_layout(const char* first, const char* last, std::size_t& dot, std::size_t& mantissa_end) noexcept +{ + const auto digit = [last](const char* q) + { + return q != last && is_digit(static_cast(*q)); + }; + const char* p = first; + p += (p != last && *p == '-') ? 1 : 0; + if (!digit(p) || (*p == '0' && digit(p + 1))) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + dot = std::string::npos; + if (p != last && *p == '.') + { + dot = static_cast(p - first); + if (!digit(++p)) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + } + mantissa_end = static_cast(p - first); + if (p != last && (*p == 'e' || *p == 'E')) + { + ++p; + p += (p != last && (*p == '+' || *p == '-')) ? 1 : 0; + if (!digit(p)) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + } + return p == last; +} + /*! @brief the value of the float token of a node, as parse() converts it Uses the lexer's conversion (detail::convert_float), so that the values are bit-identical to parse(): float and double are converted without allocation -and independent of the locale. The digit layout recorded while parsing locates -the decimal point and the exponent without scanning the token. +and independent of the locale. A token that is not a JSON number (only in a +damaged image loaded with image_check::bounds) yields 0. */ template NLOHMANN_VIEW_NOINLINE FloatType float_value(const char* first, const node& n) { const char* const last = first + n.len; - const std::size_t neg = first[0] == '-' ? 1 : 0; - const std::size_t int_digits = n.extra & 0xFFu; - const std::size_t frac_digits = n.extra >> 8u; - std::size_t dot = std::string::npos; - std::size_t mantissa_end = n.len; - if (int_digits != 255 && frac_digits != 255) + std::size_t dot = 0; + std::size_t mantissa_end = 0; + if (NLOHMANN_VIEW_UNLIKELY(!float_token_layout(first, last, dot, mantissa_end))) { - dot = frac_digits != 0 ? neg + int_digits : std::string::npos; - mantissa_end = neg + int_digits + (frac_digits != 0 ? 1 + frac_digits : 0); - } - else - { - // more digits than the layout records: locate them - for (std::size_t i = 0; i < n.len; ++i) - { - if (first[i] == '.') - { - dot = i; - } - else if (first[i] == 'e' || first[i] == 'E') - { - mantissa_end = i; - break; - } - } + return FloatType{}; } return convert_float(first, last, dot, mantissa_end); } @@ -93,16 +129,20 @@ NLOHMANN_VIEW_ALWAYS_INLINE float_significand layout_decimal(const unsigned char } if (p != e) { - // [eE][+-]digits; huge exponents saturate (the parser rejected overflow) + // [eE][+-]digits; huge exponents saturate (the parser rejected + // overflow). The token is not read beyond e, and the digits are taken + // as unsigned, so that a token that is not well-formed (a damaged + // image loaded with image_check::bounds) yields a wrong value, but no + // overflow. ++p; - const bool exp_negative = *p == '-'; - p += (*p == '-' || *p == '+') ? 1 : 0; + const bool exp_negative = p != e && *p == '-'; + p += (p != e && (*p == '-' || *p == '+')) ? 1 : 0; std::int64_t exp_value = 0; for (; p != e; ++p) { if (exp_value < 0x10000000) { - exp_value = (exp_value * 10) + (*p - '0'); + exp_value = (exp_value * 10) + static_cast(*p - '0'); } } q += exp_negative ? -exp_value : exp_value; diff --git a/include/nlohmann/detail/view/serializer.hpp b/include/nlohmann/detail/view/serializer.hpp index 5409b477b..6a6719a65 100644 --- a/include/nlohmann/detail/view/serializer.hpp +++ b/include/nlohmann/detail/view/serializer.hpp @@ -327,7 +327,9 @@ class view_serializer m_out.put('"'); } - /// as serializer::dump_escaped() for valid UTF-8 (the view has no other) + /// as serializer::dump_escaped(); strings of a document are valid UTF-8, + /// except in a damaged image loaded with image_check::bounds, for which + /// this throws what basic_json::dump() throws for the string template void write_escaped(const unsigned char* s, std::size_t n) { @@ -351,12 +353,13 @@ class view_serializer } std::uint32_t codepoint = s[i]; std::size_t len = 1; - if (codepoint >= 0xC0) + if (codepoint >= 0x80) { - len = 2; - if (codepoint >= 0xE0) + len = validate_one_utf8(s + i, n - i); + if (NLOHMANN_VIEW_UNLIKELY(len == 0)) { - len = codepoint >= 0xF0 ? 4 : 3; + invalid_utf8(s, n); + return; } codepoint &= 0xFFu >> (len + 1); for (std::size_t k = 1; k < len; ++k) @@ -369,6 +372,13 @@ class view_serializer } } + /// throw what basic_json::dump() throws for a string that is not valid UTF-8 + NLOHMANN_VIEW_NOINLINE static void invalid_utf8(const unsigned char* s, std::size_t n) + { + const string_t dumped = BasicJsonType(string_t(reinterpret_cast(s), n)).dump(); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + static_cast(dumped); + } + template void write_codepoint(std::uint32_t codepoint, const unsigned char* bytes, std::size_t len) { diff --git a/include/nlohmann/json_view.hpp b/include/nlohmann/json_view.hpp index da18cd698..fd43d3990 100644 --- a/include/nlohmann/json_view.hpp +++ b/include/nlohmann/json_view.hpp @@ -29,7 +29,7 @@ #include // all_of, min #include // array #include // nullptr_t, size_t // IWYU pragma: keep -#include // uint32_t +#include // uint8_t, uint32_t #include // memcpy, strlen #include // istream // IWYU pragma: keep #include // distance, input_iterator_tag, iterator_traits @@ -71,6 +71,7 @@ #include #include #include +#include #include #include #include @@ -1028,7 +1029,7 @@ class basic_json_document /// whether the document holds its own copy of the text bool owns_source() const noexcept { - return m_data && !m_data->owned.empty() && m_data->src == m_data->owned.data(); + return m_data && ((!m_data->owned.empty() && m_data->src == m_data->owned.data()) || !m_data->owned_image.empty()); } /// number of index nodes (values plus object keys) @@ -1037,7 +1038,7 @@ class basic_json_document return m_data ? m_data->node_count : 0; } - /// bytes held by the document (index, decoded strings, owned text) + /// bytes held by the document (index, decoded strings, and owned text or image) std::size_t memory_usage() const noexcept { if (!m_data) @@ -1046,7 +1047,7 @@ class basic_json_document } return sizeof(document_data) + (m_data->inline_capacity * sizeof(detail::view::node)) + (m_data->nodes != m_data->inline_nodes ? m_data->node_capacity * sizeof(detail::view::node) : 0) - + m_data->arena.capacity() + m_data->owned.capacity() + + m_data->arena.capacity() + m_data->owned.capacity() + m_data->owned_image.capacity() + (m_data->indexes.capacity() * sizeof(document_data::object_index)) + (m_data->index_slots.capacity() * sizeof(std::uint32_t)) + (m_data->large_objects.capacity() * sizeof(std::uint32_t)) + (m_data->edits != nullptr ? m_data->edits->bytes : 0); @@ -1066,8 +1067,10 @@ class basic_json_document // allocate everything first, so that an exception leaves the document // unchanged + // (the decoded strings of a loaded image stay in the image) + const bool arena_in_use = d.base[1] == d.arena.data(); const bool shrink_arena = d.arena.capacity() > d.arena.size(); - std::vector arena(shrink_arena ? d.arena : std::vector()); + std::vector arena(shrink_arena && arena_in_use ? d.arena : std::vector()); // (edits link to the nodes of the index, which then stays in place) const bool shrink_nodes = d.nodes != d.inline_nodes && d.node_count != d.node_capacity && d.edits == nullptr; const bool into_header = d.node_count <= d.inline_capacity; @@ -1088,7 +1091,10 @@ class basic_json_document if (shrink_arena) { d.arena.swap(arena); - d.base[1] = d.arena.data(); + if (arena_in_use) + { + d.base[1] = d.arena.data(); + } } if (d.indexes.capacity() > d.indexes.size()) { @@ -1100,6 +1106,55 @@ class basic_json_document } } + //////////// + // images // + //////////// + + /// how load() checks an image (full, bounds, or none) + using image_check = detail::view::image_check; + + /// The document as an image that load() reads without parsing: the node + /// index, the text, and the decoded strings. An edited document is + /// written in its current state (floats that are not finite become null, + /// as in dump()). + std::vector save() const + { + if (NLOHMANN_VIEW_UNLIKELY(!m_data || m_data->discarded)) + { + detail::view::throw_type_error(320, "cannot save a discarded json_document"); + } + return detail::view::save_image(*m_data); + } + + /// Read an image written by save(). The image is borrowed: it must stay + /// alive and unchanged while the document is used. + NLOHMANN_VIEW_NODISCARD + static basic_json_document load(const std::uint8_t* image, std::size_t size, const image_check check = image_check::full) + { + basic_json_document d; + d.ensure_data(nullptr, 0); + detail::view::load_image(*d.m_data, image, size, check); + return d; + } + + /// read an image (borrowed) + NLOHMANN_VIEW_NODISCARD + static basic_json_document load(const std::vector& image, const image_check check = image_check::full) + { + return load(image.data(), image.size(), check); + } + + /// read an image and keep it (no copy) + NLOHMANN_VIEW_NODISCARD + static basic_json_document load(std::vector&& image, const image_check check = image_check::full) + { + basic_json_document d; + d.ensure_data(nullptr, 0); + d.m_data->owned_image = std::move(image); + detail::view::load_image(*d.m_data, d.m_data->owned_image.data(), d.m_data->owned_image.size(), check); + return d; + } + /////////// // edits // /////////// @@ -1334,6 +1389,7 @@ class basic_json_document { d.owned.clear(); } + d.owned_image.clear(); d.src = src; d.size = size; d.node_count = 0; @@ -1358,6 +1414,7 @@ class basic_json_document { d.base[0] = d.src; d.base[1] = d.arena.data(); + d.arena_size = d.arena.size(); detail::view::build_object_indexes(d); std::vector().swap(d.large_objects); // (only needed while parsing) d.discarded = false; diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 7453e82d3..0682c08be 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -29,7 +29,7 @@ #include // all_of, min #include // array #include // nullptr_t, size_t // IWYU pragma: keep -#include // uint32_t +#include // uint8_t, uint32_t #include // memcpy, strlen #include // istream // IWYU pragma: keep #include // distance, input_iterator_tag, iterator_traits @@ -101,7 +101,7 @@ #include // array #include // size_t -#include // uint32_t +#include // uint8_t, uint32_t #include // memcmp, memcpy #include // numeric_limits #include // map @@ -356,7 +356,9 @@ struct document_data std::size_t inline_capacity = 0; std::size_t max_depth = 0; ///< deepest nesting of arrays and objects (the root container is depth 1) std::vector arena; ///< decoded strings that contained escapes (no terminator: they are handed out with their length) + std::size_t arena_size = 0; ///< bytes of decoded strings at base[1] (the arena, or those of a loaded image) std::string owned; ///< owned copy of the input, if any (a std::string, so that its NUL serves as the parser's sentinel) + std::vector owned_image; ///< a loaded image the document owns (the text and the decoded strings point into it) // hash indexes of the keys of large objects (see object_index.hpp) static constexpr std::uint32_t index_min_members = 128; @@ -433,6 +435,7 @@ struct document_data document_data() noexcept : arena() // NOLINT(readability-redundant-member-init) , owned() // NOLINT(readability-redundant-member-init) + , owned_image() // NOLINT(readability-redundant-member-init) , indexes() // NOLINT(readability-redundant-member-init) , index_slots() // NOLINT(readability-redundant-member-init) , large_objects() // NOLINT(readability-redundant-member-init) @@ -4326,6 +4329,985 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // min, sort +#include // array +#include // offsetof, size_t +#include // int64_t, uint8_t, uint16_t, uint32_t, uint64_t +#include // memcmp, memcpy, memset +#include // numeric_limits +#include // string +#include // vector + +// #include +// #include + +// #include + +// #include + +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // size_t +#include // int64_t, uint64_t +#include // numeric_limits +#include // string +#include // integral_constant + +// #include +// #include + +// #include + +// #include + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ +namespace view +{ + +/*! +@brief locate the decimal point and the end of the mantissa of a float token + +Also checks that the token is a JSON number. Tokens of the parser and of edits +always are; an image loaded with image_check::bounds can hold any bytes, which +must not reach the conversion (it expects a well-formed token). +*/ +inline bool float_token_layout(const char* first, const char* last, std::size_t& dot, std::size_t& mantissa_end) noexcept +{ + const auto digit = [last](const char* q) + { + return q != last && is_digit(static_cast(*q)); + }; + const char* p = first; + p += (p != last && *p == '-') ? 1 : 0; + if (!digit(p) || (*p == '0' && digit(p + 1))) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + dot = std::string::npos; + if (p != last && *p == '.') + { + dot = static_cast(p - first); + if (!digit(++p)) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + } + mantissa_end = static_cast(p - first); + if (p != last && (*p == 'e' || *p == 'E')) + { + ++p; + p += (p != last && (*p == '+' || *p == '-')) ? 1 : 0; + if (!digit(p)) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + } + return p == last; +} + +/*! +@brief the value of the float token of a node, as parse() converts it + +Uses the lexer's conversion (detail::convert_float), so that the values are +bit-identical to parse(): float and double are converted without allocation +and independent of the locale. A token that is not a JSON number (only in a +damaged image loaded with image_check::bounds) yields 0. +*/ +template +NLOHMANN_VIEW_NOINLINE FloatType float_value(const char* first, const node& n) +{ + const char* const last = first + n.len; + std::size_t dot = 0; + std::size_t mantissa_end = 0; + if (NLOHMANN_VIEW_UNLIKELY(!float_token_layout(first, last, dot, mantissa_end))) + { + return FloatType{}; + } + return convert_float(first, last, dot, mantissa_end); +} + +/*! +@brief the digits of a float token with at most 19 digits, from its layout + +The digit layout recorded while parsing says where the integer digits, the +fraction digits, and the exponent are, so the digits are read eight at a +time without scanning. + +@param[in] p first character of the token +@param[in] e end of the token +@param[in] limit end of the readable memory (the source text) +*/ +NLOHMANN_VIEW_ALWAYS_INLINE float_significand layout_decimal(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept +{ + const bool negative = *p == '-'; + p += negative ? 1 : 0; + std::uint64_t w = parse_upto19(p, int_digits, limit); + p += int_digits; + std::int64_t q = 0; + if (frac_digits != 0) + { + w = (w * int_pow10(frac_digits)) + parse_upto19(p + 1, frac_digits, limit); + p += 1 + frac_digits; + q = -static_cast(frac_digits); + } + if (p != e) + { + // [eE][+-]digits; huge exponents saturate (the parser rejected + // overflow). The token is not read beyond e, and the digits are taken + // as unsigned, so that a token that is not well-formed (a damaged + // image loaded with image_check::bounds) yields a wrong value, but no + // overflow. + ++p; + const bool exp_negative = p != e && *p == '-'; + p += (p != e && (*p == '-' || *p == '+')) ? 1 : 0; + std::int64_t exp_value = 0; + for (; p != e; ++p) + { + if (exp_value < 0x10000000) + { + exp_value = (exp_value * 10) + static_cast(*p - '0'); + } + } + q += exp_negative ? -exp_value : exp_value; + } + + float_significand d; + d.w = w; + d.exponent = q; + d.negative = negative; + return d; +} + +/*! +@brief the value of a float token with at most 19 digits, from its layout + +The result is correctly rounded by the lexer's conversion +(detail::decimal_to_float(): Clinger's fast path where both operands are +exact, else the Eisel-Lemire algorithm, which needs no fallback for up to 19 +digits), so it is the value parse() produces. +*/ +template +NLOHMANN_VIEW_ALWAYS_INLINE FloatType layout_float(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept +{ + return decimal_to_float(layout_decimal(p, e, int_digits, frac_digits, limit)); +} + +/// the value of a float set by an edit: its token (the shortest round-trip +/// text, or "nan", "inf", "-inf") in the edit arena +template +NLOHMANN_VIEW_NOINLINE FloatType edited_float(const char* token, const node& n) +{ + if (token[0] == 'n') + { + return std::numeric_limits::quiet_NaN(); + } + if (token[0] == 'i' || (token[0] == '-' && token[1] == 'i')) + { + return token[0] == 'i' ? std::numeric_limits::infinity() : -std::numeric_limits::infinity(); + } + return float_value(token, n); +} + +/// the value of the float token of a node, as parse() converts it; floats and +/// doubles with at most 19 digits are converted from the digit layout +template +FloatType float_value(const document_data& d, const node& n) +{ + if (NLOHMANN_VIEW_UNLIKELY((n.flags & node_flags::storage) == node_flags::edited)) + { + return edited_float(d.str(n), n); + } + return float_value(d, n, std::integral_constant::value> {}); +} + +template +FloatType float_value(const document_data& d, const node& n, std::true_type /*binary32 or binary64*/) +{ + const unsigned int_digits = n.extra & 0xFFu; + const unsigned frac_digits = n.extra >> 8u; + if (NLOHMANN_VIEW_LIKELY(int_digits + frac_digits <= 19)) // (255 marks "many") + { + // (a float token not written by an edit is in the text) + const auto* const first = reinterpret_cast(d.src + n.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + return layout_float(first, first + n.len, int_digits, frac_digits, reinterpret_cast(d.src + d.size)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + } + return float_value(d.str(n), n); +} + +template +FloatType float_value(const document_data& d, const node& n, std::false_type /*other*/) +{ + return float_value(d.str(n), n); +} + +} // namespace view +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include + +// #include + + +// Images: a document stored so that loading it needs no parsing. +// +// Layout (little-endian): a 64-byte header, the nodes, the text (the source, +// followed by the number tokens written by edits), a NUL, the decoded strings +// (followed by the strings written by edits), a NUL. The idea is that of +// zero-copy formats such as FlatBuffers (https://github.com/google/flatbuffers) +// and YaFF (https://github.com/yandex/yaff); no code is taken from them. +// check_image follows the idea of FlatBuffers' Verifier (bounds and +// structure) and also checks what the parser guarantees about strings and +// numbers, so that reading and serializing a checked image is safe and yields +// valid JSON. + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ +namespace view +{ + +/// how load() checks an image +enum class image_check +{ + /// everything the parser guarantees: structure and bounds, strings (valid + /// UTF-8; source strings without quotes, backslashes, and control + /// characters), and numbers (well-formed, matching the stored values) + full, + /// structure and bounds only: reading and serializing are safe, but a + /// crafted image can yield invalid UTF-8, strings that serialize to + /// invalid JSON, or numbers that differ from their text + bounds, + /// none: for images from a trusted source only (a damaged image is + /// undefined behavior) + none, +}; + +struct image_header +{ + std::array magic; ///< "NJVI" + std::uint32_t version; ///< 1 + std::uint64_t node_count; + std::uint64_t text_size; + std::uint64_t arena_size; + std::array reserved; ///< zero (for later versions) +}; +static_assert(sizeof(image_header) == 64, "the image header must be 64 bytes"); + +constexpr std::uint32_t image_version = 1; + +/// the largest node count and text or string size of an image (as for parsed +/// documents, offsets and counts must fit 32 bits) +constexpr std::uint64_t image_limit = 0xFFFFFFF0u; + +/// Copy the current structure of an edited document into nodes in document +/// order, as the parser would have written them. Text written by edits is +/// appended to text_tail (number tokens) and arena_tail (strings); floats that +/// are not finite become null, as dump() writes them. +inline void compact_nodes(const document_data& d, std::size_t arena_size, std::vector& out, std::string& text_tail, std::string& arena_tail) +{ + struct frame + { + const node* cur; + const node* end; + std::size_t index; ///< the container's node in out + std::uint32_t count; + bool object; + }; + std::vector stack; + const auto string_node = [&](const node & s) + { + node r = s; + r.extra = 0; + r.flags = static_cast(s.flags & node_flags::storage); + if (r.flags == node_flags::edited) + { + r.off = static_cast(arena_size + arena_tail.size()); + arena_tail.append(d.str(s), s.len); + r.flags = node_flags::escaped; + } + return r; + }; + const auto copy_node = [&](const node * v) + { + node r = *v; + switch (static_cast(v->kind)) + { + case value_t::object: + case value_t::array: + r.flags = 0; + r.extra = 0; + r.off = (v->flags & (node_flags::moved | node_flags::is_new)) != 0 ? 0 : v->off; + r.len = 0; // counted below + r.next = 0; // set when the container is complete + stack.push_back(frame{d.first_child_edited(v), d.child_end_edited(v), out.size(), 0, v->kind == static_cast(value_t::object)}); + break; + case value_t::string: + r = string_node(*v); + break; + case value_t::number_integer: + case value_t::number_unsigned: + if ((v->flags & node_flags::storage) == node_flags::edited) + { + r.off = static_cast(d.size + text_tail.size()); + text_tail.append(d.str(*v), number_length(*v)); + } + r.flags = 0; + break; + case value_t::number_float: + if ((v->flags & node_flags::storage) == node_flags::edited) + { + const char* const t = d.str(*v); + if (t[0] == 'n' || t[0] == 'i' || (v->len > 1 && t[1] == 'i')) + { + r = node{}; // nan and infinity: null, as dump() writes them + r.kind = static_cast(value_t::null); + break; + } + r.off = static_cast(d.size + text_tail.size()); + text_tail.append(t, v->len); + r.extra = 0xFFFFu; // the digit layout is not recorded + } + r.flags = 0; + break; + case value_t::boolean: + r.flags = static_cast(v->flags & node_flags::is_true); + break; + case value_t::null: + case value_t::binary: + case value_t::discarded: + default: + r.flags = 0; + break; + } + out.push_back(r); + }; + copy_node(d.nodes); + while (!stack.empty()) + { + const frame top = stack.back(); + if (top.cur == top.end) + { + node& c = out[top.index]; + c.len = top.count; + c.next = static_cast(out.size() - top.index); + stack.pop_back(); + continue; + } + if (top.object) + { + out.push_back(string_node(*top.cur)); + } + const node* const value = top.cur + (top.object ? 1 : 0); + stack.back().cur = document_data::after(value); + ++stack.back().count; + copy_node(document_data::deref(value)); // may grow the stack + } +} + +/// the document as an image +inline std::vector save_image(const document_data& d) +{ +#if !NLOHMANN_VIEW_LITTLE_ENDIAN + throw_type_error(320, "json_document images need a little-endian target"); // LCOV_EXCL_LINE +#endif + const std::size_t arena_size = d.arena_size; + std::vector compacted; + std::string text_tail; + std::string arena_tail; + if (d.edits) + { + compact_nodes(d, arena_size, compacted, text_tail, arena_tail); + } + const node* const nodes = d.edits ? compacted.data() : d.nodes; + const std::size_t count = d.edits ? compacted.size() : d.node_count; + const std::size_t text_size = d.size + text_tail.size(); + const std::size_t total_arena = arena_size + arena_tail.size(); + if (NLOHMANN_VIEW_UNLIKELY(text_size >= image_limit || total_arena >= image_limit || count >= image_limit)) + { + // LCOV_EXCL_START (4 GiB) + throw_out_of_range(416, "images of 4 GiB or more are not supported by json_document"); + // LCOV_EXCL_STOP + } + image_header h{}; + h.magic = {{'N', 'J', 'V', 'I'}}; + h.version = image_version; + h.node_count = count; + h.text_size = text_size; + h.arena_size = total_arena; + std::vector image(sizeof(h) + (count * sizeof(node)) + text_size + 1 + total_arena + 1); + std::uint8_t* o = image.data(); + std::memcpy(o, &h, sizeof(h)); + o += sizeof(h); + std::memcpy(o, nodes, count * sizeof(node)); + // the hash indexes are rebuilt by load(), whose check rejects any other + // extra of an object, and the same document saves to the same bytes + for (std::size_t i = 0; i < count; ++i) + { + if (nodes[i].kind == static_cast(value_t::object) && nodes[i].extra != 0) + { + std::memset(o + (i * sizeof(node)) + offsetof(node, extra), 0, sizeof(nodes[i].extra)); + } + } + o += count * sizeof(node); + const auto append = [&o](const char* s, std::size_t n) + { + if (n != 0) + { + std::memcpy(o, s, n); + o += n; + } + }; + append(d.src, d.size); + append(text_tail.data(), text_tail.size()); + *o++ = 0; + append(d.base[1], arena_size); + append(arena_tail.data(), arena_tail.size()); + *o = 0; + return image; +} + +/// the parts of a number token that the checks need +struct number_token +{ + const unsigned char* int_start; + std::size_t int_digits; + std::size_t frac_digits; + std::int64_t exponent; + bool negative; + bool is_float; ///< a fraction or an exponent +}; + +/// whether [s, s + len) is a JSON number (the grammar the parser accepts) +inline bool scan_number_token(const unsigned char* s, std::size_t len, number_token& t) +{ + const unsigned char* const e = s + len; + const unsigned char* p = s; + t.negative = p != e && *p == '-'; + p += t.negative ? 1 : 0; + if (p == e) + { + return false; + } + t.int_start = p; + if (*p == '0') + { + ++p; + } + else if (*p >= '1' && *p <= '9') + { + do + { + ++p; + } + while (p != e && is_digit(*p)); + } + else + { + return false; + } + t.int_digits = static_cast(p - t.int_start); + t.frac_digits = 0; + t.is_float = false; + if (p != e && *p == '.') + { + const unsigned char* const f0 = ++p; + while (p != e && is_digit(*p)) + { + ++p; + } + if (p == f0) + { + return false; + } + t.frac_digits = static_cast(p - f0); + t.is_float = true; + } + t.exponent = 0; + if (p != e && (*p | 0x20u) == 'e') + { + ++p; + const bool exp_negative = p != e && *p == '-'; + p += (p != e && (*p == '+' || *p == '-')) ? 1 : 0; + if (p == e || !is_digit(*p)) + { + return false; + } + do + { + t.exponent = t.exponent < 100000 ? (t.exponent * 10) + (*p - '0') : t.exponent; + ++p; + } + while (p != e && is_digit(*p)); + t.exponent = exp_negative ? -t.exponent : t.exponent; + t.is_float = true; + } + return p == e; +} + +/// the digit layout the parser records for a float token (compaction +/// writes "many" instead; the caller accepts both) +inline std::uint16_t float_layout(const number_token& t) +{ + return static_cast((std::min)(t.int_digits, std::size_t{255}) | ((std::min)(t.frac_digits, std::size_t{255}) << 8u)); +} + +/// whether a float token (well-formed, per scan_number_token) is finite as +/// double; parse() rejects floats that overflow, and as there, only a number +/// whose magnitude could reach 1e308 needs the conversion +inline bool float_token_finite(const unsigned char* s, std::size_t len, const number_token& t) +{ + if (static_cast(t.int_digits) + t.exponent > 300) + { + node n{}; + n.kind = static_cast(value_t::number_float); + n.len = static_cast(len); + const auto v = float_value(reinterpret_cast(s), n); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + return v <= (std::numeric_limits::max)() && v >= -(std::numeric_limits::max)(); + } + return true; +} + +/// whether an integer node matches its token the way the parser records it +/// (after the bounds check; the token has at most 256 characters) +inline bool check_integer(const node& n, const unsigned char* text) +{ + const std::size_t len = number_length(n); + const unsigned char* const s = text + n.off; + number_token t{}; + if (!scan_number_token(s, len, t)) + { + return false; + } + // the token's value is the stored one; number_integer nodes of edits can + // be non-negative (as basic_json keeps the type of a value) + const bool integer = n.kind == static_cast(value_t::number_integer); + if (t.is_float || t.int_digits > 20 || (t.negative && !integer)) + { + return false; + } + // (at most 19 digits cannot overflow; 20 digits are compared with 2^64 - 1) + if (t.int_digits == 20 && std::memcmp(t.int_start, "18446744073709551615", 20) > 0) + { + return false; + } + std::uint64_t m = 0; + for (const unsigned char* d = t.int_start; d != t.int_start + t.int_digits; ++d) + { + m = (m * 10) + static_cast(*d - '0'); + } + if (integer && m > (t.negative ? std::uint64_t{1} << 63u : (std::uint64_t{1} << 63u) - 1)) + { + return false; + } + return integer_bits(n) == (t.negative ? 0 - m : m); +} + +/// a byte range of the text or of the decoded strings +struct byte_range +{ + std::uint32_t off; + std::uint32_t len; +}; + +/// a float token and the digit layout its node records +struct float_range +{ + std::uint32_t off; + std::uint32_t len; + std::uint16_t extra; +}; + +/// Run check once for each distinct range of ranges (which are within bounds +/// and not empty). Ranges that are not identical must not overlap: save() +/// writes every string and token to a place of its own, and nodes that share +/// a value (copies within a document) share the whole range. This bounds the +/// work by the size of the text, however many nodes point to the same bytes. +template +bool check_distinct_ranges(std::vector& ranges, Check check) +{ + std::sort(ranges.begin(), ranges.end(), [](const byte_range & a, const byte_range & b) + { + return a.off != b.off ? a.off < b.off : a.len < b.len; + }); + std::size_t end = 0; + for (std::size_t i = 0; i < ranges.size(); ++i) + { + const byte_range r = ranges[i]; + if (i != 0 && r.off == ranges[i - 1].off && r.len == ranges[i - 1].len) + { + continue; + } + if (r.off < end || !check(r)) + { + return false; + } + end = static_cast(r.off) + r.len; + } + return true; +} + +/// Check the float nodes: each token once (nodes of the same token must +/// record layouts that match it), tokens must not overlap. +inline bool check_float_ranges(std::vector& ranges, const unsigned char* text) +{ + std::sort(ranges.begin(), ranges.end(), [](const float_range & a, const float_range & b) + { + if (a.off != b.off) + { + return a.off < b.off; + } + if (a.len != b.len) + { + return a.len < b.len; + } + return a.extra < b.extra; + }); + std::size_t end = 0; + std::size_t i = 0; + while (i < ranges.size()) + { + const float_range r = ranges[i]; + if (r.off < end) + { + return false; + } + number_token t{}; + const unsigned char* const s = text + r.off; + if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t)) + { + return false; + } + // the digit layout the parser records (or "many", as compaction writes it) + const std::uint16_t layout = float_layout(t); + for (; i < ranges.size() && ranges[i].off == r.off && ranges[i].len == r.len; ++i) + { + if (ranges[i].extra != layout && ranges[i].extra != 0xFFFFu) + { + return false; + } + } + end = static_cast(r.off) + r.len; + } + return true; +} + +/// the content checks of check_image: source strings as the parser leaves +/// them (no quotes, backslashes, or control characters), decoded strings +/// (valid UTF-8), and float tokens +inline bool check_contents(const unsigned char* text, const unsigned char* arena, + std::vector& source_strings, std::vector& decoded_strings, + std::vector& floats) +{ + return check_distinct_ranges(source_strings, [text](const byte_range & r) + { + const unsigned char* const b = text + r.off; + return scan_string_run(b, b + r.len) == b + r.len; + }) + && check_distinct_ranges(decoded_strings, [arena](const byte_range & r) + { + return valid_utf8_prefix(arena + r.off, r.len) == r.len; + }) + && check_float_ranges(floats, text); +} + +/// Check the nodes of a loaded image against its text and decoded strings: +/// kinds, flags, and `extra`; extents and element counts of arrays and +/// objects; keys; bounds; string contents (source strings as the parser +/// leaves them: no quotes, backslashes, or control characters; all strings +/// valid UTF-8); and number tokens. +/// +/// The structure and the bounds are checked node by node. The contents of +/// strings and of float tokens are checked afterwards, once for each distinct +/// range (see check_distinct_ranges), so that the full check is linear in the +/// size of the image plus the sorting of the ranges, and not in the number of +/// nodes times the size of the text. +inline bool check_image(const node* nodes, std::size_t count, const unsigned char* text, std::size_t text_size, + const unsigned char* arena, std::size_t arena_size, bool full) +{ + struct frame + { + std::size_t end; + std::uint32_t len; + std::uint32_t seen; + bool object; + bool expect_key; + }; + std::vector stack; + std::vector source_strings; + std::vector decoded_strings; + std::vector floats; + const auto check_string = [&](const node & n) -> bool + { + if ((n.flags & ~node_flags::escaped) != 0 || n.extra != 0) + { + return false; + } + const bool decoded = (n.flags & node_flags::escaped) != 0; + const std::size_t limit = decoded ? arena_size : text_size; + if (n.off > limit || n.len > limit - n.off) + { + return false; + } + if (full && n.len != 0) + { + (decoded ? decoded_strings : source_strings).push_back(byte_range{n.off, n.len}); + } + return true; + }; + // bounds of a number token; the recorded digit layout must lie within it + const auto number_in_bounds = [&](const node & n) -> bool + { + const std::size_t len = number_length(n); + if (len == 0 || n.off > text_size || len > text_size - n.off) + { + return false; + } + if (n.kind != static_cast(value_t::number_float)) + { + return (n.extra >> 8u) == 0; + } + // float_value() reads the sign, the integer digits, and the point and + // fraction digits the layout records (a layout of more than 19 digits + // means the general conversion, which stays within the token) + const std::size_t int_digits = n.extra & 0xFFu; + const std::size_t frac_digits = n.extra >> 8u; + const std::size_t need = (text[n.off] == '-' ? 1u : 0u) + int_digits + (frac_digits != 0 ? frac_digits + 1 : 0); + return int_digits + frac_digits > 19 || need <= len; + }; + std::size_t i = 0; + for (;;) + { + // close finished arrays and objects + while (!stack.empty() && i == stack.back().end) + { + const frame f = stack.back(); + if (f.seen != f.len || (f.object && !f.expect_key)) + { + return false; + } + stack.pop_back(); + if (!stack.empty()) + { + ++stack.back().seen; + stack.back().expect_key = true; + } + } + if (i == count) + { + return stack.empty() && (!full || check_contents(text, arena, source_strings, decoded_strings, floats)); + } + if (i != 0 && stack.empty()) + { + return false; // nodes after the root + } + const node& n = nodes[i]; + if (!stack.empty() && stack.back().object && stack.back().expect_key) + { + if (n.kind != static_cast(value_t::string) || !check_string(n)) + { + return false; + } + stack.back().expect_key = false; + ++i; + continue; + } + bool complete = true; + switch (static_cast(n.kind)) + { + case value_t::null: + // (the offset of a literal is read to size the output of dump()) + if (n.flags != 0 || n.extra != 0 || n.off > text_size) + { + return false; + } + break; + case value_t::boolean: + if ((n.flags & ~node_flags::is_true) != 0 || n.extra != 0 || n.off > text_size) + { + return false; + } + break; + case value_t::string: + if (!check_string(n)) + { + return false; + } + break; + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + if (n.flags != 0 || !number_in_bounds(n)) + { + return false; + } + if (full) + { + // (the contents of float tokens are checked later; the token of an + // integer has at most 256 characters) + if (n.kind == static_cast(value_t::number_float)) + { + floats.push_back(float_range{n.off, n.len, n.extra}); + } + else if (!check_integer(n, text)) + { + return false; + } + } + break; + case value_t::array: + case value_t::object: + { + const std::size_t limit = stack.empty() ? count : stack.back().end; + if (n.flags != 0 || n.extra != 0 || n.next == 0 || n.next > limit - i || n.off > text_size) + { + return false; + } + stack.push_back(frame{i + n.next, n.len, 0, n.kind == static_cast(value_t::object), true}); + complete = false; + break; + } + case value_t::binary: + case value_t::discarded: + default: + return false; + } + ++i; + if (complete && !stack.empty()) + { + ++stack.back().seen; + stack.back().expect_key = true; + } + } +} + +[[noreturn]] NLOHMANN_VIEW_NOINLINE inline void throw_invalid_image(const char* what) +{ + throw_parse_error(116, concat("invalid json_document image: ", what)); +} + +/// Read an image into d. The text and the decoded strings stay in the image; +/// the nodes are copied (so that they are aligned, and edits can change them). +inline void load_image(document_data& d, const std::uint8_t* image, std::size_t size, image_check check) +{ +#if !NLOHMANN_VIEW_LITTLE_ENDIAN + throw_type_error(320, "json_document images need a little-endian target"); // LCOV_EXCL_LINE +#endif + if (image == nullptr || size < sizeof(image_header)) + { + throw_invalid_image("too short"); + } + image_header h{}; + std::memcpy(&h, image, sizeof(h)); + // (the reserved fields are for later versions) + if (std::memcmp(h.magic.data(), "NJVI", 4) != 0 || h.version != image_version + || (h.reserved[0] | h.reserved[1] | h.reserved[2] | h.reserved[3]) != 0) + { + throw_invalid_image("unknown format"); + } + const std::size_t room = size - sizeof(h); + if (h.node_count == 0 || h.node_count > room / sizeof(node) || h.node_count >= image_limit || h.text_size >= image_limit || h.arena_size >= image_limit) + { + throw_invalid_image("sizes out of range"); + } + const auto count = static_cast(h.node_count); + const auto text_size = static_cast(h.text_size); + const auto arena_size = static_cast(h.arena_size); + const std::size_t text_at = sizeof(h) + (count * sizeof(node)); + // the text, a NUL, the decoded strings, a NUL, and nothing after them + if (size - text_at < 2 || text_size > size - text_at - 2 || arena_size != size - text_at - text_size - 2 + || image[text_at + text_size] != 0 || image[size - 1] != 0) + { + throw_invalid_image("sizes out of range"); + } + + d.discarded = true; + d.edits.reset(); + d.base[2] = nullptr; + d.owned.clear(); + if (d.owned_image.empty() || image != d.owned_image.data()) + { + d.owned_image.clear(); + } + d.arena.clear(); + d.indexes.clear(); + d.index_slots.clear(); + d.large_objects.clear(); + d.node_count = 0; + d.reserve(count); + std::memcpy(d.nodes, image + sizeof(h), count * sizeof(node)); + d.node_count = count; + const std::uint8_t* const text = image + text_at; + const std::uint8_t* const arena = text + text_size + 1; + d.src = reinterpret_cast(text); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + d.size = text_size; + d.base[0] = d.src; + d.base[1] = reinterpret_cast(arena); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + d.arena_size = arena_size; + if (check != image_check::none && !check_image(d.nodes, count, text, text_size, arena, arena_size, check == image_check::full)) + { + throw_invalid_image("the check failed"); + } + // the hash indexes of large objects, as after parsing + for (std::size_t i = 0; i < count; ++i) + { + const node& n = d.nodes[i]; + if (n.kind == static_cast(value_t::object) && n.len >= document_data::index_min_members) + { + d.large_objects.push_back(static_cast(i)); + } + } + build_object_indexes(d); + std::vector().swap(d.large_objects); // (only needed while building) + d.discarded = false; +} + +} // namespace view +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -4786,192 +5768,6 @@ NLOHMANN_JSON_NAMESPACE_END // #include // #include -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - - - -#include // size_t -#include // int64_t, uint64_t -#include // numeric_limits -#include // string -#include // integral_constant - -// #include -// #include - -// #include - -// #include - -// #include - - -NLOHMANN_JSON_NAMESPACE_BEGIN -namespace detail -{ -namespace view -{ - -/*! -@brief the value of the float token of a node, as parse() converts it - -Uses the lexer's conversion (detail::convert_float), so that the values are -bit-identical to parse(): float and double are converted without allocation -and independent of the locale. The digit layout recorded while parsing locates -the decimal point and the exponent without scanning the token. -*/ -template -NLOHMANN_VIEW_NOINLINE FloatType float_value(const char* first, const node& n) -{ - const char* const last = first + n.len; - const std::size_t neg = first[0] == '-' ? 1 : 0; - const std::size_t int_digits = n.extra & 0xFFu; - const std::size_t frac_digits = n.extra >> 8u; - std::size_t dot = std::string::npos; - std::size_t mantissa_end = n.len; - if (int_digits != 255 && frac_digits != 255) - { - dot = frac_digits != 0 ? neg + int_digits : std::string::npos; - mantissa_end = neg + int_digits + (frac_digits != 0 ? 1 + frac_digits : 0); - } - else - { - // more digits than the layout records: locate them - for (std::size_t i = 0; i < n.len; ++i) - { - if (first[i] == '.') - { - dot = i; - } - else if (first[i] == 'e' || first[i] == 'E') - { - mantissa_end = i; - break; - } - } - } - return convert_float(first, last, dot, mantissa_end); -} - -/*! -@brief the digits of a float token with at most 19 digits, from its layout - -The digit layout recorded while parsing says where the integer digits, the -fraction digits, and the exponent are, so the digits are read eight at a -time without scanning. - -@param[in] p first character of the token -@param[in] e end of the token -@param[in] limit end of the readable memory (the source text) -*/ -NLOHMANN_VIEW_ALWAYS_INLINE float_significand layout_decimal(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept -{ - const bool negative = *p == '-'; - p += negative ? 1 : 0; - std::uint64_t w = parse_upto19(p, int_digits, limit); - p += int_digits; - std::int64_t q = 0; - if (frac_digits != 0) - { - w = (w * int_pow10(frac_digits)) + parse_upto19(p + 1, frac_digits, limit); - p += 1 + frac_digits; - q = -static_cast(frac_digits); - } - if (p != e) - { - // [eE][+-]digits; huge exponents saturate (the parser rejected overflow) - ++p; - const bool exp_negative = *p == '-'; - p += (*p == '-' || *p == '+') ? 1 : 0; - std::int64_t exp_value = 0; - for (; p != e; ++p) - { - if (exp_value < 0x10000000) - { - exp_value = (exp_value * 10) + (*p - '0'); - } - } - q += exp_negative ? -exp_value : exp_value; - } - - float_significand d; - d.w = w; - d.exponent = q; - d.negative = negative; - return d; -} - -/*! -@brief the value of a float token with at most 19 digits, from its layout - -The result is correctly rounded by the lexer's conversion -(detail::decimal_to_float(): Clinger's fast path where both operands are -exact, else the Eisel-Lemire algorithm, which needs no fallback for up to 19 -digits), so it is the value parse() produces. -*/ -template -NLOHMANN_VIEW_ALWAYS_INLINE FloatType layout_float(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept -{ - return decimal_to_float(layout_decimal(p, e, int_digits, frac_digits, limit)); -} - -/// the value of a float set by an edit: its token (the shortest round-trip -/// text, or "nan", "inf", "-inf") in the edit arena -template -NLOHMANN_VIEW_NOINLINE FloatType edited_float(const char* token, const node& n) -{ - if (token[0] == 'n') - { - return std::numeric_limits::quiet_NaN(); - } - if (token[0] == 'i' || (token[0] == '-' && token[1] == 'i')) - { - return token[0] == 'i' ? std::numeric_limits::infinity() : -std::numeric_limits::infinity(); - } - return float_value(token, n); -} - -/// the value of the float token of a node, as parse() converts it; floats and -/// doubles with at most 19 digits are converted from the digit layout -template -FloatType float_value(const document_data& d, const node& n) -{ - if (NLOHMANN_VIEW_UNLIKELY((n.flags & node_flags::storage) == node_flags::edited)) - { - return edited_float(d.str(n), n); - } - return float_value(d, n, std::integral_constant::value> {}); -} - -template -FloatType float_value(const document_data& d, const node& n, std::true_type /*binary32 or binary64*/) -{ - const unsigned int_digits = n.extra & 0xFFu; - const unsigned frac_digits = n.extra >> 8u; - if (NLOHMANN_VIEW_LIKELY(int_digits + frac_digits <= 19)) // (255 marks "many") - { - // (a float token not written by an edit is in the text) - const auto* const first = reinterpret_cast(d.src + n.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - return layout_float(first, first + n.len, int_digits, frac_digits, reinterpret_cast(d.src + d.size)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - } - return float_value(d.str(n), n); -} - -template -FloatType float_value(const document_data& d, const node& n, std::false_type /*other*/) -{ - return float_value(d.str(n), n); -} - -} // namespace view -} // namespace detail -NLOHMANN_JSON_NAMESPACE_END NLOHMANN_JSON_NAMESPACE_BEGIN @@ -5617,7 +6413,9 @@ class view_serializer m_out.put('"'); } - /// as serializer::dump_escaped() for valid UTF-8 (the view has no other) + /// as serializer::dump_escaped(); strings of a document are valid UTF-8, + /// except in a damaged image loaded with image_check::bounds, for which + /// this throws what basic_json::dump() throws for the string template void write_escaped(const unsigned char* s, std::size_t n) { @@ -5641,12 +6439,13 @@ class view_serializer } std::uint32_t codepoint = s[i]; std::size_t len = 1; - if (codepoint >= 0xC0) + if (codepoint >= 0x80) { - len = 2; - if (codepoint >= 0xE0) + len = validate_one_utf8(s + i, n - i); + if (NLOHMANN_VIEW_UNLIKELY(len == 0)) { - len = codepoint >= 0xF0 ? 4 : 3; + invalid_utf8(s, n); + return; } codepoint &= 0xFFu >> (len + 1); for (std::size_t k = 1; k < len; ++k) @@ -5659,6 +6458,13 @@ class view_serializer } } + /// throw what basic_json::dump() throws for a string that is not valid UTF-8 + NLOHMANN_VIEW_NOINLINE static void invalid_utf8(const unsigned char* s, std::size_t n) + { + const string_t dumped = BasicJsonType(string_t(reinterpret_cast(s), n)).dump(); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + static_cast(dumped); + } + template void write_codepoint(std::uint32_t codepoint, const unsigned char* bytes, std::size_t len) { @@ -6933,7 +7739,7 @@ class basic_json_document /// whether the document holds its own copy of the text bool owns_source() const noexcept { - return m_data && !m_data->owned.empty() && m_data->src == m_data->owned.data(); + return m_data && ((!m_data->owned.empty() && m_data->src == m_data->owned.data()) || !m_data->owned_image.empty()); } /// number of index nodes (values plus object keys) @@ -6942,7 +7748,7 @@ class basic_json_document return m_data ? m_data->node_count : 0; } - /// bytes held by the document (index, decoded strings, owned text) + /// bytes held by the document (index, decoded strings, and owned text or image) std::size_t memory_usage() const noexcept { if (!m_data) @@ -6951,7 +7757,7 @@ class basic_json_document } return sizeof(document_data) + (m_data->inline_capacity * sizeof(detail::view::node)) + (m_data->nodes != m_data->inline_nodes ? m_data->node_capacity * sizeof(detail::view::node) : 0) - + m_data->arena.capacity() + m_data->owned.capacity() + + m_data->arena.capacity() + m_data->owned.capacity() + m_data->owned_image.capacity() + (m_data->indexes.capacity() * sizeof(document_data::object_index)) + (m_data->index_slots.capacity() * sizeof(std::uint32_t)) + (m_data->large_objects.capacity() * sizeof(std::uint32_t)) + (m_data->edits != nullptr ? m_data->edits->bytes : 0); @@ -6971,8 +7777,10 @@ class basic_json_document // allocate everything first, so that an exception leaves the document // unchanged + // (the decoded strings of a loaded image stay in the image) + const bool arena_in_use = d.base[1] == d.arena.data(); const bool shrink_arena = d.arena.capacity() > d.arena.size(); - std::vector arena(shrink_arena ? d.arena : std::vector()); + std::vector arena(shrink_arena && arena_in_use ? d.arena : std::vector()); // (edits link to the nodes of the index, which then stays in place) const bool shrink_nodes = d.nodes != d.inline_nodes && d.node_count != d.node_capacity && d.edits == nullptr; const bool into_header = d.node_count <= d.inline_capacity; @@ -6993,7 +7801,10 @@ class basic_json_document if (shrink_arena) { d.arena.swap(arena); - d.base[1] = d.arena.data(); + if (arena_in_use) + { + d.base[1] = d.arena.data(); + } } if (d.indexes.capacity() > d.indexes.size()) { @@ -7005,6 +7816,55 @@ class basic_json_document } } + //////////// + // images // + //////////// + + /// how load() checks an image (full, bounds, or none) + using image_check = detail::view::image_check; + + /// The document as an image that load() reads without parsing: the node + /// index, the text, and the decoded strings. An edited document is + /// written in its current state (floats that are not finite become null, + /// as in dump()). + std::vector save() const + { + if (NLOHMANN_VIEW_UNLIKELY(!m_data || m_data->discarded)) + { + detail::view::throw_type_error(320, "cannot save a discarded json_document"); + } + return detail::view::save_image(*m_data); + } + + /// Read an image written by save(). The image is borrowed: it must stay + /// alive and unchanged while the document is used. + NLOHMANN_VIEW_NODISCARD + static basic_json_document load(const std::uint8_t* image, std::size_t size, const image_check check = image_check::full) + { + basic_json_document d; + d.ensure_data(nullptr, 0); + detail::view::load_image(*d.m_data, image, size, check); + return d; + } + + /// read an image (borrowed) + NLOHMANN_VIEW_NODISCARD + static basic_json_document load(const std::vector& image, const image_check check = image_check::full) + { + return load(image.data(), image.size(), check); + } + + /// read an image and keep it (no copy) + NLOHMANN_VIEW_NODISCARD + static basic_json_document load(std::vector&& image, const image_check check = image_check::full) + { + basic_json_document d; + d.ensure_data(nullptr, 0); + d.m_data->owned_image = std::move(image); + detail::view::load_image(*d.m_data, d.m_data->owned_image.data(), d.m_data->owned_image.size(), check); + return d; + } + /////////// // edits // /////////// @@ -7239,6 +8099,7 @@ class basic_json_document { d.owned.clear(); } + d.owned_image.clear(); d.src = src; d.size = size; d.node_count = 0; @@ -7263,6 +8124,7 @@ class basic_json_document { d.base[0] = d.src; d.base[1] = d.arena.data(); + d.arena_size = d.arena.size(); detail::view::build_object_indexes(d); std::vector().swap(d.large_objects); // (only needed while parsing) d.discarded = false; diff --git a/tests/Makefile b/tests/Makefile index e8e61e603..e03b9dc7d 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -10,7 +10,7 @@ CXXFLAGS += -std=c++11 CPPFLAGS += -I ../single_include FUZZER_ENGINE = src/fuzzer-driver_afl.cpp -FUZZERS = parse_afl_fuzzer parse_bson_fuzzer parse_cbor_fuzzer parse_msgpack_fuzzer parse_ubjson_fuzzer parse_bjdata_fuzzer parse_bon8_fuzzer parse_json_view_fuzzer +FUZZERS = parse_afl_fuzzer parse_bson_fuzzer parse_cbor_fuzzer parse_msgpack_fuzzer parse_ubjson_fuzzer parse_bjdata_fuzzer parse_bon8_fuzzer parse_json_view_fuzzer json_view_image_fuzzer fuzzers: $(FUZZERS) parse_afl_fuzzer: @@ -19,6 +19,9 @@ parse_afl_fuzzer: parse_json_view_fuzzer: $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(FUZZER_ENGINE) src/fuzzer-parse_json_view.cpp -o $@ +json_view_image_fuzzer: + $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(FUZZER_ENGINE) src/fuzzer-json_view_image.cpp -o $@ + parse_bson_fuzzer: $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(FUZZER_ENGINE) src/fuzzer-parse_bson.cpp -o $@ diff --git a/tests/fuzzing.md b/tests/fuzzing.md index feafbfe76..2399bdfd2 100644 --- a/tests/fuzzing.md +++ b/tests/fuzzing.md @@ -12,6 +12,12 @@ produces, and that a rejected input makes both parsers throw with an identical ` are taken from the low bits of the first input byte (the byte stays part of the text). It takes plain JSON text, so it reuses the `corpus_json` corpus rather than a format of its own. +`json_view_image_fuzzer` (`tests/src/fuzzer-json_view_image.cpp`) tests the images of `json_document` (`save()` and +`load()`). It uses each input twice: as an image, which `load()` must either reject with `parse_error.116` or read +safely (with `image_check::full`, the document must also serialize to the JSON it reads as), and as a JSON text, whose +image must load and serialize to the same text. A corpus of images can be made from JSON files with a small program +that calls `json_document::parse(text).save()`; plain JSON files work as well. + ## What the fuzzers check Each fuzzer driver (`tests/src/fuzzer-parse_*.cpp`) parses its input twice: once with `allow_exceptions = false` and diff --git a/tests/src/fuzzer-json_view_image.cpp b/tests/src/fuzzer-json_view_image.cpp new file mode 100644 index 000000000..8495468a4 --- /dev/null +++ b/tests/src/fuzzer-json_view_image.cpp @@ -0,0 +1,95 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +/* +This file implements a test of json_document images suitable for fuzz +testing. The input is used twice: + +- as an image: json_document::load() with image_check::full must either throw + a parse_error or yield a document that serializes to the JSON text it reads + as; with image_check::bounds, reading and serializing must be safe (checked + by the sanitizers), and serializing may only throw type_error.316 +- as a JSON text: if json_document::parse() accepts it, the image of the + document must load (with every check) and serialize to the same text + +The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer +drivers. +*/ + +#include +#include +#include +#include +#include +#include + +// the checks below are assertions; NDEBUG would compile them away +#ifdef NDEBUG + #error "the fuzzer drivers must be built without NDEBUG" +#endif + +using json = nlohmann::json; +using json_document = nlohmann::json_document; +using image_check = json_document::image_check; + +// see http://llvm.org/docs/LibFuzzer.html +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) +{ + // the input as an image + for (const image_check check : + { + image_check::full, image_check::bounds + }) + { + json_document d; + try + { + d = json_document::load(data, size, check); + } + catch (const json::parse_error& e) + { + assert(e.id == 116); + continue; + } + std::string dumped; + try + { + dumped = d.root().dump(); + } + catch (const json::type_error& e) + { + // invalid UTF-8 can only pass the bounds check + assert(check == image_check::bounds && e.id == 316); + continue; + } + const json j = d.root().materialize(); + if (check == image_check::full) + { + assert(json::parse(dumped) == j); + // an image of the loaded document is the input + assert(d.save() == std::vector(data, data + size)); + } + } + + // the input as a JSON text + const std::string text(reinterpret_cast(data), size); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const json_document parsed = json_document::parse(text, false); + if (!parsed.is_discarded()) + { + const std::vector image = parsed.save(); + for (const image_check check : + { + image_check::full, image_check::bounds, image_check::none + }) + { + const json_document loaded = json_document::load(image, check); + assert(loaded.root().dump() == parsed.root().dump()); + } + } + return 0; +} diff --git a/tests/src/unit-json_view_image.cpp b/tests/src/unit-json_view_image.cpp new file mode 100644 index 000000000..994bd211c --- /dev/null +++ b/tests/src/unit-json_view_image.cpp @@ -0,0 +1,1237 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; +using nlohmann::ordered_json; +using nlohmann::json_document; +using nlohmann::json_editable_document; +using nlohmann::ordered_json_document; +using nlohmann::ordered_json_editable_document; +using image_check = json_document::image_check; +using nlohmann::detail::view::node; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if !(defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + +namespace +{ +#if !defined(JSON_NOEXCEPTION) +std::string exception_of(const std::function& f) +{ + try + { + f(); + } + catch (const json::exception& e) + { + return e.what(); + } + return ""; +} + +const char* const check_failed = "[json.exception.parse_error.116] parse error: invalid json_document image: the check failed"; +#endif + +std::string read_file(const std::string& name) +{ + const std::ifstream f(std::string(TEST_DATA_DIRECTORY) + name, std::ios::binary); + std::stringstream ss; + ss << f.rdbuf(); + return ss.str(); +} + +// the dump of a loaded image, through a named document (root() of a temporary +// document does not compile, and its views would dangle) +std::string loaded_dump(const std::vector& image, image_check check = image_check::full) +{ + const json_document d = json_document::load(image, check); + return d.root().dump(); +} + +// the offsets of the parts of an image +constexpr std::size_t header_size = 64; + +std::uint64_t header_field(const std::vector& image, std::size_t offset) +{ + std::uint64_t v = 0; + std::memcpy(&v, image.data() + offset, sizeof(v)); + return v; +} + +// a header field as a size; the cast is from a variable, which GCC's +// -Wuseless-cast does not flag where std::uint64_t and std::size_t coincide +std::size_t header_size_field(const std::vector& image, std::size_t offset) +{ + const std::uint64_t v = header_field(image, offset); + return static_cast(v); +} + +void set_header_field(std::vector& image, std::size_t offset, std::uint64_t v) +{ + std::memcpy(image.data() + offset, &v, sizeof(v)); +} + +std::size_t node_count(const std::vector& image) +{ + return header_size_field(image, 8); +} + +std::size_t text_at(const std::vector& image) +{ + return header_size + (node_count(image) * sizeof(node)); +} + +node node_at(const std::vector& image, std::size_t i) +{ + node n{}; + std::memcpy(&n, image.data() + header_size + (i * sizeof(node)), sizeof(node)); + return n; +} + +void set_node(std::vector& image, std::size_t i, const node& n) +{ + std::memcpy(image.data() + header_size + (i * sizeof(node)), &n, sizeof(node)); +} + +#if !defined(JSON_NOEXCEPTION) +/// the result of loading an image with a check: "" or the exception message +std::string load_result(const std::vector& image, image_check check) +{ + return exception_of([&] + { + const json_document d = json_document::load(image, check); + static_cast(d); + }); +} + +/// a copy of the image with node i changed by f +template +std::vector corrupted(const std::vector& image, std::size_t i, F f) +{ + std::vector b = image; + node n = node_at(b, i); + f(n); + set_node(b, i, n); + return b; +} +#endif + +/// a document and the documents loaded from its image must be equal +template +void check_round_trip(const Document& d) +{ + const std::vector image = d.save(); + for (const image_check check : + { + image_check::full, image_check::bounds, image_check::none + }) + { + const json_document l = json_document::load(image, check); + CHECK(l.root().dump() == d.root().dump()); + CHECK(l.root().dump(2) == d.root().dump(2)); + CHECK(l.root().materialize() == json(d.root().materialize())); + // an image of a loaded document is the same image + CHECK(l.save() == image); + } + // an editable document can be loaded, too + const ordered_json_editable_document e = ordered_json_editable_document::load(image); + CHECK(e.root().dump() == d.root().dump()); +} + +std::uint32_t rng() +{ + static std::mt19937 generator(5295); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible + // result_type is std::uint_fast32_t, which may be wider than 32 bits + const std::mt19937::result_type value = generator(); + return static_cast(value); +} +} // namespace + +TEST_CASE("json_view images: round trips") +{ + SECTION("small documents") + { + for (const char* text : + { + "null", "true", "false", "0", "-0", "42", "-42", "18446744073709551615", "-9223372036854775808", + "123456789012345678901234567890", "1.5", "-1.25e-300", "1E308", "0.1000000000000000000000000001", + "\"\"", "\"text\"", R"("esc\"aped\n\u00e9\ud83d\ude00")", "\"\xc3\xa9\xe3\x81\x82\"", + "[]", "{}", "[[]]", "[{}]", "{\"\":{}}", + R"({"a": [1, 2.5, "x\ty", true, null, {"b": []}], "c": {"d": -3, "eA": "f"}})", + R"({"k": 1, "k": 2, "l": [], "k": 3})", + " [1 , 2 ] " + }) + { + CAPTURE(text) + // false positive: parse() returns a document with a root + // @infer-ignore NULLPTR_DEREFERENCE + check_round_trip(json_document::parse(text)); + // false positive: parse() returns a document with a root + // @infer-ignore NULLPTR_DEREFERENCE + check_round_trip(ordered_json_document::parse(text)); + } + } + + SECTION("files") + { + for (const char* name : + { + "/json_testsuite/sample.json", "/nativejson-benchmark/canada.json", "/nativejson-benchmark/citm_catalog.json", + "/nativejson-benchmark/twitter.json", "/json_tests/pass1.json", "/json_tests/pass2.json", "/json_tests/pass3.json" + }) + { + CAPTURE(name) + const std::string text = read_file(name); + const json_document d = json_document::parse(text); + check_round_trip(d); + // what a loaded document reads is what parse() produces + const json_document l = json_document::load(d.save()); + CHECK(l.root().materialize() == json::parse(text)); + } + } + + SECTION("images are deterministic") + { + const std::string text = R"({"b": [1, 2, {"c": "\u00e9"}], "a": 1.5})"; + const json_document d = json_document::parse(text); + CHECK(d.save() == json_document::parse(text).save()); + CHECK(d.save() == json_editable_document::parse(text).save()); + CHECK(d.save() == ordered_json_document::parse(text).save()); + const json_document copy = json_document::parse_copy(text); + CHECK(copy.save() == d.save()); + } + + SECTION("large objects get their hash index again") + { + std::string text = "{"; + for (int i = 0; i < 1000; ++i) + { + text += (i != 0 ? ",\"k" : "\"k") + std::to_string(i) + "\":" + std::to_string(i); + } + text += R"(,"k7":"a duplicate","inner":{)"; + for (int i = 0; i < 200; ++i) + { + text += (i != 0 ? ",\"m" : "\"m") + std::to_string(i) + "\":" + std::to_string(-i); + } + text += "}}"; + const json_document d = json_document::parse(text); + const std::vector image = d.save(); + for (const image_check check : + { + image_check::full, image_check::none + }) + { + const json_document l = json_document::load(image, check); + for (int i = 0; i < 1000; ++i) + { + CHECK(l.root()["k" + std::to_string(i)] == d.root()["k" + std::to_string(i)]); + } + CHECK(l.root()["k7"].get() == 7); // the first of duplicate keys + CHECK(l.root()["inner"]["m199"].get() == -199); + CHECK(!l.root().contains("k1000")); + // the index is not part of the image + CHECK(l.save() == image); + } + // the nodes of objects in the image do not carry the number of an index + CHECK(node_at(image, 0).extra == 0); + } + + SECTION("duplicate keys of a large object: lookups return the first member") + { + std::string text = "{"; + for (int i = 0; i < 200; ++i) + { + text += (i != 0 ? ",\"k" : "\"k") + std::to_string(i) + "\":" + std::to_string(i); + } + // three members for k5 (the first is a number), and duplicates of k0 and k199 + text += R"(,"k5":"two","k0":null,"k199":[],"k5":"three"})"; + const json_document d = json_document::parse(text); + REQUIRE(d.root().size() == 204); + CHECK(d.root()["k5"] == 5); + const std::vector image = d.save(); + for (const image_check check : + { + image_check::full, image_check::bounds, image_check::none + }) + { + const json_document l = json_document::load(image, check); + CHECK(l.root().size() == 204); + CHECK(l.root()["k5"] == 5); + CHECK(l.root().at("k5") == 5); + CHECK(l.root().find("k5").value() == 5); + CHECK(l.root()["k0"] == 0); + CHECK(l.root()["k199"] == 199); + CHECK(l.root()["k100"] == 100); + // materialize() keeps the semantics of parse(): the last value + CHECK(l.root().materialize() == d.root().materialize()); + CHECK(l.root().materialize()["k5"] == "three"); + } + } + + SECTION("objects with colliding keys") + { + // the hash is not seeded: keys can be found that land in one slot of + // a table (see json_view: "colliding keys") + const auto keys_for = [](std::size_t table_size, std::size_t colliding_count, std::size_t spread_count) + { + std::vector colliding; + std::vector spread; + for (std::uint64_t counter = 0; colliding.size() < colliding_count || spread.size() < spread_count; ++counter) + { + std::string key(8, 'a'); + std::uint64_t x = counter; + for (std::size_t i = 0; i < 8; ++i, x /= 26) + { + key[i] = static_cast('a' + (x % 26)); + } + const bool lands_in_slot_zero = (nlohmann::detail::view::key_hash(key.data(), key.size()) & (table_size - 1)) == 0; + if (lands_in_slot_zero && colliding.size() < colliding_count) + { + colliding.push_back(key); + } + else if (!lands_in_slot_zero && spread.size() < spread_count) + { + spread.push_back(key); + } + } + colliding.insert(colliding.end(), spread.begin(), spread.end()); + return colliding; + }; + const auto make_text = [](const std::vector& keys) + { + std::string text = "{"; + for (std::size_t i = 0; i < keys.size(); ++i) + { + text += (i != 0 ? ",\"" : "\"") + keys[i] + "\":" + std::to_string(i); + } + return text + "}"; + }; + + // 300 keys in one slot (the table is 1024 slots): the table would + // exceed the probe limit, so the object has none and is searched + // linearly; 40 of 200 keys in one slot (512 slots): a table with a + // long chain + const struct // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays): an unnamed type + { + std::size_t table_size; + std::size_t colliding; + std::size_t spread; + } cases[] = {{1024, 300, 0}, {512, 40, 160}}; + for (const auto& c : cases) + { + CAPTURE(c.table_size) + std::vector keys = keys_for(c.table_size, c.colliding, c.spread); + const std::size_t members = keys.size(); + // a duplicate of a colliding key, of the first and of a spread key + const std::vector duplicated = {keys[c.colliding - 1], keys[0], keys[members - 1]}; + std::string text = make_text(keys); + text.pop_back(); + for (const std::string& key : duplicated) + { + text += ",\"" + key + R"(":"last")"; + } + text += "}"; + const json_document d = json_document::parse(text); + const std::vector image = d.save(); + for (const image_check check : + { + image_check::full, image_check::bounds, image_check::none + }) + { + const json_document l = json_document::load(image, check); + REQUIRE(l.root().size() == members + duplicated.size()); + for (std::size_t i = 0; i < members; ++i) + { + CAPTURE(i) + // lookups find the first member, also of the duplicated keys + CHECK(l.root()[keys[i]] == i); + CHECK(l.root().at(keys[i]) == l.root()[keys[i]]); + CHECK(l.root().find(keys[i]).key() == keys[i]); + CHECK(!l.root().contains(keys[i] + "x")); + } + CHECK(!l.root().contains("missing")); + CHECK(l.root().materialize() == json::parse(text)); + } + } + } + + SECTION("loading releases the list of large objects") + { + std::string text = "["; + for (int object = 0; object < 400; ++object) + { + text += object != 0 ? ",{" : "{"; + for (int i = 0; i < 128; ++i) + { + text += (i != 0 ? ",\"" : "\"") + std::to_string(i) + "\":" + std::to_string(i); + } + text += '}'; + } + text += "]"; + json_document parsed = json_document::parse(text); + const std::vector image = parsed.save(); + json_document loaded = json_document::load(image); + CHECK(loaded.root()[399]["127"] == 127); + // Parsing and loading build the same tables, and keep nothing else: + // not the positions of the objects to index (2 KiB here). The slack + // covers the nodes in the header of the document, which are sized + // differently. + parsed.shrink_to_fit(); + loaded.shrink_to_fit(); + CHECK(loaded.memory_usage() <= parsed.memory_usage() + 512); + } +} + +TEST_CASE("json_view images: edited documents") +{ + const std::string text = R"({"name": "x", "n": 1, "f": 2.5, "list": [1, 2, 3], "obj": {"a": "\u00e9", "b": [true]}, "s": "a\"b"})"; + + SECTION("every kind of edit") + { + ordered_json_editable_document d = ordered_json_editable_document::parse(text); + d.set(d.root()["name"], "a new \"name\""); // string in the edit arena + d.set(d.root()["n"], -17); // negative integer + d.set(d.root(), "p", 5); // non-negative number_integer + d.set(d.root(), "u", 18446744073709551615u); // unsigned + d.set(d.root()["f"], 0.1); // float token + d.set(d.root(), "nan", std::numeric_limits::quiet_NaN()); + d.set(d.root(), "inf", -std::numeric_limits::infinity()); + d.push_back(d.root()["list"], "pushed"); // moved array + d.insert(d.root()["list"], 0, ordered_json::object({{"new", {1, 2}}})); + d.erase(d.root()["list"], 2); + d.erase(d.root(), "s"); + d.set(d.root()["obj"], "c", ordered_json::array({1, "two", 3.5, nullptr, false})); // new object member with a new array + d.set(d.root(), "copy", d.root()["obj"]); // a copy of a subtree + d.set(d.root(), "key \xc3\xa9", true); // a key in the edit arena + + const std::vector image = d.save(); + const ordered_json expected = ordered_json::parse(d.root().dump()); + for (const image_check check : + { + image_check::full, image_check::bounds, image_check::none + }) + { + const ordered_json_document l = ordered_json_document::load(image, check); + CHECK(l.root().dump() == d.root().dump()); + CHECK(l.root().materialize() == expected); + CHECK(l.root()["nan"].is_null()); + CHECK(l.root()["inf"].is_null()); + CHECK(l.root()["p"].is_number_integer()); + CHECK(l.root()["p"].get() == 5); + CHECK(l.root()["f"].get() == 0.1); + CHECK(l.root()["u"].get() == 18446744073709551615u); + } + // the node index is in document order again: an image of the loaded + // document is the same image + CHECK(ordered_json_document::load(image).save() == image); + // number tokens of edits follow the source; the text is the source's + // prefix + const ordered_json_document l = ordered_json_document::load(image); + REQUIRE(l.source().size() > text.size()); + CHECK(std::string(l.source().data(), text.size()) == text); // NOLINT(bugprone-suspicious-stringview-data-usage): the size is given and checked above + } + + SECTION("a loaded document can be edited and saved again") + { + const std::vector first = json_editable_document::parse(text).save(); + json_editable_document d = json_editable_document::load(first); + d.set(d.root()["obj"]["a"], "changed"); + d.push_back(d.root()["list"], 4); + d.set(d.root(), "z", json::array({json::object()})); + const std::vector second = d.save(); + const json_document l = json_document::load(second); + CHECK(l.root().dump() == d.root().dump()); + CHECK(l.root()["obj"]["a"] == "changed"); + CHECK(l.root()["list"].size() == 4); + } + + SECTION("the targets of links do not reach the image") + { + // Edits that make an entry of a moved sequence link to a value mark the + // value as linked (node_flags::linked). An image has no links and no + // such flag: the full and the bounds check reject it. + const std::string nested = R"({"a": [1, 2, {"x": [3]}, "s", 1.5, true, null, [4, 5]], "b": {"c": 1, "d": [1, 2, 3], "e": {"f": "g"}}, "h": "str"})"; + const auto check_image = [](const json_editable_document & d) + { + const std::vector image = d.save(); + for (std::size_t i = 0; i < node_count(image); ++i) + { + CAPTURE(i) + CHECK((node_at(image, i).flags & nlohmann::detail::view::node_flags::linked) == 0); + CHECK(node_at(image, i).kind != nlohmann::detail::view::kind_link); + } + for (const image_check check : + { + image_check::full, image_check::bounds + }) + { +#if !defined(JSON_NOEXCEPTION) + CHECK(load_result(image, check).empty()); +#else + const json_document loaded = json_document::load(image, check); + CHECK(loaded.root().dump() == d.root().dump()); +#endif + } + check_round_trip(d); + // a loaded document is edited and saved again + json_editable_document e = json_editable_document::load(image); + CHECK(e.root().dump() == d.root().dump()); + e.set(e.root(), "after", 1); + CHECK(loaded_dump(e.save()) == e.root().dump()); + }; + + SECTION("a container replaced by a scalar after an earlier edit") + { + json_editable_document d = json_editable_document::parse(nested); + d.set(d.root()["a"][0], 9); + d.set(d.root()["a"][2], 5); + check_image(d); + d.set(d.root()["a"][2], "now a string"); + check_image(d); + d.set(d.root()["a"][7], 0.25); + check_image(d); + d.set(d.root()["b"]["e"], true); + d.set(d.root()["b"]["d"], 7); + check_image(d); + } + + SECTION("insert and push_back into parsed arrays") + { + json_editable_document d = json_editable_document::parse(nested); + d.push_back(d.root()["a"], 6); + check_image(d); + d.insert(d.root()["a"], 0, "first"); + check_image(d); + d.insert(d.root()["b"]["d"], 1, json::array({1, 2})); + d.push_back(d.root()["b"]["d"], json::object({{"k", nullptr}})); + check_image(d); + // links to values that are replaced afterwards + d.set(d.root()["a"][3], json::object()); + d.set(d.root()["a"][4], false); + d.set(d.root()["a"][5], "replaced"); + d.set(d.root()["a"][9], 1e300); + check_image(d); + } + + SECTION("members set in parsed objects") + { + json_editable_document d = json_editable_document::parse(nested); + d.set(d.root(), "new", 1); + d.set(d.root()["b"], "c", "changed"); + d.set(d.root()["b"], "e", json::array({1, {{"z", 2}}})); + d.set(d.root()["b"], "more", d.root()["a"]); + check_image(d); + d.set(d.root()["a"][2], 3); // a link target in a copied subtree + d.erase(d.root()["b"], "c"); + d.set(d.root(), "h", json::object()); + check_image(d); + } + + SECTION("erase") + { + json_editable_document d = json_editable_document::parse(nested); + d.push_back(d.root()["a"], 6); + d.erase(d.root()["a"], 0); + d.erase(d.root()["a"], 1); + d.set(d.root()["a"][0], 1); + d.erase(d.root()["b"], "d"); + check_image(d); + } + } + +#if !defined(JSON_NOEXCEPTION) + SECTION("invalid UTF-8 of a loaded image is not copied into an editable document") + { + // loading with the bounds check (or none) does not look at the encoding + // of strings: dump() of such a view throws, and an editable document + // must not take the string over + const auto patched = [](const std::string & json_text) + { + std::vector image = json_document::parse(json_text).save(); + bool found = false; + for (std::size_t i = 0; i + 1 < image.size(); ++i) + { + if (image[i] == 'Q' && image[i + 1] == 'Z') + { + image[i] = 0xC3; + image[i + 1] = 0x28; // an invalid sequence + found = true; + } + } + REQUIRE(found); + return image; + }; + + for (const image_check check : + { + image_check::bounds, image_check::none + }) + { + // as a value + { + const json_document loaded = json_document::load(patched(R"(["abQZ"])"), check); + json_editable_document e = json_editable_document::parse("[]"); + CHECK_THROWS_WITH_AS(e.push_back(e.root(), loaded.root()[0]), "[json.exception.type_error.316] invalid UTF-8 byte at index 3: 0x28", json::type_error&); + CHECK_THROWS_WITH_AS(e.insert(e.root(), 0, loaded.root()[0]), "[json.exception.type_error.316] invalid UTF-8 byte at index 3: 0x28", json::type_error&); + CHECK_THROWS_WITH_AS(e.set(e.root(), loaded.root()[0]), "[json.exception.type_error.316] invalid UTF-8 byte at index 3: 0x28", json::type_error&); + CHECK_THROWS_WITH_AS(e.push_back(e.root(), loaded.root()), "[json.exception.type_error.316] invalid UTF-8 byte at index 3: 0x28", json::type_error&); + // nothing changed + CHECK(e.root().dump() == "[]"); + CHECK(e.root().size() == 0); + CHECK(loaded_dump(e.save()) == "[]"); + // the valid part of the same document can be copied + e.push_back(e.root(), loaded.root().size()); + CHECK(e.root().dump() == "[1]"); + } + // as a key, and in a nested value + { + const json_document loaded = json_document::load(patched(R"([{"abQZ": 1}, [["x", "QZ"]]])"), check); + json_editable_document e = json_editable_document::parse(R"({"keep": [1]})"); + CHECK_THROWS_WITH_AS(e.set(e.root(), "k", loaded.root()[0]), "[json.exception.type_error.316] invalid UTF-8 byte at index 3: 0x28", json::type_error&); + CHECK_THROWS_WITH_AS(e.set(e.root(), "k", loaded.root()[1]), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0x28", json::type_error&); + CHECK_THROWS_WITH_AS(e.push_back(e.root()["keep"], loaded.root()[1]), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0x28", json::type_error&); + CHECK(e.root().dump() == R"({"keep":[1]})"); + CHECK(loaded_dump(e.save()) == R"({"keep":[1]})"); + } + } + } +#endif + + SECTION("the root replaced") + { + json_editable_document d = json_editable_document::parse(text); + d.set(d.root(), json::array({1, "x"})); + check_round_trip(d); + d.set(d.root(), 3.5); + check_round_trip(d); + d.set(d.root(), "text"); + check_round_trip(d); + } +} + +TEST_CASE("json_view images: ownership") +{ + const std::string text = R"({"a": "esc\u00e9aped", "b": [1, 2]})"; + const std::vector image = json_document::parse(text).save(); + + SECTION("borrowed") + { + const json_document d = json_document::load(image); + CHECK(!d.owns_source()); + CHECK(d.root()["a"] == "esc\xc3\xa9" "aped"); + const json_document p = json_document::load(image.data(), image.size()); + CHECK(!p.owns_source()); + CHECK(p.root() == d.root()); + // the text is the image's + CHECK(d.source().data() == reinterpret_cast(image.data() + text_at(image))); + } + + SECTION("owned") + { + std::vector copy = image; + const std::uint8_t* const data = copy.data(); + json_document d = json_document::load(std::move(copy)); + CHECK(d.owns_source()); + CHECK(d.source().data() == reinterpret_cast(data + text_at(image))); + CHECK(d.memory_usage() >= image.size()); + CHECK(d.root()["b"][1] == 2); + // read() replaces the image + d.read(std::string("[1]")); + CHECK(d.owns_source()); + CHECK(d.root().dump() == "[1]"); + const std::string borrowed = "[2]"; + d.read(borrowed); + CHECK(!d.owns_source()); + } + + SECTION("shrink_to_fit keeps the decoded strings of the image") + { + json_document d = json_document::parse(R"(["\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9"])"); + d.shrink_to_fit(); + const std::vector img = d.save(); + json_document l = json_document::load(img); + l.shrink_to_fit(); + CHECK(l.root().dump() == d.root().dump()); + CHECK(l.save() == img); + } +} + +// the remaining tests are about the exceptions of load() and save() +#if !defined(JSON_NOEXCEPTION) +TEST_CASE("json_view images: errors") +{ + SECTION("a literal as the root: dump() after loading") + { + for (const char* text : + { + "null", "true", "false" + }) + { + std::vector image = json_document::parse(text).save(); + node n = node_at(image, 0); + n.off = static_cast(image.size()); + set_node(image, 0, n); + CHECK(load_result(image, image_check::full) == check_failed); + CHECK(load_result(image, image_check::bounds) == check_failed); + } + } + + SECTION("saving a discarded document") + { + const json_document empty{}; + CHECK(exception_of([&] { static_cast(empty.save()); }) == "[json.exception.type_error.320] cannot save a discarded json_document"); + const json_document failed = json_document::parse("[1,", false); + CHECK(exception_of([&] { static_cast(failed.save()); }) == "[json.exception.type_error.320] cannot save a discarded json_document"); + } + + const std::vector image = json_document::parse(R"({"a": [1, "\u00e9"]})").save(); + const std::string prefix = "[json.exception.parse_error.116] parse error: invalid json_document image: "; + + SECTION("header and sizes") + { + CHECK(exception_of([] + { + const json_document d = json_document::load(nullptr, 0); + static_cast(d); + }) == prefix + "too short"); + CHECK(exception_of([&] + { + const json_document d = json_document::load(image.data(), 63); + static_cast(d); + }) == prefix + "too short"); + + std::vector bad = image; + bad[0] = 'X'; + CHECK(load_result(bad, image_check::full) == prefix + "unknown format"); + bad = image; + bad[4] = 2; // version + CHECK(load_result(bad, image_check::full) == prefix + "unknown format"); + for (std::size_t reserved = 32; reserved < 64; reserved += 8) + { + bad = image; + bad[reserved + 3] = 1; + CHECK(load_result(bad, image_check::none) == prefix + "unknown format"); + } + + const auto sizes = [&](std::size_t offset, std::uint64_t v) + { + std::vector b = image; + set_header_field(b, offset, v); + return load_result(b, image_check::none); + }; + CHECK(sizes(8, 0) == prefix + "sizes out of range"); // no nodes + CHECK(sizes(8, 1000) == prefix + "sizes out of range"); // more nodes than bytes + CHECK(sizes(8, 0xFFFFFFF0u) == prefix + "sizes out of range"); + CHECK(sizes(16, 0xFFFFFFF0u) == prefix + "sizes out of range"); // text size + CHECK(sizes(16, header_field(image, 16) + 1) == prefix + "sizes out of range"); + CHECK(sizes(16, image.size()) == prefix + "sizes out of range"); + CHECK(sizes(24, 0xFFFFFFF0u) == prefix + "sizes out of range"); // decoded string size + CHECK(sizes(24, header_field(image, 24) - 1) == prefix + "sizes out of range"); + + // the NULs after the text and the decoded strings + bad = image; + bad[text_at(image) + header_size_field(image, 16)] = 'x'; + CHECK(load_result(bad, image_check::none) == prefix + "sizes out of range"); + bad = image; + bad.back() = 'x'; + CHECK(load_result(bad, image_check::none) == prefix + "sizes out of range"); + // nothing after the image + bad = image; + bad.push_back(0); + CHECK(load_result(bad, image_check::none) == prefix + "sizes out of range"); + // nodes, but not even room for the NULs + bad.assign(image.begin(), image.begin() + static_cast(text_at(image))); + CHECK(load_result(bad, image_check::none) == prefix + "sizes out of range"); + } +} + +TEST_CASE("json_view images: check") +{ + // nodes: 0 { 1 "s" 2 "x\"y" (escaped) 3 "i" 4 -12 5 "u" 6 7 7 "f" 8 1.5e300 9 "b" 10 true 11 "n" 12 null + // 13 "a" 14 [ 15 "t" 16 {} ] + const std::string text = R"({"s":"x\"y","i":-12,"u":7,"f":1.5e300,"b":true,"n":null,"a":["t",{}]})"; + const std::vector image = json_document::parse(text).save(); + REQUIRE(load_result(image, image_check::full).empty()); + REQUIRE(node_count(image) == 17); + + // bounds: rejected by both checks; content: only by the full one + const auto rejected = [&](const std::vector& b, bool bounds) + { + CHECK(load_result(b, image_check::full) == check_failed); + CHECK(load_result(b, image_check::bounds) == (bounds ? check_failed : "")); + }; + + SECTION("kinds") + { + const std::array kinds = {{8, 9, 10, 200}}; // binary, discarded, link, unknown + for (const std::uint8_t kind : kinds) + { + rejected(corrupted(image, 12, [&](node & n) + { + n.kind = kind; + }), true); + } + // a key that is not a string + rejected(corrupted(image, 1, [](node & n) + { + n.kind = 0; + n.len = 0; + n.off = 0; + }), true); + } + + SECTION("flags and extra") + { + rejected(corrupted(image, 12, [](node & n) + { + n.flags = 4; + }), true); + rejected(corrupted(image, 12, [](node & n) + { + n.extra = 1; + }), true); + rejected(corrupted(image, 10, [](node & n) + { + n.flags = 5; + }), true); + rejected(corrupted(image, 10, [](node & n) + { + n.extra = 1; + }), true); + rejected(corrupted(image, 1, [](node & n) + { + n.flags = 2; // a string in the edit arena + }), true); + rejected(corrupted(image, 1, [](node & n) + { + n.extra = 3; + }), true); + rejected(corrupted(image, 4, [](node & n) + { + n.flags = 2; + }), true); + rejected(corrupted(image, 4, [](node & n) + { + n.extra = static_cast(n.extra | 0x100u); // an integer with fraction digits + }), true); + rejected(corrupted(image, 0, [](node & n) + { + n.flags = 8; // moved + }), true); + rejected(corrupted(image, 0, [](node & n) + { + n.extra = 1; // a hash index + }), true); + // the flag of the targets of links (editable documents) is not part of an image + for (std::size_t i = 0; i < node_count(image); ++i) + { + CAPTURE(i) + rejected(corrupted(image, i, [](node & n) + { + n.flags = static_cast(n.flags | nlohmann::detail::view::node_flags::linked); + }), true); + } + } + + SECTION("bounds") + { + const std::size_t text_size = header_size_field(image, 16); + const std::size_t arena_size = header_size_field(image, 24); + rejected(corrupted(image, 1, [&](node & n) + { + n.off = static_cast(text_size + 1); + }), true); + rejected(corrupted(image, 1, [&](node & n) + { + n.len = static_cast(text_size); + }), true); + rejected(corrupted(image, 2, [&](node & n) + { + n.len = static_cast(arena_size + 1); + }), true); + rejected(corrupted(image, 6, [&](node & n) + { + n.off = static_cast(text_size); + }), true); + rejected(corrupted(image, 6, [&](node & n) + { + n.off = static_cast(text_size + 5); + }), true); + rejected(corrupted(image, 6, [](node & n) + { + n.extra = 0; // no digits + }), true); + rejected(corrupted(image, 8, [&](node & n) + { + n.len = static_cast(text_size); + }), true); + rejected(corrupted(image, 8, [](node & n) + { + n.len = 2; // shorter than the recorded digits + }), true); + rejected(corrupted(image, 14, [&](node & n) + { + n.off = static_cast(text_size + 1); + }), true); + // literals: their offset sizes the output of dump() + rejected(corrupted(image, 10, [&](node & n) + { + n.off = static_cast(text_size + 1); + }), true); + rejected(corrupted(image, 12, [&](node & n) + { + n.off = 0xFFFFFFFFu; + }), true); + } + + SECTION("structure") + { + rejected(corrupted(image, 0, [](node & n) + { + n.next = 0; + }), true); + rejected(corrupted(image, 0, [](node & n) + { + n.next = 18; // beyond the image + }), true); + rejected(corrupted(image, 14, [](node & n) + { + n.next = 4; // beyond the enclosing object + }), true); + rejected(corrupted(image, 0, [](node & n) + { + n.len = 6; // member count + }), true); + rejected(corrupted(image, 14, [](node & n) + { + n.len = 3; // element count + }), true); + rejected(corrupted(image, 0, [](node & n) + { + n.next = 14; // the object ends after the key "a" + n.len = 7; + }), true); + rejected(corrupted(image, 0, [](node & n) + { + n.next = 13; // nodes after the root + n.len = 6; + }), true); + rejected(corrupted(image, 0, [](node & n) + { + n.kind = 2; // an array: the "keys" are values, and the counts do not match + }), true); + const std::vector as_array = corrupted(image, 16, [](node & n) + { + n.kind = 2; // {} as []: fine + }); + CHECK(load_result(as_array, image_check::full).empty()); + const std::string expected_dump = R"({"s":"x\"y","i":-12,"u":7,"f":1.5e+300,"b":true,"n":null,"a":["t",[]]})"; + CHECK(loaded_dump(as_array) == expected_dump); + } + + SECTION("strings") + { + // a quote in a source string (the full check only) + std::vector b = image; + const std::size_t t = text_at(image); + const node t15 = node_at(image, 15); + b[t + t15.off] = '"'; + rejected(b, false); + // a control character + b[t + t15.off] = '\n'; + rejected(b, false); + // invalid UTF-8 in a decoded string + b = image; + const node s2 = node_at(image, 2); + b[t + header_size_field(image, 16) + 1 + s2.off] = 0xFF; + rejected(b, false); + } + + SECTION("numbers") + { + const std::size_t t = text_at(image); + const node i4 = node_at(image, 4); + const node u6 = node_at(image, 6); + const node f8 = node_at(image, 8); + const auto at_token = [&](const node & n, std::size_t k, std::uint8_t c) + { + std::vector b = image; + b[t + n.off + k] = c; + return b; + }; + rejected(at_token(i4, 1, 'x'), false); // -x2 + rejected(at_token(i4, 1, '0'), false); // -02 + rejected(at_token(i4, 0, '1'), false); // 112 != -12 + rejected(at_token(f8, 1, 'x'), false); // 1x5e300 + rejected(at_token(f8, 2, 'e'), false); // 1.ee300 + rejected(at_token(f8, 4, 'x'), false); // 1.5ex00 + rejected(at_token(f8, 3, '0'), false); // 1.50300: another layout + rejected(at_token(f8, 4, '9'), false); // 1.5e900: overflow + rejected(at_token(f8, 0, 'x'), false); + rejected(at_token(u6, 0, '8'), false); // 8 != 7 + rejected(corrupted(image, 4, [](node & n) + { + n.kind = 6; // "-12" as unsigned: a sign + n.extra = 3; + }), false); + // a non-negative number_integer (as edits write it): fine + const std::vector positive = corrupted(image, 6, [](node & n) + { + n.kind = 5; + n.extra = 0; + }); + CHECK(load_result(positive, image_check::full).empty()); + const json_document pos = json_document::load(positive); + CHECK(pos.root()["u"].is_number_integer()); + rejected(corrupted(image, 8, [](node & n) + { + n.kind = 6; // a float token as integer + n.extra = 7; + }), false); + } + + SECTION("float tokens of an image checked for bounds only") + { + // A float node whose layout records "many" digits is converted from + // its token alone; a token that is not a JSON number reads as 0. + const std::vector img = json_document::parse("[1.5e300,2]").save(); + const std::size_t t = text_at(img); + for (const char* token : + { + "x.5e300", "01.5e30", "1.xe300", "1.5ex00", "1.5e+x0", "1.5e30x", "-.5e300", "1.5E300" + }) + { + CAPTURE(token) + std::vector b = img; + node n = node_at(b, 1); + n.extra = 0xFFFFu; + set_node(b, 1, n); + std::memcpy(b.data() + t + n.off, token, n.len); + const json_document d = json_document::load(b, image_check::bounds); + const auto v = d.root()[0].get(); + CHECK(v == (std::string(token) == "1.5E300" ? 1.5e300 : 0.0)); + CHECK(load_result(b, image_check::full) == (std::string(token) == "1.5E300" ? "" : check_failed)); + } + } + + SECTION("nodes that share a range") + { + // [big string, then n strings made to point to the big string]: every + // node shares the one range (the check reads it once) + const std::size_t n = 200; + std::string long_text = "[\"" + std::string(5000, 'a') + "\""; + for (std::size_t k = 0; k < n; ++k) + { + long_text += ",\"x\""; + } + long_text += "]"; + const std::vector img = json_document::parse(long_text).save(); + const node big = node_at(img, 1); + std::vector b = img; + for (std::size_t k = 0; k < n; ++k) + { + set_node(b, 2 + k, big); + } + CHECK(load_result(b, image_check::full).empty()); + const json_document d = json_document::load(b); + CHECK(d.root().size() == n + 1); + CHECK(d.root()[n].get() == std::string(5000, 'a')); + CHECK(d.root().dump() == loaded_dump(b, image_check::none)); + + // the same for decoded strings and float tokens, with a node that + // records another digit layout than its token + const std::vector img2 = json_document::parse(R"(["a\"b", "a\"b", 1.25, 1.25, 1.25e3])").save(); + CHECK(load_result(img2, image_check::full).empty()); + std::vector same = img2; + set_node(same, 2, node_at(img2, 1)); + set_node(same, 4, node_at(img2, 3)); + CHECK(load_result(same, image_check::full).empty()); + // (a raw string with a backslash must not be a macro argument: MSVC C2017) + const std::string expected_same = R"(["a\"b","a\"b",1.25,1.25,1250.0])"; + CHECK(loaded_dump(same) == expected_same); + std::vector layout = same; + node f4 = node_at(layout, 4); + f4.extra = 0x0100u; // the layout of "1.", and not that of "1.25" + set_node(layout, 4, f4); + rejected(layout, false); + f4.extra = 0xFFFFu; // "many" digits: fine, as compaction writes it + set_node(layout, 4, f4); + CHECK(load_result(layout, image_check::full).empty()); + } + + SECTION("ranges that overlap") + { + // save() writes every string and every token to a place of its own + // (nodes that share a value share the whole range): ranges that + // overlap without being identical are a damaged image, though each + // range is a valid string or token + // nodes: 0 [ 1 "abcdef" 2 "ghijkl" 3 1.2525 4 9.9 5 "a\"bcd" (decoded) 6 "e\"fgh" (decoded) + const std::vector img = json_document::parse(R"(["abcdef", "ghijkl", 1.2525, 9.9, "a\"bcd", "e\"fgh"])").save(); + REQUIRE(load_result(img, image_check::full).empty()); + // node i with the range (off of node of + shift, length), and extra + const auto aliased = [&](std::size_t i, std::size_t of, std::uint32_t shift, std::uint32_t length, std::uint16_t extra) + { + std::vector b = img; + node n = node_at(b, of); + n.off += shift; + n.len = length; + n.extra = extra; + set_node(b, i, n); + return b; + }; + // source strings + rejected(aliased(2, 1, 0, 4, 0), false); // "abcd": the start of another string + rejected(aliased(2, 1, 1, 4, 0), false); // "bcde": inside + rejected(aliased(2, 1, 2, 4, 0), false); // "cdef": the end + CHECK(load_result(aliased(2, 1, 0, 6, 0), image_check::full).empty()); // the whole range: identical + // decoded strings: inside, and partially overlapping (the arena holds a"bcde"fgh) + rejected(aliased(6, 5, 1, 3, 0), false); + rejected(aliased(6, 5, 3, 5, 0), false); + CHECK(load_result(aliased(6, 5, 0, 5, 0), image_check::full).empty()); + // float tokens: "1.25" and "525" (an integer token as a float) inside "1.2525" + rejected(aliased(4, 3, 0, 4, 0x0201u), false); + rejected(aliased(4, 3, 3, 3, 0x0003u), false); + CHECK(load_result(aliased(4, 3, 0, 6, 0x0401u), image_check::full).empty()); + // a string and a float token may use the same bytes (each is checked by its own kind) + std::vector shared = img; + node as_string = node_at(img, 2); + as_string.off = node_at(img, 3).off; + as_string.len = 6; + set_node(shared, 2, as_string); + CHECK(load_result(shared, image_check::full).empty()); + // empty strings inside others are not ranges + CHECK(load_result(aliased(2, 1, 2, 0, 0), image_check::full).empty()); + } + + SECTION("copies within an edited document") + { + // copies within a document share the value: identical ranges + json_editable_document d = json_editable_document::parse(R"({"s": "ab", "e": "x\"y", "f": 1.5, "g": 1.5e300, "a": [1.5, "ab"]})"); + d.set(d.root(), "c1", d.root()["a"]); + d.set(d.root(), "c2", d.root()["a"]); + d.set(d.root(), "c3", d.root()); + d.push_back(d.root()["a"], d.root()["e"]); + d.push_back(d.root()["a"], d.root()["g"]); + d.push_back(d.root()["a"], d.root()["g"]); + d.set(d.root()["s"], "changed"); + d.set(d.root()["f"], 7.25); + const std::vector saved = d.save(); + CHECK(load_result(saved, image_check::full).empty()); + CHECK(loaded_dump(saved) == d.root().dump()); + check_round_trip(d); + } + + SECTION("integer ranges") + { + // tokens of many digits, which the parser stores as floats + const std::string big = R"([123456789012345678901234, 99999999999999999999, 9223372036854775808])"; + const std::vector img = json_document::parse(big).save(); + const auto as_integer = [&](std::size_t i, std::uint8_t kind, std::uint16_t extra) + { + std::vector b = img; + node n = node_at(b, i); + n.kind = kind; + n.extra = extra; + set_node(b, i, n); + return load_result(b, image_check::full); + }; + CHECK(as_integer(1, 6, 24) == check_failed); // more than 20 digits + CHECK(as_integer(2, 6, 20) == check_failed); // more than 2^64 - 1 + CHECK(as_integer(3, 5, 18) == check_failed); // more than 2^63 - 1 as number_integer + } +} + +TEST_CASE("json_view images: damaged images") +{ + // A damaged image must be rejected, or read safely; with the full check, + // it also serializes to the JSON it reads as. + const std::vector texts = + { + R"({"a": [1, -2, 3.25, "x\u00e9y", true, null], "b": {"c": "\"q\"", "d": 1e10}, "e": ""})", + R"([[[[]]], {"k": {"k": {"k": 12345678901234567890}}}, "\ud83d\ude00", -0.0, 0])", + }; + for (const std::string& text : texts) + { + const std::vector image = json_document::parse(text).save(); + for (int round = 0; round < 3000; ++round) + { + std::vector b = image; + const std::uint32_t flips = 1 + (rng() % 3); + for (std::uint32_t k = 0; k < flips; ++k) + { + // mostly the nodes, where the damage matters most + const std::size_t at = rng() % 4 != 0 ? header_size + (rng() % (b.size() - header_size)) : rng() % b.size(); + b[at] = static_cast(rng() % 3 == 0 ? rng() : b[at] ^ (1u << (rng() % 8))); + } + for (const image_check check : + { + image_check::full, image_check::bounds + }) + { + json_document d; + try + { + d = json_document::load(b, check); + } + catch (const json::parse_error& e) + { + CHECK(e.id == 116); + continue; + } + std::string dumped; + std::string dumped_ascii; + try + { + dumped = d.root().dump(); + dumped_ascii = d.root().dump(-1, ' ', true); + } + catch (const json::type_error& e) + { + // invalid UTF-8 (the bounds check only) + CHECK(check == image_check::bounds); + CHECK(e.id == 316); + continue; + } + const json j = d.root().materialize(); + if (check == image_check::full) + { + CHECK(json::parse(dumped) == j); + CHECK(json::parse(dumped_ascii) == j); + } + } + } + } +} +#endif + +#else + +TEST_CASE("json_view images: big-endian targets") +{ + const json_document d = json_document::parse("[1]"); + CHECK_THROWS_WITH_AS(d.save(), "[json.exception.type_error.320] json_document images need a little-endian target", json::type_error&); +} + +#endif