diff --git a/docs/mkdocs/docs/api/basic_json_view/materialize.md b/docs/mkdocs/docs/api/basic_json_view/materialize.md index ba57e4f47..982b1597f 100644 --- a/docs/mkdocs/docs/api/basic_json_view/materialize.md +++ b/docs/mkdocs/docs/api/basic_json_view/materialize.md @@ -43,9 +43,13 @@ and [`end_pos()`](../basic_json/end_pos.md) return `std::string::npos` for - an array or object whose elements were changed (its elements keep theirs), and - an array or object whose last element has no positions (its end is found from that element). -A `basic_json` value, in contrast, keeps its positions when it is changed. Documents [loaded](../basic_json_document/load.md) -from an image have the positions of the saved document, except that its strings with escapes have none if it was saved -without `JSON_DIAGNOSTIC_POSITIONS`. +This differs from a `basic_json` value, whose positions are not updated when it is changed: they stay as they were +parsed, even if they no longer match the value (see [`start_pos()`](../basic_json/start_pos.md)). + +Documents [loaded](../basic_json_document/load.md) from an image have the positions of the saved document. Its strings +with escapes have positions only if the document was parsed with `JSON_DIAGNOSTIC_POSITIONS`: the position of their +opening quote is recorded while parsing, and `save()` and `load()` keep it whether or not they are compiled with the +macro. Calling `materialize()` on the same view repeatedly builds a new, independent `BasicJsonType` value each time; it never caches the result. diff --git a/docs/mkdocs/docs/home/architecture.md b/docs/mkdocs/docs/home/architecture.md index f27b45b97..76b3e1af1 100644 --- a/docs/mkdocs/docs/home/architecture.md +++ b/docs/mkdocs/docs/home/architecture.md @@ -203,7 +203,7 @@ packet-beta | 2-3 | `extra` | `uint16_t` | numbers: the number of integer digits (low byte) and fraction digits (high byte), 255 for more; objects: the number of their key hash index (1-based), or 0; otherwise 0 | | 4-7 | `off` | `uint32_t` | where the value starts: the first byte after a string's opening quote (or its position in the buffer of decoded strings), the first byte of a number or literal, the bracket of an array or object | | 8-11 | `len` | `uint32_t` | strings: the length after decoding; floats and literals: the length of the token; arrays and objects: the number of elements | -| 12-15 | `next` | `uint32_t` | arrays and objects: the number of nodes of the subtree, including the node itself; strings with escapes: the position of the opening quote in the source plus one, or 0 (only with [`JSON_DIAGNOSTIC_POSITIONS`](../api/macros/json_diagnostic_positions.md)) | +| 12-15 | `next` | `uint32_t` | arrays and objects: the number of nodes of the subtree, including the node itself; strings with escapes: the position of the opening quote in the source plus one if the document was parsed with [`JSON_DIAGNOSTIC_POSITIONS`](../api/macros/json_diagnostic_positions.md), else 0; otherwise 0 |

Fields of a node, by their byte offset within the 16-byte node. Integers use bytes 8-15 differently, see below.

@@ -219,8 +219,9 @@ packet-beta - **Offsets** are 32 bits wide, so a document is limited to 4294967279 bytes, 4 GiB minus 16 bytes (a margin below 2^32 for positions one scanner step past the end of the text; `out_of_range.416`). - **Source positions** ([`JSON_DIAGNOSTIC_POSITIONS`](../api/macros/json_diagnostic_positions.md)) are derived by - [`materialize()`](../api/basic_json_view/materialize.md) and cost nothing while parsing: a scalar's range follows - from `off` and its length, a string with escapes is scanned from its opening quote (`next`) to the closing one, + [`materialize()`](../api/basic_json_view/materialize.md) and cost almost nothing while parsing: the only work is + storing the offset of the opening quote of a string with escapes in its `next`, which is otherwise unused. A + scalar's range follows from `off` and its length, a string with escapes is scanned from its opening quote (`next`) to the closing one, and the end of an array or object is found by scanning from the end of its last element over whitespace, comments, and a trailing comma. `materialize()` hands these positions to the SAX builder of `parse()` in place of its lexer. - **Large objects** (128 members or more) get a hash index for their keys after parsing diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 1a528e36b..0ceec61b7 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -233,7 +233,7 @@ struct diagnostic_positions // (npos: a value without a source position, see materialize() of // json_view) const std::size_t read = lexer->get_chars_read_total(); - v.start_position = read == std::string::npos ? read : read - 1; + v.start_position = read == std::string::npos ? std::string::npos : read - 1; } } @@ -252,7 +252,7 @@ struct diagnostic_positions { // a range has both ends or none (exception::diagnostics() // prints only complete ones) - v.start_position = v.end_position; + v.start_position = std::string::npos; } } } @@ -273,7 +273,7 @@ struct diagnostic_positions if (v.end_position == std::string::npos) { // a value without a source position - v.start_position = v.end_position; + v.start_position = std::string::npos; return; } diff --git a/include/nlohmann/detail/view/builder.hpp b/include/nlohmann/detail/view/builder.hpp index 34fb0f304..aa8605966 100644 --- a/include/nlohmann/detail/view/builder.hpp +++ b/include/nlohmann/detail/view/builder.hpp @@ -1196,10 +1196,11 @@ indent_done: #if JSON_DIAGNOSTIC_POSITIONS // the decoded string is in the arena: next keeps the source offset // of the opening quote plus one (0: unknown) for materialize() - push_node(node_kind::string, node_flags::escaped, 0, r.start, static_cast(r.len) | (static_cast(s - b) << 32)); + const std::uint64_t second = static_cast(r.len) | (static_cast(s - b) << 32); #else - push_node(node_kind::string, node_flags::escaped, 0, r.start, r.len); + const std::uint64_t second = r.len; #endif + push_node(node_kind::string, node_flags::escaped, 0, r.start, second); return true; } }; diff --git a/include/nlohmann/detail/view/materialize.hpp b/include/nlohmann/detail/view/materialize.hpp index ef403dfdd..0b7d91d71 100644 --- a/include/nlohmann/detail/view/materialize.hpp +++ b/include/nlohmann/detail/view/materialize.hpp @@ -32,7 +32,9 @@ namespace view Stands in for the lexer of json_sax_dom_parser, which reads the positions of the values it stores from these three functions (diagnostic_positions). npos -is a value without a source position. +is a value without a source position. One position_source serves a whole +materialize() call: before each value is handed to the handler, the range of +that value is set (or cleared), overwriting that of the value before. */ struct position_source { @@ -65,9 +67,20 @@ struct position_source /// the range [first, last), or none if either is npos void set(std::size_t first, std::size_t last) noexcept { - const bool known = first != std::string::npos && last != std::string::npos; - start = known ? first : std::string::npos; - end = known ? last : std::string::npos; + if (first == std::string::npos || last == std::string::npos) + { + clear(); + return; + } + start = first; + end = last; + } + + /// no range: the next value has no source position + void clear() noexcept + { + start = std::string::npos; + end = std::string::npos; } /// one past the closing quote of the string whose opening quote is at @@ -140,58 +153,64 @@ struct position_source /// the range of the scalar n of document d void set_scalar(const document_data& d, const node& n) noexcept { - const std::size_t size = d.size; - const std::size_t off = n.off; + if ((n.flags & node_flags::is_new) != 0) + { + clear(); + return; + } std::size_t first = std::string::npos; std::size_t last = std::string::npos; - if ((n.flags & node_flags::is_new) == 0) + switch (n.kind) { - switch (static_cast(n.kind)) - { - case value_t::string: - if ((n.flags & node_flags::storage) == 0) + case node_kind::string: + if ((n.flags & node_flags::storage) == 0) + { + // off is the first byte after the opening quote + if (n.off != 0) { - // off is the first byte after the opening quote - if (off != 0) - { - first = off - 1; - last = off + std::size_t{n.len} + 1; - } + first = n.off - 1; + last = n.off + std::size_t{n.len} + 1; } - else if ((n.flags & node_flags::storage) == node_flags::escaped && n.next != 0) - { - // next is the offset of the opening quote plus one - first = n.next - 1; - last = string_end(d.base[0], size, first); - } - break; - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - if ((n.flags & node_flags::storage) == 0) - { - first = off; - last = off + number_length(n); - } - break; - case value_t::boolean: - first = off; - last = off + ((n.flags & node_flags::is_true) != 0 ? 4u : 5u); - break; - case value_t::null: - first = off; - last = off + 4u; - break; - case value_t::object: - case value_t::array: - case value_t::binary: - case value_t::discarded: - default: - break; - } + } + else if ((n.flags & node_flags::storage) == node_flags::escaped && n.next != 0) + { + // next is the offset of the opening quote plus one + first = n.next - 1; + last = string_end(d.base[0], d.size, first); + } + break; + case node_kind::number_integer: + case node_kind::number_unsigned: + case node_kind::number_float: + if ((n.flags & node_flags::storage) == 0) + { + first = n.off; + last = n.off + number_length(n); + } + break; + case node_kind::boolean: + first = n.off; + last = n.off + ((n.flags & node_flags::is_true) != 0 ? 4u : 5u); + break; + case node_kind::null: + first = n.off; + last = n.off + 4u; + break; + case node_kind::object: + case node_kind::array: + case node_kind::link: + default: + break; } // (a crafted image can point past the text) - set(first, last != std::string::npos && last <= size ? last : std::string::npos); + if (last == std::string::npos || last > d.size) + { + clear(); + } + else + { + set(first, last); + } } }; #endif @@ -214,11 +233,13 @@ template BasicJsonType materialize(const document_data& d, const node* n) { using string_t = typename BasicJsonType::string_t; + using input_t = iterator_input_adapter; #if JSON_DIAGNOSTIC_POSITIONS - using sax_t = json_sax_dom_parser, true, position_source>; + using lexer_t = position_source; #else - using sax_t = json_sax_dom_parser, true>; + using lexer_t = lexer; // (the default: unused, as no lexer is passed) #endif + using sax_t = json_sax_dom_parser; using nav = navigation; // the open containers: their next element (or key of the next member), @@ -306,7 +327,14 @@ BasicJsonType materialize(const document_data& d, const node* n) // (moved containers are also new) const std::size_t start = (n->flags & node_flags::is_new) == 0 && n->off < d.size ? std::size_t{n->off} : std::string::npos; - positions.set(start, start + 1); + if (start == std::string::npos) + { + positions.clear(); + } + else + { + positions.set(start, start + 1); + } #endif if (object) { @@ -324,14 +352,14 @@ BasicJsonType materialize(const document_data& d, const node* n) open = deep.data(); capacity = deep.size(); } + // (with positions: until an element is read, the container's + // own bracket is the last thing read) + open[depth++] = frame{nav::first(d, n), nav::end(d, n), object #if JSON_DIAGNOSTIC_POSITIONS - // (until an element is read, the container's own bracket is the last thing read) - open[depth++] = frame{nav::first(d, n), nav::end(d, n), object, start, positions.end}; - break; -#else - open[depth++] = frame {nav::first(d, n), nav::end(d, n), object}; - break; + , start, positions.end #endif + }; + break; } case value_t::string: { @@ -372,9 +400,19 @@ BasicJsonType materialize(const document_data& d, const node* n) if (f.pos == f.end) { #if JSON_DIAGNOSTIC_POSITIONS - const std::size_t end = f.start == std::string::npos || f.last_end == std::string::npos ? std::string::npos : position_source::container_end(d.base[0], d.size, f.last_end); - // (an end of npos clears the start, see diagnostic_positions) - positions.set(end == std::string::npos ? end : end - 1, end); + const std::size_t end = f.start == std::string::npos || f.last_end == std::string::npos ? std::string::npos : + position_source::container_end(d.base[0], d.size, f.last_end); + if (end == std::string::npos) + { + positions.clear(); + } + else + { + // the closing bracket, as the lexer reports it (end_object() + // and end_array() only read its end; the start of the + // container was set when it was opened) + positions.set(end - 1, end); + } if (depth > 1) { open[depth - 2].last_end = end; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index c04ca7ac6..5f829a88a 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -15632,7 +15632,7 @@ struct diagnostic_positions // (npos: a value without a source position, see materialize() of // json_view) const std::size_t read = lexer->get_chars_read_total(); - v.start_position = read == std::string::npos ? read : read - 1; + v.start_position = read == std::string::npos ? std::string::npos : read - 1; } } @@ -15651,7 +15651,7 @@ struct diagnostic_positions { // a range has both ends or none (exception::diagnostics() // prints only complete ones) - v.start_position = v.end_position; + v.start_position = std::string::npos; } } } @@ -15672,7 +15672,7 @@ struct diagnostic_positions if (v.end_position == std::string::npos) { // a value without a source position - v.start_position = v.end_position; + v.start_position = std::string::npos; return; } diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index e6b958f19..41eb15940 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -2405,10 +2405,11 @@ indent_done: #if JSON_DIAGNOSTIC_POSITIONS // the decoded string is in the arena: next keeps the source offset // of the opening quote plus one (0: unknown) for materialize() - push_node(node_kind::string, node_flags::escaped, 0, r.start, static_cast(r.len) | (static_cast(s - b) << 32)); + const std::uint64_t second = static_cast(r.len) | (static_cast(s - b) << 32); #else - push_node(node_kind::string, node_flags::escaped, 0, r.start, r.len); + const std::uint64_t second = r.len; #endif + push_node(node_kind::string, node_flags::escaped, 0, r.start, second); return true; } }; @@ -6126,7 +6127,9 @@ namespace view Stands in for the lexer of json_sax_dom_parser, which reads the positions of the values it stores from these three functions (diagnostic_positions). npos -is a value without a source position. +is a value without a source position. One position_source serves a whole +materialize() call: before each value is handed to the handler, the range of +that value is set (or cleared), overwriting that of the value before. */ struct position_source { @@ -6159,9 +6162,20 @@ struct position_source /// the range [first, last), or none if either is npos void set(std::size_t first, std::size_t last) noexcept { - const bool known = first != std::string::npos && last != std::string::npos; - start = known ? first : std::string::npos; - end = known ? last : std::string::npos; + if (first == std::string::npos || last == std::string::npos) + { + clear(); + return; + } + start = first; + end = last; + } + + /// no range: the next value has no source position + void clear() noexcept + { + start = std::string::npos; + end = std::string::npos; } /// one past the closing quote of the string whose opening quote is at @@ -6234,58 +6248,64 @@ struct position_source /// the range of the scalar n of document d void set_scalar(const document_data& d, const node& n) noexcept { - const std::size_t size = d.size; - const std::size_t off = n.off; + if ((n.flags & node_flags::is_new) != 0) + { + clear(); + return; + } std::size_t first = std::string::npos; std::size_t last = std::string::npos; - if ((n.flags & node_flags::is_new) == 0) + switch (n.kind) { - switch (static_cast(n.kind)) - { - case value_t::string: - if ((n.flags & node_flags::storage) == 0) + case node_kind::string: + if ((n.flags & node_flags::storage) == 0) + { + // off is the first byte after the opening quote + if (n.off != 0) { - // off is the first byte after the opening quote - if (off != 0) - { - first = off - 1; - last = off + std::size_t{n.len} + 1; - } + first = n.off - 1; + last = n.off + std::size_t{n.len} + 1; } - else if ((n.flags & node_flags::storage) == node_flags::escaped && n.next != 0) - { - // next is the offset of the opening quote plus one - first = n.next - 1; - last = string_end(d.base[0], size, first); - } - break; - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - if ((n.flags & node_flags::storage) == 0) - { - first = off; - last = off + number_length(n); - } - break; - case value_t::boolean: - first = off; - last = off + ((n.flags & node_flags::is_true) != 0 ? 4u : 5u); - break; - case value_t::null: - first = off; - last = off + 4u; - break; - case value_t::object: - case value_t::array: - case value_t::binary: - case value_t::discarded: - default: - break; - } + } + else if ((n.flags & node_flags::storage) == node_flags::escaped && n.next != 0) + { + // next is the offset of the opening quote plus one + first = n.next - 1; + last = string_end(d.base[0], d.size, first); + } + break; + case node_kind::number_integer: + case node_kind::number_unsigned: + case node_kind::number_float: + if ((n.flags & node_flags::storage) == 0) + { + first = n.off; + last = n.off + number_length(n); + } + break; + case node_kind::boolean: + first = n.off; + last = n.off + ((n.flags & node_flags::is_true) != 0 ? 4u : 5u); + break; + case node_kind::null: + first = n.off; + last = n.off + 4u; + break; + case node_kind::object: + case node_kind::array: + case node_kind::link: + default: + break; } // (a crafted image can point past the text) - set(first, last != std::string::npos && last <= size ? last : std::string::npos); + if (last == std::string::npos || last > d.size) + { + clear(); + } + else + { + set(first, last); + } } }; #endif @@ -6308,11 +6328,13 @@ template BasicJsonType materialize(const document_data& d, const node* n) { using string_t = typename BasicJsonType::string_t; + using input_t = iterator_input_adapter; #if JSON_DIAGNOSTIC_POSITIONS - using sax_t = json_sax_dom_parser, true, position_source>; + using lexer_t = position_source; #else - using sax_t = json_sax_dom_parser, true>; + using lexer_t = lexer; // (the default: unused, as no lexer is passed) #endif + using sax_t = json_sax_dom_parser; using nav = navigation; // the open containers: their next element (or key of the next member), @@ -6400,7 +6422,14 @@ BasicJsonType materialize(const document_data& d, const node* n) // (moved containers are also new) const std::size_t start = (n->flags & node_flags::is_new) == 0 && n->off < d.size ? std::size_t{n->off} : std::string::npos; - positions.set(start, start + 1); + if (start == std::string::npos) + { + positions.clear(); + } + else + { + positions.set(start, start + 1); + } #endif if (object) { @@ -6418,14 +6447,14 @@ BasicJsonType materialize(const document_data& d, const node* n) open = deep.data(); capacity = deep.size(); } + // (with positions: until an element is read, the container's + // own bracket is the last thing read) + open[depth++] = frame{nav::first(d, n), nav::end(d, n), object #if JSON_DIAGNOSTIC_POSITIONS - // (until an element is read, the container's own bracket is the last thing read) - open[depth++] = frame{nav::first(d, n), nav::end(d, n), object, start, positions.end}; - break; -#else - open[depth++] = frame {nav::first(d, n), nav::end(d, n), object}; - break; + , start, positions.end #endif + }; + break; } case value_t::string: { @@ -6466,9 +6495,19 @@ BasicJsonType materialize(const document_data& d, const node* n) if (f.pos == f.end) { #if JSON_DIAGNOSTIC_POSITIONS - const std::size_t end = f.start == std::string::npos || f.last_end == std::string::npos ? std::string::npos : position_source::container_end(d.base[0], d.size, f.last_end); - // (an end of npos clears the start, see diagnostic_positions) - positions.set(end == std::string::npos ? end : end - 1, end); + const std::size_t end = f.start == std::string::npos || f.last_end == std::string::npos ? std::string::npos : + position_source::container_end(d.base[0], d.size, f.last_end); + if (end == std::string::npos) + { + positions.clear(); + } + else + { + // the closing bracket, as the lexer reports it (end_object() + // and end_array() only read its end; the start of the + // container was set when it was opened) + positions.set(end - 1, end); + } if (depth > 1) { open[depth - 2].last_end = end;