Niels Lohmann
54f6b3ef06
* json_view: source positions in materialize() with JSON_DIAGNOSTIC_POSITIONS With JSON_DIAGNOSTIC_POSITIONS, materialize() now gives every value the start_pos()/end_pos() that basic_json::parse() would give it, so exceptions about materialized values name the same bytes (#5864). - json_sax_dom_parser takes the lexer type as a defaulted template parameter; materialize() passes position_source, which reports the positions it derives from the index. diagnostic_positions treats npos as "no position" and keeps ranges all-or-nothing. - Strings with escapes keep the offset of their opening quote plus one in `next` (only with the macro); materialize() scans from there to the closing quote. The end of an array or object is found by scanning from the end of its last element over whitespace, comments and a trailing comma. - In editable documents, values written by edits, arrays and objects whose elements changed, and containers whose last element has no position get none. Images keep the flag is_new, and load() checks that `next` of a string is within the text. Nothing changes on the parse path without the macro. The view's own exceptions still carry no byte range, so the tests that compare them with those of materialized values skip it under the macro, as they skip the path under JSON_DIAGNOSTICS. Signed-off-by: Niels Lohmann <mail@nlohmann.me> * Indent comment in unit-json_view_from_json.cpp Signed-off-by: Niels Lohmann <mail@nlohmann.me> * Address review: clearer position code and docs - position_source gets clear(); set_scalar() returns early for new values, switches on node_kind, and uses no local copies; the one position_source per materialize() call is documented - no npos + 1 for containers without a position; a comment on why the end of a container is reported as its closing bracket - less duplication under #if in materialize() and the builder - diagnostic_positions sets npos explicitly - docs: positions of basic_json values after changes, positions in images depend on how the document was parsed, when next is set, and the cost of positions while parsing Signed-off-by: Niels Lohmann <mail@nlohmann.me> * Share the comment scanners of the view builder with materialize() skip_line_comment() and find_block_comment_end() in scan.hpp are used by the builder's comment() and by position_source::container_end(), which had its own copy. Signed-off-by: Niels Lohmann <mail@nlohmann.me> * Address review: if/else for container positions, clearer next field - the opening bracket of a container sets positions with an if/else instead of a ternary, and the frame takes its start from positions - the closing bracket of a container is found in nested if/else; the parent's last_end is taken from positions, so it is npos whenever no range is set - architecture.md: the next field reads "or 0 if it was not; all other values: 0" instead of "else 0; otherwise 0" Signed-off-by: Niels Lohmann <mail@nlohmann.me> * Fix GCC unused-result and clang-tidy raw-string findings in unit-json_view_positions.cpp Signed-off-by: Niels Lohmann <mail@nlohmann.me> --------- Signed-off-by: Niels Lohmann <mail@nlohmann.me>
448 lines
15 KiB
C++
448 lines
15 KiB
C++
// __ _____ _____ _____
|
|
// __| | __| | | | JSON for Modern C++
|
|
// | | |__ | | | | | | version 3.12.0
|
|
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
|
|
//
|
|
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
#pragma once
|
|
|
|
#include <array> // array
|
|
#include <cstddef> // size_t
|
|
#include <cstdint> // int64_t, uint8_t
|
|
#include <string> // string
|
|
#include <vector> // vector
|
|
|
|
#include <nlohmann/json.hpp>
|
|
#include <nlohmann/detail/view/document_data.hpp>
|
|
#include <nlohmann/detail/view/macro_scope.hpp>
|
|
#include <nlohmann/detail/view/node.hpp>
|
|
#include <nlohmann/detail/view/number.hpp>
|
|
#include <nlohmann/detail/view/scan.hpp>
|
|
|
|
NLOHMANN_JSON_NAMESPACE_BEGIN
|
|
namespace detail
|
|
{
|
|
namespace view
|
|
{
|
|
|
|
#if JSON_DIAGNOSTIC_POSITIONS
|
|
/*!
|
|
@brief the source positions of the values materialize() replays
|
|
|
|
Stands in for the lexer of json_sax_dom_parser, which reads the positions of
|
|
the values it stores from these three functions (diagnostic_positions). npos
|
|
is a value without a source position. One position_source serves a whole
|
|
materialize() call: before each value is handed to the handler, the range of
|
|
that value is set (or cleared), overwriting that of the value before.
|
|
*/
|
|
struct position_source
|
|
{
|
|
/// the token, of which diagnostic_positions only reads the length
|
|
struct token
|
|
{
|
|
std::size_t length;
|
|
std::size_t size() const noexcept
|
|
{
|
|
return length;
|
|
}
|
|
};
|
|
|
|
std::size_t start = std::string::npos; ///< the first byte of the next value
|
|
std::size_t end = std::string::npos; ///< one past the next value (start_object/start_array: one past the bracket)
|
|
|
|
std::size_t get_chars_read_total() const noexcept
|
|
{
|
|
return end;
|
|
}
|
|
std::size_t get_token_start_position() const noexcept
|
|
{
|
|
return start;
|
|
}
|
|
token get_string() const noexcept
|
|
{
|
|
return token{end - start};
|
|
}
|
|
|
|
/// the range [first, last), or none if either is npos
|
|
void set(std::size_t first, std::size_t last) noexcept
|
|
{
|
|
if (first == std::string::npos || last == std::string::npos)
|
|
{
|
|
clear();
|
|
return;
|
|
}
|
|
start = first;
|
|
end = last;
|
|
}
|
|
|
|
/// no range: the next value has no source position
|
|
void clear() noexcept
|
|
{
|
|
start = std::string::npos;
|
|
end = std::string::npos;
|
|
}
|
|
|
|
/// one past the closing quote of the string whose opening quote is at
|
|
/// src[quote], or npos (the scan is bounded by size, as images can be
|
|
/// loaded without a check)
|
|
static std::size_t string_end(const char* src, std::size_t size, std::size_t quote) noexcept
|
|
{
|
|
for (std::size_t i = quote + 1; i < size; ++i)
|
|
{
|
|
if (src[i] == '\\')
|
|
{
|
|
++i;
|
|
}
|
|
else if (src[i] == '"')
|
|
{
|
|
return i + 1;
|
|
}
|
|
}
|
|
return std::string::npos;
|
|
}
|
|
|
|
/// one past the closing bracket of the array or object whose last element
|
|
/// ends at src[pos] (or whose opening bracket is at src[pos - 1]), or npos;
|
|
/// in between, there can only be whitespace, comments, and a trailing comma
|
|
/// (the source was validated when it was parsed)
|
|
static std::size_t container_end(const char* src, std::size_t size, std::size_t pos) noexcept
|
|
{
|
|
const auto* const b = reinterpret_cast<const unsigned char*>(src); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the text as bytes, as the builder reads it
|
|
const unsigned char* const e = b + size;
|
|
while (pos < size)
|
|
{
|
|
switch (src[pos])
|
|
{
|
|
case ' ':
|
|
case '\t':
|
|
case '\n':
|
|
case '\r':
|
|
case ',':
|
|
++pos;
|
|
break;
|
|
case ']':
|
|
case '}':
|
|
return pos + 1;
|
|
case '/':
|
|
if (pos + 1 < size && src[pos + 1] == '/')
|
|
{
|
|
pos = static_cast<std::size_t>(skip_line_comment(b + pos + 2, e) - b);
|
|
break;
|
|
}
|
|
if (pos + 1 < size && src[pos + 1] == '*')
|
|
{
|
|
// (unterminated: pos is size, which ends the loop)
|
|
pos = static_cast<std::size_t>(find_block_comment_end(b + pos + 2, e) - b) + 2;
|
|
break;
|
|
}
|
|
return std::string::npos;
|
|
default:
|
|
return std::string::npos;
|
|
}
|
|
}
|
|
return std::string::npos;
|
|
}
|
|
|
|
/// the range of the scalar n of document d
|
|
void set_scalar(const document_data& d, const node& n) noexcept
|
|
{
|
|
if ((n.flags & node_flags::is_new) != 0)
|
|
{
|
|
clear();
|
|
return;
|
|
}
|
|
std::size_t first = std::string::npos;
|
|
std::size_t last = std::string::npos;
|
|
switch (n.kind)
|
|
{
|
|
case node_kind::string:
|
|
if ((n.flags & node_flags::storage) == 0)
|
|
{
|
|
// off is the first byte after the opening quote
|
|
if (n.off != 0)
|
|
{
|
|
first = n.off - 1;
|
|
last = n.off + std::size_t{n.len} + 1;
|
|
}
|
|
}
|
|
else if ((n.flags & node_flags::storage) == node_flags::escaped && n.next != 0)
|
|
{
|
|
// next is the offset of the opening quote plus one
|
|
first = n.next - 1;
|
|
last = string_end(d.base[0], d.size, first);
|
|
}
|
|
break;
|
|
case node_kind::number_integer:
|
|
case node_kind::number_unsigned:
|
|
case node_kind::number_float:
|
|
if ((n.flags & node_flags::storage) == 0)
|
|
{
|
|
first = n.off;
|
|
last = n.off + number_length(n);
|
|
}
|
|
break;
|
|
case node_kind::boolean:
|
|
first = n.off;
|
|
last = n.off + ((n.flags & node_flags::is_true) != 0 ? 4u : 5u);
|
|
break;
|
|
case node_kind::null:
|
|
first = n.off;
|
|
last = n.off + 4u;
|
|
break;
|
|
case node_kind::object:
|
|
case node_kind::array:
|
|
case node_kind::link:
|
|
default:
|
|
break;
|
|
}
|
|
// (a crafted image can point past the text)
|
|
if (last == std::string::npos || last > d.size)
|
|
{
|
|
clear();
|
|
}
|
|
else
|
|
{
|
|
set(first, last);
|
|
}
|
|
}
|
|
};
|
|
#endif
|
|
|
|
/*!
|
|
@brief the basic_json value of the subtree at n
|
|
|
|
The subtree is replayed into the SAX handler that parse() uses to build its
|
|
values, so the result is the value parse() would produce: duplicate keys keep
|
|
the last value, and with JSON_DIAGNOSTICS the parent pointers are set. It is
|
|
iterative, so the nesting depth is limited by memory only, as for parse().
|
|
|
|
With JSON_DIAGNOSTIC_POSITIONS, the values get their source positions as with
|
|
parse(): position_source stands in for the lexer. Values written by edits have
|
|
none (npos), and neither has an array or object whose elements changed, or
|
|
whose last element has none (the end of its closing bracket is found by
|
|
scanning from the end of the last element).
|
|
*/
|
|
template<typename BasicJsonType, bool Editable>
|
|
BasicJsonType materialize(const document_data& d, const node* n)
|
|
{
|
|
using string_t = typename BasicJsonType::string_t;
|
|
using input_t = iterator_input_adapter<const char*>;
|
|
#if JSON_DIAGNOSTIC_POSITIONS
|
|
using lexer_t = position_source;
|
|
#else
|
|
using lexer_t = lexer<BasicJsonType, input_t>; // (the default: unused, as no lexer is passed)
|
|
#endif
|
|
using sax_t = json_sax_dom_parser<BasicJsonType, input_t, true, lexer_t>;
|
|
using nav = navigation<Editable>;
|
|
|
|
// the open containers: their next element (or key of the next member),
|
|
// their end, and whether they are objects; for the first 16 levels on the
|
|
// stack, else sized once from the document's deepest nesting (for a
|
|
// subtree an upper bound)
|
|
struct frame
|
|
{
|
|
const node* pos;
|
|
const node* end;
|
|
bool object;
|
|
#if JSON_DIAGNOSTIC_POSITIONS
|
|
std::size_t start; ///< the opening bracket (npos: unknown)
|
|
std::size_t last_end; ///< the end of the last element read (npos: unknown)
|
|
#endif
|
|
};
|
|
|
|
#if !JSON_DIAGNOSTIC_POSITIONS
|
|
// a scalar needs neither the handler nor the stack of open containers
|
|
// (with positions, it goes through the handler, which sets them)
|
|
switch (static_cast<value_t>(n->kind))
|
|
{
|
|
case value_t::string:
|
|
return BasicJsonType(string_t(d.str(*n), n->len));
|
|
case value_t::number_integer:
|
|
return BasicJsonType(static_cast<typename BasicJsonType::number_integer_t>(static_cast<std::int64_t>(integer_bits(*n))));
|
|
case value_t::number_unsigned:
|
|
return BasicJsonType(static_cast<typename BasicJsonType::number_unsigned_t>(integer_bits(*n)));
|
|
case value_t::number_float:
|
|
return BasicJsonType(float_value<typename BasicJsonType::number_float_t>(d, *n));
|
|
case value_t::boolean:
|
|
return BasicJsonType((n->flags & node_flags::is_true) != 0);
|
|
case value_t::null:
|
|
return BasicJsonType(nullptr);
|
|
case value_t::object:
|
|
case value_t::array:
|
|
case value_t::binary:
|
|
case value_t::discarded:
|
|
default:
|
|
break;
|
|
}
|
|
#endif
|
|
|
|
BasicJsonType result;
|
|
#if JSON_DIAGNOSTIC_POSITIONS
|
|
position_source positions;
|
|
sax_t sax(result, true, &positions);
|
|
#else
|
|
sax_t sax(result, true);
|
|
#endif
|
|
const string_t no_token {};
|
|
// (shallow subtrees on the stack: a call for a small value then allocates only for the value)
|
|
std::array<frame, 16> shallow; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): filled as containers open
|
|
std::vector<frame> deep{};
|
|
frame* open = shallow.data();
|
|
std::size_t capacity = shallow.size();
|
|
if (d.max_depth > capacity)
|
|
{
|
|
deep.resize(d.max_depth);
|
|
open = deep.data();
|
|
capacity = deep.size();
|
|
}
|
|
std::size_t depth = 0;
|
|
for (;;)
|
|
{
|
|
#if JSON_DIAGNOSTIC_POSITIONS
|
|
if (!is_container(*n))
|
|
{
|
|
positions.set_scalar(d, *n);
|
|
if (depth != 0)
|
|
{
|
|
open[depth - 1].last_end = positions.end;
|
|
}
|
|
}
|
|
#endif
|
|
// false positive: n comes from nav::value(), which never returns null for a valid index
|
|
// @infer-ignore NULLPTR_DEREFERENCE
|
|
switch (static_cast<value_t>(n->kind))
|
|
{
|
|
case value_t::object:
|
|
case value_t::array:
|
|
{
|
|
const bool object = n->kind == node_kind::object;
|
|
#if JSON_DIAGNOSTIC_POSITIONS
|
|
// (moved containers are also new)
|
|
if ((n->flags & node_flags::is_new) != 0 || n->off >= d.size)
|
|
{
|
|
positions.clear();
|
|
}
|
|
else
|
|
{
|
|
positions.set(std::size_t{n->off}, std::size_t{n->off} + 1);
|
|
}
|
|
#endif
|
|
if (object)
|
|
{
|
|
sax.start_object(n->len);
|
|
}
|
|
else
|
|
{
|
|
sax.start_array(n->len);
|
|
}
|
|
if (NLOHMANN_VIEW_UNLIKELY(depth == capacity))
|
|
{
|
|
// (deeper than the recorded nesting, e.g. after edits)
|
|
deep.assign(open, open + depth);
|
|
deep.resize(depth * 2);
|
|
open = deep.data();
|
|
capacity = deep.size();
|
|
}
|
|
// (with positions: until an element is read, the container's
|
|
// own bracket is the last thing read)
|
|
open[depth++] = frame{nav::first(d, n), nav::end(d, n), object
|
|
#if JSON_DIAGNOSTIC_POSITIONS
|
|
, positions.start, positions.end
|
|
#endif
|
|
};
|
|
break;
|
|
}
|
|
case value_t::string:
|
|
{
|
|
string_t s(d.str(*n), n->len);
|
|
sax.string(s);
|
|
break;
|
|
}
|
|
case value_t::number_integer:
|
|
sax.number_integer(static_cast<typename BasicJsonType::number_integer_t>(static_cast<std::int64_t>(integer_bits(*n))));
|
|
break;
|
|
case value_t::number_unsigned:
|
|
sax.number_unsigned(static_cast<typename BasicJsonType::number_unsigned_t>(integer_bits(*n)));
|
|
break;
|
|
case value_t::number_float:
|
|
sax.number_float(float_value<typename BasicJsonType::number_float_t>(d, *n), no_token);
|
|
break;
|
|
case value_t::boolean:
|
|
sax.boolean((n->flags & node_flags::is_true) != 0);
|
|
break;
|
|
case value_t::null:
|
|
case value_t::binary:
|
|
case value_t::discarded:
|
|
default:
|
|
// false positive: the parser's frame stack points into its own storage (never null)
|
|
// @infer-ignore NULLPTR_DEREFERENCE
|
|
sax.null();
|
|
break;
|
|
}
|
|
|
|
// the next value: close finished containers, then read the key
|
|
for (;;)
|
|
{
|
|
if (depth == 0)
|
|
{
|
|
return result;
|
|
}
|
|
frame& f = open[depth - 1];
|
|
if (f.pos == f.end)
|
|
{
|
|
#if JSON_DIAGNOSTIC_POSITIONS
|
|
if (f.start == std::string::npos || f.last_end == std::string::npos)
|
|
{
|
|
positions.clear();
|
|
}
|
|
else
|
|
{
|
|
const std::size_t end = position_source::container_end(d.base[0], d.size, f.last_end);
|
|
if (end == std::string::npos)
|
|
{
|
|
positions.clear();
|
|
}
|
|
else
|
|
{
|
|
// the closing bracket, as the lexer reports it (end_object()
|
|
// and end_array() only read its end; the start of the
|
|
// container was set when it was opened)
|
|
positions.set(end - 1, end);
|
|
}
|
|
}
|
|
if (depth > 1)
|
|
{
|
|
open[depth - 2].last_end = positions.end;
|
|
}
|
|
#endif
|
|
if (f.object)
|
|
{
|
|
sax.end_object();
|
|
}
|
|
else
|
|
{
|
|
sax.end_array();
|
|
}
|
|
--depth;
|
|
continue;
|
|
}
|
|
const node* entry = f.pos;
|
|
if (f.object)
|
|
{
|
|
string_t key(d.str(*entry), entry->len);
|
|
sax.key(key);
|
|
++entry;
|
|
}
|
|
n = nav::value(entry);
|
|
f.pos = document_data::after(entry);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
} // namespace view
|
|
} // namespace detail
|
|
NLOHMANN_JSON_NAMESPACE_END
|