Compare commits

..
Author SHA1 Message Date
Niels Lohmann 74f5ecbd35 accept() checks strings without copying them
json::accept() parses through json_sax_acceptor, which ignores the
values of strings and object keys, but the lexer still copied every
string token into its token buffer. That copy was 18 % of the
instructions of json::accept() on twitter.json (x86, GCC 16).

The lexer's flag for accept(), renamed from discard_number_values to
discard_values, now also selects a second instantiation of
scan_string() that performs the same checks (escapes, surrogates,
UTF-8, control characters, with the same error messages) without
writing to the token buffer. scan() chooses the instantiation once per
string: a run-time check inside the string loop made number-heavy
files up to 3 % slower on Apple M1.

json::accept() over 19 files, time after/before (geomean): 0.876 on x86
with GCC 16.2, 0.884 with Clang 23, 0.932 on Apple M1; up to 0.74 on
single files; files with few strings (canada, mesh, numbers) unchanged.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-31 00:07:57 +01:00
10 changed files with 296 additions and 558 deletions

No files matched your search

@@ -13,15 +13,9 @@ function never throws an exception for invalid input, and the returned `#!cpp bo
is returned. is returned.
The result is the same as that of `BasicJsonType::accept()` by design; the difference is how the check is made. This The result is the same as that of `BasicJsonType::accept()` by design; the difference is how the check is made. This
function runs the document's builder, which checks the text as [`parse()`](parse.md) does, while function runs the document's index builder (and discards the index), while `BasicJsonType::accept()` runs the SAX
`BasicJsonType::accept()` runs the SAX parser with a handler that ignores every event. Use this one if you go on to parser with a handler that ignores every event. Use this one if you go on to parse the same input with
parse the same input with [`parse()`](parse.md) anyway, so that both use the same code; either is fine for a plain [`parse()`](parse.md) anyway, so that both use the same code; either is fine for a plain yes/no answer.
yes/no answer.
When both `ignore_comments` and `ignore_trailing_commas` are `#!cpp false` (the default), the builder writes no index.
With either option set, it builds the index as [`parse()`](parse.md) does and discards it: the result is the same, only
the time differs. (Neither option needs the index; the variant that skips it exists only for the default options to
keep the code size down.)
## Template parameters ## Template parameters
+83 -41
View File
@@ -268,10 +268,10 @@ class lexer : public lexer_base<BasicJsonType>
public: public:
using token_type = typename lexer_base<BasicJsonType>::token_type; using token_type = typename lexer_base<BasicJsonType>::token_type;
explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_values_ = false) noexcept
: ia(std::move(adapter)) : ia(std::move(adapter))
, ignore_comments(ignore_comments_) , ignore_comments(ignore_comments_)
, discard_number_values(discard_number_values_) , discard_values(discard_values_)
{ {
// the token string of an error before the first token is everything // the token string of an error before the first token is everything
// read so far // read so far
@@ -388,6 +388,8 @@ class lexer : public lexer_base<BasicJsonType>
checks if it is inside the range. If a violation was detected, set up an checks if it is inside the range. If a violation was detected, set up an
error message and return false. Otherwise, return true. error message and return false. Otherwise, return true.
@param[in] keep whether the bytes are added to token_buffer (false for
accept(), see scan())
@param[in] ranges list of integers; interpreted as list of pairs of @param[in] ranges list of integers; interpreted as list of pairs of
inclusive lower and upper bound, respectively inclusive lower and upper bound, respectively
@@ -396,17 +398,18 @@ class lexer : public lexer_base<BasicJsonType>
@return true if and only if no range violation was detected @return true if and only if no range violation was detected
*/ */
bool next_byte_in_range(std::initializer_list<char_int_type> ranges) template<bool Keep>
bool next_byte_in_range(std::integral_constant<bool, Keep> keep, std::initializer_list<char_int_type> ranges)
{ {
JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6); JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6);
add(current); add_string_char(keep, current);
for (auto range = ranges.begin(); range != ranges.end(); ++range) for (auto range = ranges.begin(); range != ranges.end(); ++range)
{ {
get(); get();
if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions) if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions)
{ {
add(current); add_string_char(keep, current);
} }
else else
{ {
@@ -419,14 +422,16 @@ class lexer : public lexer_base<BasicJsonType>
} }
/// contiguous input: bulk-append the run of ordinary characters and complete /// contiguous input: bulk-append the run of ordinary characters and complete
/// well-formed UTF-8 sequences starting at the current read position, then /// well-formed UTF-8 sequences starting at the current read position (only
/// skip it if @a keep is false), then
/// read the first byte that needs individual handling (the closing quote, an /// read the first byte that needs individual handling (the closing quote, an
/// escape, a control character, or an ill-formed UTF-8 byte) like get(). /// escape, a control character, or an ill-formed UTF-8 byte) like get().
/// That byte is taken from the buffer that was just scanned. Through get(), /// That byte is taken from the buffer that was just scanned. Through get(),
/// Apple Clang merged this read with the other get() calls of the scan /// Apple Clang merged this read with the other get() calls of the scan
/// function, which load the read position again right after it was stored /// function, which load the read position again right after it was stored
/// (3% of json::accept() on jeopardy.json on Apple M1). /// (3% of json::accept() on jeopardy.json on Apple M1).
char_int_type get_after_string_run(std::true_type /*bulk*/) template<bool Keep>
char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant<bool, Keep> keep)
{ {
// a pending unget must be consumed through the normal path first // a pending unget must be consumed through the normal path first
if (next_unget) if (next_unget)
@@ -444,7 +449,7 @@ class lexer : public lexer_base<BasicJsonType>
const std::size_t pos = lexer_string_bulk_run(data, remaining); const std::size_t pos = lexer_string_bulk_run(data, remaining);
if (pos != 0) if (pos != 0)
{ {
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), pos); append_string_run(keep, data, pos);
bulk_consume(reinterpret_cast<const char*>(data), pos); bulk_consume(reinterpret_cast<const char*>(data), pos);
} }
if (pos == remaining) if (pos == remaining)
@@ -488,7 +493,8 @@ class lexer : public lexer_base<BasicJsonType>
} }
/// streaming input: no bulk fast path /// streaming input: no bulk fast path
char_int_type get_after_string_run(std::false_type /*bulk*/) template<bool Keep>
char_int_type get_after_string_run(std::false_type /*bulk*/, std::integral_constant<bool, Keep> /*keep*/)
{ {
return get(); return get();
} }
@@ -502,13 +508,18 @@ class lexer : public lexer_base<BasicJsonType>
may contain \0 bytes), and token_buffer.size() is the number of bytes in the may contain \0 bytes), and token_buffer.size() is the number of bytes in the
string. string.
@param[in] keep whether the string is copied into token_buffer; if false
(accept(), see scan()), it is only checked, with the same
errors, and token_buffer stays empty
@return token_type::value_string if string could be successfully scanned, @return token_type::value_string if string could be successfully scanned,
token_type::parse_error otherwise token_type::parse_error otherwise
@note In case of errors, variable error_message contains a textual @note In case of errors, variable error_message contains a textual
description. description.
*/ */
token_type scan_string() template<bool Keep>
token_type scan_string(std::integral_constant<bool, Keep> keep)
{ {
// reset token_buffer (ignore opening quote) // reset token_buffer (ignore opening quote)
reset(); reset();
@@ -520,7 +531,7 @@ class lexer : public lexer_base<BasicJsonType>
{ {
// bulk-consume ordinary characters from contiguous input, then // bulk-consume ordinary characters from contiguous input, then
// get the next character, the one that ends the run // get the next character, the one that ends the run
switch (get_after_string_run(std::integral_constant<bool, bulk_scan>())) switch (get_after_string_run(std::integral_constant<bool, bulk_scan>(), keep))
{ {
// end of file while parsing the string // end of file while parsing the string
case char_traits<char_type>::eof(): case char_traits<char_type>::eof():
@@ -542,35 +553,35 @@ class lexer : public lexer_base<BasicJsonType>
{ {
// quotation mark // quotation mark
case '\"': case '\"':
add('\"'); add_string_char(keep, '\"');
break; break;
// reverse solidus // reverse solidus
case '\\': case '\\':
add('\\'); add_string_char(keep, '\\');
break; break;
// solidus // solidus
case '/': case '/':
add('/'); add_string_char(keep, '/');
break; break;
// backspace // backspace
case 'b': case 'b':
add('\b'); add_string_char(keep, '\b');
break; break;
// form feed // form feed
case 'f': case 'f':
add('\f'); add_string_char(keep, '\f');
break; break;
// line feed // line feed
case 'n': case 'n':
add('\n'); add_string_char(keep, '\n');
break; break;
// carriage return // carriage return
case 'r': case 'r':
add('\r'); add_string_char(keep, '\r');
break; break;
// tab // tab
case 't': case 't':
add('\t'); add_string_char(keep, '\t');
break; break;
// unicode escapes // unicode escapes
@@ -638,10 +649,7 @@ class lexer : public lexer_base<BasicJsonType>
JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF); JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF);
// translate codepoint into bytes // translate codepoint into bytes
encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte) add_codepoint(keep, codepoint);
{
add(static_cast<char_int_type>(byte));
});
break; break;
} }
@@ -824,7 +832,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0x7E: case 0x7E:
case 0x7F: case 0x7F:
{ {
add(current); add_string_char(keep, current);
break; break;
} }
@@ -860,7 +868,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xDE: case 0xDE:
case 0xDF: case 0xDF:
{ {
if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF}))) if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range(keep, {0x80, 0xBF})))
{ {
return token_type::parse_error; return token_type::parse_error;
} }
@@ -870,7 +878,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+0800..U+0FFF: bytes E0 A0..BF 80..BF // U+0800..U+0FFF: bytes E0 A0..BF 80..BF
case 0xE0: case 0xE0:
{ {
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF})))) if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0xA0, 0xBF, 0x80, 0xBF}))))
{ {
return token_type::parse_error; return token_type::parse_error;
} }
@@ -894,7 +902,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xEE: case 0xEE:
case 0xEF: case 0xEF:
{ {
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF})))) if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF}))))
{ {
return token_type::parse_error; return token_type::parse_error;
} }
@@ -904,7 +912,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+D000..U+D7FF: bytes ED 80..9F 80..BF // U+D000..U+D7FF: bytes ED 80..9F 80..BF
case 0xED: case 0xED:
{ {
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF})))) if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x9F, 0x80, 0xBF}))))
{ {
return token_type::parse_error; return token_type::parse_error;
} }
@@ -914,7 +922,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+10000..U+3FFFF F0 90..BF 80..BF 80..BF // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
case 0xF0: case 0xF0:
{ {
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
{ {
return token_type::parse_error; return token_type::parse_error;
} }
@@ -926,7 +934,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xF2: case 0xF2:
case 0xF3: case 0xF3:
{ {
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
{ {
return token_type::parse_error; return token_type::parse_error;
} }
@@ -936,7 +944,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+100000..U+10FFFF F4 80..8F 80..BF 80..BF // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
case 0xF4: case 0xF4:
{ {
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
{ {
return token_type::parse_error; return token_type::parse_error;
} }
@@ -1427,7 +1435,7 @@ scan_number_done:
token_type convert_number(token_type number_type, std::size_t mantissa_end) token_type convert_number(token_type number_type, std::size_t mantissa_end)
{ {
// accept() only needs to know whether the input is valid, so it sets // accept() only needs to know whether the input is valid, so it sets
// discard_number_values (see json.hpp), and an integer token whose // discard_values (see json.hpp), and an integer token whose
// digit count shows that it fits is reported without calling // digit count shows that it fits is reported without calling
// convert_integer(). A number with up to 18 digits always fits into // convert_integer(). A number with up to 18 digits always fits into
// both std::uint64_t and std::int64_t (18 nines is about 1e18, below // both std::uint64_t and std::int64_t (18 nines is about 1e18, below
@@ -1442,7 +1450,7 @@ scan_number_done:
// json_sax_acceptor, whose number callbacks discard their argument and // json_sax_acceptor, whose number callbacks discard their argument and
// return true, and the parser rejects neither integers nor finite // return true, and the parser rejects neither integers nor finite
// floats. value_unsigned/value_integer are left unset here, so a caller // floats. value_unsigned/value_integer are left unset here, so a caller
// that reads the converted value must not set discard_number_values. // that reads the converted value must not set discard_values.
// //
// On contiguous input, scan_number_bulk_contiguous() converts integer // On contiguous input, scan_number_bulk_contiguous() converts integer
// tokens itself and does not pass them to this function, unless // tokens itself and does not pass them to this function, unless
@@ -1450,7 +1458,7 @@ scan_number_done:
// reached for input without bulk access (e.g. streams), with // reached for input without bulk access (e.g. streams), with
// JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous() // JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous()
// falls back to scan_number(). // falls back to scan_number().
if (discard_number_values) if (discard_values)
{ {
constexpr std::size_t safe_digit_count = 18; constexpr std::size_t safe_digit_count = 18;
if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count) if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count)
@@ -1701,7 +1709,7 @@ scan_number_done:
// w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a // w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a
// value that underflows is a finite number as well. Only a token that // value that underflows is a finite number as well. Only a token that
// may overflow has to be converted to find out. // may overflow has to be converted to find out.
if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10)) if (JSON_HEDLEY_UNLIKELY(discard_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
{ {
value_float = 0; value_float = 0;
return true; return true;
@@ -2033,6 +2041,34 @@ scan_number_done:
token_buffer.push_back(static_cast<typename string_t::value_type>(c)); token_buffer.push_back(static_cast<typename string_t::value_type>(c));
} }
/// add a character of a string token; accept() checks the string but does
/// not keep its value (@a keep is false)
void add_string_char(std::true_type /*keep*/, char_int_type c)
{
token_buffer.push_back(static_cast<typename string_t::value_type>(c));
}
static void add_string_char(std::false_type /*keep*/, char_int_type /*c*/) noexcept {}
/// add the UTF-8 bytes of a code point from a \\u escape to a string token
void add_codepoint(std::true_type /*keep*/, int codepoint)
{
encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte)
{
add(static_cast<char_int_type>(byte));
});
}
static void add_codepoint(std::false_type /*keep*/, int /*codepoint*/) noexcept {}
/// append a run of ordinary string characters (see get_after_string_run())
void append_string_run(std::true_type /*keep*/, const unsigned char* data, std::size_t n)
{
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), n);
}
static void append_string_run(std::false_type /*keep*/, const unsigned char* /*data*/, std::size_t /*n*/) noexcept {}
public: public:
///////////////////// /////////////////////
// value getters // value getters
@@ -2427,7 +2463,11 @@ scan_number_done:
// string // string
case '\"': case '\"':
return scan_string(); // accept() (discard_values) checks strings without copying them
// into token_buffer. The choice is made once per string: a
// check inside the string loop made number-heavy documents up
// to 3 % slower (mesh.json, Apple M1)
return discard_values ? scan_string(std::false_type {}) : scan_string(std::true_type {});
// number // number
case '-': case '-':
@@ -2512,11 +2552,13 @@ scan_number_done:
std::size_t decimal_point_position = std::string::npos; std::size_t decimal_point_position = std::string::npos;
/// whether the caller only needs the token types and never looks at the /// whether the caller only needs the token types and never looks at the
/// converted numeric values; set only by accept(), which parses through /// values of number and string tokens; set only by accept(), which parses
/// json_sax_acceptor. When set, convert_number() skips converting integer /// through json_sax_acceptor. When set, convert_number() skips converting
/// tokens whose digit count guarantees that they fit into 64 bits (see /// integer tokens whose digit count guarantees that they fit into 64 bits
/// there) /// (see there), floats are converted only when they may overflow (see
const bool discard_number_values = false; /// convert_accumulated_float()), and strings are checked without being
/// copied into token_buffer (see scan())
const bool discard_values = false;
}; };
} // namespace detail } // namespace detail
+2 -2
View File
@@ -89,9 +89,9 @@ class parser
const bool allow_exceptions_ = true, const bool allow_exceptions_ = true,
const bool ignore_comments = false, const bool ignore_comments = false,
const bool ignore_trailing_commas_ = false, const bool ignore_trailing_commas_ = false,
const bool discard_number_values_ = false) const bool discard_values_ = false)
: callback(std::move(cb)) : callback(std::move(cb))
, m_lexer(std::move(adapter), ignore_comments, discard_number_values_) , m_lexer(std::move(adapter), ignore_comments, discard_values_)
, allow_exceptions(allow_exceptions_) , allow_exceptions(allow_exceptions_)
, ignore_trailing_commas(ignore_trailing_commas_) , ignore_trailing_commas(ignore_trailing_commas_)
{ {
+18 -120
View File
@@ -69,8 +69,7 @@ struct parse_failure
}; };
/// FloatType: the number_float_t of the document, whose overflow parse() rejects /// FloatType: the number_float_t of the document, whose overflow parse() rejects
/// Store: false for accept(): the grammar, number, string and UTF-8 checks run as for parse(), but no node is written template<typename FloatType, bool Comments, bool TrailingCommas, bool NulIsEnd, bool Sentinel>
template<typename FloatType, bool Comments, bool TrailingCommas, bool NulIsEnd, bool Sentinel, bool Store>
class builder class builder
{ {
public: public:
@@ -246,87 +245,11 @@ class builder
return p; return p;
} }
/// the string from p, the first backslash, control character, ill-formed UTF-8 byte, or the end, checked and not
/// decoded (accept() stores nothing, so the arena is not written): the checks, errors and error positions of
/// slow_string(), for everything up to the closing quote
NLOHMANN_VIEW_NOINLINE decoded check_string(const unsigned char* p)
{
const unsigned char* const end = m_end;
for (;;)
{
if (p == end)
{
return failed(error_code::string_missing_quote, p);
}
const unsigned char c = *p;
if (c == '"')
{
++p;
return decoded{p, 0, 0};
}
if (c != '\\')
{
return failed(c < 0x20 ? error_code::string_control_character : error_code::string_utf8, p);
}
++p;
if (p == end)
{
return failed(error_code::string_missing_quote, p);
}
const unsigned char d = *p++;
if (d == 'u')
{
const int cp = hex4(p); // (moves p past the four digits)
if (NLOHMANN_VIEW_UNLIKELY(cp < 0))
{
return failed(error_code::string_unicode_hex, p);
}
if (NLOHMANN_VIEW_UNLIKELY((cp & 0xF800) == 0xD800)) // a surrogate
{
if (cp >= 0xDC00)
{
return failed(error_code::string_surrogate_low, p);
}
if (end - p < 2 || p[0] != '\\' || p[1] != 'u')
{
return failed(error_code::string_surrogate_high, p);
}
p += 2;
const int lo = hex4(p); // (moves p past the four digits)
if (lo < 0)
{
return failed(error_code::string_unicode_hex, p);
}
if (lo < 0xDC00 || lo > 0xDFFF)
{
return failed(error_code::string_surrogate_high, p);
}
}
}
else if (!(d == '"' || d == '\\' || d == '/' || d == 'b' || d == 'f' || d == 'n' || d == 'r' || d == 't'))
{
--p;
return failed(error_code::string_escape, p);
}
if (p != end && *p == '\\')
{
continue; // (escapes in a row: no run of text between them)
}
p = scan_string_run(p, end); // the text up to the next backslash, quote, control character, or ill-formed UTF-8
}
}
/// s points to the start of the string, p to a backslash, a control /// s points to the start of the string, p to a backslash, a control
/// character, an ill-formed UTF-8 byte, or the end: decode the string into /// character, an ill-formed UTF-8 byte, or the end: decode the string into
/// the arena /// the arena
NLOHMANN_VIEW_NOINLINE decoded slow_string(const unsigned char* s, const unsigned char* p) NLOHMANN_VIEW_NOINLINE decoded slow_string(const unsigned char* s, const unsigned char* p)
{ {
if (!enabled(Store))
{
// (accept(): the string is checked, not decoded)
static_cast<void>(s);
return check_string(p);
}
// single-character escapes; 0: invalid (and 'u', handled separately) // single-character escapes; 0: invalid (and 'u', handled separately)
static const std::array<char, 128> simple_escape = static const std::array<char, 128> simple_escape =
{ {
@@ -687,13 +610,10 @@ run_found:
NLOHMANN_VIEW_ALWAYS_INLINE bool run() NLOHMANN_VIEW_ALWAYS_INLINE bool run()
{ {
if (enabled(Store)) cold.m_doc.reserve(estimate_nodes(reinterpret_cast<const char*>(b), static_cast<std::size_t>(e - b)));
{ base = cold.m_doc.nodes;
cold.m_doc.reserve(estimate_nodes(reinterpret_cast<const char*>(b), static_cast<std::size_t>(e - b))); out = base;
base = cold.m_doc.nodes; cap = base + cold.m_doc.node_capacity;
out = base;
cap = base + cold.m_doc.node_capacity;
}
if (e - p >= 3 && p[0] == 0xEF && p[1] == 0xBB && p[2] == 0xBF) if (e - p >= 3 && p[0] == 0xEF && p[1] == 0xBB && p[2] == 0xBF)
{ {
@@ -877,10 +797,7 @@ close_object:
// an array pays nothing for this) // an array pays nothing for this)
if (NLOHMANN_VIEW_UNLIKELY(cur_count >= document_data::index_min_members)) if (NLOHMANN_VIEW_UNLIKELY(cur_count >= document_data::index_min_members))
{ {
if (enabled(Store)) cold.note_large_object(cur_idx);
{
cold.note_large_object(cur_idx);
}
} }
close_container: close_container:
@@ -1040,12 +957,6 @@ indent_done:
/// an integer's value); two stores on little-endian targets /// an integer's value); two stores on little-endian targets
NLOHMANN_VIEW_ALWAYS_INLINE node* push_node(node_kind k, std::uint8_t flags, std::uint16_t extra, std::size_t off, std::uint64_t second) NLOHMANN_VIEW_ALWAYS_INLINE node* push_node(node_kind k, std::uint8_t flags, std::uint16_t extra, std::size_t off, std::uint64_t second)
{ {
if (!enabled(Store))
{
// accept(): nothing is stored, so what computes the arguments is not run either. base and out stay
// null: their difference, which open() takes, is 0.
return out;
}
if (NLOHMANN_VIEW_UNLIKELY(out == cap)) if (NLOHMANN_VIEW_UNLIKELY(out == cap))
{ {
const auto n = static_cast<std::size_t>(out - base); const auto n = static_cast<std::size_t>(out - base);
@@ -1104,12 +1015,9 @@ indent_done:
NLOHMANN_VIEW_ALWAYS_INLINE void close() NLOHMANN_VIEW_ALWAYS_INLINE void close()
{ {
if (enabled(Store)) node& n = base[cur_idx];
{ n.len = cur_count;
node& n = base[cur_idx]; n.next = static_cast<std::uint32_t>(out - base) - cur_idx;
n.len = cur_count;
n.next = static_cast<std::uint32_t>(out - base) - cur_idx;
}
if (--depth != 0) if (--depth != 0)
{ {
if (NLOHMANN_VIEW_LIKELY(depth <= 64)) if (NLOHMANN_VIEW_LIKELY(depth <= 64))
@@ -1284,44 +1192,34 @@ indent_done:
}; };
/// run the builder with compile-time options /// run the builder with compile-time options
template<typename FloatType, bool Comments, bool TrailingCommas, bool NulIsEnd, bool Store> template<typename FloatType, bool Comments, bool TrailingCommas, bool NulIsEnd>
inline bool build_with(document_data& d, const char* src, std::size_t size, bool sentinel, parse_failure& failure) inline bool build_with(document_data& d, const char* src, std::size_t size, bool sentinel, parse_failure& failure)
{ {
if (sentinel) if (sentinel)
{ {
builder<FloatType, Comments, TrailingCommas, NulIsEnd, true, Store> bld(d, src, size); builder<FloatType, Comments, TrailingCommas, NulIsEnd, true> bld(d, src, size);
const bool ok = bld.run(); const bool ok = bld.run();
failure = bld.failure(); failure = bld.failure();
return ok; return ok;
} }
builder<FloatType, Comments, TrailingCommas, NulIsEnd, false, Store> bld(d, src, size); builder<FloatType, Comments, TrailingCommas, NulIsEnd, false> bld(d, src, size);
const bool ok = bld.run(); const bool ok = bld.run();
failure = bld.failure(); failure = bld.failure();
return ok; return ok;
} }
/// sentinel: src[size] is readable and 0 (e.g. std::string); FloatType: the /// sentinel: src[size] is readable and 0 (e.g. std::string); FloatType: the
/// number_float_t of the document; validate_only: accept(), only the result is /// number_float_t of the document
/// used
template<typename FloatType, bool NulIsEnd> template<typename FloatType, bool NulIsEnd>
inline bool build(document_data& d, const char* src, std::size_t size, bool comments, bool trailing_commas, bool sentinel, bool validate_only, inline bool build(document_data& d, const char* src, std::size_t size, bool comments, bool trailing_commas, bool sentinel, parse_failure& failure)
parse_failure& failure)
{ {
if (validate_only && !comments && !trailing_commas)
{
// accept() of plain JSON: nothing is stored. The other option sets
// could skip the stores just as well; they keep the builder that
// stores only so that there are two more instantiations of the parse
// loop, not eight.
return build_with<FloatType, false, false, NulIsEnd, false>(d, src, size, sentinel, failure);
}
if (comments) if (comments)
{ {
return trailing_commas ? build_with<FloatType, true, true, NulIsEnd, true>(d, src, size, sentinel, failure) return trailing_commas ? build_with<FloatType, true, true, NulIsEnd>(d, src, size, sentinel, failure)
: build_with<FloatType, true, false, NulIsEnd, true>(d, src, size, sentinel, failure); : build_with<FloatType, true, false, NulIsEnd>(d, src, size, sentinel, failure);
} }
return trailing_commas ? build_with<FloatType, false, true, NulIsEnd, true>(d, src, size, sentinel, failure) return trailing_commas ? build_with<FloatType, false, true, NulIsEnd>(d, src, size, sentinel, failure)
: build_with<FloatType, false, false, NulIsEnd, true>(d, src, size, sentinel, failure); : build_with<FloatType, false, false, NulIsEnd>(d, src, size, sentinel, failure);
} }
} // namespace view } // namespace view
+2 -2
View File
@@ -184,11 +184,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
const bool allow_exceptions = true, const bool allow_exceptions = true,
const bool ignore_comments = false, const bool ignore_comments = false,
const bool ignore_trailing_commas = false, const bool ignore_trailing_commas = false,
const bool discard_number_values = false const bool discard_values = false
) )
{ {
return ::nlohmann::detail::parser<basic_json, InputAdapterType>(std::move(adapter), return ::nlohmann::detail::parser<basic_json, InputAdapterType>(std::move(adapter),
std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values); std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_values);
} }
private: private:
+19 -37
View File
@@ -1060,7 +1060,7 @@ class basic_json_document
const bool ignore_trailing_commas = false) const bool ignore_trailing_commas = false)
{ {
basic_json_document d; basic_json_document d;
d.read_range(first, last, error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas); d.read_range(first, last, allow_exceptions, ignore_comments, ignore_trailing_commas);
return d; return d;
} }
@@ -1075,7 +1075,7 @@ class basic_json_document
basic_json_document d; basic_json_document d;
std::string buf; std::string buf;
append_input(buf, std::forward<InputType>(input)); append_input(buf, std::forward<InputType>(input));
d.build_owned(std::move(buf), error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas); d.build_owned(std::move(buf), allow_exceptions, ignore_comments, ignore_trailing_commas);
return d; return d;
} }
@@ -1088,8 +1088,7 @@ class basic_json_document
static bool accept(InputType&& input, const bool ignore_comments = false, const bool ignore_trailing_commas = false) static bool accept(InputType&& input, const bool ignore_comments = false, const bool ignore_trailing_commas = false)
{ {
basic_json_document d; basic_json_document d;
d.read_kind(std::forward<InputType>(input), on_error::validate, ignore_comments, ignore_trailing_commas, d.read(std::forward<InputType>(input), false, ignore_comments, ignore_trailing_commas);
std::integral_constant<detail::view::input_kind, detail::view::classify_input<InputType>::value> {});
return !d.is_discarded(); return !d.is_discarded();
} }
@@ -1105,7 +1104,7 @@ class basic_json_document
const bool ignore_comments = false, const bool ignore_comments = false,
const bool ignore_trailing_commas = false) const bool ignore_trailing_commas = false)
{ {
read_kind(std::forward<InputType>(input), error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas, read_kind(std::forward<InputType>(input), allow_exceptions, ignore_comments, ignore_trailing_commas,
std::integral_constant<detail::view::input_kind, detail::view::classify_input<InputType>::value> {}); std::integral_constant<detail::view::input_kind, detail::view::classify_input<InputType>::value> {});
} }
@@ -1464,22 +1463,6 @@ class basic_json_document
return idx; return idx;
} }
/// what a failed parse does
enum class on_error : std::uint8_t
{
discard, ///< the document is discarded
raise, ///< parse_error is thrown
validate ///< accept(): the document is discarded, and no node is stored while parsing
};
static on_error error_mode(bool allow_exceptions) noexcept
{
return allow_exceptions ? on_error::raise : on_error::discard;
}
/// NulIsEnd of the builder: a null byte ends the input unless strict_nul_handling
static constexpr bool nul_is_end = !detail::abi_config::strict_nul_handling;
/// create the storage (sized for the input) on first use /// create the storage (sized for the input) on first use
void ensure_data(const char* src, std::size_t size) void ensure_data(const char* src, std::size_t size)
{ {
@@ -1491,7 +1474,7 @@ class basic_json_document
/// parse a buffer the document takes ownership of (it replaces the /// parse a buffer the document takes ownership of (it replaces the
/// previous owned copy) /// previous owned copy)
void build_owned(std::string&& buf, on_error allow_exceptions, bool comments, bool trailing_commas) void build_owned(std::string&& buf, bool allow_exceptions, bool comments, bool trailing_commas)
{ {
ensure_data(buf.data(), buf.size()); ensure_data(buf.data(), buf.size());
m_data->owned = std::move(buf); m_data->owned = std::move(buf);
@@ -1501,7 +1484,7 @@ class basic_json_document
/// parse a copy of the input; the copy is appended to the cleared owned /// parse a copy of the input; the copy is appended to the cleared owned
/// buffer, so that its capacity is reused /// buffer, so that its capacity is reused
template<typename T> template<typename T>
void build_copy(T&& input, on_error allow_exceptions, bool comments, bool trailing_commas) void build_copy(T&& input, bool allow_exceptions, bool comments, bool trailing_commas)
{ {
if (!m_data) if (!m_data)
{ {
@@ -1522,7 +1505,7 @@ class basic_json_document
} }
/// sentinel: src[size] is readable and 0 (std::string, C strings) /// sentinel: src[size] is readable and 0 (std::string, C strings)
void build(const char* src, std::size_t size, on_error allow_exceptions, bool comments, bool trailing_commas, bool owned, bool sentinel) void build(const char* src, std::size_t size, bool allow_exceptions, bool comments, bool trailing_commas, bool owned, bool sentinel)
{ {
ensure_data(src, size); ensure_data(src, size);
document_data& d = *m_data; document_data& d = *m_data;
@@ -1549,8 +1532,7 @@ class basic_json_document
} }
else else
{ {
ok = detail::view::build<typename BasicJsonType::number_float_t, nul_is_end>(d, src, size, comments, trailing_commas, sentinel, ok = detail::view::build < typename BasicJsonType::number_float_t, !detail::abi_config::strict_nul_handling > (d, src, size, comments, trailing_commas, sentinel, failure);
allow_exceptions == on_error::validate, failure);
} }
if (NLOHMANN_VIEW_LIKELY(ok)) if (NLOHMANN_VIEW_LIKELY(ok))
{ {
@@ -1563,7 +1545,7 @@ class basic_json_document
return; return;
} }
std::vector<std::uint32_t>().swap(d.large_objects); std::vector<std::uint32_t>().swap(d.large_objects);
if (allow_exceptions == on_error::raise) if (allow_exceptions)
{ {
detail::view::throw_parse_failure<BasicJsonType>(failure, src, size, comments, trailing_commas); detail::view::throw_parse_failure<BasicJsonType>(failure, src, size, comments, trailing_commas);
} }
@@ -1571,13 +1553,13 @@ class basic_json_document
// --- input dispatch (see detail::view::input_kind) --- // --- input dispatch (see detail::view::input_kind) ---
void read_kind(std::string&& s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::move_string> /*unused*/) void read_kind(std::string&& s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::move_string> /*unused*/)
{ {
build_owned(std::move(s), ae, c, tc); build_owned(std::move(s), ae, c, tc);
} }
template<typename CharT> template<typename CharT>
void read_kind(CharT* s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::c_string> /*unused*/) void read_kind(CharT* s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::c_string> /*unused*/)
{ {
if (s == nullptr) if (s == nullptr)
{ {
@@ -1591,7 +1573,7 @@ class basic_json_document
} }
template<typename Array> template<typename Array>
void read_kind(Array& a, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::char_array> /*unused*/) void read_kind(Array& a, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::char_array> /*unused*/)
{ {
using CharT = typename std::remove_cv<typename std::remove_extent<Array>::type>::type; using CharT = typename std::remove_cv<typename std::remove_extent<Array>::type>::type;
const std::size_t n = std::extent<Array>::value; const std::size_t n = std::extent<Array>::value;
@@ -1602,7 +1584,7 @@ class basic_json_document
} }
template<typename T> template<typename T>
void read_kind(const T& s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::borrow_range> /*unused*/) void read_kind(const T& s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::borrow_range> /*unused*/)
{ {
// std::basic_string guarantees data()[size()] == 0: use it as sentinel // std::basic_string guarantees data()[size()] == 0: use it as sentinel
const auto size = static_cast<std::size_t>(s.size()); const auto size = static_cast<std::size_t>(s.size());
@@ -1611,13 +1593,13 @@ class basic_json_document
} }
template<typename T> template<typename T>
void read_kind(const T& s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::copy_range> /*unused*/) void read_kind(const T& s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::copy_range> /*unused*/)
{ {
build_copy(s, ae, c, tc); build_copy(s, ae, c, tc);
} }
template<typename CharT> template<typename CharT>
void read_kind(CharT* s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::wide_c_string> /*unused*/) void read_kind(CharT* s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::wide_c_string> /*unused*/)
{ {
if (s == nullptr) if (s == nullptr)
{ {
@@ -1629,13 +1611,13 @@ class basic_json_document
} }
template<typename T> template<typename T>
void read_kind(T&& input, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::adapter> /*unused*/) void read_kind(T&& input, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::adapter> /*unused*/)
{ {
build_copy(std::forward<T>(input), ae, c, tc); build_copy(std::forward<T>(input), ae, c, tc);
} }
template<typename IteratorType> template<typename IteratorType>
void read_range(IteratorType first, IteratorType last, on_error ae, bool c, bool tc) void read_range(IteratorType first, IteratorType last, bool ae, bool c, bool tc)
{ {
using value_type = typename std::remove_cv<typename std::iterator_traits<IteratorType>::value_type>::type; using value_type = typename std::remove_cv<typename std::iterator_traits<IteratorType>::value_type>::type;
// borrow the range where the library's input adapter scans it as one // borrow the range where the library's input adapter scans it as one
@@ -1646,14 +1628,14 @@ class basic_json_document
} }
template<typename IteratorType> template<typename IteratorType>
void read_range_impl(IteratorType first, IteratorType last, on_error ae, bool c, bool tc, std::true_type /*contiguous bytes*/) void read_range_impl(IteratorType first, IteratorType last, bool ae, bool c, bool tc, std::true_type /*contiguous bytes*/)
{ {
const auto size = static_cast<std::size_t>(std::distance(first, last)); const auto size = static_cast<std::size_t>(std::distance(first, last));
build(size == 0 ? "" : reinterpret_cast<const char*>(&*first), size, ae, c, tc, false, size == 0); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) build(size == 0 ? "" : reinterpret_cast<const char*>(&*first), size, ae, c, tc, false, size == 0); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
} }
template<typename IteratorType> template<typename IteratorType>
void read_range_impl(IteratorType first, IteratorType last, on_error ae, bool c, bool tc, std::false_type /*other*/) void read_range_impl(IteratorType first, IteratorType last, bool ae, bool c, bool tc, std::false_type /*other*/)
{ {
build_copy(detail::view::make_adapter_input(detail::input_adapter(first, last)), ae, c, tc); build_copy(detail::view::make_adapter_input(detail::input_adapter(first, last)), ae, c, tc);
} }
+87 -45
View File
@@ -13124,10 +13124,10 @@ class lexer : public lexer_base<BasicJsonType>
public: public:
using token_type = typename lexer_base<BasicJsonType>::token_type; using token_type = typename lexer_base<BasicJsonType>::token_type;
explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_values_ = false) noexcept
: ia(std::move(adapter)) : ia(std::move(adapter))
, ignore_comments(ignore_comments_) , ignore_comments(ignore_comments_)
, discard_number_values(discard_number_values_) , discard_values(discard_values_)
{ {
// the token string of an error before the first token is everything // the token string of an error before the first token is everything
// read so far // read so far
@@ -13244,6 +13244,8 @@ class lexer : public lexer_base<BasicJsonType>
checks if it is inside the range. If a violation was detected, set up an checks if it is inside the range. If a violation was detected, set up an
error message and return false. Otherwise, return true. error message and return false. Otherwise, return true.
@param[in] keep whether the bytes are added to token_buffer (false for
accept(), see scan())
@param[in] ranges list of integers; interpreted as list of pairs of @param[in] ranges list of integers; interpreted as list of pairs of
inclusive lower and upper bound, respectively inclusive lower and upper bound, respectively
@@ -13252,17 +13254,18 @@ class lexer : public lexer_base<BasicJsonType>
@return true if and only if no range violation was detected @return true if and only if no range violation was detected
*/ */
bool next_byte_in_range(std::initializer_list<char_int_type> ranges) template<bool Keep>
bool next_byte_in_range(std::integral_constant<bool, Keep> keep, std::initializer_list<char_int_type> ranges)
{ {
JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6); JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6);
add(current); add_string_char(keep, current);
for (auto range = ranges.begin(); range != ranges.end(); ++range) for (auto range = ranges.begin(); range != ranges.end(); ++range)
{ {
get(); get();
if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions) if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions)
{ {
add(current); add_string_char(keep, current);
} }
else else
{ {
@@ -13275,14 +13278,16 @@ class lexer : public lexer_base<BasicJsonType>
} }
/// contiguous input: bulk-append the run of ordinary characters and complete /// contiguous input: bulk-append the run of ordinary characters and complete
/// well-formed UTF-8 sequences starting at the current read position, then /// well-formed UTF-8 sequences starting at the current read position (only
/// skip it if @a keep is false), then
/// read the first byte that needs individual handling (the closing quote, an /// read the first byte that needs individual handling (the closing quote, an
/// escape, a control character, or an ill-formed UTF-8 byte) like get(). /// escape, a control character, or an ill-formed UTF-8 byte) like get().
/// That byte is taken from the buffer that was just scanned. Through get(), /// That byte is taken from the buffer that was just scanned. Through get(),
/// Apple Clang merged this read with the other get() calls of the scan /// Apple Clang merged this read with the other get() calls of the scan
/// function, which load the read position again right after it was stored /// function, which load the read position again right after it was stored
/// (3% of json::accept() on jeopardy.json on Apple M1). /// (3% of json::accept() on jeopardy.json on Apple M1).
char_int_type get_after_string_run(std::true_type /*bulk*/) template<bool Keep>
char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant<bool, Keep> keep)
{ {
// a pending unget must be consumed through the normal path first // a pending unget must be consumed through the normal path first
if (next_unget) if (next_unget)
@@ -13300,7 +13305,7 @@ class lexer : public lexer_base<BasicJsonType>
const std::size_t pos = lexer_string_bulk_run(data, remaining); const std::size_t pos = lexer_string_bulk_run(data, remaining);
if (pos != 0) if (pos != 0)
{ {
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), pos); append_string_run(keep, data, pos);
bulk_consume(reinterpret_cast<const char*>(data), pos); bulk_consume(reinterpret_cast<const char*>(data), pos);
} }
if (pos == remaining) if (pos == remaining)
@@ -13344,7 +13349,8 @@ class lexer : public lexer_base<BasicJsonType>
} }
/// streaming input: no bulk fast path /// streaming input: no bulk fast path
char_int_type get_after_string_run(std::false_type /*bulk*/) template<bool Keep>
char_int_type get_after_string_run(std::false_type /*bulk*/, std::integral_constant<bool, Keep> /*keep*/)
{ {
return get(); return get();
} }
@@ -13358,13 +13364,18 @@ class lexer : public lexer_base<BasicJsonType>
may contain \0 bytes), and token_buffer.size() is the number of bytes in the may contain \0 bytes), and token_buffer.size() is the number of bytes in the
string. string.
@param[in] keep whether the string is copied into token_buffer; if false
(accept(), see scan()), it is only checked, with the same
errors, and token_buffer stays empty
@return token_type::value_string if string could be successfully scanned, @return token_type::value_string if string could be successfully scanned,
token_type::parse_error otherwise token_type::parse_error otherwise
@note In case of errors, variable error_message contains a textual @note In case of errors, variable error_message contains a textual
description. description.
*/ */
token_type scan_string() template<bool Keep>
token_type scan_string(std::integral_constant<bool, Keep> keep)
{ {
// reset token_buffer (ignore opening quote) // reset token_buffer (ignore opening quote)
reset(); reset();
@@ -13376,7 +13387,7 @@ class lexer : public lexer_base<BasicJsonType>
{ {
// bulk-consume ordinary characters from contiguous input, then // bulk-consume ordinary characters from contiguous input, then
// get the next character, the one that ends the run // get the next character, the one that ends the run
switch (get_after_string_run(std::integral_constant<bool, bulk_scan>())) switch (get_after_string_run(std::integral_constant<bool, bulk_scan>(), keep))
{ {
// end of file while parsing the string // end of file while parsing the string
case char_traits<char_type>::eof(): case char_traits<char_type>::eof():
@@ -13398,35 +13409,35 @@ class lexer : public lexer_base<BasicJsonType>
{ {
// quotation mark // quotation mark
case '\"': case '\"':
add('\"'); add_string_char(keep, '\"');
break; break;
// reverse solidus // reverse solidus
case '\\': case '\\':
add('\\'); add_string_char(keep, '\\');
break; break;
// solidus // solidus
case '/': case '/':
add('/'); add_string_char(keep, '/');
break; break;
// backspace // backspace
case 'b': case 'b':
add('\b'); add_string_char(keep, '\b');
break; break;
// form feed // form feed
case 'f': case 'f':
add('\f'); add_string_char(keep, '\f');
break; break;
// line feed // line feed
case 'n': case 'n':
add('\n'); add_string_char(keep, '\n');
break; break;
// carriage return // carriage return
case 'r': case 'r':
add('\r'); add_string_char(keep, '\r');
break; break;
// tab // tab
case 't': case 't':
add('\t'); add_string_char(keep, '\t');
break; break;
// unicode escapes // unicode escapes
@@ -13494,10 +13505,7 @@ class lexer : public lexer_base<BasicJsonType>
JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF); JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF);
// translate codepoint into bytes // translate codepoint into bytes
encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte) add_codepoint(keep, codepoint);
{
add(static_cast<char_int_type>(byte));
});
break; break;
} }
@@ -13680,7 +13688,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0x7E: case 0x7E:
case 0x7F: case 0x7F:
{ {
add(current); add_string_char(keep, current);
break; break;
} }
@@ -13716,7 +13724,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xDE: case 0xDE:
case 0xDF: case 0xDF:
{ {
if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF}))) if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range(keep, {0x80, 0xBF})))
{ {
return token_type::parse_error; return token_type::parse_error;
} }
@@ -13726,7 +13734,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+0800..U+0FFF: bytes E0 A0..BF 80..BF // U+0800..U+0FFF: bytes E0 A0..BF 80..BF
case 0xE0: case 0xE0:
{ {
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF})))) if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0xA0, 0xBF, 0x80, 0xBF}))))
{ {
return token_type::parse_error; return token_type::parse_error;
} }
@@ -13750,7 +13758,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xEE: case 0xEE:
case 0xEF: case 0xEF:
{ {
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF})))) if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF}))))
{ {
return token_type::parse_error; return token_type::parse_error;
} }
@@ -13760,7 +13768,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+D000..U+D7FF: bytes ED 80..9F 80..BF // U+D000..U+D7FF: bytes ED 80..9F 80..BF
case 0xED: case 0xED:
{ {
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF})))) if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x9F, 0x80, 0xBF}))))
{ {
return token_type::parse_error; return token_type::parse_error;
} }
@@ -13770,7 +13778,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+10000..U+3FFFF F0 90..BF 80..BF 80..BF // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
case 0xF0: case 0xF0:
{ {
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
{ {
return token_type::parse_error; return token_type::parse_error;
} }
@@ -13782,7 +13790,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xF2: case 0xF2:
case 0xF3: case 0xF3:
{ {
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
{ {
return token_type::parse_error; return token_type::parse_error;
} }
@@ -13792,7 +13800,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+100000..U+10FFFF F4 80..8F 80..BF 80..BF // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
case 0xF4: case 0xF4:
{ {
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
{ {
return token_type::parse_error; return token_type::parse_error;
} }
@@ -14283,7 +14291,7 @@ scan_number_done:
token_type convert_number(token_type number_type, std::size_t mantissa_end) token_type convert_number(token_type number_type, std::size_t mantissa_end)
{ {
// accept() only needs to know whether the input is valid, so it sets // accept() only needs to know whether the input is valid, so it sets
// discard_number_values (see json.hpp), and an integer token whose // discard_values (see json.hpp), and an integer token whose
// digit count shows that it fits is reported without calling // digit count shows that it fits is reported without calling
// convert_integer(). A number with up to 18 digits always fits into // convert_integer(). A number with up to 18 digits always fits into
// both std::uint64_t and std::int64_t (18 nines is about 1e18, below // both std::uint64_t and std::int64_t (18 nines is about 1e18, below
@@ -14298,7 +14306,7 @@ scan_number_done:
// json_sax_acceptor, whose number callbacks discard their argument and // json_sax_acceptor, whose number callbacks discard their argument and
// return true, and the parser rejects neither integers nor finite // return true, and the parser rejects neither integers nor finite
// floats. value_unsigned/value_integer are left unset here, so a caller // floats. value_unsigned/value_integer are left unset here, so a caller
// that reads the converted value must not set discard_number_values. // that reads the converted value must not set discard_values.
// //
// On contiguous input, scan_number_bulk_contiguous() converts integer // On contiguous input, scan_number_bulk_contiguous() converts integer
// tokens itself and does not pass them to this function, unless // tokens itself and does not pass them to this function, unless
@@ -14306,7 +14314,7 @@ scan_number_done:
// reached for input without bulk access (e.g. streams), with // reached for input without bulk access (e.g. streams), with
// JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous() // JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous()
// falls back to scan_number(). // falls back to scan_number().
if (discard_number_values) if (discard_values)
{ {
constexpr std::size_t safe_digit_count = 18; constexpr std::size_t safe_digit_count = 18;
if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count) if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count)
@@ -14557,7 +14565,7 @@ scan_number_done:
// w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a // w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a
// value that underflows is a finite number as well. Only a token that // value that underflows is a finite number as well. Only a token that
// may overflow has to be converted to find out. // may overflow has to be converted to find out.
if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10)) if (JSON_HEDLEY_UNLIKELY(discard_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
{ {
value_float = 0; value_float = 0;
return true; return true;
@@ -14889,6 +14897,34 @@ scan_number_done:
token_buffer.push_back(static_cast<typename string_t::value_type>(c)); token_buffer.push_back(static_cast<typename string_t::value_type>(c));
} }
/// add a character of a string token; accept() checks the string but does
/// not keep its value (@a keep is false)
void add_string_char(std::true_type /*keep*/, char_int_type c)
{
token_buffer.push_back(static_cast<typename string_t::value_type>(c));
}
static void add_string_char(std::false_type /*keep*/, char_int_type /*c*/) noexcept {}
/// add the UTF-8 bytes of a code point from a \\u escape to a string token
void add_codepoint(std::true_type /*keep*/, int codepoint)
{
encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte)
{
add(static_cast<char_int_type>(byte));
});
}
static void add_codepoint(std::false_type /*keep*/, int /*codepoint*/) noexcept {}
/// append a run of ordinary string characters (see get_after_string_run())
void append_string_run(std::true_type /*keep*/, const unsigned char* data, std::size_t n)
{
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), n);
}
static void append_string_run(std::false_type /*keep*/, const unsigned char* /*data*/, std::size_t /*n*/) noexcept {}
public: public:
///////////////////// /////////////////////
// value getters // value getters
@@ -15283,7 +15319,11 @@ scan_number_done:
// string // string
case '\"': case '\"':
return scan_string(); // accept() (discard_values) checks strings without copying them
// into token_buffer. The choice is made once per string: a
// check inside the string loop made number-heavy documents up
// to 3 % slower (mesh.json, Apple M1)
return discard_values ? scan_string(std::false_type {}) : scan_string(std::true_type {});
// number // number
case '-': case '-':
@@ -15368,11 +15408,13 @@ scan_number_done:
std::size_t decimal_point_position = std::string::npos; std::size_t decimal_point_position = std::string::npos;
/// whether the caller only needs the token types and never looks at the /// whether the caller only needs the token types and never looks at the
/// converted numeric values; set only by accept(), which parses through /// values of number and string tokens; set only by accept(), which parses
/// json_sax_acceptor. When set, convert_number() skips converting integer /// through json_sax_acceptor. When set, convert_number() skips converting
/// tokens whose digit count guarantees that they fit into 64 bits (see /// integer tokens whose digit count guarantees that they fit into 64 bits
/// there) /// (see there), floats are converted only when they may overflow (see
const bool discard_number_values = false; /// convert_accumulated_float()), and strings are checked without being
/// copied into token_buffer (see scan())
const bool discard_values = false;
}; };
} // namespace detail } // namespace detail
@@ -22581,9 +22623,9 @@ class parser
const bool allow_exceptions_ = true, const bool allow_exceptions_ = true,
const bool ignore_comments = false, const bool ignore_comments = false,
const bool ignore_trailing_commas_ = false, const bool ignore_trailing_commas_ = false,
const bool discard_number_values_ = false) const bool discard_values_ = false)
: callback(std::move(cb)) : callback(std::move(cb))
, m_lexer(std::move(adapter), ignore_comments, discard_number_values_) , m_lexer(std::move(adapter), ignore_comments, discard_values_)
, allow_exceptions(allow_exceptions_) , allow_exceptions(allow_exceptions_)
, ignore_trailing_commas(ignore_trailing_commas_) , ignore_trailing_commas(ignore_trailing_commas_)
{ {
@@ -36980,11 +37022,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
const bool allow_exceptions = true, const bool allow_exceptions = true,
const bool ignore_comments = false, const bool ignore_comments = false,
const bool ignore_trailing_commas = false, const bool ignore_trailing_commas = false,
const bool discard_number_values = false const bool discard_values = false
) )
{ {
return ::nlohmann::detail::parser<basic_json, InputAdapterType>(std::move(adapter), return ::nlohmann::detail::parser<basic_json, InputAdapterType>(std::move(adapter),
std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values); std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_values);
} }
private: private:
+37 -157
View File
@@ -1311,8 +1311,7 @@ struct parse_failure
}; };
/// FloatType: the number_float_t of the document, whose overflow parse() rejects /// FloatType: the number_float_t of the document, whose overflow parse() rejects
/// Store: false for accept(): the grammar, number, string and UTF-8 checks run as for parse(), but no node is written template<typename FloatType, bool Comments, bool TrailingCommas, bool NulIsEnd, bool Sentinel>
template<typename FloatType, bool Comments, bool TrailingCommas, bool NulIsEnd, bool Sentinel, bool Store>
class builder class builder
{ {
public: public:
@@ -1488,87 +1487,11 @@ class builder
return p; return p;
} }
/// the string from p, the first backslash, control character, ill-formed UTF-8 byte, or the end, checked and not
/// decoded (accept() stores nothing, so the arena is not written): the checks, errors and error positions of
/// slow_string(), for everything up to the closing quote
NLOHMANN_VIEW_NOINLINE decoded check_string(const unsigned char* p)
{
const unsigned char* const end = m_end;
for (;;)
{
if (p == end)
{
return failed(error_code::string_missing_quote, p);
}
const unsigned char c = *p;
if (c == '"')
{
++p;
return decoded{p, 0, 0};
}
if (c != '\\')
{
return failed(c < 0x20 ? error_code::string_control_character : error_code::string_utf8, p);
}
++p;
if (p == end)
{
return failed(error_code::string_missing_quote, p);
}
const unsigned char d = *p++;
if (d == 'u')
{
const int cp = hex4(p); // (moves p past the four digits)
if (NLOHMANN_VIEW_UNLIKELY(cp < 0))
{
return failed(error_code::string_unicode_hex, p);
}
if (NLOHMANN_VIEW_UNLIKELY((cp & 0xF800) == 0xD800)) // a surrogate
{
if (cp >= 0xDC00)
{
return failed(error_code::string_surrogate_low, p);
}
if (end - p < 2 || p[0] != '\\' || p[1] != 'u')
{
return failed(error_code::string_surrogate_high, p);
}
p += 2;
const int lo = hex4(p); // (moves p past the four digits)
if (lo < 0)
{
return failed(error_code::string_unicode_hex, p);
}
if (lo < 0xDC00 || lo > 0xDFFF)
{
return failed(error_code::string_surrogate_high, p);
}
}
}
else if (!(d == '"' || d == '\\' || d == '/' || d == 'b' || d == 'f' || d == 'n' || d == 'r' || d == 't'))
{
--p;
return failed(error_code::string_escape, p);
}
if (p != end && *p == '\\')
{
continue; // (escapes in a row: no run of text between them)
}
p = scan_string_run(p, end); // the text up to the next backslash, quote, control character, or ill-formed UTF-8
}
}
/// s points to the start of the string, p to a backslash, a control /// s points to the start of the string, p to a backslash, a control
/// character, an ill-formed UTF-8 byte, or the end: decode the string into /// character, an ill-formed UTF-8 byte, or the end: decode the string into
/// the arena /// the arena
NLOHMANN_VIEW_NOINLINE decoded slow_string(const unsigned char* s, const unsigned char* p) NLOHMANN_VIEW_NOINLINE decoded slow_string(const unsigned char* s, const unsigned char* p)
{ {
if (!enabled(Store))
{
// (accept(): the string is checked, not decoded)
static_cast<void>(s);
return check_string(p);
}
// single-character escapes; 0: invalid (and 'u', handled separately) // single-character escapes; 0: invalid (and 'u', handled separately)
static const std::array<char, 128> simple_escape = static const std::array<char, 128> simple_escape =
{ {
@@ -1929,13 +1852,10 @@ run_found:
NLOHMANN_VIEW_ALWAYS_INLINE bool run() NLOHMANN_VIEW_ALWAYS_INLINE bool run()
{ {
if (enabled(Store)) cold.m_doc.reserve(estimate_nodes(reinterpret_cast<const char*>(b), static_cast<std::size_t>(e - b)));
{ base = cold.m_doc.nodes;
cold.m_doc.reserve(estimate_nodes(reinterpret_cast<const char*>(b), static_cast<std::size_t>(e - b))); out = base;
base = cold.m_doc.nodes; cap = base + cold.m_doc.node_capacity;
out = base;
cap = base + cold.m_doc.node_capacity;
}
if (e - p >= 3 && p[0] == 0xEF && p[1] == 0xBB && p[2] == 0xBF) if (e - p >= 3 && p[0] == 0xEF && p[1] == 0xBB && p[2] == 0xBF)
{ {
@@ -2119,10 +2039,7 @@ close_object:
// an array pays nothing for this) // an array pays nothing for this)
if (NLOHMANN_VIEW_UNLIKELY(cur_count >= document_data::index_min_members)) if (NLOHMANN_VIEW_UNLIKELY(cur_count >= document_data::index_min_members))
{ {
if (enabled(Store)) cold.note_large_object(cur_idx);
{
cold.note_large_object(cur_idx);
}
} }
close_container: close_container:
@@ -2282,12 +2199,6 @@ indent_done:
/// an integer's value); two stores on little-endian targets /// an integer's value); two stores on little-endian targets
NLOHMANN_VIEW_ALWAYS_INLINE node* push_node(node_kind k, std::uint8_t flags, std::uint16_t extra, std::size_t off, std::uint64_t second) NLOHMANN_VIEW_ALWAYS_INLINE node* push_node(node_kind k, std::uint8_t flags, std::uint16_t extra, std::size_t off, std::uint64_t second)
{ {
if (!enabled(Store))
{
// accept(): nothing is stored, so what computes the arguments is not run either. base and out stay
// null: their difference, which open() takes, is 0.
return out;
}
if (NLOHMANN_VIEW_UNLIKELY(out == cap)) if (NLOHMANN_VIEW_UNLIKELY(out == cap))
{ {
const auto n = static_cast<std::size_t>(out - base); const auto n = static_cast<std::size_t>(out - base);
@@ -2346,12 +2257,9 @@ indent_done:
NLOHMANN_VIEW_ALWAYS_INLINE void close() NLOHMANN_VIEW_ALWAYS_INLINE void close()
{ {
if (enabled(Store)) node& n = base[cur_idx];
{ n.len = cur_count;
node& n = base[cur_idx]; n.next = static_cast<std::uint32_t>(out - base) - cur_idx;
n.len = cur_count;
n.next = static_cast<std::uint32_t>(out - base) - cur_idx;
}
if (--depth != 0) if (--depth != 0)
{ {
if (NLOHMANN_VIEW_LIKELY(depth <= 64)) if (NLOHMANN_VIEW_LIKELY(depth <= 64))
@@ -2526,44 +2434,34 @@ indent_done:
}; };
/// run the builder with compile-time options /// run the builder with compile-time options
template<typename FloatType, bool Comments, bool TrailingCommas, bool NulIsEnd, bool Store> template<typename FloatType, bool Comments, bool TrailingCommas, bool NulIsEnd>
inline bool build_with(document_data& d, const char* src, std::size_t size, bool sentinel, parse_failure& failure) inline bool build_with(document_data& d, const char* src, std::size_t size, bool sentinel, parse_failure& failure)
{ {
if (sentinel) if (sentinel)
{ {
builder<FloatType, Comments, TrailingCommas, NulIsEnd, true, Store> bld(d, src, size); builder<FloatType, Comments, TrailingCommas, NulIsEnd, true> bld(d, src, size);
const bool ok = bld.run(); const bool ok = bld.run();
failure = bld.failure(); failure = bld.failure();
return ok; return ok;
} }
builder<FloatType, Comments, TrailingCommas, NulIsEnd, false, Store> bld(d, src, size); builder<FloatType, Comments, TrailingCommas, NulIsEnd, false> bld(d, src, size);
const bool ok = bld.run(); const bool ok = bld.run();
failure = bld.failure(); failure = bld.failure();
return ok; return ok;
} }
/// sentinel: src[size] is readable and 0 (e.g. std::string); FloatType: the /// sentinel: src[size] is readable and 0 (e.g. std::string); FloatType: the
/// number_float_t of the document; validate_only: accept(), only the result is /// number_float_t of the document
/// used
template<typename FloatType, bool NulIsEnd> template<typename FloatType, bool NulIsEnd>
inline bool build(document_data& d, const char* src, std::size_t size, bool comments, bool trailing_commas, bool sentinel, bool validate_only, inline bool build(document_data& d, const char* src, std::size_t size, bool comments, bool trailing_commas, bool sentinel, parse_failure& failure)
parse_failure& failure)
{ {
if (validate_only && !comments && !trailing_commas)
{
// accept() of plain JSON: nothing is stored. The other option sets
// could skip the stores just as well; they keep the builder that
// stores only so that there are two more instantiations of the parse
// loop, not eight.
return build_with<FloatType, false, false, NulIsEnd, false>(d, src, size, sentinel, failure);
}
if (comments) if (comments)
{ {
return trailing_commas ? build_with<FloatType, true, true, NulIsEnd, true>(d, src, size, sentinel, failure) return trailing_commas ? build_with<FloatType, true, true, NulIsEnd>(d, src, size, sentinel, failure)
: build_with<FloatType, true, false, NulIsEnd, true>(d, src, size, sentinel, failure); : build_with<FloatType, true, false, NulIsEnd>(d, src, size, sentinel, failure);
} }
return trailing_commas ? build_with<FloatType, false, true, NulIsEnd, true>(d, src, size, sentinel, failure) return trailing_commas ? build_with<FloatType, false, true, NulIsEnd>(d, src, size, sentinel, failure)
: build_with<FloatType, false, false, NulIsEnd, true>(d, src, size, sentinel, failure); : build_with<FloatType, false, false, NulIsEnd>(d, src, size, sentinel, failure);
} }
} // namespace view } // namespace view
@@ -10203,7 +10101,7 @@ class basic_json_document
const bool ignore_trailing_commas = false) const bool ignore_trailing_commas = false)
{ {
basic_json_document d; basic_json_document d;
d.read_range(first, last, error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas); d.read_range(first, last, allow_exceptions, ignore_comments, ignore_trailing_commas);
return d; return d;
} }
@@ -10218,7 +10116,7 @@ class basic_json_document
basic_json_document d; basic_json_document d;
std::string buf; std::string buf;
append_input(buf, std::forward<InputType>(input)); append_input(buf, std::forward<InputType>(input));
d.build_owned(std::move(buf), error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas); d.build_owned(std::move(buf), allow_exceptions, ignore_comments, ignore_trailing_commas);
return d; return d;
} }
@@ -10231,8 +10129,7 @@ class basic_json_document
static bool accept(InputType&& input, const bool ignore_comments = false, const bool ignore_trailing_commas = false) static bool accept(InputType&& input, const bool ignore_comments = false, const bool ignore_trailing_commas = false)
{ {
basic_json_document d; basic_json_document d;
d.read_kind(std::forward<InputType>(input), on_error::validate, ignore_comments, ignore_trailing_commas, d.read(std::forward<InputType>(input), false, ignore_comments, ignore_trailing_commas);
std::integral_constant<detail::view::input_kind, detail::view::classify_input<InputType>::value> {});
return !d.is_discarded(); return !d.is_discarded();
} }
@@ -10248,7 +10145,7 @@ class basic_json_document
const bool ignore_comments = false, const bool ignore_comments = false,
const bool ignore_trailing_commas = false) const bool ignore_trailing_commas = false)
{ {
read_kind(std::forward<InputType>(input), error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas, read_kind(std::forward<InputType>(input), allow_exceptions, ignore_comments, ignore_trailing_commas,
std::integral_constant<detail::view::input_kind, detail::view::classify_input<InputType>::value> {}); std::integral_constant<detail::view::input_kind, detail::view::classify_input<InputType>::value> {});
} }
@@ -10607,22 +10504,6 @@ class basic_json_document
return idx; return idx;
} }
/// what a failed parse does
enum class on_error : std::uint8_t
{
discard, ///< the document is discarded
raise, ///< parse_error is thrown
validate ///< accept(): the document is discarded, and no node is stored while parsing
};
static on_error error_mode(bool allow_exceptions) noexcept
{
return allow_exceptions ? on_error::raise : on_error::discard;
}
/// NulIsEnd of the builder: a null byte ends the input unless strict_nul_handling
static constexpr bool nul_is_end = !detail::abi_config::strict_nul_handling;
/// create the storage (sized for the input) on first use /// create the storage (sized for the input) on first use
void ensure_data(const char* src, std::size_t size) void ensure_data(const char* src, std::size_t size)
{ {
@@ -10634,7 +10515,7 @@ class basic_json_document
/// parse a buffer the document takes ownership of (it replaces the /// parse a buffer the document takes ownership of (it replaces the
/// previous owned copy) /// previous owned copy)
void build_owned(std::string&& buf, on_error allow_exceptions, bool comments, bool trailing_commas) void build_owned(std::string&& buf, bool allow_exceptions, bool comments, bool trailing_commas)
{ {
ensure_data(buf.data(), buf.size()); ensure_data(buf.data(), buf.size());
m_data->owned = std::move(buf); m_data->owned = std::move(buf);
@@ -10644,7 +10525,7 @@ class basic_json_document
/// parse a copy of the input; the copy is appended to the cleared owned /// parse a copy of the input; the copy is appended to the cleared owned
/// buffer, so that its capacity is reused /// buffer, so that its capacity is reused
template<typename T> template<typename T>
void build_copy(T&& input, on_error allow_exceptions, bool comments, bool trailing_commas) void build_copy(T&& input, bool allow_exceptions, bool comments, bool trailing_commas)
{ {
if (!m_data) if (!m_data)
{ {
@@ -10665,7 +10546,7 @@ class basic_json_document
} }
/// sentinel: src[size] is readable and 0 (std::string, C strings) /// sentinel: src[size] is readable and 0 (std::string, C strings)
void build(const char* src, std::size_t size, on_error allow_exceptions, bool comments, bool trailing_commas, bool owned, bool sentinel) void build(const char* src, std::size_t size, bool allow_exceptions, bool comments, bool trailing_commas, bool owned, bool sentinel)
{ {
ensure_data(src, size); ensure_data(src, size);
document_data& d = *m_data; document_data& d = *m_data;
@@ -10692,8 +10573,7 @@ class basic_json_document
} }
else else
{ {
ok = detail::view::build<typename BasicJsonType::number_float_t, nul_is_end>(d, src, size, comments, trailing_commas, sentinel, ok = detail::view::build < typename BasicJsonType::number_float_t, !detail::abi_config::strict_nul_handling > (d, src, size, comments, trailing_commas, sentinel, failure);
allow_exceptions == on_error::validate, failure);
} }
if (NLOHMANN_VIEW_LIKELY(ok)) if (NLOHMANN_VIEW_LIKELY(ok))
{ {
@@ -10706,7 +10586,7 @@ class basic_json_document
return; return;
} }
std::vector<std::uint32_t>().swap(d.large_objects); std::vector<std::uint32_t>().swap(d.large_objects);
if (allow_exceptions == on_error::raise) if (allow_exceptions)
{ {
detail::view::throw_parse_failure<BasicJsonType>(failure, src, size, comments, trailing_commas); detail::view::throw_parse_failure<BasicJsonType>(failure, src, size, comments, trailing_commas);
} }
@@ -10714,13 +10594,13 @@ class basic_json_document
// --- input dispatch (see detail::view::input_kind) --- // --- input dispatch (see detail::view::input_kind) ---
void read_kind(std::string&& s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::move_string> /*unused*/) void read_kind(std::string&& s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::move_string> /*unused*/)
{ {
build_owned(std::move(s), ae, c, tc); build_owned(std::move(s), ae, c, tc);
} }
template<typename CharT> template<typename CharT>
void read_kind(CharT* s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::c_string> /*unused*/) void read_kind(CharT* s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::c_string> /*unused*/)
{ {
if (s == nullptr) if (s == nullptr)
{ {
@@ -10734,7 +10614,7 @@ class basic_json_document
} }
template<typename Array> template<typename Array>
void read_kind(Array& a, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::char_array> /*unused*/) void read_kind(Array& a, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::char_array> /*unused*/)
{ {
using CharT = typename std::remove_cv<typename std::remove_extent<Array>::type>::type; using CharT = typename std::remove_cv<typename std::remove_extent<Array>::type>::type;
const std::size_t n = std::extent<Array>::value; const std::size_t n = std::extent<Array>::value;
@@ -10745,7 +10625,7 @@ class basic_json_document
} }
template<typename T> template<typename T>
void read_kind(const T& s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::borrow_range> /*unused*/) void read_kind(const T& s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::borrow_range> /*unused*/)
{ {
// std::basic_string guarantees data()[size()] == 0: use it as sentinel // std::basic_string guarantees data()[size()] == 0: use it as sentinel
const auto size = static_cast<std::size_t>(s.size()); const auto size = static_cast<std::size_t>(s.size());
@@ -10754,13 +10634,13 @@ class basic_json_document
} }
template<typename T> template<typename T>
void read_kind(const T& s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::copy_range> /*unused*/) void read_kind(const T& s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::copy_range> /*unused*/)
{ {
build_copy(s, ae, c, tc); build_copy(s, ae, c, tc);
} }
template<typename CharT> template<typename CharT>
void read_kind(CharT* s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::wide_c_string> /*unused*/) void read_kind(CharT* s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::wide_c_string> /*unused*/)
{ {
if (s == nullptr) if (s == nullptr)
{ {
@@ -10772,13 +10652,13 @@ class basic_json_document
} }
template<typename T> template<typename T>
void read_kind(T&& input, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::adapter> /*unused*/) void read_kind(T&& input, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::adapter> /*unused*/)
{ {
build_copy(std::forward<T>(input), ae, c, tc); build_copy(std::forward<T>(input), ae, c, tc);
} }
template<typename IteratorType> template<typename IteratorType>
void read_range(IteratorType first, IteratorType last, on_error ae, bool c, bool tc) void read_range(IteratorType first, IteratorType last, bool ae, bool c, bool tc)
{ {
using value_type = typename std::remove_cv<typename std::iterator_traits<IteratorType>::value_type>::type; using value_type = typename std::remove_cv<typename std::iterator_traits<IteratorType>::value_type>::type;
// borrow the range where the library's input adapter scans it as one // borrow the range where the library's input adapter scans it as one
@@ -10789,14 +10669,14 @@ class basic_json_document
} }
template<typename IteratorType> template<typename IteratorType>
void read_range_impl(IteratorType first, IteratorType last, on_error ae, bool c, bool tc, std::true_type /*contiguous bytes*/) void read_range_impl(IteratorType first, IteratorType last, bool ae, bool c, bool tc, std::true_type /*contiguous bytes*/)
{ {
const auto size = static_cast<std::size_t>(std::distance(first, last)); const auto size = static_cast<std::size_t>(std::distance(first, last));
build(size == 0 ? "" : reinterpret_cast<const char*>(&*first), size, ae, c, tc, false, size == 0); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) build(size == 0 ? "" : reinterpret_cast<const char*>(&*first), size, ae, c, tc, false, size == 0); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
} }
template<typename IteratorType> template<typename IteratorType>
void read_range_impl(IteratorType first, IteratorType last, on_error ae, bool c, bool tc, std::false_type /*other*/) void read_range_impl(IteratorType first, IteratorType last, bool ae, bool c, bool tc, std::false_type /*other*/)
{ {
build_copy(detail::view::make_adapter_input(detail::input_adapter(first, last)), ae, c, tc); build_copy(detail::view::make_adapter_input(detail::input_adapter(first, last)), ae, c, tc);
} }
+44
View File
@@ -2982,6 +2982,50 @@ TEST_CASE("accept() agrees with parse() on floats around the overflow limit")
} }
} }
TEST_CASE("accept() agrees with parse() on strings, which it checks without copying them")
{
// accept() scans strings and object keys with a separate instantiation of
// the string scanner that does not copy them into the token buffer; it
// must accept and reject exactly what parse() does
const std::vector<std::string> pieces =
{
"", "abc", "\\\"\\\\\\/\\b\\f\\n\\r\\t", "\\u0041\\u00e4\\u20AC", "\\ud83d\\ude00", "\\uD83D\\uDE00",
"\\ud800", "\\udc00", "\\ud800\\u0041", "\\ud800x", "\\udbff\\udfff", "\\u12", "\\u12g4", "\\x", "\\",
"\xc3\xa4", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xf4\x8f\xbf\xbf", "\xc0\xaf", "\xed\xa0\x80", "\xf4\x90\x80\x80",
"\xff", "\xe2\x82", "\xc3", "\x01", "\x1f", "\x7f", "\n", std::string(1, '\0')
};
for (std::size_t run = 0; run <= 70; ++run)
{
for (const std::string& piece : pieces)
{
// a run of ordinary characters puts the piece at every offset of the bulk scanners
const std::string str = std::string(run, 'a') + piece;
for (const std::string& text :
{
"\"" + str + "\"", "[\"" + str + "\"]", "{\"" + str + "\":1}", "{\"k\":\"" + str + "\",\"" + str + "\":[\"" + str + "\"]}",
"[\"" + str
})
{
CAPTURE(text)
for (int options = 0; options < 4; ++options)
{
const bool ignore_comments = (options & 1) != 0;
const bool ignore_trailing_commas = (options & 2) != 0;
const bool parses = !json::parse(text, nullptr, false, ignore_comments, ignore_trailing_commas).is_discarded();
CHECK(json::accept(text, ignore_comments, ignore_trailing_commas) == parses);
std::istringstream stream(text);
CHECK(json::accept(stream, ignore_comments, ignore_trailing_commas) == parses);
const std::list<char> chars(text.begin(), text.end());
CHECK(json::accept(chars.begin(), chars.end(), ignore_comments, ignore_trailing_commas) == parses);
}
}
}
}
}
namespace namespace
{ {
// a DOM parser that reports parse errors without throwing, as in the sax_no_exception example // a DOM parser that reports parse errors without throwing, as in the sax_no_exception example
+1 -145
View File
@@ -59,8 +59,7 @@ built build(const std::string& text, bool comments, bool trailing_commas, bool s
r.copy.assign(text.begin(), text.end()); r.copy.assign(text.begin(), text.end());
src = r.copy.data(); src = r.copy.data();
} }
constexpr bool nul_is_end = !nlohmann::detail::abi_config::strict_nul_handling; r.ok = nlohmann::detail::view::build < FloatType, !nlohmann::detail::abi_config::strict_nul_handling > (*r.data, src, text.size(), comments, trailing_commas, sentinel, r.failure);
r.ok = nlohmann::detail::view::build<FloatType, nul_is_end>(*r.data, src, text.size(), comments, trailing_commas, sentinel, false, r.failure);
r.data->src = src; r.data->src = src;
r.data->base[0] = src; r.data->base[0] = src;
r.data->base[1] = r.data->arena.data(); r.data->base[1] = r.data->arena.data();
@@ -173,22 +172,6 @@ void check_string(const std::string& content)
} }
} }
// accept() stores no nodes: it must accept and reject what json::accept does, for all options and for inputs with
// and without a terminating NUL
void check_accept(const std::string& text)
{
CAPTURE(text)
const std::vector<char> bytes(text.begin(), text.end());
for (int options = 0; options < 4; ++options)
{
const bool comments = (options & 1) != 0;
const bool trailing_commas = (options & 2) != 0;
const bool accepted = json::accept(text, comments, trailing_commas);
CHECK(nlohmann::json_document::accept(text, comments, trailing_commas) == accepted);
CHECK(nlohmann::json_document::accept(bytes, comments, trailing_commas) == accepted);
}
}
// a small deterministic generator of documents // a small deterministic generator of documents
struct generator struct generator
{ {
@@ -440,133 +423,6 @@ TEST_CASE("json_view builder")
} }
} }
SECTION("accept() without an index")
{
for (const char* text :
{
"", " ", "\n", "null", "nul", "true", "[", "]", "[]", "{}", "[1,2,3]", "[1,2,3", "[1,2,3,]", "{\"a\":1,}", "{\"a\"}", "{\"a\":}",
"[1] x", "[1][2]", "\xEF\xBB\xBF[1]", "[\"a\\", "[\"\\u12\"]", "[\"\\ud800\"]", "[\"\x01\"]", "[\"\xff\"]", "[01]", "[1.]",
"[.5]", "[-]", "[1e]", "// c\n[1]", "/* c */ {\"a\" /* x */ : 1}", "/* c", "[1] // c", "1e400", "-1e400", "123456789012345678901234567890",
"{\"a\":1,\"a\":2}", "[[[]]]", "\"\\u00e4\\n\\ud83d\\ude00\""
})
{
check_accept(text);
}
// a null byte inside the text, and after it
check_accept(std::string("[1,\0]", 5));
check_accept(std::string("[1]\0", 4));
check_accept(std::string("\0", 1));
// generated documents and damaged copies, as for the builder above
generator g;
for (int i = 0; i < 2000; ++i)
{
std::string text;
g.value(text, 0);
check_accept(text);
std::string damaged = text;
const auto at = static_cast<std::size_t>(g.r(static_cast<int>(damaged.size())));
static const char replacements[] = {'x', '"', '\\', ',', ':', ']', '}', '[', '{', '1', '-', '.', 'e', '\0', '\n', '/'}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
damaged[at] = replacements[g.r(16)];
check_accept(damaged);
check_accept(text.substr(0, at));
}
// nesting beyond the 64 levels that are kept on the builder's own stack, closed and not closed
for (const std::size_t depth :
{
63, 64, 65, 66, 200, 1000
})
{
check_accept(std::string(depth, '[') + std::string(depth, ']'));
check_accept(std::string(depth, '[') + std::string(depth - 1, ']'));
std::string objects;
for (std::size_t i = 0; i < depth; ++i)
{
objects += "{\"k\":";
}
objects += "1" + std::string(depth, '}');
check_accept(objects);
check_accept(objects.substr(0, objects.size() - 1));
}
// objects with enough members to get a hash index when they are indexed (and duplicate keys)
std::string large = "{";
for (int i = 0; i < 300; ++i)
{
large += (i == 0 ? "\"k" : ",\"k") + std::to_string(i % 200) + "\":" + std::to_string(i);
}
check_accept(large + "}");
check_accept(large);
check_accept("[" + large + "},{" + large.substr(1) + "}]");
// the overflow check uses the floating-point type of the document
using float_json = nlohmann::basic_json<std::map, std::vector, std::string, bool, std::int64_t, std::uint64_t, float>;
for (const char* text :
{
"1e39", "-1e39", "3.4028235e38", "3.4028236e38", "[1.5e38, 3.5e38]", "{\"a\": 1e-50, \"b\": 1e39}", "340282356779733661637539395458142568448"
})
{
CAPTURE(text)
CHECK(nlohmann::basic_json_document<float_json>::accept(text) == float_json::accept(text));
}
// strings are checked and not decoded: runs of text of every length from 0 to 40 between valid and invalid
// escapes, UTF-8 sequences and control characters, in values and in keys, cut near the end of the text
const std::array<const char*, 9> valid = {{"\\n", "\\\"", "\\\\", "\\/", "\\t", "\\u00e4", "\\u20ac", "\\ud83d\\ude00", "\\u0000"}};
const std::array<const char*, 12> invalid = {{"\\x", "\\ud800", "\\udc00", "\\u12G4", "\\u12", "\\ud83dx", "\\ud83d\\u0041", "\\ud83d\\ud83d", "\\", "\x01", "\x1f", "\x7f"}};
const std::array<const char*, 6> utf8 = {{"\xc3\xa4", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xc3", "\xff", "\x80"}};
std::mt19937 rng(2029); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible
for (std::size_t run = 0; run <= 40; ++run)
{
for (int i = 0; i < 30; ++i)
{
std::string content = std::string(rng() % 20, 'a');
const auto pieces = 1 + (rng() % 6);
for (unsigned k = 0; k < pieces; ++k)
{
content += valid[rng() % valid.size()];
content += std::string(k == 0 ? run : rng() % 41, static_cast<char>('b' + (rng() % 20)));
switch (rng() % 8)
{
case 0:
content += utf8[rng() % 3];
break;
case 1:
content += utf8[rng() % utf8.size()];
break;
case 2:
content += invalid[rng() % invalid.size()];
break;
default:
break;
}
}
for (const std::string& text :
{
"[\"" + content + "\"]", "{\"" + content + "\":1}", "[\"" + content, "[\"" + content.substr(0, content.size() - (rng() % 4)) + "\"]"
})
{
check_accept(text);
}
}
}
}
SECTION("escapes in a row")
{
for (const char* content :
{
"\\n", "\\n\\t\\\\", "\\u00e9\\u00e9\\u4e2d", "\\u00e9\\u00e", "\\u00e9\\", "\\n\\q", "\\n\\ud83d\\ude00\\n",
"\\ud83d\\ude00\\ud83d", "\\n\\n\\n abc \\n\\n", "\\n\x01", "\\n\xC3\x28", "\\n\\"
})
{
check_accept(std::string("[\"") + content + "\"]");
check_accept(std::string("[\"") + content);
check_accept(std::string("{\"") + content + "\":1}");
}
}
SECTION("test files") SECTION("test files")
{ {
for (const char* name : for (const char* name :