Compare commits

..
Author SHA1 Message Date
Niels Lohmann 74f5ecbd35 accept() checks strings without copying them
json::accept() parses through json_sax_acceptor, which ignores the
values of strings and object keys, but the lexer still copied every
string token into its token buffer. That copy was 18 % of the
instructions of json::accept() on twitter.json (x86, GCC 16).

The lexer's flag for accept(), renamed from discard_number_values to
discard_values, now also selects a second instantiation of
scan_string() that performs the same checks (escapes, surrogates,
UTF-8, control characters, with the same error messages) without
writing to the token buffer. scan() chooses the instantiation once per
string: a run-time check inside the string loop made number-heavy
files up to 3 % slower on Apple M1.

json::accept() over 19 files, time after/before (geomean): 0.876 on x86
with GCC 16.2, 0.884 with Clang 23, 0.932 on Apple M1; up to 0.74 on
single files; files with few strings (canada, mesh, numbers) unchanged.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-31 00:07:57 +01:00
11 changed files with 218 additions and 311 deletions

No files matched your search

+83 -41
View File
@@ -268,10 +268,10 @@ class lexer : public lexer_base<BasicJsonType>
public:
using token_type = typename lexer_base<BasicJsonType>::token_type;
explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept
explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_values_ = false) noexcept
: ia(std::move(adapter))
, ignore_comments(ignore_comments_)
, discard_number_values(discard_number_values_)
, discard_values(discard_values_)
{
// the token string of an error before the first token is everything
// read so far
@@ -388,6 +388,8 @@ class lexer : public lexer_base<BasicJsonType>
checks if it is inside the range. If a violation was detected, set up an
error message and return false. Otherwise, return true.
@param[in] keep whether the bytes are added to token_buffer (false for
accept(), see scan())
@param[in] ranges list of integers; interpreted as list of pairs of
inclusive lower and upper bound, respectively
@@ -396,17 +398,18 @@ class lexer : public lexer_base<BasicJsonType>
@return true if and only if no range violation was detected
*/
bool next_byte_in_range(std::initializer_list<char_int_type> ranges)
template<bool Keep>
bool next_byte_in_range(std::integral_constant<bool, Keep> keep, std::initializer_list<char_int_type> ranges)
{
JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6);
add(current);
add_string_char(keep, current);
for (auto range = ranges.begin(); range != ranges.end(); ++range)
{
get();
if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions)
{
add(current);
add_string_char(keep, current);
}
else
{
@@ -419,14 +422,16 @@ class lexer : public lexer_base<BasicJsonType>
}
/// contiguous input: bulk-append the run of ordinary characters and complete
/// well-formed UTF-8 sequences starting at the current read position, then
/// well-formed UTF-8 sequences starting at the current read position (only
/// skip it if @a keep is false), then
/// read the first byte that needs individual handling (the closing quote, an
/// escape, a control character, or an ill-formed UTF-8 byte) like get().
/// That byte is taken from the buffer that was just scanned. Through get(),
/// Apple Clang merged this read with the other get() calls of the scan
/// function, which load the read position again right after it was stored
/// (3% of json::accept() on jeopardy.json on Apple M1).
char_int_type get_after_string_run(std::true_type /*bulk*/)
template<bool Keep>
char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant<bool, Keep> keep)
{
// a pending unget must be consumed through the normal path first
if (next_unget)
@@ -444,7 +449,7 @@ class lexer : public lexer_base<BasicJsonType>
const std::size_t pos = lexer_string_bulk_run(data, remaining);
if (pos != 0)
{
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), pos);
append_string_run(keep, data, pos);
bulk_consume(reinterpret_cast<const char*>(data), pos);
}
if (pos == remaining)
@@ -488,7 +493,8 @@ class lexer : public lexer_base<BasicJsonType>
}
/// streaming input: no bulk fast path
char_int_type get_after_string_run(std::false_type /*bulk*/)
template<bool Keep>
char_int_type get_after_string_run(std::false_type /*bulk*/, std::integral_constant<bool, Keep> /*keep*/)
{
return get();
}
@@ -502,13 +508,18 @@ class lexer : public lexer_base<BasicJsonType>
may contain \0 bytes), and token_buffer.size() is the number of bytes in the
string.
@param[in] keep whether the string is copied into token_buffer; if false
(accept(), see scan()), it is only checked, with the same
errors, and token_buffer stays empty
@return token_type::value_string if string could be successfully scanned,
token_type::parse_error otherwise
@note In case of errors, variable error_message contains a textual
description.
*/
token_type scan_string()
template<bool Keep>
token_type scan_string(std::integral_constant<bool, Keep> keep)
{
// reset token_buffer (ignore opening quote)
reset();
@@ -520,7 +531,7 @@ class lexer : public lexer_base<BasicJsonType>
{
// bulk-consume ordinary characters from contiguous input, then
// get the next character, the one that ends the run
switch (get_after_string_run(std::integral_constant<bool, bulk_scan>()))
switch (get_after_string_run(std::integral_constant<bool, bulk_scan>(), keep))
{
// end of file while parsing the string
case char_traits<char_type>::eof():
@@ -542,35 +553,35 @@ class lexer : public lexer_base<BasicJsonType>
{
// quotation mark
case '\"':
add('\"');
add_string_char(keep, '\"');
break;
// reverse solidus
case '\\':
add('\\');
add_string_char(keep, '\\');
break;
// solidus
case '/':
add('/');
add_string_char(keep, '/');
break;
// backspace
case 'b':
add('\b');
add_string_char(keep, '\b');
break;
// form feed
case 'f':
add('\f');
add_string_char(keep, '\f');
break;
// line feed
case 'n':
add('\n');
add_string_char(keep, '\n');
break;
// carriage return
case 'r':
add('\r');
add_string_char(keep, '\r');
break;
// tab
case 't':
add('\t');
add_string_char(keep, '\t');
break;
// unicode escapes
@@ -638,10 +649,7 @@ class lexer : public lexer_base<BasicJsonType>
JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF);
// translate codepoint into bytes
encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte)
{
add(static_cast<char_int_type>(byte));
});
add_codepoint(keep, codepoint);
break;
}
@@ -824,7 +832,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0x7E:
case 0x7F:
{
add(current);
add_string_char(keep, current);
break;
}
@@ -860,7 +868,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xDE:
case 0xDF:
{
if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF})))
if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range(keep, {0x80, 0xBF})))
{
return token_type::parse_error;
}
@@ -870,7 +878,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+0800..U+0FFF: bytes E0 A0..BF 80..BF
case 0xE0:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0xA0, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -894,7 +902,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xEE:
case 0xEF:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -904,7 +912,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+D000..U+D7FF: bytes ED 80..9F 80..BF
case 0xED:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x9F, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -914,7 +922,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
case 0xF0:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -926,7 +934,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xF2:
case 0xF3:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -936,7 +944,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
case 0xF4:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -1427,7 +1435,7 @@ scan_number_done:
token_type convert_number(token_type number_type, std::size_t mantissa_end)
{
// accept() only needs to know whether the input is valid, so it sets
// discard_number_values (see json.hpp), and an integer token whose
// discard_values (see json.hpp), and an integer token whose
// digit count shows that it fits is reported without calling
// convert_integer(). A number with up to 18 digits always fits into
// both std::uint64_t and std::int64_t (18 nines is about 1e18, below
@@ -1442,7 +1450,7 @@ scan_number_done:
// json_sax_acceptor, whose number callbacks discard their argument and
// return true, and the parser rejects neither integers nor finite
// floats. value_unsigned/value_integer are left unset here, so a caller
// that reads the converted value must not set discard_number_values.
// that reads the converted value must not set discard_values.
//
// On contiguous input, scan_number_bulk_contiguous() converts integer
// tokens itself and does not pass them to this function, unless
@@ -1450,7 +1458,7 @@ scan_number_done:
// reached for input without bulk access (e.g. streams), with
// JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous()
// falls back to scan_number().
if (discard_number_values)
if (discard_values)
{
constexpr std::size_t safe_digit_count = 18;
if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count)
@@ -1701,7 +1709,7 @@ scan_number_done:
// w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a
// value that underflows is a finite number as well. Only a token that
// may overflow has to be converted to find out.
if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
if (JSON_HEDLEY_UNLIKELY(discard_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
{
value_float = 0;
return true;
@@ -2033,6 +2041,34 @@ scan_number_done:
token_buffer.push_back(static_cast<typename string_t::value_type>(c));
}
/// add a character of a string token; accept() checks the string but does
/// not keep its value (@a keep is false)
void add_string_char(std::true_type /*keep*/, char_int_type c)
{
token_buffer.push_back(static_cast<typename string_t::value_type>(c));
}
static void add_string_char(std::false_type /*keep*/, char_int_type /*c*/) noexcept {}
/// add the UTF-8 bytes of a code point from a \\u escape to a string token
void add_codepoint(std::true_type /*keep*/, int codepoint)
{
encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte)
{
add(static_cast<char_int_type>(byte));
});
}
static void add_codepoint(std::false_type /*keep*/, int /*codepoint*/) noexcept {}
/// append a run of ordinary string characters (see get_after_string_run())
void append_string_run(std::true_type /*keep*/, const unsigned char* data, std::size_t n)
{
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), n);
}
static void append_string_run(std::false_type /*keep*/, const unsigned char* /*data*/, std::size_t /*n*/) noexcept {}
public:
/////////////////////
// value getters
@@ -2427,7 +2463,11 @@ scan_number_done:
// string
case '\"':
return scan_string();
// accept() (discard_values) checks strings without copying them
// into token_buffer. The choice is made once per string: a
// check inside the string loop made number-heavy documents up
// to 3 % slower (mesh.json, Apple M1)
return discard_values ? scan_string(std::false_type {}) : scan_string(std::true_type {});
// number
case '-':
@@ -2512,11 +2552,13 @@ scan_number_done:
std::size_t decimal_point_position = std::string::npos;
/// whether the caller only needs the token types and never looks at the
/// converted numeric values; set only by accept(), which parses through
/// json_sax_acceptor. When set, convert_number() skips converting integer
/// tokens whose digit count guarantees that they fit into 64 bits (see
/// there)
const bool discard_number_values = false;
/// values of number and string tokens; set only by accept(), which parses
/// through json_sax_acceptor. When set, convert_number() skips converting
/// integer tokens whose digit count guarantees that they fit into 64 bits
/// (see there), floats are converted only when they may overflow (see
/// convert_accumulated_float()), and strings are checked without being
/// copied into token_buffer (see scan())
const bool discard_values = false;
};
} // namespace detail
+2 -2
View File
@@ -89,9 +89,9 @@ class parser
const bool allow_exceptions_ = true,
const bool ignore_comments = false,
const bool ignore_trailing_commas_ = false,
const bool discard_number_values_ = false)
const bool discard_values_ = false)
: callback(std::move(cb))
, m_lexer(std::move(adapter), ignore_comments, discard_number_values_)
, m_lexer(std::move(adapter), ignore_comments, discard_values_)
, allow_exceptions(allow_exceptions_)
, ignore_trailing_commas(ignore_trailing_commas_)
{
@@ -1977,23 +1977,6 @@ class serializer
return;
}
// 17 to 20 digits: the first one to four, then twice eight
if (magnitude_fits_64_bits(abs_value, std::integral_constant<bool, magnitude_has_at_most_64_bits>()))
{
const auto value = static_cast<std::uint64_t>(abs_value);
const std::uint64_t top = value / 10000000000000000u; // 1 to 1844
const std::uint64_t low16 = value - (top * 10000000000000000u);
const std::uint64_t middle = low16 / 100000000u;
const std::uint64_t first = dtoa_impl::eight_digit_bytes(top);
const auto leading = static_cast<unsigned>(count_leading_zeros(first) / 8); // (first is not 0)
char* const p = buffer_ptr + n_chars;
dtoa_impl::store_msb_first(p, (first << (8 * leading)) + 0x3030303030303030u);
dtoa_impl::store_msb_first(p + 8 - leading, dtoa_impl::eight_digit_bytes(middle) + 0x3030303030303030u);
dtoa_impl::store_msb_first(p + 16 - leading, dtoa_impl::eight_digit_bytes(low16 - (middle * 100000000u)) + 0x3030303030303030u);
write_buffer_pos += n_chars + 24 - leading;
return;
}
n_chars += count_digits(abs_value);
// spare 1 byte for '\0'
@@ -2139,23 +2122,6 @@ class serializer
}
}
/// whether every number_magnitude_t fits 64 bits
static constexpr bool magnitude_has_at_most_64_bits = std::numeric_limits<number_magnitude_t>::is_specialized
&& std::numeric_limits<number_magnitude_t>::digits <= 64;
/// whether @a x fits 64 bits: always if number_magnitude_t has at most
/// 64 bits (selected by type, as the comparison would be reported as
/// always true), else compared
static bool magnitude_fits_64_bits(const number_magnitude_t& /*unused*/, std::true_type /*unused*/) noexcept
{
return true;
}
static bool magnitude_fits_64_bits(const number_magnitude_t& x, std::false_type /*unused*/) noexcept
{
return x <= 18446744073709551615u;
}
/*
* Helper function for dump_integer
*
-15
View File
@@ -343,21 +343,6 @@ class builder
}
arena_ensure(w, w_end, 4);
w = put_utf8(w, cp);
// A chain of unicode escapes ("\u3042\u3044\u3046", as Python's json.dumps writes non-ASCII text) is
// decoded here, without going round the loop above for each of them. Anything else (a surrogate, a
// bad digit, fewer than six bytes) leaves p at the backslash: the loop above handles it, and reports
// the same error at the same position as it would without this loop.
while (end - p >= 6 && p[0] == '\\' && p[1] == 'u')
{
const int next = hex_codepoint(p + 2);
if (next < 0 || (next & 0xF800) == 0xD800)
{
break;
}
p += 6;
arena_ensure(w, w_end, 4);
w = put_utf8(w, next);
}
}
else if (NLOHMANN_VIEW_LIKELY(d < 128 && simple_escape[d] != 0))
{
+2 -2
View File
@@ -184,11 +184,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
const bool allow_exceptions = true,
const bool ignore_comments = false,
const bool ignore_trailing_commas = false,
const bool discard_number_values = false
const bool discard_values = false
)
{
return ::nlohmann::detail::parser<basic_json, InputAdapterType>(std::move(adapter),
std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values);
std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_values);
}
private:
+87 -79
View File
@@ -13124,10 +13124,10 @@ class lexer : public lexer_base<BasicJsonType>
public:
using token_type = typename lexer_base<BasicJsonType>::token_type;
explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept
explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_values_ = false) noexcept
: ia(std::move(adapter))
, ignore_comments(ignore_comments_)
, discard_number_values(discard_number_values_)
, discard_values(discard_values_)
{
// the token string of an error before the first token is everything
// read so far
@@ -13244,6 +13244,8 @@ class lexer : public lexer_base<BasicJsonType>
checks if it is inside the range. If a violation was detected, set up an
error message and return false. Otherwise, return true.
@param[in] keep whether the bytes are added to token_buffer (false for
accept(), see scan())
@param[in] ranges list of integers; interpreted as list of pairs of
inclusive lower and upper bound, respectively
@@ -13252,17 +13254,18 @@ class lexer : public lexer_base<BasicJsonType>
@return true if and only if no range violation was detected
*/
bool next_byte_in_range(std::initializer_list<char_int_type> ranges)
template<bool Keep>
bool next_byte_in_range(std::integral_constant<bool, Keep> keep, std::initializer_list<char_int_type> ranges)
{
JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6);
add(current);
add_string_char(keep, current);
for (auto range = ranges.begin(); range != ranges.end(); ++range)
{
get();
if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions)
{
add(current);
add_string_char(keep, current);
}
else
{
@@ -13275,14 +13278,16 @@ class lexer : public lexer_base<BasicJsonType>
}
/// contiguous input: bulk-append the run of ordinary characters and complete
/// well-formed UTF-8 sequences starting at the current read position, then
/// well-formed UTF-8 sequences starting at the current read position (only
/// skip it if @a keep is false), then
/// read the first byte that needs individual handling (the closing quote, an
/// escape, a control character, or an ill-formed UTF-8 byte) like get().
/// That byte is taken from the buffer that was just scanned. Through get(),
/// Apple Clang merged this read with the other get() calls of the scan
/// function, which load the read position again right after it was stored
/// (3% of json::accept() on jeopardy.json on Apple M1).
char_int_type get_after_string_run(std::true_type /*bulk*/)
template<bool Keep>
char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant<bool, Keep> keep)
{
// a pending unget must be consumed through the normal path first
if (next_unget)
@@ -13300,7 +13305,7 @@ class lexer : public lexer_base<BasicJsonType>
const std::size_t pos = lexer_string_bulk_run(data, remaining);
if (pos != 0)
{
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), pos);
append_string_run(keep, data, pos);
bulk_consume(reinterpret_cast<const char*>(data), pos);
}
if (pos == remaining)
@@ -13344,7 +13349,8 @@ class lexer : public lexer_base<BasicJsonType>
}
/// streaming input: no bulk fast path
char_int_type get_after_string_run(std::false_type /*bulk*/)
template<bool Keep>
char_int_type get_after_string_run(std::false_type /*bulk*/, std::integral_constant<bool, Keep> /*keep*/)
{
return get();
}
@@ -13358,13 +13364,18 @@ class lexer : public lexer_base<BasicJsonType>
may contain \0 bytes), and token_buffer.size() is the number of bytes in the
string.
@param[in] keep whether the string is copied into token_buffer; if false
(accept(), see scan()), it is only checked, with the same
errors, and token_buffer stays empty
@return token_type::value_string if string could be successfully scanned,
token_type::parse_error otherwise
@note In case of errors, variable error_message contains a textual
description.
*/
token_type scan_string()
template<bool Keep>
token_type scan_string(std::integral_constant<bool, Keep> keep)
{
// reset token_buffer (ignore opening quote)
reset();
@@ -13376,7 +13387,7 @@ class lexer : public lexer_base<BasicJsonType>
{
// bulk-consume ordinary characters from contiguous input, then
// get the next character, the one that ends the run
switch (get_after_string_run(std::integral_constant<bool, bulk_scan>()))
switch (get_after_string_run(std::integral_constant<bool, bulk_scan>(), keep))
{
// end of file while parsing the string
case char_traits<char_type>::eof():
@@ -13398,35 +13409,35 @@ class lexer : public lexer_base<BasicJsonType>
{
// quotation mark
case '\"':
add('\"');
add_string_char(keep, '\"');
break;
// reverse solidus
case '\\':
add('\\');
add_string_char(keep, '\\');
break;
// solidus
case '/':
add('/');
add_string_char(keep, '/');
break;
// backspace
case 'b':
add('\b');
add_string_char(keep, '\b');
break;
// form feed
case 'f':
add('\f');
add_string_char(keep, '\f');
break;
// line feed
case 'n':
add('\n');
add_string_char(keep, '\n');
break;
// carriage return
case 'r':
add('\r');
add_string_char(keep, '\r');
break;
// tab
case 't':
add('\t');
add_string_char(keep, '\t');
break;
// unicode escapes
@@ -13494,10 +13505,7 @@ class lexer : public lexer_base<BasicJsonType>
JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF);
// translate codepoint into bytes
encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte)
{
add(static_cast<char_int_type>(byte));
});
add_codepoint(keep, codepoint);
break;
}
@@ -13680,7 +13688,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0x7E:
case 0x7F:
{
add(current);
add_string_char(keep, current);
break;
}
@@ -13716,7 +13724,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xDE:
case 0xDF:
{
if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF})))
if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range(keep, {0x80, 0xBF})))
{
return token_type::parse_error;
}
@@ -13726,7 +13734,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+0800..U+0FFF: bytes E0 A0..BF 80..BF
case 0xE0:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0xA0, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -13750,7 +13758,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xEE:
case 0xEF:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -13760,7 +13768,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+D000..U+D7FF: bytes ED 80..9F 80..BF
case 0xED:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x9F, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -13770,7 +13778,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
case 0xF0:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -13782,7 +13790,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xF2:
case 0xF3:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -13792,7 +13800,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
case 0xF4:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -14283,7 +14291,7 @@ scan_number_done:
token_type convert_number(token_type number_type, std::size_t mantissa_end)
{
// accept() only needs to know whether the input is valid, so it sets
// discard_number_values (see json.hpp), and an integer token whose
// discard_values (see json.hpp), and an integer token whose
// digit count shows that it fits is reported without calling
// convert_integer(). A number with up to 18 digits always fits into
// both std::uint64_t and std::int64_t (18 nines is about 1e18, below
@@ -14298,7 +14306,7 @@ scan_number_done:
// json_sax_acceptor, whose number callbacks discard their argument and
// return true, and the parser rejects neither integers nor finite
// floats. value_unsigned/value_integer are left unset here, so a caller
// that reads the converted value must not set discard_number_values.
// that reads the converted value must not set discard_values.
//
// On contiguous input, scan_number_bulk_contiguous() converts integer
// tokens itself and does not pass them to this function, unless
@@ -14306,7 +14314,7 @@ scan_number_done:
// reached for input without bulk access (e.g. streams), with
// JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous()
// falls back to scan_number().
if (discard_number_values)
if (discard_values)
{
constexpr std::size_t safe_digit_count = 18;
if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count)
@@ -14557,7 +14565,7 @@ scan_number_done:
// w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a
// value that underflows is a finite number as well. Only a token that
// may overflow has to be converted to find out.
if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
if (JSON_HEDLEY_UNLIKELY(discard_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
{
value_float = 0;
return true;
@@ -14889,6 +14897,34 @@ scan_number_done:
token_buffer.push_back(static_cast<typename string_t::value_type>(c));
}
/// add a character of a string token; accept() checks the string but does
/// not keep its value (@a keep is false)
void add_string_char(std::true_type /*keep*/, char_int_type c)
{
token_buffer.push_back(static_cast<typename string_t::value_type>(c));
}
static void add_string_char(std::false_type /*keep*/, char_int_type /*c*/) noexcept {}
/// add the UTF-8 bytes of a code point from a \\u escape to a string token
void add_codepoint(std::true_type /*keep*/, int codepoint)
{
encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte)
{
add(static_cast<char_int_type>(byte));
});
}
static void add_codepoint(std::false_type /*keep*/, int /*codepoint*/) noexcept {}
/// append a run of ordinary string characters (see get_after_string_run())
void append_string_run(std::true_type /*keep*/, const unsigned char* data, std::size_t n)
{
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), n);
}
static void append_string_run(std::false_type /*keep*/, const unsigned char* /*data*/, std::size_t /*n*/) noexcept {}
public:
/////////////////////
// value getters
@@ -15283,7 +15319,11 @@ scan_number_done:
// string
case '\"':
return scan_string();
// accept() (discard_values) checks strings without copying them
// into token_buffer. The choice is made once per string: a
// check inside the string loop made number-heavy documents up
// to 3 % slower (mesh.json, Apple M1)
return discard_values ? scan_string(std::false_type {}) : scan_string(std::true_type {});
// number
case '-':
@@ -15368,11 +15408,13 @@ scan_number_done:
std::size_t decimal_point_position = std::string::npos;
/// whether the caller only needs the token types and never looks at the
/// converted numeric values; set only by accept(), which parses through
/// json_sax_acceptor. When set, convert_number() skips converting integer
/// tokens whose digit count guarantees that they fit into 64 bits (see
/// there)
const bool discard_number_values = false;
/// values of number and string tokens; set only by accept(), which parses
/// through json_sax_acceptor. When set, convert_number() skips converting
/// integer tokens whose digit count guarantees that they fit into 64 bits
/// (see there), floats are converted only when they may overflow (see
/// convert_accumulated_float()), and strings are checked without being
/// copied into token_buffer (see scan())
const bool discard_values = false;
};
} // namespace detail
@@ -22581,9 +22623,9 @@ class parser
const bool allow_exceptions_ = true,
const bool ignore_comments = false,
const bool ignore_trailing_commas_ = false,
const bool discard_number_values_ = false)
const bool discard_values_ = false)
: callback(std::move(cb))
, m_lexer(std::move(adapter), ignore_comments, discard_number_values_)
, m_lexer(std::move(adapter), ignore_comments, discard_values_)
, allow_exceptions(allow_exceptions_)
, ignore_trailing_commas(ignore_trailing_commas_)
{
@@ -36224,23 +36266,6 @@ class serializer
return;
}
// 17 to 20 digits: the first one to four, then twice eight
if (magnitude_fits_64_bits(abs_value, std::integral_constant<bool, magnitude_has_at_most_64_bits>()))
{
const auto value = static_cast<std::uint64_t>(abs_value);
const std::uint64_t top = value / 10000000000000000u; // 1 to 1844
const std::uint64_t low16 = value - (top * 10000000000000000u);
const std::uint64_t middle = low16 / 100000000u;
const std::uint64_t first = dtoa_impl::eight_digit_bytes(top);
const auto leading = static_cast<unsigned>(count_leading_zeros(first) / 8); // (first is not 0)
char* const p = buffer_ptr + n_chars;
dtoa_impl::store_msb_first(p, (first << (8 * leading)) + 0x3030303030303030u);
dtoa_impl::store_msb_first(p + 8 - leading, dtoa_impl::eight_digit_bytes(middle) + 0x3030303030303030u);
dtoa_impl::store_msb_first(p + 16 - leading, dtoa_impl::eight_digit_bytes(low16 - (middle * 100000000u)) + 0x3030303030303030u);
write_buffer_pos += n_chars + 24 - leading;
return;
}
n_chars += count_digits(abs_value);
// spare 1 byte for '\0'
@@ -36386,23 +36411,6 @@ class serializer
}
}
/// whether every number_magnitude_t fits 64 bits
static constexpr bool magnitude_has_at_most_64_bits = std::numeric_limits<number_magnitude_t>::is_specialized
&& std::numeric_limits<number_magnitude_t>::digits <= 64;
/// whether @a x fits 64 bits: always if number_magnitude_t has at most
/// 64 bits (selected by type, as the comparison would be reported as
/// always true), else compared
static bool magnitude_fits_64_bits(const number_magnitude_t& /*unused*/, std::true_type /*unused*/) noexcept
{
return true;
}
static bool magnitude_fits_64_bits(const number_magnitude_t& x, std::false_type /*unused*/) noexcept
{
return x <= 18446744073709551615u;
}
/*
* Helper function for dump_integer
*
@@ -37014,11 +37022,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
const bool allow_exceptions = true,
const bool ignore_comments = false,
const bool ignore_trailing_commas = false,
const bool discard_number_values = false
const bool discard_values = false
)
{
return ::nlohmann::detail::parser<basic_json, InputAdapterType>(std::move(adapter),
std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values);
std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_values);
}
private:
-15
View File
@@ -1585,21 +1585,6 @@ class builder
}
arena_ensure(w, w_end, 4);
w = put_utf8(w, cp);
// A chain of unicode escapes ("\u3042\u3044\u3046", as Python's json.dumps writes non-ASCII text) is
// decoded here, without going round the loop above for each of them. Anything else (a surrogate, a
// bad digit, fewer than six bytes) leaves p at the backslash: the loop above handles it, and reports
// the same error at the same position as it would without this loop.
while (end - p >= 6 && p[0] == '\\' && p[1] == 'u')
{
const int next = hex_codepoint(p + 2);
if (next < 0 || (next & 0xF800) == 0xD800)
{
break;
}
p += 6;
arena_ensure(w, w_end, 4);
w = put_utf8(w, next);
}
}
else if (NLOHMANN_VIEW_LIKELY(d < 128 && simple_escape[d] != 0))
{
+44
View File
@@ -2982,6 +2982,50 @@ TEST_CASE("accept() agrees with parse() on floats around the overflow limit")
}
}
TEST_CASE("accept() agrees with parse() on strings, which it checks without copying them")
{
// accept() scans strings and object keys with a separate instantiation of
// the string scanner that does not copy them into the token buffer; it
// must accept and reject exactly what parse() does
const std::vector<std::string> pieces =
{
"", "abc", "\\\"\\\\\\/\\b\\f\\n\\r\\t", "\\u0041\\u00e4\\u20AC", "\\ud83d\\ude00", "\\uD83D\\uDE00",
"\\ud800", "\\udc00", "\\ud800\\u0041", "\\ud800x", "\\udbff\\udfff", "\\u12", "\\u12g4", "\\x", "\\",
"\xc3\xa4", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xf4\x8f\xbf\xbf", "\xc0\xaf", "\xed\xa0\x80", "\xf4\x90\x80\x80",
"\xff", "\xe2\x82", "\xc3", "\x01", "\x1f", "\x7f", "\n", std::string(1, '\0')
};
for (std::size_t run = 0; run <= 70; ++run)
{
for (const std::string& piece : pieces)
{
// a run of ordinary characters puts the piece at every offset of the bulk scanners
const std::string str = std::string(run, 'a') + piece;
for (const std::string& text :
{
"\"" + str + "\"", "[\"" + str + "\"]", "{\"" + str + "\":1}", "{\"k\":\"" + str + "\",\"" + str + "\":[\"" + str + "\"]}",
"[\"" + str
})
{
CAPTURE(text)
for (int options = 0; options < 4; ++options)
{
const bool ignore_comments = (options & 1) != 0;
const bool ignore_trailing_commas = (options & 2) != 0;
const bool parses = !json::parse(text, nullptr, false, ignore_comments, ignore_trailing_commas).is_discarded();
CHECK(json::accept(text, ignore_comments, ignore_trailing_commas) == parses);
std::istringstream stream(text);
CHECK(json::accept(stream, ignore_comments, ignore_trailing_commas) == parses);
const std::list<char> chars(text.begin(), text.end());
CHECK(json::accept(chars.begin(), chars.end(), ignore_comments, ignore_trailing_commas) == parses);
}
}
}
}
}
namespace
{
// a DOM parser that reports parse errors without throwing, as in the sax_no_exception example
-53
View File
@@ -575,59 +575,6 @@ TEST_CASE("json_view builder: escapes between runs of text")
}
}
TEST_CASE("json_view builder: chains of unicode escapes")
{
// Consecutive \uXXXX escapes are decoded in a loop of their own. Chains of
// every length from 1 to 40 of escapes that give one, two, and three
// bytes of UTF-8, in upper and lower case, followed by what ends a chain:
// the end of the string, text, another escape, UTF-8, and everything that
// is not a valid escape (a bad digit at every position of the last
// escape, a lone surrogate, a surrogate pair, a cut in every place).
const std::array<const char*, 8> chain = {{"\\u0041", "\\u00e4", "\\u00E4", "\\u20ac", "\\u3042", "\\uFFFF", "\\u0000", "\\u07ff"}};
const std::array<const char*, 12> ends = {{"", "abc", "\\n", "\\u00e9", "\\ud83d\\ude00", "\\ud83d", "\\ude00", "\\ud83d\\u0041", "\\u12G4", "\\u12", "\xc3\xa4", "\\"}};
std::mt19937 rng(2027); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible
for (std::size_t length = 1; length <= 40; ++length)
{
for (int i = 0; i < 20; ++i)
{
std::string content = std::string(rng() % 20, 'a');
for (std::size_t k = 0; k < length; ++k)
{
content += chain[rng() % chain.size()];
}
const std::string whole = content;
for (const char* end : ends)
{
check_string(whole + end);
check_string(whole + end + std::string(rng() % 19, 'z'));
}
// a bad digit at every position of every escape near the end of the chain
for (std::size_t back = 1; back <= 6 && back <= length; ++back)
{
for (std::size_t digit = 2; digit < 6; ++digit)
{
std::string broken = whole;
broken[broken.size() - (6 * back) + digit] = "gG\"\x01 z"[rng() % 6];
check_string(broken);
}
}
// the text cut in every place of the last three escapes
for (std::size_t cut = 0; cut <= 18 && cut <= whole.size(); ++cut)
{
check_string(whole.substr(0, whole.size() - cut));
}
}
}
// a chain longer than the first growth of the arena, in a key and in a value
std::string long_chain;
for (int i = 0; i < 3000; ++i)
{
long_chain += chain[static_cast<std::size_t>(i) % chain.size()];
}
check_string(long_chain);
check_string(long_chain + "\\ud83d\\ude00" + long_chain);
}
TEST_CASE("json_view node integer bits")
{
using nlohmann::detail::view::integer_bits;
-24
View File
@@ -443,30 +443,6 @@ TEST_CASE("json_view from_json")
CHECK(d.root().materialize().get<point>().x == 3);
}
SECTION("escaped keys: members are found by their decoded names")
{
// keys are decoded before they are compared, as by parse(); a lookup
// that compares the raw bytes of the source would miss these members
const json_document d = json_document::parse(R"([{"\u0078": 1, "y": 2}, {"V\u0061lue": 3, "Te\u0078t": "t"}, {"\u0062": "escaped"}, {"x": 1, "\u0078": 3, "y": 2}])");
const json j = d.root().materialize();
CHECK(d.root()[0].get<point>().x == 1);
CHECK(d.root()[0].get<point>().y == 2);
CHECK((d.root()[0].get<point>() == j[0].get<point>()));
CHECK(d.root()[1].get<with_names>().value == 3);
CHECK(d.root()[1].get<with_names>().text == "t");
CHECK(d.root()[2].get<with_defaults>().b == "escaped");
CHECK(d.root()[2].get<with_defaults>().a == 1);
// a duplicate key written once plain and once escaped: still the first member
CHECK(d.root()[3].get<point>().x == 1);
CHECK(j[3].get<point>().x == 3);
#if !defined(JSON_NOEXCEPTION) && !JSON_DIAGNOSTICS && !JSON_DIAGNOSTIC_POSITIONS
// an escaped backslash is part of the decoded key: "x\\" is not "x"
const json_document e = json_document::parse(R"({"x\\": 1, "y": 2})");
CHECK(exception_of_call([&] { static_cast<void>(e.root().get<point>()); }) == exception_of_call([&] { static_cast<void>(e.root().materialize().get<point>()); }));
CHECK(exception_of_call([&] { static_cast<void>(e.root().get<point>()); }) == "[json.exception.out_of_range.403] key 'x' not found");
#endif
}
SECTION("editable documents and ordered_json")
{
json_editable_document e = json_editable_document::parse(R"({"x": 1, "y": 2})");
-46
View File
@@ -23,7 +23,6 @@ using nlohmann::json;
#include <cstdint>
#include <sstream>
#include <iomanip>
#include <limits>
#include <string>
#include <vector>
@@ -249,51 +248,6 @@ TEST_CASE_TEMPLATE("serialization for extreme integer values", T, int32_t, uint3
}
}
TEST_CASE("integers of every digit count are written like std::to_string")
{
// dump_integer() writes up to 16 digits in two groups of eight and 17 to
// 20 digits as one to four digits and two groups of eight
std::vector<std::uint64_t> values;
std::uint64_t power = 1;
for (int digits = 1; digits <= 20; ++digits)
{
values.push_back(power); // 1 followed by zeros
values.push_back(power + 1);
values.push_back((power * 2) - 1);
if (digits < 20)
{
values.push_back((power * 10) - 1); // all 9s
values.push_back((power * 10) - power); // 9 followed by zeros
power *= 10;
}
}
values.push_back((std::numeric_limits<std::uint64_t>::max)());
values.push_back(static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()));
values.push_back(static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()) + 1);
std::uint64_t state = 0x9E3779B97F4A7C15u;
for (int i = 0; i < 20000; ++i)
{
state = (state * 6364136223846793005u) + 1442695040888963407u;
values.push_back(state >> (state % 61)); // every bit length
}
for (const auto v : values)
{
CAPTURE(v)
CHECK(json(v).dump() == std::to_string(v));
if (v <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()))
{
const auto s = static_cast<std::int64_t>(v);
CHECK(json(s).dump() == std::to_string(s));
CHECK(json(-s).dump() == std::to_string(-s));
}
}
CHECK(json((std::numeric_limits<std::int64_t>::min)()).dump() == "-9223372036854775808");
// in a container, so the digits are followed by other output
CHECK(json::array({std::uint64_t{18446744073709551615u}, std::int64_t{-1234567890123456789}, 1}).dump() == "[18446744073709551615,-1234567890123456789,1]");
}
TEST_CASE("dump with binary values")
{
auto binary = json::binary({1, 2, 3, 4});