wt/json
1
0 Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-10-31 13:00:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

accept() checks strings without copying them

Browse Source 
json::accept() parses through json_sax_acceptor, which ignores the
values of strings and object keys, but the lexer still copied every
string token into its token buffer. That copy was 18 % of the
instructions of json::accept() on twitter.json (x86, GCC 16).

The lexer's flag for accept(), renamed from discard_number_values to
discard_values, now also selects a second instantiation of
scan_string() that performs the same checks (escapes, surrogates,
UTF-8, control characters, with the same error messages) without
writing to the token buffer. scan() chooses the instantiation once per
string: a run-time check inside the string loop made number-heavy
files up to 3 % slower on Apple M1.

json::accept() over 19 files, time after/before (geomean): 0.876 on x86
with GCC 16.2, 0.884 with Clang 23, 0.932 on Apple M1; up to 0.74 on
single files; files with few strings (canada, mesh, numbers) unchanged.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann committed 2026-10-31 00:07:57 +01:00
1 parent 755bb16d7a
commit 74f5ecbd35
5 files changed
+218 -90
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

include/nlohmann/detail/input/lexer.hpp
+83 -41
View File
@@ -268,10 +268,10 @@ class lexer : public lexer_base<BasicJsonType>
public:
using token_type = typename lexer_base<BasicJsonType>::token_type;
explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept
explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_values_ = false) noexcept
: ia(std::move(adapter))
, ignore_comments(ignore_comments_)
, discard_number_values(discard_number_values_)
, discard_values(discard_values_)
{
// the token string of an error before the first token is everything
// read so far
@@ -388,6 +388,8 @@ class lexer : public lexer_base<BasicJsonType>
checks if it is inside the range. If a violation was detected, set up an
error message and return false. Otherwise, return true.
@param[in] keep whether the bytes are added to token_buffer (false for
accept(), see scan())
@param[in] ranges list of integers; interpreted as list of pairs of
inclusive lower and upper bound, respectively
@@ -396,17 +398,18 @@ class lexer : public lexer_base<BasicJsonType>
@return true if and only if no range violation was detected
*/
bool next_byte_in_range(std::initializer_list<char_int_type> ranges)
template<bool Keep>
bool next_byte_in_range(std::integral_constant<bool, Keep> keep, std::initializer_list<char_int_type> ranges)
{
JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6);
add(current);
add_string_char(keep, current);
for (auto range = ranges.begin(); range != ranges.end(); ++range)
{
get();
if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions)
{
add(current);
add_string_char(keep, current);
}
else
{
@@ -419,14 +422,16 @@ class lexer : public lexer_base<BasicJsonType>
}
/// contiguous input: bulk-append the run of ordinary characters and complete
/// well-formed UTF-8 sequences starting at the current read position, then
/// well-formed UTF-8 sequences starting at the current read position (only
/// skip it if @a keep is false), then
/// read the first byte that needs individual handling (the closing quote, an
/// escape, a control character, or an ill-formed UTF-8 byte) like get().
/// That byte is taken from the buffer that was just scanned. Through get(),
/// Apple Clang merged this read with the other get() calls of the scan
/// function, which load the read position again right after it was stored
/// (3% of json::accept() on jeopardy.json on Apple M1).
char_int_type get_after_string_run(std::true_type /*bulk*/)
template<bool Keep>
char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant<bool, Keep> keep)
{
// a pending unget must be consumed through the normal path first
if (next_unget)
@@ -444,7 +449,7 @@ class lexer : public lexer_base<BasicJsonType>
const std::size_t pos = lexer_string_bulk_run(data, remaining);
if (pos != 0)
{
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), pos);
append_string_run(keep, data, pos);
bulk_consume(reinterpret_cast<const char*>(data), pos);
}
if (pos == remaining)
@@ -488,7 +493,8 @@ class lexer : public lexer_base<BasicJsonType>
}
/// streaming input: no bulk fast path
char_int_type get_after_string_run(std::false_type /*bulk*/)
template<bool Keep>
char_int_type get_after_string_run(std::false_type /*bulk*/, std::integral_constant<bool, Keep> /*keep*/)
{
return get();
}
@@ -502,13 +508,18 @@ class lexer : public lexer_base<BasicJsonType>
may contain \0 bytes), and token_buffer.size() is the number of bytes in the
string.
@param[in] keep whether the string is copied into token_buffer; if false
(accept(), see scan()), it is only checked, with the same
errors, and token_buffer stays empty
@return token_type::value_string if string could be successfully scanned,
token_type::parse_error otherwise
@note In case of errors, variable error_message contains a textual
description.
*/
token_type scan_string()
template<bool Keep>
token_type scan_string(std::integral_constant<bool, Keep> keep)
{
// reset token_buffer (ignore opening quote)
reset();
@@ -520,7 +531,7 @@ class lexer : public lexer_base<BasicJsonType>
{
// bulk-consume ordinary characters from contiguous input, then
// get the next character, the one that ends the run
switch (get_after_string_run(std::integral_constant<bool, bulk_scan>()))
switch (get_after_string_run(std::integral_constant<bool, bulk_scan>(), keep))
{
// end of file while parsing the string
case char_traits<char_type>::eof():
@@ -542,35 +553,35 @@ class lexer : public lexer_base<BasicJsonType>
{
// quotation mark
case '\"':
add('\"');
add_string_char(keep, '\"');
break;
// reverse solidus
case '\\':
add('\\');
add_string_char(keep, '\\');
break;
// solidus
case '/':
add('/');
add_string_char(keep, '/');
break;
// backspace
case 'b':
add('\b');
add_string_char(keep, '\b');
break;
// form feed
case 'f':
add('\f');
add_string_char(keep, '\f');
break;
// line feed
case 'n':
add('\n');
add_string_char(keep, '\n');
break;
// carriage return
case 'r':
add('\r');
add_string_char(keep, '\r');
break;
// tab
case 't':
add('\t');
add_string_char(keep, '\t');
break;
// unicode escapes
@@ -638,10 +649,7 @@ class lexer : public lexer_base<BasicJsonType>
JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF);
// translate codepoint into bytes
encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte)
{
add(static_cast<char_int_type>(byte));
});
add_codepoint(keep, codepoint);
break;
}
@@ -824,7 +832,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0x7E:
case 0x7F:
{
add(current);
add_string_char(keep, current);
break;
}
@@ -860,7 +868,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xDE:
case 0xDF:
{
if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF})))
if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range(keep, {0x80, 0xBF})))
{
return token_type::parse_error;
}
@@ -870,7 +878,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+0800..U+0FFF: bytes E0 A0..BF 80..BF
case 0xE0:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0xA0, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -894,7 +902,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xEE:
case 0xEF:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -904,7 +912,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+D000..U+D7FF: bytes ED 80..9F 80..BF
case 0xED:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x9F, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -914,7 +922,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
case 0xF0:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -926,7 +934,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xF2:
case 0xF3:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -936,7 +944,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
case 0xF4:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -1427,7 +1435,7 @@ scan_number_done:
token_type convert_number(token_type number_type, std::size_t mantissa_end)
{
// accept() only needs to know whether the input is valid, so it sets
// discard_number_values (see json.hpp), and an integer token whose
// discard_values (see json.hpp), and an integer token whose
// digit count shows that it fits is reported without calling
// convert_integer(). A number with up to 18 digits always fits into
// both std::uint64_t and std::int64_t (18 nines is about 1e18, below
@@ -1442,7 +1450,7 @@ scan_number_done:
// json_sax_acceptor, whose number callbacks discard their argument and
// return true, and the parser rejects neither integers nor finite
// floats. value_unsigned/value_integer are left unset here, so a caller
// that reads the converted value must not set discard_number_values.
// that reads the converted value must not set discard_values.
//
// On contiguous input, scan_number_bulk_contiguous() converts integer
// tokens itself and does not pass them to this function, unless
@@ -1450,7 +1458,7 @@ scan_number_done:
// reached for input without bulk access (e.g. streams), with
// JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous()
// falls back to scan_number().
if (discard_number_values)
if (discard_values)
{
constexpr std::size_t safe_digit_count = 18;
if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count)
@@ -1701,7 +1709,7 @@ scan_number_done:
// w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a
// value that underflows is a finite number as well. Only a token that
// may overflow has to be converted to find out.
if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
if (JSON_HEDLEY_UNLIKELY(discard_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
{
value_float = 0;
return true;
@@ -2033,6 +2041,34 @@ scan_number_done:
token_buffer.push_back(static_cast<typename string_t::value_type>(c));
}
/// add a character of a string token; accept() checks the string but does
/// not keep its value (@a keep is false)
void add_string_char(std::true_type /*keep*/, char_int_type c)
{
token_buffer.push_back(static_cast<typename string_t::value_type>(c));
}
static void add_string_char(std::false_type /*keep*/, char_int_type /*c*/) noexcept {}
/// add the UTF-8 bytes of a code point from a \\u escape to a string token
void add_codepoint(std::true_type /*keep*/, int codepoint)
{
encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte)
{
add(static_cast<char_int_type>(byte));
});
}
static void add_codepoint(std::false_type /*keep*/, int /*codepoint*/) noexcept {}
/// append a run of ordinary string characters (see get_after_string_run())
void append_string_run(std::true_type /*keep*/, const unsigned char* data, std::size_t n)
{
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), n);
}
static void append_string_run(std::false_type /*keep*/, const unsigned char* /*data*/, std::size_t /*n*/) noexcept {}
public:
/////////////////////
// value getters
@@ -2427,7 +2463,11 @@ scan_number_done:
// string
case '\"':
return scan_string();
// accept() (discard_values) checks strings without copying them
// into token_buffer. The choice is made once per string: a
// check inside the string loop made number-heavy documents up
// to 3 % slower (mesh.json, Apple M1)
return discard_values ? scan_string(std::false_type {}) : scan_string(std::true_type {});
// number
case '-':
@@ -2512,11 +2552,13 @@ scan_number_done:
std::size_t decimal_point_position = std::string::npos;
/// whether the caller only needs the token types and never looks at the
/// converted numeric values; set only by accept(), which parses through
/// json_sax_acceptor. When set, convert_number() skips converting integer
/// tokens whose digit count guarantees that they fit into 64 bits (see
/// there)
const bool discard_number_values = false;
/// values of number and string tokens; set only by accept(), which parses
/// through json_sax_acceptor. When set, convert_number() skips converting
/// integer tokens whose digit count guarantees that they fit into 64 bits
/// (see there), floats are converted only when they may overflow (see
/// convert_accumulated_float()), and strings are checked without being
/// copied into token_buffer (see scan())
const bool discard_values = false;
};
} // namespace detail
include/nlohmann/detail/input/parser.hpp
+2 -2
View File
@@ -89,9 +89,9 @@ class parser
const bool allow_exceptions_ = true,
const bool ignore_comments = false,
const bool ignore_trailing_commas_ = false,
const bool discard_number_values_ = false)
const bool discard_values_ = false)
: callback(std::move(cb))
, m_lexer(std::move(adapter), ignore_comments, discard_number_values_)
, m_lexer(std::move(adapter), ignore_comments, discard_values_)
, allow_exceptions(allow_exceptions_)
, ignore_trailing_commas(ignore_trailing_commas_)
{
include/nlohmann/json.hpp
+2 -2
View File
@@ -184,11 +184,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
const bool allow_exceptions = true,
const bool ignore_comments = false,
const bool ignore_trailing_commas = false,
const bool discard_number_values = false
const bool discard_values = false
)
{
return ::nlohmann::detail::parser<basic_json, InputAdapterType>(std::move(adapter),
std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values);
std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_values);
}
private: