Merge branch 'develop' into claude/fix-issue-3989-db7e45

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann committed 2026-10-21 12:49:18 +02:00
commit 5da0e9181f
9 files changed
+1240 -111

No files matched your search

+26 -19
View File
@@ -20,32 +20,39 @@ below apply on every supported standard. This adapter exposes two optimizations
- it can reconstruct already-consumed input on demand for error messages, instead of copying every character as it is - it can reconstruct already-consumed input on demand for error messages, instead of copying every character as it is
read, and read, and
- the lexer can scan ordinary string characters directly out of the buffer, several bytes at a time, rather than one - the lexer can scan ordinary string characters, numbers, and indentation directly out of the buffer, several bytes at
character (and one function call) at a time. a time, rather than one character (and one function call) at a time.
A `#!cpp std::istream` (including `#!cpp std::ifstream`) or `#!cpp FILE*`, by contrast, is read through A `#!cpp std::istream` (including `#!cpp std::ifstream`) gets the same bulk scanning: `input_stream_adapter` scans the
`input_stream_adapter` or `file_input_adapter`, which read one character (or one block, for binary formats) at a time characters its stream buffer already holds (the *get area*) and asks the stream buffer for more only when they are used
and expose neither optimization -- the lexer falls back to the same byte-at-a-time path it uses for any up. Nothing is read ahead, so the stream is left exactly where it would be after reading one character at a time, and
non-contiguous, general-purpose iterator range. An iterator pair over non-contiguous but random-access storage (e.g. [`operator>>`](../api/operator_gtgt.md) can still read several values from one stream. A stream buffer without a get
`#!cpp std::deque<char>::iterator`) gets the first optimization but not the second, since the byte-scanning fast path area is read one character at a time. A `#!cpp FILE*` is still read one character at a time, but `file_input_adapter`
additionally requires contiguous storage. locks the file once per parse instead of once per character. See
[input sources and performance](parsing/index.md#input-sources-and-performance) for the full table. According to the
description of [#5828](https://github.com/nlohmann/json/pull/5828), parsing from a stream is within about 10 to 20 % of
parsing an equivalent `#!cpp std::string`.
Practically: if the JSON text is already in memory, or small enough to read into memory, prefer passing a Inputs that are still read one character at a time: iterator ranges that are not contiguous (e.g.
`#!cpp std::string`, a `#!cpp std::vector<char>`, or a pointer range to `parse` over a `#!cpp std::istream`. For a `#!cpp std::list<char>` or `#!cpp std::istreambuf_iterator<char>`), wide strings, and user-defined adapters. An
file, that means reading it into a string first and then parsing the string, rather than passing a iterator pair over non-contiguous but random-access storage (e.g. `#!cpp std::deque<char>::iterator`) gets the first
`#!cpp std::ifstream` directly to `parse` -- the latter never benefits from either optimization: optimization but not the second, since the byte-scanning fast path requires contiguous storage.
Practically: a file can be passed directly to `parse`, which also avoids the memory for a copy of the file. Reading it
into a string first is no longer necessary for speed:
```cpp ```cpp
// gets the contiguous fast paths // fast: the stream buffer's get area is scanned in bulk
std::ifstream f("example.json"); std::ifstream f("example.json");
std::string contents((std::istreambuf_iterator<char>(f)), std::istreambuf_iterator<char>()); json j = json::parse(f);
json j = json::parse(contents);
// does not: input_stream_adapter has no fast path
std::ifstream f2("example.json");
json j2 = json::parse(f2);
``` ```
Two macros switch the stream optimizations off without changing any result:
[`JSON_DISABLE_STREAM_BULK_SCAN`](../api/macros/json_disable_stream_bulk_scan.md) for stream buffers that do not follow
the usual `#!cpp std::streambuf` contract, and
[`JSON_DISABLE_UNLOCKED_FILE_IO`](../api/macros/json_disable_unlocked_file_io.md) for programs in which another thread
must use the same `#!cpp FILE` object while it is parsed.
For contiguous input with many non-ASCII characters, [`JSON_USE_SIMDUTF`](../api/macros/json_use_simdutf.md) can For contiguous input with many non-ASCII characters, [`JSON_USE_SIMDUTF`](../api/macros/json_use_simdutf.md) can
additionally speed up UTF-8 validation by using the [simdutf](https://github.com/simdutf/simdutf) library instead of additionally speed up UTF-8 validation by using the [simdutf](https://github.com/simdutf/simdutf) library instead of
the built-in scalar validator; streaming inputs (files, `#!cpp std::istream`, wide strings, user-defined adapters) the built-in scalar validator; streaming inputs (files, `#!cpp std::istream`, wide strings, user-defined adapters)
+111 -14
View File
@@ -1690,9 +1690,12 @@ scan_number_done:
const std::size_t avail = rem + 1; const std::size_t avail = rem + 1;
// validate + classify the number extent (mirrors scan_number()'s grammar) // validate + classify the number extent (mirrors scan_number()'s grammar)
// and accumulate the mantissa digits on the way, so that tokens with up
// to 19 digits need no second pass for their value
std::size_t i = 0; std::size_t i = 0;
std::size_t dot_index = std::string::npos; std::size_t dot_index = std::string::npos;
token_type number_type = token_type::value_unsigned; token_type number_type = token_type::value_unsigned;
std::uint64_t w = 0;
if (data[0] == '-') if (data[0] == '-')
{ {
number_type = token_type::value_integer; number_type = token_type::value_integer;
@@ -1702,22 +1705,21 @@ scan_number_done:
return token_type::uninitialized; return token_type::uninitialized;
} }
} }
const std::size_t integer_begin = i;
if (data[i] == '0') if (data[i] == '0')
{ {
++i; ++i;
} }
else if (data[i] >= '1' && data[i] <= '9') else if (data[i] >= '1' && data[i] <= '9')
{ {
++i; i = accumulate_digits(data, i, avail, w);
while (i < avail && data[i] >= '0' && data[i] <= '9')
{
++i;
}
} }
else else
{ {
return token_type::uninitialized; return token_type::uninitialized;
} }
std::size_t digit_count = i - integer_begin;
std::size_t fraction_digits = 0;
if (i < avail && data[i] == '.') if (i < avail && data[i] == '.')
{ {
number_type = token_type::value_float; number_type = token_type::value_float;
@@ -1727,29 +1729,39 @@ scan_number_done:
{ {
return token_type::uninitialized; return token_type::uninitialized;
} }
while (i < avail && data[i] >= '0' && data[i] <= '9') const std::size_t fraction_begin = i;
{ i = accumulate_digits(data, i, avail, w);
++i; fraction_digits = i - fraction_begin;
} digit_count += fraction_digits;
} }
// the mantissa ends here, whether or not an exponent part follows // the mantissa ends here, whether or not an exponent part follows
const std::size_t mantissa_end = i; const std::size_t mantissa_end = i;
std::int64_t exponent = 0;
if (i < avail && (data[i] == 'e' || data[i] == 'E')) if (i < avail && (data[i] == 'e' || data[i] == 'E'))
{ {
number_type = token_type::value_float; number_type = token_type::value_float;
++i; ++i;
bool negative_exponent = false;
if (i < avail && (data[i] == '+' || data[i] == '-')) if (i < avail && (data[i] == '+' || data[i] == '-'))
{ {
negative_exponent = data[i] == '-';
++i; ++i;
} }
if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) if (i >= avail || !(data[i] >= '0' && data[i] <= '9'))
{ {
return token_type::uninitialized; return token_type::uninitialized;
} }
// saturated far beyond every range, see parse_float_exponent()
constexpr std::int64_t saturation = 100000000000000000; // 10^17
while (i < avail && data[i] >= '0' && data[i] <= '9') while (i < avail && data[i] >= '0' && data[i] <= '9')
{ {
if (exponent < saturation)
{
exponent = (exponent * 10) + (data[i] - '0');
}
++i; ++i;
} }
exponent = negative_exponent ? -exponent : exponent;
} }
const std::size_t len = i; const std::size_t len = i;
if (number_may_continue(len == avail, std::integral_constant<bool, bulk_window> {})) if (number_may_continue(len == avail, std::integral_constant<bool, bulk_window> {}))
@@ -1770,7 +1782,10 @@ scan_number_done:
#if !JSON_DIAGNOSTIC_POSITIONS #if !JSON_DIAGNOSTIC_POSITIONS
if (number_type != token_type::value_float) if (number_type != token_type::value_float)
{ {
const token_type integer_result = convert_integer(number_type, data, data + len); // up to 18 digits always fit into std::uint64_t and std::int64_t
const token_type integer_result = (digit_count <= 18)
? store_integer(number_type, w)
: convert_integer(number_type, data, data + len);
if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized)) if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized))
{ {
bulk_consume(data + 1, len - 1); bulk_consume(data + 1, len - 1);
@@ -1784,15 +1799,42 @@ scan_number_done:
} }
#endif #endif
// materialize the token exactly as scan_number() would. reset() already
// cleared token_buffer, so append() fills it (assign() is avoided
// because custom string_t types need not provide it)
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), len);
decimal_point_position = dot_index; decimal_point_position = dot_index;
bulk_consume(data + 1, len - 1); bulk_consume(data + 1, len - 1);
read_number_terminator(std::integral_constant<bool, bulk_window> {}); read_number_terminator(std::integral_constant<bool, bulk_window> {});
// a float whose digits all fit into w converts from the accumulated
// value; longer ones (and other float formats) take the general path
if (number_type == token_type::value_float && digit_count <= 19)
{
float_significand s;
s.w = w;
s.exponent = exponent - static_cast<std::int64_t>(fraction_digits);
s.negative = data[0] == '-';
if (convert_accumulated_float(s, std::integral_constant<bool, has_native_float_format<number_float_t>::value> {}))
{
// the token text is only copied into token_buffer when
// get_string() asks for it; the library's own SAX handlers
// ignore it. A window into a stream may be refilled by the
// next read, so there the text is copied right away.
if (bulk_window)
{
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), len);
}
else
{
pending_number_text = data;
pending_number_length = len;
}
return token_type::value_float;
}
}
// materialize the token exactly as scan_number() would. reset() already
// cleared token_buffer, so append() fills it (assign() is avoided
// because custom string_t types need not provide it)
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), len);
return convert_number(number_type, mantissa_end); return convert_number(number_type, mantissa_end);
} }
@@ -1821,6 +1863,35 @@ scan_number_done:
/// the input is not handed back, so the terminator can stay unread /// the input is not handed back, so the terminator can stay unread
void read_number_terminator(std::false_type /*window*/) const noexcept {} void read_number_terminator(std::false_type /*window*/) const noexcept {}
/// binary32/binary64: the value of a float token from its accumulated
/// significand (see scan_number_bulk_contiguous())
bool convert_accumulated_float(const float_significand& s, std::true_type /*native*/) noexcept
{
value_float = decimal_to_float<number_float_t>(s);
return true;
}
/// other formats (e.g. x87 long double): not instantiated for them, as
/// decimal_to_float() only supports binary32/binary64
static bool convert_accumulated_float(const float_significand& /*s*/, std::false_type /*native*/) noexcept
{
return false;
}
/// the value of an integer token with at most 18 digits, accumulated in @a w
token_type store_integer(token_type number_type, std::uint64_t w) noexcept
{
if (number_type == token_type::value_unsigned)
{
value_unsigned = static_cast<number_unsigned_t>(w);
// reject values that do not round-trip into a narrower type
return (static_cast<std::uint64_t>(value_unsigned) == w) ? token_type::value_unsigned : token_type::uninitialized;
}
const std::int64_t x = -static_cast<std::int64_t>(w);
value_integer = static_cast<number_integer_t>(x);
return (static_cast<std::int64_t>(value_integer) == x) ? token_type::value_integer : token_type::uninitialized;
}
/// contiguous input: try the number fast path, else the byte-path scanner /// contiguous input: try the number fast path, else the byte-path scanner
token_type scan_number_dispatch(std::true_type /*bulk*/) token_type scan_number_dispatch(std::true_type /*bulk*/)
{ {
@@ -1864,6 +1935,7 @@ scan_number_done:
void reset() noexcept void reset() noexcept
{ {
token_buffer.clear(); token_buffer.clear();
pending_number_text = nullptr;
decimal_point_position = std::string::npos; decimal_point_position = std::string::npos;
#if JSON_DIAGNOSTIC_POSITIONS #if JSON_DIAGNOSTIC_POSITIONS
@@ -2123,10 +2195,30 @@ scan_number_done:
/// return current string value /// return current string value
string_t& get_string() string_t& get_string()
{ {
if (pending_number_text != nullptr)
{
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(pending_number_text), pending_number_length);
pending_number_text = nullptr;
}
// a number token holds '.' regardless of the locale (#4084) // a number token holds '.' regardless of the locale (#4084)
return token_buffer; return token_buffer;
} }
/// the value of a string token (key or string), which never has pending
/// number text, so get_string()'s check is not needed
string_t& get_string_token() noexcept
{
JSON_ASSERT(pending_number_text == nullptr);
return token_buffer;
}
/// the text of a float token for a SAX handler that ignores it (the
/// library's own handlers): empty, without copying the token
const string_t& get_ignored_number_text() const noexcept
{
return token_buffer;
}
///////////////////// /////////////////////
// diagnostics // diagnostics
///////////////////// /////////////////////
@@ -3104,6 +3196,11 @@ scan_number_done:
/// whether the next get() call should just return current /// whether the next get() call should just return current
bool next_unget = false; bool next_unget = false;
/// the text of the current float token in the input, not yet copied into
/// token_buffer (see get_string()), or nullptr
const char* pending_number_text = nullptr;
std::size_t pending_number_length = 0;
/// the start position of the current token /// the start position of the current token
position_t position {}; position_t position {};
+72 -26
View File
@@ -275,13 +275,77 @@ JSON_HEDLEY_ALWAYS_INLINE std::uint32_t parse_eight_digits(std::uint64_t v) noex
return static_cast<std::uint32_t>(((v & 0x0000FFFF0000FFFFu) * 42949672960001u) >> 32u); return static_cast<std::uint32_t>(((v & 0x0000FFFF0000FFFFu) * 42949672960001u) >> 32u);
} }
/// whether [first, last) contains a digit other than '0' /// whether the eight bytes in @a v (see read_eight_bytes()) are all ASCII
inline bool has_nonzero_digit(const char* first, const char* last) noexcept /// digits (after fast_float)
JSON_HEDLEY_ALWAYS_INLINE bool is_eight_digits(std::uint64_t v) noexcept
{ {
return std::any_of(first, last, [](char c) noexcept return ((v & 0xF0F0F0F0F0F0F0F0u) | (((v + 0x0606060606060606u) & 0xF0F0F0F0F0F0F0F0u) >> 4u)) == 0x3333333333333333u;
}
/*!
@brief skip the ASCII digits starting at data[i] and accumulate their value
Eight digits are taken at a time while at least eight bytes remain. @a w wraps
around after 19 digits; the caller only uses it for shorter digit runs.
@return the index of the first byte at or after @a i that is not a digit
*/
JSON_HEDLEY_ALWAYS_INLINE std::size_t accumulate_digits(const char* data, std::size_t i, std::size_t avail, std::uint64_t& w) noexcept
{
while (avail - i >= 8)
{ {
return c != '0'; const std::uint64_t v = read_eight_bytes(data + i);
}); if (!is_eight_digits(v))
{
break;
}
w = (w * 100000000u) + parse_eight_digits(v);
i += 8;
}
while (i < avail && data[i] >= '0' && data[i] <= '9')
{
w = (w * 10u) + static_cast<std::uint64_t>(data[i] - '0');
++i;
}
return i;
}
/// whether [first, last) contains a digit other than '0'; a plain loop, as
/// GCC calls std::any_of out of line even for the usual empty range
JSON_HEDLEY_ALWAYS_INLINE bool has_nonzero_digit(const char* first, const char* last) noexcept
{
for (; first != last; ++first)
{
if (*first != '0')
{
return true;
}
}
return false;
}
/*!
@brief accumulate the validated digits [p, last) into @a w, as long as
@a remaining digits still fit into it
Eight digits are taken at a time while at least eight remain, and @a remaining
is decreased by the number of digits taken.
@return a pointer past the last digit taken
*/
JSON_HEDLEY_ALWAYS_INLINE const char* accumulate_significant_digits(const char* p, const char* last, int& remaining, std::uint64_t& w) noexcept
{
while (remaining >= 8 && last - p >= 8)
{
w = (w * 100000000u) + parse_eight_digits(read_eight_bytes(p));
p += 8;
remaining -= 8;
}
for (; remaining > 0 && p != last; ++p, --remaining)
{
w = (w * 10u) + static_cast<std::uint64_t>(*p - '0');
}
return p;
} }
/// the value of the validated exponent digits [+-]?[0-9]+ in [first, last), /// the value of the validated exponent digits [+-]?[0-9]+ in [first, last),
@@ -327,7 +391,7 @@ truncated is set.
std::string::npos if there is none std::string::npos if there is none
@param[in] mantissa_end index of the 'e'/'E', or the token length @param[in] mantissa_end index of the 'e'/'E', or the token length
*/ */
inline float_significand parse_float_significand(const char* first, const char* last, JSON_HEDLEY_ALWAYS_INLINE float_significand parse_float_significand(const char* first, const char* last,
std::size_t decimal_point_position, std::size_t mantissa_end) noexcept std::size_t decimal_point_position, std::size_t mantissa_end) noexcept
{ {
float_significand s; float_significand s;
@@ -342,16 +406,7 @@ inline float_significand parse_float_significand(const char* first, const char*
int remaining = 19; // digits that still fit into w int remaining = 19; // digits that still fit into w
if (*p != '0') // the integer part is "0" or [1-9][0-9]* if (*p != '0') // the integer part is "0" or [1-9][0-9]*
{ {
while (remaining >= 8 && integer_last - p >= 8) p = accumulate_significant_digits(p, integer_last, remaining, w);
{
w = (w * 100000000u) + parse_eight_digits(read_eight_bytes(p));
p += 8;
remaining -= 8;
}
for (; remaining > 0 && p != integer_last; ++p, --remaining)
{
w = (w * 10u) + static_cast<std::uint64_t>(*p - '0');
}
s.exponent = integer_last - p; s.exponent = integer_last - p;
s.truncated = has_nonzero_digit(p, integer_last); s.truncated = has_nonzero_digit(p, integer_last);
} }
@@ -370,16 +425,7 @@ inline float_significand parse_float_significand(const char* first, const char*
s.exponent -= p - zeros; s.exponent -= p - zeros;
} }
const char* const digits = p; const char* const digits = p;
while (remaining >= 8 && mantissa_last - p >= 8) p = accumulate_significant_digits(p, mantissa_last, remaining, w);
{
w = (w * 100000000u) + parse_eight_digits(read_eight_bytes(p));
p += 8;
remaining -= 8;
}
for (; remaining > 0 && p != mantissa_last; ++p, --remaining)
{
w = (w * 10u) + static_cast<std::uint64_t>(*p - '0');
}
s.exponent -= p - digits; s.exponent -= p - digits;
s.truncated = s.truncated || has_nonzero_digit(p, mantissa_last); s.truncated = s.truncated || has_nonzero_digit(p, mantissa_last);
} }
+37 -6
View File
@@ -51,6 +51,20 @@ template<typename BasicJsonType>
using parser_callback_t = using parser_callback_t =
std::function<bool(int /*depth*/, parse_event_t /*event*/, BasicJsonType& /*parsed*/)>; std::function<bool(int /*depth*/, parse_event_t /*event*/, BasicJsonType& /*parsed*/)>;
/// whether a SAX handler ignores the text of float tokens (the library's own
/// handlers do), so that the parser need not materialize it
template<typename SAX>
struct sax_ignores_number_text : std::false_type {};
template<typename BasicJsonType, typename InputAdapterType>
struct sax_ignores_number_text<json_sax_dom_parser<BasicJsonType, InputAdapterType>> : std::true_type {};
template<typename BasicJsonType, typename InputAdapterType>
struct sax_ignores_number_text<json_sax_dom_callback_parser<BasicJsonType, InputAdapterType>> : std::true_type {};
template<typename BasicJsonType>
struct sax_ignores_number_text<json_sax_acceptor<BasicJsonType>> : std::true_type {};
/*! /*!
@brief syntax analysis @brief syntax analysis
@@ -205,6 +219,23 @@ class parser
return result && !error_reported; return result && !error_reported;
} }
/// the text of the current float token for @a sax (see sax_ignores_number_text)
template<typename SAX>
const string_t& number_float_text(SAX* /*sax*/)
{
return number_float_text(std::integral_constant<bool, sax_ignores_number_text<SAX>::value> {});
}
const string_t& number_float_text(std::true_type /*ignored*/) const noexcept
{
return m_lexer.get_ignored_number_text();
}
string_t& number_float_text(std::false_type /*ignored*/)
{
return m_lexer.get_string();
}
/*! /*!
@brief run a DOM SAX parser to completion and position the lexer @brief run a DOM SAX parser to completion and position the lexer
@@ -299,7 +330,7 @@ class parser
} }
continue; continue;
} }
if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string_token())))
{ {
return false; return false;
} }
@@ -356,7 +387,7 @@ class parser
break; break;
} }
if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, m_lexer.get_string()))) if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, number_float_text(sax))))
{ {
return false; return false;
} }
@@ -402,7 +433,7 @@ class parser
case token_type::value_string: case token_type::value_string:
{ {
if (JSON_HEDLEY_UNLIKELY(!sax->string(m_lexer.get_string()))) if (JSON_HEDLEY_UNLIKELY(!sax->string(m_lexer.get_string_token())))
{ {
return false; return false;
} }
@@ -612,7 +643,7 @@ class parser
} }
continue; continue;
} }
if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string_token())))
{ {
return false; return false;
} }
@@ -725,7 +756,7 @@ class parser
return key_error(sax, allow_recovery, false); return key_error(sax, allow_recovery, false);
} }
if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string_token())))
{ {
return next_step::stop; return next_step::stop;
} }
@@ -760,7 +791,7 @@ class parser
{ {
return false; return false;
} }
return sax->number_float(value, m_lexer.get_string()); return sax->number_float(value, number_float_text(sax));
} }
/// @copydoc overflow_error /// @copydoc overflow_error
+220 -46
View File
@@ -10472,13 +10472,77 @@ JSON_HEDLEY_ALWAYS_INLINE std::uint32_t parse_eight_digits(std::uint64_t v) noex
return static_cast<std::uint32_t>(((v & 0x0000FFFF0000FFFFu) * 42949672960001u) >> 32u); return static_cast<std::uint32_t>(((v & 0x0000FFFF0000FFFFu) * 42949672960001u) >> 32u);
} }
/// whether [first, last) contains a digit other than '0' /// whether the eight bytes in @a v (see read_eight_bytes()) are all ASCII
inline bool has_nonzero_digit(const char* first, const char* last) noexcept /// digits (after fast_float)
JSON_HEDLEY_ALWAYS_INLINE bool is_eight_digits(std::uint64_t v) noexcept
{ {
return std::any_of(first, last, [](char c) noexcept return ((v & 0xF0F0F0F0F0F0F0F0u) | (((v + 0x0606060606060606u) & 0xF0F0F0F0F0F0F0F0u) >> 4u)) == 0x3333333333333333u;
}
/*!
@brief skip the ASCII digits starting at data[i] and accumulate their value
Eight digits are taken at a time while at least eight bytes remain. @a w wraps
around after 19 digits; the caller only uses it for shorter digit runs.
@return the index of the first byte at or after @a i that is not a digit
*/
JSON_HEDLEY_ALWAYS_INLINE std::size_t accumulate_digits(const char* data, std::size_t i, std::size_t avail, std::uint64_t& w) noexcept
{
while (avail - i >= 8)
{ {
return c != '0'; const std::uint64_t v = read_eight_bytes(data + i);
}); if (!is_eight_digits(v))
{
break;
}
w = (w * 100000000u) + parse_eight_digits(v);
i += 8;
}
while (i < avail && data[i] >= '0' && data[i] <= '9')
{
w = (w * 10u) + static_cast<std::uint64_t>(data[i] - '0');
++i;
}
return i;
}
/// whether [first, last) contains a digit other than '0'; a plain loop, as
/// GCC calls std::any_of out of line even for the usual empty range
JSON_HEDLEY_ALWAYS_INLINE bool has_nonzero_digit(const char* first, const char* last) noexcept
{
for (; first != last; ++first)
{
if (*first != '0')
{
return true;
}
}
return false;
}
/*!
@brief accumulate the validated digits [p, last) into @a w, as long as
@a remaining digits still fit into it
Eight digits are taken at a time while at least eight remain, and @a remaining
is decreased by the number of digits taken.
@return a pointer past the last digit taken
*/
JSON_HEDLEY_ALWAYS_INLINE const char* accumulate_significant_digits(const char* p, const char* last, int& remaining, std::uint64_t& w) noexcept
{
while (remaining >= 8 && last - p >= 8)
{
w = (w * 100000000u) + parse_eight_digits(read_eight_bytes(p));
p += 8;
remaining -= 8;
}
for (; remaining > 0 && p != last; ++p, --remaining)
{
w = (w * 10u) + static_cast<std::uint64_t>(*p - '0');
}
return p;
} }
/// the value of the validated exponent digits [+-]?[0-9]+ in [first, last), /// the value of the validated exponent digits [+-]?[0-9]+ in [first, last),
@@ -10524,7 +10588,7 @@ truncated is set.
std::string::npos if there is none std::string::npos if there is none
@param[in] mantissa_end index of the 'e'/'E', or the token length @param[in] mantissa_end index of the 'e'/'E', or the token length
*/ */
inline float_significand parse_float_significand(const char* first, const char* last, JSON_HEDLEY_ALWAYS_INLINE float_significand parse_float_significand(const char* first, const char* last,
std::size_t decimal_point_position, std::size_t mantissa_end) noexcept std::size_t decimal_point_position, std::size_t mantissa_end) noexcept
{ {
float_significand s; float_significand s;
@@ -10539,16 +10603,7 @@ inline float_significand parse_float_significand(const char* first, const char*
int remaining = 19; // digits that still fit into w int remaining = 19; // digits that still fit into w
if (*p != '0') // the integer part is "0" or [1-9][0-9]* if (*p != '0') // the integer part is "0" or [1-9][0-9]*
{ {
while (remaining >= 8 && integer_last - p >= 8) p = accumulate_significant_digits(p, integer_last, remaining, w);
{
w = (w * 100000000u) + parse_eight_digits(read_eight_bytes(p));
p += 8;
remaining -= 8;
}
for (; remaining > 0 && p != integer_last; ++p, --remaining)
{
w = (w * 10u) + static_cast<std::uint64_t>(*p - '0');
}
s.exponent = integer_last - p; s.exponent = integer_last - p;
s.truncated = has_nonzero_digit(p, integer_last); s.truncated = has_nonzero_digit(p, integer_last);
} }
@@ -10567,16 +10622,7 @@ inline float_significand parse_float_significand(const char* first, const char*
s.exponent -= p - zeros; s.exponent -= p - zeros;
} }
const char* const digits = p; const char* const digits = p;
while (remaining >= 8 && mantissa_last - p >= 8) p = accumulate_significant_digits(p, mantissa_last, remaining, w);
{
w = (w * 100000000u) + parse_eight_digits(read_eight_bytes(p));
p += 8;
remaining -= 8;
}
for (; remaining > 0 && p != mantissa_last; ++p, --remaining)
{
w = (w * 10u) + static_cast<std::uint64_t>(*p - '0');
}
s.exponent -= p - digits; s.exponent -= p - digits;
s.truncated = s.truncated || has_nonzero_digit(p, mantissa_last); s.truncated = s.truncated || has_nonzero_digit(p, mantissa_last);
} }
@@ -13292,9 +13338,12 @@ scan_number_done:
const std::size_t avail = rem + 1; const std::size_t avail = rem + 1;
// validate + classify the number extent (mirrors scan_number()'s grammar) // validate + classify the number extent (mirrors scan_number()'s grammar)
// and accumulate the mantissa digits on the way, so that tokens with up
// to 19 digits need no second pass for their value
std::size_t i = 0; std::size_t i = 0;
std::size_t dot_index = std::string::npos; std::size_t dot_index = std::string::npos;
token_type number_type = token_type::value_unsigned; token_type number_type = token_type::value_unsigned;
std::uint64_t w = 0;
if (data[0] == '-') if (data[0] == '-')
{ {
number_type = token_type::value_integer; number_type = token_type::value_integer;
@@ -13304,22 +13353,21 @@ scan_number_done:
return token_type::uninitialized; return token_type::uninitialized;
} }
} }
const std::size_t integer_begin = i;
if (data[i] == '0') if (data[i] == '0')
{ {
++i; ++i;
} }
else if (data[i] >= '1' && data[i] <= '9') else if (data[i] >= '1' && data[i] <= '9')
{ {
++i; i = accumulate_digits(data, i, avail, w);
while (i < avail && data[i] >= '0' && data[i] <= '9')
{
++i;
}
} }
else else
{ {
return token_type::uninitialized; return token_type::uninitialized;
} }
std::size_t digit_count = i - integer_begin;
std::size_t fraction_digits = 0;
if (i < avail && data[i] == '.') if (i < avail && data[i] == '.')
{ {
number_type = token_type::value_float; number_type = token_type::value_float;
@@ -13329,29 +13377,39 @@ scan_number_done:
{ {
return token_type::uninitialized; return token_type::uninitialized;
} }
while (i < avail && data[i] >= '0' && data[i] <= '9') const std::size_t fraction_begin = i;
{ i = accumulate_digits(data, i, avail, w);
++i; fraction_digits = i - fraction_begin;
} digit_count += fraction_digits;
} }
// the mantissa ends here, whether or not an exponent part follows // the mantissa ends here, whether or not an exponent part follows
const std::size_t mantissa_end = i; const std::size_t mantissa_end = i;
std::int64_t exponent = 0;
if (i < avail && (data[i] == 'e' || data[i] == 'E')) if (i < avail && (data[i] == 'e' || data[i] == 'E'))
{ {
number_type = token_type::value_float; number_type = token_type::value_float;
++i; ++i;
bool negative_exponent = false;
if (i < avail && (data[i] == '+' || data[i] == '-')) if (i < avail && (data[i] == '+' || data[i] == '-'))
{ {
negative_exponent = data[i] == '-';
++i; ++i;
} }
if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) if (i >= avail || !(data[i] >= '0' && data[i] <= '9'))
{ {
return token_type::uninitialized; return token_type::uninitialized;
} }
// saturated far beyond every range, see parse_float_exponent()
constexpr std::int64_t saturation = 100000000000000000; // 10^17
while (i < avail && data[i] >= '0' && data[i] <= '9') while (i < avail && data[i] >= '0' && data[i] <= '9')
{ {
if (exponent < saturation)
{
exponent = (exponent * 10) + (data[i] - '0');
}
++i; ++i;
} }
exponent = negative_exponent ? -exponent : exponent;
} }
const std::size_t len = i; const std::size_t len = i;
if (number_may_continue(len == avail, std::integral_constant<bool, bulk_window> {})) if (number_may_continue(len == avail, std::integral_constant<bool, bulk_window> {}))
@@ -13372,7 +13430,10 @@ scan_number_done:
#if !JSON_DIAGNOSTIC_POSITIONS #if !JSON_DIAGNOSTIC_POSITIONS
if (number_type != token_type::value_float) if (number_type != token_type::value_float)
{ {
const token_type integer_result = convert_integer(number_type, data, data + len); // up to 18 digits always fit into std::uint64_t and std::int64_t
const token_type integer_result = (digit_count <= 18)
? store_integer(number_type, w)
: convert_integer(number_type, data, data + len);
if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized)) if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized))
{ {
bulk_consume(data + 1, len - 1); bulk_consume(data + 1, len - 1);
@@ -13386,15 +13447,42 @@ scan_number_done:
} }
#endif #endif
// materialize the token exactly as scan_number() would. reset() already
// cleared token_buffer, so append() fills it (assign() is avoided
// because custom string_t types need not provide it)
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), len);
decimal_point_position = dot_index; decimal_point_position = dot_index;
bulk_consume(data + 1, len - 1); bulk_consume(data + 1, len - 1);
read_number_terminator(std::integral_constant<bool, bulk_window> {}); read_number_terminator(std::integral_constant<bool, bulk_window> {});
// a float whose digits all fit into w converts from the accumulated
// value; longer ones (and other float formats) take the general path
if (number_type == token_type::value_float && digit_count <= 19)
{
float_significand s;
s.w = w;
s.exponent = exponent - static_cast<std::int64_t>(fraction_digits);
s.negative = data[0] == '-';
if (convert_accumulated_float(s, std::integral_constant<bool, has_native_float_format<number_float_t>::value> {}))
{
// the token text is only copied into token_buffer when
// get_string() asks for it; the library's own SAX handlers
// ignore it. A window into a stream may be refilled by the
// next read, so there the text is copied right away.
if (bulk_window)
{
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), len);
}
else
{
pending_number_text = data;
pending_number_length = len;
}
return token_type::value_float;
}
}
// materialize the token exactly as scan_number() would. reset() already
// cleared token_buffer, so append() fills it (assign() is avoided
// because custom string_t types need not provide it)
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), len);
return convert_number(number_type, mantissa_end); return convert_number(number_type, mantissa_end);
} }
@@ -13423,6 +13511,35 @@ scan_number_done:
/// the input is not handed back, so the terminator can stay unread /// the input is not handed back, so the terminator can stay unread
void read_number_terminator(std::false_type /*window*/) const noexcept {} void read_number_terminator(std::false_type /*window*/) const noexcept {}
/// binary32/binary64: the value of a float token from its accumulated
/// significand (see scan_number_bulk_contiguous())
bool convert_accumulated_float(const float_significand& s, std::true_type /*native*/) noexcept
{
value_float = decimal_to_float<number_float_t>(s);
return true;
}
/// other formats (e.g. x87 long double): not instantiated for them, as
/// decimal_to_float() only supports binary32/binary64
static bool convert_accumulated_float(const float_significand& /*s*/, std::false_type /*native*/) noexcept
{
return false;
}
/// the value of an integer token with at most 18 digits, accumulated in @a w
token_type store_integer(token_type number_type, std::uint64_t w) noexcept
{
if (number_type == token_type::value_unsigned)
{
value_unsigned = static_cast<number_unsigned_t>(w);
// reject values that do not round-trip into a narrower type
return (static_cast<std::uint64_t>(value_unsigned) == w) ? token_type::value_unsigned : token_type::uninitialized;
}
const std::int64_t x = -static_cast<std::int64_t>(w);
value_integer = static_cast<number_integer_t>(x);
return (static_cast<std::int64_t>(value_integer) == x) ? token_type::value_integer : token_type::uninitialized;
}
/// contiguous input: try the number fast path, else the byte-path scanner /// contiguous input: try the number fast path, else the byte-path scanner
token_type scan_number_dispatch(std::true_type /*bulk*/) token_type scan_number_dispatch(std::true_type /*bulk*/)
{ {
@@ -13466,6 +13583,7 @@ scan_number_done:
void reset() noexcept void reset() noexcept
{ {
token_buffer.clear(); token_buffer.clear();
pending_number_text = nullptr;
decimal_point_position = std::string::npos; decimal_point_position = std::string::npos;
#if JSON_DIAGNOSTIC_POSITIONS #if JSON_DIAGNOSTIC_POSITIONS
@@ -13725,10 +13843,30 @@ scan_number_done:
/// return current string value /// return current string value
string_t& get_string() string_t& get_string()
{ {
if (pending_number_text != nullptr)
{
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(pending_number_text), pending_number_length);
pending_number_text = nullptr;
}
// a number token holds '.' regardless of the locale (#4084) // a number token holds '.' regardless of the locale (#4084)
return token_buffer; return token_buffer;
} }
/// the value of a string token (key or string), which never has pending
/// number text, so get_string()'s check is not needed
string_t& get_string_token() noexcept
{
JSON_ASSERT(pending_number_text == nullptr);
return token_buffer;
}
/// the text of a float token for a SAX handler that ignores it (the
/// library's own handlers): empty, without copying the token
const string_t& get_ignored_number_text() const noexcept
{
return token_buffer;
}
///////////////////// /////////////////////
// diagnostics // diagnostics
///////////////////// /////////////////////
@@ -14706,6 +14844,11 @@ scan_number_done:
/// whether the next get() call should just return current /// whether the next get() call should just return current
bool next_unget = false; bool next_unget = false;
/// the text of the current float token in the input, not yet copied into
/// token_buffer (see get_string()), or nullptr
const char* pending_number_text = nullptr;
std::size_t pending_number_length = 0;
/// the start position of the current token /// the start position of the current token
position_t position {}; position_t position {};
@@ -21611,6 +21754,20 @@ template<typename BasicJsonType>
using parser_callback_t = using parser_callback_t =
std::function<bool(int /*depth*/, parse_event_t /*event*/, BasicJsonType& /*parsed*/)>; std::function<bool(int /*depth*/, parse_event_t /*event*/, BasicJsonType& /*parsed*/)>;
/// whether a SAX handler ignores the text of float tokens (the library's own
/// handlers do), so that the parser need not materialize it
template<typename SAX>
struct sax_ignores_number_text : std::false_type {};
template<typename BasicJsonType, typename InputAdapterType>
struct sax_ignores_number_text<json_sax_dom_parser<BasicJsonType, InputAdapterType>> : std::true_type {};
template<typename BasicJsonType, typename InputAdapterType>
struct sax_ignores_number_text<json_sax_dom_callback_parser<BasicJsonType, InputAdapterType>> : std::true_type {};
template<typename BasicJsonType>
struct sax_ignores_number_text<json_sax_acceptor<BasicJsonType>> : std::true_type {};
/*! /*!
@brief syntax analysis @brief syntax analysis
@@ -21765,6 +21922,23 @@ class parser
return result && !error_reported; return result && !error_reported;
} }
/// the text of the current float token for @a sax (see sax_ignores_number_text)
template<typename SAX>
const string_t& number_float_text(SAX* /*sax*/)
{
return number_float_text(std::integral_constant<bool, sax_ignores_number_text<SAX>::value> {});
}
const string_t& number_float_text(std::true_type /*ignored*/) const noexcept
{
return m_lexer.get_ignored_number_text();
}
string_t& number_float_text(std::false_type /*ignored*/)
{
return m_lexer.get_string();
}
/*! /*!
@brief run a DOM SAX parser to completion and position the lexer @brief run a DOM SAX parser to completion and position the lexer
@@ -21859,7 +22033,7 @@ class parser
} }
continue; continue;
} }
if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string_token())))
{ {
return false; return false;
} }
@@ -21916,7 +22090,7 @@ class parser
break; break;
} }
if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, m_lexer.get_string()))) if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, number_float_text(sax))))
{ {
return false; return false;
} }
@@ -21962,7 +22136,7 @@ class parser
case token_type::value_string: case token_type::value_string:
{ {
if (JSON_HEDLEY_UNLIKELY(!sax->string(m_lexer.get_string()))) if (JSON_HEDLEY_UNLIKELY(!sax->string(m_lexer.get_string_token())))
{ {
return false; return false;
} }
@@ -22172,7 +22346,7 @@ class parser
} }
continue; continue;
} }
if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string_token())))
{ {
return false; return false;
} }
@@ -22285,7 +22459,7 @@ class parser
return key_error(sax, allow_recovery, false); return key_error(sax, allow_recovery, false);
} }
if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string_token())))
{ {
return next_step::stop; return next_step::stop;
} }
@@ -22320,7 +22494,7 @@ class parser
{ {
return false; return false;
} }
return sax->number_float(value, m_lexer.get_string()); return sax->number_float(value, number_float_text(sax));
} }
/// @copydoc overflow_error /// @copydoc overflow_error
+9
View File
@@ -22,6 +22,15 @@ Micro-benchmarks for parsing, serialization and the binary formats, written with
| `ViewAccept` | validate with `json_document::accept`; compare with `Accept` | | `ViewAccept` | validate with `json_document::accept`; compare with `Accept` |
| `ViewMaterialize` | convert a parsed `json_document` into a `json` value | | `ViewMaterialize` | convert a parsed `json_document` into a `json` value |
| `ViewDump` | serialize a parsed `json_document`; compare with `Dump` | | `ViewDump` | serialize a parsed `json_document`; compare with `Dump` |
| `SaxParse` | parse with a SAX handler that only counts events, so no values are built |
| `Copy`, `CopyAssign`, `CopyRing` | copy a parsed document and destroy a copy, both timed: a temporary copy, an assignment over an earlier copy, and replacing the oldest of eight live copies |
| `CopyOnly`, `DestroyOnly` | only the copy or only the destruction, the other half untimed; a fast destructor can leave work to the next allocations, so these tell the halves apart but do not compare versions |
| `Equal`, `Iterate` | compare a parsed document with a copy, or visit every value with range-based `for` loops |
| `Flatten`, `FlattenOrdered`, `Unflatten` | `flatten()` for `json` and `ordered_json`, and `unflatten()` |
| `Diff`, `Patch` | `diff()` against the document with every eighth scalar changed, and applying that patch |
| `JsonPointerParse`, `JsonPointerToString`, `JsonPointerAt`, `JsonPointerContains` | the JSON pointers of every scalar in a document: parse them from strings, turn them back into strings, look them up with `at()` and `contains()` |
| `GetVectorInt`, `GetVectorDouble`, `GetMap` | `get<std::vector<std::int64_t>>()`, `get<std::vector<double>>()` and `get<std::map<std::string, json>>()` on large arrays and a large flat object |
| `ToJsonStrings` | create a `json` array from a `std::vector` of 100,000 strings, copied or moved, and destroy it again |
The input files are those of [nativejson-benchmark](https://github.com/miloyip/nativejson-benchmark) (`canada`, The input files are those of [nativejson-benchmark](https://github.com/miloyip/nativejson-benchmark) (`canada`,
`citm_catalog`, `twitter`), a large `jeopardy` file, and number-heavy files (`floats`, `signed_ints`, ...). `citm_catalog`, `twitter`), a large `jeopardy` file, and number-heavy files (`floats`, `signed_ints`, ...).
+581
View File
@@ -9,12 +9,15 @@
#include <benchmark/benchmark.h> #include <benchmark/benchmark.h>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include <array> #include <array>
#include <cstdint>
#include <cstdio> #include <cstdio>
#include <deque> #include <deque>
#include <fstream> #include <fstream>
#include <map>
#include <numeric> #include <numeric>
#include <set> #include <set>
#include <sstream> #include <sstream>
#include <string>
#include <vector> #include <vector>
#include <test_data.hpp> #include <test_data.hpp>
@@ -710,6 +713,584 @@ static void FromCborChunkedString(benchmark::State& state, const std::size_t chu
BENCHMARK_CAPTURE(FromCborChunkedString, 10000 chunks, 10000); BENCHMARK_CAPTURE(FromCborChunkedString, 10000 chunks, 10000);
//////////////////////////////////////////////////////////////////////////////
// SAX-parse JSON from string
//
// sax_parse() with a handler that does next to nothing runs the lexer and
// parser without building values, so the gap to ParseString above is the cost
// of building the DOM.
//////////////////////////////////////////////////////////////////////////////
static std::string read_file(const char* filename)
{
std::ifstream f(filename);
return std::string((std::istreambuf_iterator<char>(f)), std::istreambuf_iterator<char>());
}
static json load_json(const char* filename)
{
return json::parse(read_file(filename));
}
/// SAX handler that only counts the events, so that the handler costs next to nothing
class counting_sax : public nlohmann::json_sax<json>
{
public:
std::size_t events = 0;
bool null() override
{
return count();
}
bool boolean(bool /*val*/) override
{
return count();
}
bool number_integer(number_integer_t /*val*/) override
{
return count();
}
bool number_unsigned(number_unsigned_t /*val*/) override
{
return count();
}
bool number_float(number_float_t /*val*/, const string_t& /*s*/) override
{
return count();
}
bool string(string_t& /*val*/) override
{
return count();
}
bool binary(binary_t& /*val*/) override
{
return count();
}
bool start_object(std::size_t /*elements*/) override
{
return count();
}
bool key(string_t& /*val*/) override
{
return count();
}
bool end_object() override
{
return count();
}
bool start_array(std::size_t /*elements*/) override
{
return count();
}
bool end_array() override
{
return count();
}
bool parse_error(std::size_t /*position*/, const std::string& /*last_token*/, const json::exception& /*ex*/) override
{
return false;
}
private:
bool count()
{
++events;
return true;
}
};
static void SaxParse(benchmark::State& state, const char* filename)
{
const std::string str = read_file(filename);
for (auto _ : state)
{
counting_sax sax;
bool result = json::sax_parse(str, &sax);
benchmark::DoNotOptimize(result);
benchmark::DoNotOptimize(sax.events);
}
state.SetBytesProcessed(state.iterations() * str.size());
}
BENCHMARK_CAPTURE(SaxParse, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
BENCHMARK_CAPTURE(SaxParse, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(SaxParse, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
//////////////////////////////////////////////////////////////////////////////
// work with values: copy, destroy, compare, iterate
//
// The parsed test files stand in for a document an application keeps around.
// Each benchmark times one whole-document operation; setup and cleanup run
// outside the timed section.
//
// A copy is always destroyed again, and how a document is freed shapes the
// heap the next copy allocates from: a destructor that frees quickly can leave
// work to the allocator's next calls. Copy, CopyAssign and CopyRing therefore
// time the copy together with the destruction of a copy; CopyOnly and
// DestroyOnly time one half each and are meant for telling the halves apart,
// not for comparing versions.
//////////////////////////////////////////////////////////////////////////////
/// copy a document and destroy the copy again, as a temporary copy is used
static void Copy(benchmark::State& state, const char* filename)
{
const json j = load_json(filename);
for (auto _ : state)
{
json copy = j;
benchmark::DoNotOptimize(copy);
}
}
BENCHMARK_CAPTURE(Copy, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
BENCHMARK_CAPTURE(Copy, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(Copy, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
/// copy-assign a document over an earlier copy of it, which is freed in the process
static void CopyAssign(benchmark::State& state, const char* filename)
{
const json j = load_json(filename);
json target = j;
for (auto _ : state)
{
target = j;
benchmark::DoNotOptimize(target);
}
}
BENCHMARK_CAPTURE(CopyAssign, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
BENCHMARK_CAPTURE(CopyAssign, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(CopyAssign, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
/// keep eight copies alive and replace the oldest one, as a cache or an undo
/// history does, so that the copy is not made from memory just freed by an
/// identical document
static void CopyRing(benchmark::State& state, const char* filename)
{
const json j = load_json(filename);
std::vector<json> ring(8, j);
std::size_t oldest = 0;
for (auto _ : state)
{
ring[oldest] = j;
benchmark::DoNotOptimize(ring[oldest]);
oldest = (oldest + 1) % ring.size();
}
}
BENCHMARK_CAPTURE(CopyRing, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
BENCHMARK_CAPTURE(CopyRing, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(CopyRing, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
/// only the copy; the copy is destroyed with the timer paused (diagnostic)
static void CopyOnly(benchmark::State& state, const char* filename)
{
const json j = load_json(filename);
for (auto _ : state)
{
auto* copy = new json(j);
state.PauseTiming();
delete copy;
state.ResumeTiming();
}
}
BENCHMARK_CAPTURE(CopyOnly, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
BENCHMARK_CAPTURE(CopyOnly, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(CopyOnly, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
/// only the destruction; the copy is made with the timer paused (diagnostic)
static void DestroyOnly(benchmark::State& state, const char* filename)
{
const json j = load_json(filename);
for (auto _ : state)
{
state.PauseTiming();
auto* copy = new json(j);
state.ResumeTiming();
delete copy;
}
}
BENCHMARK_CAPTURE(DestroyOnly, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
BENCHMARK_CAPTURE(DestroyOnly, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(DestroyOnly, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
static void Equal(benchmark::State& state, const char* filename)
{
// two separate values, so that the comparison cannot stop at identical addresses
const json j = load_json(filename);
const json copy = j;
for (auto _ : state)
{
bool result = (j == copy);
benchmark::DoNotOptimize(result);
}
}
BENCHMARK_CAPTURE(Equal, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
BENCHMARK_CAPTURE(Equal, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(Equal, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
/// visit every value with range-based for loops, as client code walks a document
static std::size_t count_values(const json& j)
{
std::size_t result = 1;
if (j.is_structured())
{
for (const auto& element : j)
{
result += count_values(element);
}
}
return result;
}
static void Iterate(benchmark::State& state, const char* filename)
{
const json j = load_json(filename);
for (auto _ : state)
{
std::size_t result = count_values(j);
benchmark::DoNotOptimize(result);
}
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(count_values(j)));
}
BENCHMARK_CAPTURE(Iterate, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
BENCHMARK_CAPTURE(Iterate, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(Iterate, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
//////////////////////////////////////////////////////////////////////////////
// flatten and unflatten
//////////////////////////////////////////////////////////////////////////////
static void Flatten(benchmark::State& state, const char* filename)
{
const json j = load_json(filename);
for (auto _ : state)
{
json flat = j.flatten();
benchmark::DoNotOptimize(flat);
}
}
BENCHMARK_CAPTURE(Flatten, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
BENCHMARK_CAPTURE(Flatten, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(Flatten, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
// ordered_json objects look keys up linearly, which flatten() used to do for
// every value; canada (330,000 values in one flat object) is left out, as it
// would take minutes with such versions
static void FlattenOrdered(benchmark::State& state, const char* filename)
{
const nlohmann::ordered_json j = nlohmann::ordered_json::parse(read_file(filename));
for (auto _ : state)
{
nlohmann::ordered_json flat = j.flatten();
benchmark::DoNotOptimize(flat);
}
}
BENCHMARK_CAPTURE(FlattenOrdered, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(FlattenOrdered, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
static void Unflatten(benchmark::State& state, const char* filename)
{
const json flat = load_json(filename).flatten();
for (auto _ : state)
{
json j = flat.unflatten();
benchmark::DoNotOptimize(j);
}
}
BENCHMARK_CAPTURE(Unflatten, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
BENCHMARK_CAPTURE(Unflatten, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(Unflatten, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
//////////////////////////////////////////////////////////////////////////////
// diff and patch
//
// The target is the document with every eighth scalar changed, so the patch
// has one "replace" operation per changed value and the structure stays the
// same.
//////////////////////////////////////////////////////////////////////////////
static void change_every_eighth_scalar(json& j, std::size_t& counter)
{
if (j.is_structured())
{
for (auto& element : j)
{
change_every_eighth_scalar(element, counter);
}
return;
}
if (++counter % 8 != 0)
{
return;
}
switch (j.type())
{
case json::value_t::number_integer:
j = j.get<json::number_integer_t>() + 1;
break;
case json::value_t::number_unsigned:
j = j.get<json::number_unsigned_t>() + 1;
break;
case json::value_t::number_float:
j = j.get<json::number_float_t>() + 0.5;
break;
case json::value_t::string:
j = j.get<std::string>() + "!";
break;
case json::value_t::boolean:
j = !j.get<bool>();
break;
case json::value_t::null:
case json::value_t::object:
case json::value_t::array:
case json::value_t::binary:
case json::value_t::discarded:
default:
j = 0;
break;
}
}
static json changed_copy(const json& j)
{
json result = j;
std::size_t counter = 0;
change_every_eighth_scalar(result, counter);
return result;
}
static void Diff(benchmark::State& state, const char* filename)
{
const json source = load_json(filename);
const json target = changed_copy(source);
for (auto _ : state)
{
json patch = json::diff(source, target);
benchmark::DoNotOptimize(patch);
}
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(json::diff(source, target).size()));
}
BENCHMARK_CAPTURE(Diff, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
BENCHMARK_CAPTURE(Diff, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(Diff, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
static void Patch(benchmark::State& state, const char* filename)
{
const json source = load_json(filename);
const json patch = json::diff(source, changed_copy(source));
for (auto _ : state)
{
json result = source.patch(patch);
benchmark::DoNotOptimize(result);
}
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(patch.size()));
}
BENCHMARK_CAPTURE(Patch, canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
BENCHMARK_CAPTURE(Patch, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(Patch, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
//////////////////////////////////////////////////////////////////////////////
// JSON Pointer
//
// The pointers are those of every scalar in the document, i.e., the keys of
// its flattened form.
//////////////////////////////////////////////////////////////////////////////
static std::vector<std::string> pointer_strings(const json& j)
{
std::vector<std::string> result;
const json flat = j.flatten();
result.reserve(flat.size());
for (auto it = flat.begin(); it != flat.end(); ++it)
{
result.push_back(it.key());
}
return result;
}
static std::vector<json::json_pointer> pointers(const json& j)
{
const std::vector<std::string> strings = pointer_strings(j);
std::vector<json::json_pointer> result;
result.reserve(strings.size());
for (const auto& s : strings)
{
result.emplace_back(s);
}
return result;
}
static void JsonPointerParse(benchmark::State& state, const char* filename)
{
const std::vector<std::string> strings = pointer_strings(load_json(filename));
for (auto _ : state)
{
for (const auto& s : strings)
{
json::json_pointer ptr(s);
benchmark::DoNotOptimize(ptr);
}
}
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(strings.size()));
}
BENCHMARK_CAPTURE(JsonPointerParse, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(JsonPointerParse, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
static void JsonPointerToString(benchmark::State& state, const char* filename)
{
const std::vector<json::json_pointer> ptrs = pointers(load_json(filename));
for (auto _ : state)
{
for (const auto& ptr : ptrs)
{
std::string s = ptr.to_string();
benchmark::DoNotOptimize(s);
}
}
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(ptrs.size()));
}
BENCHMARK_CAPTURE(JsonPointerToString, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(JsonPointerToString, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
static void JsonPointerAt(benchmark::State& state, const char* filename)
{
const json j = load_json(filename);
const std::vector<json::json_pointer> ptrs = pointers(j);
for (auto _ : state)
{
for (const auto& ptr : ptrs)
{
const json& value = j.at(ptr);
benchmark::DoNotOptimize(&value);
}
}
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(ptrs.size()));
}
BENCHMARK_CAPTURE(JsonPointerAt, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(JsonPointerAt, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
static void JsonPointerContains(benchmark::State& state, const char* filename)
{
const json j = load_json(filename);
const std::vector<json::json_pointer> ptrs = pointers(j);
for (auto _ : state)
{
for (const auto& ptr : ptrs)
{
bool result = j.contains(ptr);
benchmark::DoNotOptimize(result);
}
}
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(ptrs.size()));
}
BENCHMARK_CAPTURE(JsonPointerContains, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
BENCHMARK_CAPTURE(JsonPointerContains, twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
//////////////////////////////////////////////////////////////////////////////
// conversions from and to C++ containers
//////////////////////////////////////////////////////////////////////////////
static void GetVectorInt(benchmark::State& state, const char* filename)
{
const json j = load_json(filename);
for (auto _ : state)
{
auto v = j.get<std::vector<std::int64_t>>();
benchmark::DoNotOptimize(v);
}
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(j.size()));
}
BENCHMARK_CAPTURE(GetVectorInt, signed_ints, TEST_DATA_DIRECTORY "/regression/signed_ints.json");
static void GetVectorDouble(benchmark::State& state, const char* filename)
{
const json j = load_json(filename);
for (auto _ : state)
{
auto v = j.get<std::vector<double>>();
benchmark::DoNotOptimize(v);
}
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(j.size()));
}
BENCHMARK_CAPTURE(GetVectorDouble, floats, TEST_DATA_DIRECTORY "/regression/floats.json");
// a large flat object: the flattened document, one member per scalar
static void GetMap(benchmark::State& state, const char* filename)
{
const json j = load_json(filename).flatten();
for (auto _ : state)
{
auto m = j.get<std::map<std::string, json>>();
benchmark::DoNotOptimize(m);
}
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(j.size()));
}
BENCHMARK_CAPTURE(GetMap, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
/// 100,000 strings, each too long for the small string optimization
static std::vector<std::string> make_strings()
{
std::vector<std::string> result;
result.reserve(100000);
for (std::size_t i = 0; i < 100000; ++i)
{
result.push_back("a string that does not fit in place: " + std::to_string(i));
}
return result;
}
static void ToJsonStrings(benchmark::State& state, const bool move)
{
const std::vector<std::string> strings = make_strings();
for (auto _ : state)
{
state.PauseTiming();
std::vector<std::string> input = strings;
state.ResumeTiming();
// destroyed inside the timed section, like the copies above
auto* j = move ? new json(std::move(input)) : new json(input);
delete j;
}
state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(strings.size()));
}
BENCHMARK_CAPTURE(ToJsonStrings, copy, false);
BENCHMARK_CAPTURE(ToJsonStrings, move, true);
// conversion of a JSON array into standard containers // conversion of a JSON array into standard containers
template<typename Container> template<typename Container>
static void GetArray(benchmark::State& state) static void GetArray(benchmark::State& state)
+109
View File
@@ -15,9 +15,12 @@ using nlohmann::json;
#include <array> // array #include <array> // array
#include <cstdint> // uint32_t, uint64_t #include <cstdint> // uint32_t, uint64_t
#include <cstdio> // snprintf #include <cstdio> // snprintf
#include <cmath> // isfinite
#include <cstdlib> // strtod #include <cstdlib> // strtod
#include <cstring> // memcpy #include <cstring> // memcpy
#include <limits> // numeric_limits #include <limits> // numeric_limits
#include <initializer_list> // initializer_list
#include <list> // list
#include <map> // map #include <map> // map
#include <random> // mt19937 #include <random> // mt19937
#include <sstream> // stringstream #include <sstream> // stringstream
@@ -332,6 +335,112 @@ TEST_CASE("lexer number fast path")
} }
} }
#if !defined(JSON_NOEXCEPTION)
// this section reports parse errors, which abort when exceptions are off
SECTION("digits accumulated while scanning")
{
// The fast path accumulates the digits while it validates them, eight
// at a time where possible, and converts tokens with at most 19
// digits (floats) or 18 digits (integers) from that value. Every digit
// count from 1 to 25, in the integer and in the fraction part, with
// and without sign and exponent, must give the byte path's result. A
// std::list has no contiguous storage, so parsing it takes the byte
// path.
// type and value, or the error (values beyond the range of double
// are rejected with out_of_range.406 on both paths)
const auto outcome = [](const std::string & doc, bool byte_path) -> std::string
{
JSON_TRY
{
const std::list<char> chars(doc.begin(), doc.end());
const json j = byte_path ? json::parse(chars.begin(), chars.end()) : json::parse(doc);
return std::string(j[0].type_name()) + '|' + j.dump();
}
JSON_CATCH (const json::exception & e)
{
return e.what();
}
};
// the concatenation of @a parts
const auto join = [](std::initializer_list<std::string> parts)
{
std::string result;
for (const auto& part : parts)
{
result += part;
}
return result;
};
std::vector<std::string> numbers;
for (std::size_t digits = 1; digits <= 25; ++digits)
{
std::string run;
for (std::size_t i = 0; i < digits; ++i)
{
run += static_cast<char>('1' + static_cast<char>((i * 7) % 9));
}
const std::string nines(digits, '9');
for (const char* sign_text :
{"", "-"
})
{
const std::string sign = sign_text;
numbers.push_back(join({sign, run}));
numbers.push_back(join({sign, nines}));
numbers.push_back(join({sign, "1", std::string(digits - 1, '0')}));
numbers.push_back(join({sign, "0.", run}));
numbers.push_back(join({sign, "0.", std::string(digits, '0'), "1"}));
numbers.push_back(join({sign, run, ".", run}));
numbers.push_back(join({sign, run, "e-", std::to_string(digits)}));
numbers.push_back(join({sign, "1.", run, "E+", std::to_string(digits * 13)}));
}
}
// exponents beyond every range, and zeros
numbers.insert(numbers.end(), {"1e400", "1e-400", "-1e99999999999999999999", "1e-99999999999999999999",
"0e99999999999999999999", "0.0e-5", "-0.000", "123456789e-0000000000000000000000330"
});
for (const auto& n : numbers)
{
CAPTURE(n)
const std::string doc = "[" + n + "]";
const std::string a = outcome(doc, false);
CHECK(a == outcome(doc, true));
const double expected = std::strtod(n.c_str(), nullptr);
if (a.compare(0, 6, "number") == 0 && std::isfinite(expected) && n.find_first_of(".eE") != std::string::npos)
{
CHECK(json::parse(doc)[0].get<double>() == expected);
}
}
}
#endif
SECTION("integers that do not fit a narrower type become floats")
{
// up to 18 digits, the fast path stores the accumulated value without
// an overflow check; a narrower number_integer_t/number_unsigned_t
// must still see the value rejected and converted as a float
using json32 = nlohmann::basic_json<std::map, std::vector, std::string, bool, std::int32_t, std::uint32_t, float>;
for (const char* n :
{"4294967295", "4294967296", "2147483647", "-2147483648", "-2147483649", "99999999999", "-99999999999"
})
{
CAPTURE(n)
const std::string doc = std::string("[") + n + "]";
const std::list<char> chars(doc.begin(), doc.end());
const json32 a = json32::parse(doc);
const json32 b = json32::parse(chars.begin(), chars.end());
CHECK(a[0].type() == b[0].type());
CHECK(a == b);
}
CHECK(json32::parse("[4294967295]")[0].is_number_unsigned());
CHECK(json32::parse("[4294967296]")[0].is_number_float());
CHECK(json32::parse("[-2147483648]")[0].is_number_integer());
CHECK(json32::parse("[-2147483649]")[0].is_number_float());
}
SECTION("token type classification") SECTION("token type classification")
{ {
CHECK((scan_string("0") == json::lexer::token_type::value_unsigned)); CHECK((scan_string("0") == json::lexer::token_type::value_unsigned));
+75
View File
@@ -30,7 +30,9 @@ using nlohmann::json;
#include <valarray> #include <valarray>
#include <algorithm> #include <algorithm>
#include <cstdio> #include <cstdio>
#include <cstdint>
#include <fstream> #include <fstream>
#include <iterator>
#include <list> #include <list>
#include <sstream> #include <sstream>
#include <string> #include <string>
@@ -2450,6 +2452,79 @@ std::string parse_error_message_range(IteratorType first, IteratorType last)
} }
} // namespace } // namespace
TEST_CASE("text of float tokens")
{
// For contiguous input, the lexer copies the text of a float token into
// its buffer only when a SAX handler asks for it (the library's own
// handlers do not). A user handler must still see the exact text, for
// every input adapter and in every position of a document.
const std::string doc = R"([1.5, -0.0, 0.1, 1e5, -2.5E-4, 12345678901234567890.5, 0.12345678901234567890123,
{"a": 3.25, "b": [6.02e23, 1]}, "x", true, 7e-1])";
const std::vector<std::string> expected_floats =
{
"number_float(1.5)", "number_float(-0.0)", "number_float(0.1)", "number_float(1e5)", "number_float(-2.5E-4)",
"number_float(12345678901234567890.5)", "number_float(0.12345678901234567890123)", "number_float(3.25)",
"number_float(6.02e23)", "number_float(7e-1)"
};
const auto floats = [](const std::vector<std::string>& events)
{
std::vector<std::string> result;
std::copy_if(events.begin(), events.end(), std::back_inserter(result), [](const std::string & e)
{
return e.compare(0, 13, "number_float(") == 0;
});
return result;
};
SECTION("contiguous input")
{
SaxEventLogger l;
CHECK(json::sax_parse(doc, &l));
CHECK(floats(l.events) == expected_floats);
}
SECTION("byte input")
{
const std::vector<std::uint8_t> bytes(doc.begin(), doc.end());
SaxEventLogger l;
CHECK(json::sax_parse(bytes, &l));
CHECK(floats(l.events) == expected_floats);
}
SECTION("stream input")
{
std::istringstream is(doc);
SaxEventLogger l;
CHECK(json::sax_parse(is, &l));
CHECK(floats(l.events) == expected_floats);
}
SECTION("non-contiguous iterators")
{
const std::list<char> chars(doc.begin(), doc.end());
SaxEventLogger l;
CHECK(json::sax_parse(chars.begin(), chars.end(), &l));
CHECK(floats(l.events) == expected_floats);
}
SECTION("the library's own handlers do not need the text")
{
using lexer_adapter = nlohmann::detail::iterator_input_adapter<const char*>;
CHECK(nlohmann::detail::sax_ignores_number_text<nlohmann::detail::json_sax_dom_parser<json, lexer_adapter>>::value);
CHECK(nlohmann::detail::sax_ignores_number_text<nlohmann::detail::json_sax_dom_callback_parser<json, lexer_adapter>>::value);
CHECK(nlohmann::detail::sax_ignores_number_text<nlohmann::detail::json_sax_acceptor<json>>::value);
CHECK_FALSE(nlohmann::detail::sax_ignores_number_text<SaxEventLogger>::value);
// and the values are the same as when the text is read
const json j = json::parse(doc);
CHECK(j[0] == 1.5);
CHECK(j[5].get<double>() == 12345678901234567890.5);
CHECK(j[7]["b"][0] == 6.02e23);
CHECK(json::accept(doc));
}
}
TEST_CASE("last-read diagnostics are identical across input adapters") TEST_CASE("last-read diagnostics are identical across input adapters")
{ {
// The lexer reconstructs the "last read" token lazily for seekable adapters // The lexer reconstructs the "last read" token lazily for seekable adapters