Compare commits

..
Author SHA1 Message Date
Niels LohmannandClaude Sonnet 5.5 1c588321c5 Fix UBSan implicit-conversion in BSON stream tests
std::string(first, last) over std::uint8_t iterators converts each byte
to char implicitly, which -fsanitize=implicit-conversion reports as soon
as a byte is >= 0x80 (the document size byte of the enlarged test input,
and 0xFF in the new array element name test). Build the string from a
const char* and a size instead.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01N5NasCxh8jKXBaa6HBx9Xk
Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-31 07:28:43 +00:00
Niels Lohmann cff8548496 BSON reader: find C-string terminators 8 bytes at a time, skip array element names
get_bson_cstr_bulk() searched for the 0x00 byte one byte at a time. It now uses
find_zero_byte() (string_scan.hpp), which tests 8 bytes per step with the
SWAR zero-byte test the lexer's scanners use.

The names of an array's elements ("0", "1", ...) are not passed to the SAX
parser, but were still copied into the key string and checked against the
error handler. They are now skipped without a copy, except with
error_handler_t::strict, which still rejects ill-formed UTF-8 in them with
the same error and position as before.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-30 22:17:08 +01:00
9 changed files with 213 additions and 506 deletions

No files matched your search

@@ -13,15 +13,9 @@ function never throws an exception for invalid input, and the returned `#!cpp bo
is returned.
The result is the same as that of `BasicJsonType::accept()` by design; the difference is how the check is made. This
function runs the document's builder, which checks the text as [`parse()`](parse.md) does, while
`BasicJsonType::accept()` runs the SAX parser with a handler that ignores every event. Use this one if you go on to
parse the same input with [`parse()`](parse.md) anyway, so that both use the same code; either is fine for a plain
yes/no answer.
When both `ignore_comments` and `ignore_trailing_commas` are `#!cpp false` (the default), the builder writes no index.
With either option set, it builds the index as [`parse()`](parse.md) does and discards it: the result is the same, only
the time differs. (Neither option needs the index; the variant that skips it exists only for the default options to
keep the code size down.)
function runs the document's index builder (and discards the index), while `BasicJsonType::accept()` runs the SAX
parser with a handler that ignores every event. Use this one if you go on to parse the same input with
[`parse()`](parse.md) anyway, so that both use the same code; either is fine for a plain yes/no answer.
## Template parameters
+22 -18
View File
@@ -465,15 +465,17 @@ class binary_reader
}
const std::size_t element_type_parse_position = chars_read;
// an array's elements are named "0", "1", ... in the wire format,
// and those names are not passed on: they are skipped unless
// error_handler_t::strict must reject ill-formed UTF-8 in them
const bool is_object = container_stack.back().is_object;
key.clear();
if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))
if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key, is_object || error_handler == error_handler_t::strict)))
{
return false;
}
// an array's elements are named "0", "1", ... in the wire format,
// and those names are not passed on
if (container_stack.back().is_object && !sax->key(key))
if (is_object && !sax->key(key))
{
return false;
}
@@ -489,14 +491,18 @@ class binary_reader
@brief Parses a C-style string from the BSON input.
@param[in,out] result A reference to the string variable where the read
string is to be stored.
@param[in] store whether the characters are needed; if not (the names of
an array's elements), they are skipped and @a result may
be left unchanged, and they are not checked against
@ref error_handler
@return `true` if the \\x00-byte indicating the end of the string was
encountered before the EOF; false` indicates an unexpected EOF.
*/
bool get_bson_cstr(string_t& result)
bool get_bson_cstr(string_t& result, const bool store = true)
{
if (get_bson_cstr_bulk(result, std::integral_constant<bool, bulk_scan> {}))
if (get_bson_cstr_bulk(result, store, std::integral_constant<bool, bulk_scan> {}))
{
return check_string_utf8(result, "key");
return !store || check_string_utf8(result, "key");
}
auto out = std::back_inserter(result);
@@ -509,7 +515,7 @@ class binary_reader
}
if (current == 0x00)
{
return check_string_utf8(result, "key");
return !store || check_string_utf8(result, "key");
}
*out++ = static_cast<typename string_t::value_type>(current);
}
@@ -519,10 +525,11 @@ class binary_reader
@brief read a C-style string from contiguous input in one step
@param[in,out] result the string to append to
@param[in] store whether to append the characters to @a result
@return whether the string was read; if the input has no \\x00-byte, nothing
is read, and @ref get_bson_cstr reports the end of the input
*/
bool get_bson_cstr_bulk(string_t& result, std::true_type /*bulk*/)
bool get_bson_cstr_bulk(string_t& result, const bool store, std::true_type /*bulk*/)
{
const std::size_t remaining = ia.bulk_remaining();
if (remaining == 0)
@@ -530,18 +537,15 @@ class binary_reader
return false;
}
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
// a plain loop rather than std::memchr: most keys are short (array
// indices are keys, too), and the call would cost more than it saves
std::size_t length = 0;
while (length < remaining && data[length] != 0x00)
{
++length;
}
const std::size_t length = find_zero_byte(data, remaining);
if (length == remaining)
{
return false;
}
result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
if (store)
{
result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
}
// consume the string and its \x00-byte, as the byte-wise path does
ia.bulk_skip(length + 1);
chars_read += length + 1;
@@ -550,7 +554,7 @@ class binary_reader
}
/// input that is not contiguous: C-style strings are read byte by byte
bool get_bson_cstr_bulk(string_t& /*result*/, std::false_type /*bulk*/) const noexcept
bool get_bson_cstr_bulk(string_t& /*result*/, const bool /*store*/, std::false_type /*bulk*/) const noexcept
{
return false;
}
@@ -90,6 +90,34 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t find_string_special(const unsigned char* d
return n;
}
// return the index of the first 0x00 byte in [data, data+n), or n if there is
// none; scans 8 bytes at a time (BSON's C-style strings)
JSON_HEDLEY_ALWAYS_INLINE std::size_t find_zero_byte(const unsigned char* data, std::size_t n) noexcept
{
constexpr std::uint64_t ones = 0x0101010101010101ull;
constexpr std::uint64_t high = 0x8080808080808080ull;
std::size_t i = 0;
for (; i + 8 <= n; i += 8)
{
const std::uint64_t v = read_eight_bytes(data + i);
const std::uint64_t zero = (v - ones) & ~v & high;
if (zero != 0)
{
// the lowest flagged byte is the first zero byte (see
// find_string_special())
return i + (static_cast<std::size_t>(count_trailing_zeros(zero)) / 8);
}
}
for (; i < n; ++i)
{
if (data[i] == 0x00)
{
return i;
}
}
return n;
}
// classify a byte as one the serializer must NOT copy verbatim when
// ensure_ascii is requested: the closing quote, an escape, a control character
// (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else -
+18 -120
View File
@@ -69,8 +69,7 @@ struct parse_failure
};
/// FloatType: the number_float_t of the document, whose overflow parse() rejects
/// Store: false for accept(): the grammar, number, string and UTF-8 checks run as for parse(), but no node is written
template<typename FloatType, bool Comments, bool TrailingCommas, bool NulIsEnd, bool Sentinel, bool Store>
template<typename FloatType, bool Comments, bool TrailingCommas, bool NulIsEnd, bool Sentinel>
class builder
{
public:
@@ -246,87 +245,11 @@ class builder
return p;
}
/// the string from p, the first backslash, control character, ill-formed UTF-8 byte, or the end, checked and not
/// decoded (accept() stores nothing, so the arena is not written): the checks, errors and error positions of
/// slow_string(), for everything up to the closing quote
NLOHMANN_VIEW_NOINLINE decoded check_string(const unsigned char* p)
{
const unsigned char* const end = m_end;
for (;;)
{
if (p == end)
{
return failed(error_code::string_missing_quote, p);
}
const unsigned char c = *p;
if (c == '"')
{
++p;
return decoded{p, 0, 0};
}
if (c != '\\')
{
return failed(c < 0x20 ? error_code::string_control_character : error_code::string_utf8, p);
}
++p;
if (p == end)
{
return failed(error_code::string_missing_quote, p);
}
const unsigned char d = *p++;
if (d == 'u')
{
const int cp = hex4(p); // (moves p past the four digits)
if (NLOHMANN_VIEW_UNLIKELY(cp < 0))
{
return failed(error_code::string_unicode_hex, p);
}
if (NLOHMANN_VIEW_UNLIKELY((cp & 0xF800) == 0xD800)) // a surrogate
{
if (cp >= 0xDC00)
{
return failed(error_code::string_surrogate_low, p);
}
if (end - p < 2 || p[0] != '\\' || p[1] != 'u')
{
return failed(error_code::string_surrogate_high, p);
}
p += 2;
const int lo = hex4(p); // (moves p past the four digits)
if (lo < 0)
{
return failed(error_code::string_unicode_hex, p);
}
if (lo < 0xDC00 || lo > 0xDFFF)
{
return failed(error_code::string_surrogate_high, p);
}
}
}
else if (!(d == '"' || d == '\\' || d == '/' || d == 'b' || d == 'f' || d == 'n' || d == 'r' || d == 't'))
{
--p;
return failed(error_code::string_escape, p);
}
if (p != end && *p == '\\')
{
continue; // (escapes in a row: no run of text between them)
}
p = scan_string_run(p, end); // the text up to the next backslash, quote, control character, or ill-formed UTF-8
}
}
/// s points to the start of the string, p to a backslash, a control
/// character, an ill-formed UTF-8 byte, or the end: decode the string into
/// the arena
NLOHMANN_VIEW_NOINLINE decoded slow_string(const unsigned char* s, const unsigned char* p)
{
if (!enabled(Store))
{
// (accept(): the string is checked, not decoded)
static_cast<void>(s);
return check_string(p);
}
// single-character escapes; 0: invalid (and 'u', handled separately)
static const std::array<char, 128> simple_escape =
{
@@ -687,13 +610,10 @@ run_found:
NLOHMANN_VIEW_ALWAYS_INLINE bool run()
{
if (enabled(Store))
{
cold.m_doc.reserve(estimate_nodes(reinterpret_cast<const char*>(b), static_cast<std::size_t>(e - b)));
base = cold.m_doc.nodes;
out = base;
cap = base + cold.m_doc.node_capacity;
}
cold.m_doc.reserve(estimate_nodes(reinterpret_cast<const char*>(b), static_cast<std::size_t>(e - b)));
base = cold.m_doc.nodes;
out = base;
cap = base + cold.m_doc.node_capacity;
if (e - p >= 3 && p[0] == 0xEF && p[1] == 0xBB && p[2] == 0xBF)
{
@@ -877,10 +797,7 @@ close_object:
// an array pays nothing for this)
if (NLOHMANN_VIEW_UNLIKELY(cur_count >= document_data::index_min_members))
{
if (enabled(Store))
{
cold.note_large_object(cur_idx);
}
cold.note_large_object(cur_idx);
}
close_container:
@@ -1040,12 +957,6 @@ indent_done:
/// an integer's value); two stores on little-endian targets
NLOHMANN_VIEW_ALWAYS_INLINE node* push_node(node_kind k, std::uint8_t flags, std::uint16_t extra, std::size_t off, std::uint64_t second)
{
if (!enabled(Store))
{
// accept(): nothing is stored, so what computes the arguments is not run either. base and out stay
// null: their difference, which open() takes, is 0.
return out;
}
if (NLOHMANN_VIEW_UNLIKELY(out == cap))
{
const auto n = static_cast<std::size_t>(out - base);
@@ -1104,12 +1015,9 @@ indent_done:
NLOHMANN_VIEW_ALWAYS_INLINE void close()
{
if (enabled(Store))
{
node& n = base[cur_idx];
n.len = cur_count;
n.next = static_cast<std::uint32_t>(out - base) - cur_idx;
}
node& n = base[cur_idx];
n.len = cur_count;
n.next = static_cast<std::uint32_t>(out - base) - cur_idx;
if (--depth != 0)
{
if (NLOHMANN_VIEW_LIKELY(depth <= 64))
@@ -1284,44 +1192,34 @@ indent_done:
};
/// run the builder with compile-time options
template<typename FloatType, bool Comments, bool TrailingCommas, bool NulIsEnd, bool Store>
template<typename FloatType, bool Comments, bool TrailingCommas, bool NulIsEnd>
inline bool build_with(document_data& d, const char* src, std::size_t size, bool sentinel, parse_failure& failure)
{
if (sentinel)
{
builder<FloatType, Comments, TrailingCommas, NulIsEnd, true, Store> bld(d, src, size);
builder<FloatType, Comments, TrailingCommas, NulIsEnd, true> bld(d, src, size);
const bool ok = bld.run();
failure = bld.failure();
return ok;
}
builder<FloatType, Comments, TrailingCommas, NulIsEnd, false, Store> bld(d, src, size);
builder<FloatType, Comments, TrailingCommas, NulIsEnd, false> bld(d, src, size);
const bool ok = bld.run();
failure = bld.failure();
return ok;
}
/// sentinel: src[size] is readable and 0 (e.g. std::string); FloatType: the
/// number_float_t of the document; validate_only: accept(), only the result is
/// used
/// number_float_t of the document
template<typename FloatType, bool NulIsEnd>
inline bool build(document_data& d, const char* src, std::size_t size, bool comments, bool trailing_commas, bool sentinel, bool validate_only,
parse_failure& failure)
inline bool build(document_data& d, const char* src, std::size_t size, bool comments, bool trailing_commas, bool sentinel, parse_failure& failure)
{
if (validate_only && !comments && !trailing_commas)
{
// accept() of plain JSON: nothing is stored. The other option sets
// could skip the stores just as well; they keep the builder that
// stores only so that there are two more instantiations of the parse
// loop, not eight.
return build_with<FloatType, false, false, NulIsEnd, false>(d, src, size, sentinel, failure);
}
if (comments)
{
return trailing_commas ? build_with<FloatType, true, true, NulIsEnd, true>(d, src, size, sentinel, failure)
: build_with<FloatType, true, false, NulIsEnd, true>(d, src, size, sentinel, failure);
return trailing_commas ? build_with<FloatType, true, true, NulIsEnd>(d, src, size, sentinel, failure)
: build_with<FloatType, true, false, NulIsEnd>(d, src, size, sentinel, failure);
}
return trailing_commas ? build_with<FloatType, false, true, NulIsEnd, true>(d, src, size, sentinel, failure)
: build_with<FloatType, false, false, NulIsEnd, true>(d, src, size, sentinel, failure);
return trailing_commas ? build_with<FloatType, false, true, NulIsEnd>(d, src, size, sentinel, failure)
: build_with<FloatType, false, false, NulIsEnd>(d, src, size, sentinel, failure);
}
} // namespace view
+19 -37
View File
@@ -1060,7 +1060,7 @@ class basic_json_document
const bool ignore_trailing_commas = false)
{
basic_json_document d;
d.read_range(first, last, error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas);
d.read_range(first, last, allow_exceptions, ignore_comments, ignore_trailing_commas);
return d;
}
@@ -1075,7 +1075,7 @@ class basic_json_document
basic_json_document d;
std::string buf;
append_input(buf, std::forward<InputType>(input));
d.build_owned(std::move(buf), error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas);
d.build_owned(std::move(buf), allow_exceptions, ignore_comments, ignore_trailing_commas);
return d;
}
@@ -1088,8 +1088,7 @@ class basic_json_document
static bool accept(InputType&& input, const bool ignore_comments = false, const bool ignore_trailing_commas = false)
{
basic_json_document d;
d.read_kind(std::forward<InputType>(input), on_error::validate, ignore_comments, ignore_trailing_commas,
std::integral_constant<detail::view::input_kind, detail::view::classify_input<InputType>::value> {});
d.read(std::forward<InputType>(input), false, ignore_comments, ignore_trailing_commas);
return !d.is_discarded();
}
@@ -1105,7 +1104,7 @@ class basic_json_document
const bool ignore_comments = false,
const bool ignore_trailing_commas = false)
{
read_kind(std::forward<InputType>(input), error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas,
read_kind(std::forward<InputType>(input), allow_exceptions, ignore_comments, ignore_trailing_commas,
std::integral_constant<detail::view::input_kind, detail::view::classify_input<InputType>::value> {});
}
@@ -1464,22 +1463,6 @@ class basic_json_document
return idx;
}
/// what a failed parse does
enum class on_error : std::uint8_t
{
discard, ///< the document is discarded
raise, ///< parse_error is thrown
validate ///< accept(): the document is discarded, and no node is stored while parsing
};
static on_error error_mode(bool allow_exceptions) noexcept
{
return allow_exceptions ? on_error::raise : on_error::discard;
}
/// NulIsEnd of the builder: a null byte ends the input unless strict_nul_handling
static constexpr bool nul_is_end = !detail::abi_config::strict_nul_handling;
/// create the storage (sized for the input) on first use
void ensure_data(const char* src, std::size_t size)
{
@@ -1491,7 +1474,7 @@ class basic_json_document
/// parse a buffer the document takes ownership of (it replaces the
/// previous owned copy)
void build_owned(std::string&& buf, on_error allow_exceptions, bool comments, bool trailing_commas)
void build_owned(std::string&& buf, bool allow_exceptions, bool comments, bool trailing_commas)
{
ensure_data(buf.data(), buf.size());
m_data->owned = std::move(buf);
@@ -1501,7 +1484,7 @@ class basic_json_document
/// parse a copy of the input; the copy is appended to the cleared owned
/// buffer, so that its capacity is reused
template<typename T>
void build_copy(T&& input, on_error allow_exceptions, bool comments, bool trailing_commas)
void build_copy(T&& input, bool allow_exceptions, bool comments, bool trailing_commas)
{
if (!m_data)
{
@@ -1522,7 +1505,7 @@ class basic_json_document
}
/// sentinel: src[size] is readable and 0 (std::string, C strings)
void build(const char* src, std::size_t size, on_error allow_exceptions, bool comments, bool trailing_commas, bool owned, bool sentinel)
void build(const char* src, std::size_t size, bool allow_exceptions, bool comments, bool trailing_commas, bool owned, bool sentinel)
{
ensure_data(src, size);
document_data& d = *m_data;
@@ -1549,8 +1532,7 @@ class basic_json_document
}
else
{
ok = detail::view::build<typename BasicJsonType::number_float_t, nul_is_end>(d, src, size, comments, trailing_commas, sentinel,
allow_exceptions == on_error::validate, failure);
ok = detail::view::build < typename BasicJsonType::number_float_t, !detail::abi_config::strict_nul_handling > (d, src, size, comments, trailing_commas, sentinel, failure);
}
if (NLOHMANN_VIEW_LIKELY(ok))
{
@@ -1563,7 +1545,7 @@ class basic_json_document
return;
}
std::vector<std::uint32_t>().swap(d.large_objects);
if (allow_exceptions == on_error::raise)
if (allow_exceptions)
{
detail::view::throw_parse_failure<BasicJsonType>(failure, src, size, comments, trailing_commas);
}
@@ -1571,13 +1553,13 @@ class basic_json_document
// --- input dispatch (see detail::view::input_kind) ---
void read_kind(std::string&& s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::move_string> /*unused*/)
void read_kind(std::string&& s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::move_string> /*unused*/)
{
build_owned(std::move(s), ae, c, tc);
}
template<typename CharT>
void read_kind(CharT* s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::c_string> /*unused*/)
void read_kind(CharT* s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::c_string> /*unused*/)
{
if (s == nullptr)
{
@@ -1591,7 +1573,7 @@ class basic_json_document
}
template<typename Array>
void read_kind(Array& a, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::char_array> /*unused*/)
void read_kind(Array& a, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::char_array> /*unused*/)
{
using CharT = typename std::remove_cv<typename std::remove_extent<Array>::type>::type;
const std::size_t n = std::extent<Array>::value;
@@ -1602,7 +1584,7 @@ class basic_json_document
}
template<typename T>
void read_kind(const T& s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::borrow_range> /*unused*/)
void read_kind(const T& s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::borrow_range> /*unused*/)
{
// std::basic_string guarantees data()[size()] == 0: use it as sentinel
const auto size = static_cast<std::size_t>(s.size());
@@ -1611,13 +1593,13 @@ class basic_json_document
}
template<typename T>
void read_kind(const T& s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::copy_range> /*unused*/)
void read_kind(const T& s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::copy_range> /*unused*/)
{
build_copy(s, ae, c, tc);
}
template<typename CharT>
void read_kind(CharT* s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::wide_c_string> /*unused*/)
void read_kind(CharT* s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::wide_c_string> /*unused*/)
{
if (s == nullptr)
{
@@ -1629,13 +1611,13 @@ class basic_json_document
}
template<typename T>
void read_kind(T&& input, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::adapter> /*unused*/)
void read_kind(T&& input, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::adapter> /*unused*/)
{
build_copy(std::forward<T>(input), ae, c, tc);
}
template<typename IteratorType>
void read_range(IteratorType first, IteratorType last, on_error ae, bool c, bool tc)
void read_range(IteratorType first, IteratorType last, bool ae, bool c, bool tc)
{
using value_type = typename std::remove_cv<typename std::iterator_traits<IteratorType>::value_type>::type;
// borrow the range where the library's input adapter scans it as one
@@ -1646,14 +1628,14 @@ class basic_json_document
}
template<typename IteratorType>
void read_range_impl(IteratorType first, IteratorType last, on_error ae, bool c, bool tc, std::true_type /*contiguous bytes*/)
void read_range_impl(IteratorType first, IteratorType last, bool ae, bool c, bool tc, std::true_type /*contiguous bytes*/)
{
const auto size = static_cast<std::size_t>(std::distance(first, last));
build(size == 0 ? "" : reinterpret_cast<const char*>(&*first), size, ae, c, tc, false, size == 0); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
}
template<typename IteratorType>
void read_range_impl(IteratorType first, IteratorType last, on_error ae, bool c, bool tc, std::false_type /*other*/)
void read_range_impl(IteratorType first, IteratorType last, bool ae, bool c, bool tc, std::false_type /*other*/)
{
build_copy(detail::view::make_adapter_input(detail::input_adapter(first, last)), ae, c, tc);
}
+50 -18
View File
@@ -12458,6 +12458,34 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t find_string_special(const unsigned char* d
return n;
}
// return the index of the first 0x00 byte in [data, data+n), or n if there is
// none; scans 8 bytes at a time (BSON's C-style strings)
JSON_HEDLEY_ALWAYS_INLINE std::size_t find_zero_byte(const unsigned char* data, std::size_t n) noexcept
{
constexpr std::uint64_t ones = 0x0101010101010101ull;
constexpr std::uint64_t high = 0x8080808080808080ull;
std::size_t i = 0;
for (; i + 8 <= n; i += 8)
{
const std::uint64_t v = read_eight_bytes(data + i);
const std::uint64_t zero = (v - ones) & ~v & high;
if (zero != 0)
{
// the lowest flagged byte is the first zero byte (see
// find_string_special())
return i + (static_cast<std::size_t>(count_trailing_zeros(zero)) / 8);
}
}
for (; i < n; ++i)
{
if (data[i] == 0x00)
{
return i;
}
}
return n;
}
// classify a byte as one the serializer must NOT copy verbatim when
// ensure_ascii is requested: the closing quote, an escape, a control character
// (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else -
@@ -18241,15 +18269,17 @@ class binary_reader
}
const std::size_t element_type_parse_position = chars_read;
// an array's elements are named "0", "1", ... in the wire format,
// and those names are not passed on: they are skipped unless
// error_handler_t::strict must reject ill-formed UTF-8 in them
const bool is_object = container_stack.back().is_object;
key.clear();
if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))
if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key, is_object || error_handler == error_handler_t::strict)))
{
return false;
}
// an array's elements are named "0", "1", ... in the wire format,
// and those names are not passed on
if (container_stack.back().is_object && !sax->key(key))
if (is_object && !sax->key(key))
{
return false;
}
@@ -18265,14 +18295,18 @@ class binary_reader
@brief Parses a C-style string from the BSON input.
@param[in,out] result A reference to the string variable where the read
string is to be stored.
@param[in] store whether the characters are needed; if not (the names of
an array's elements), they are skipped and @a result may
be left unchanged, and they are not checked against
@ref error_handler
@return `true` if the \\x00-byte indicating the end of the string was
encountered before the EOF; false` indicates an unexpected EOF.
*/
bool get_bson_cstr(string_t& result)
bool get_bson_cstr(string_t& result, const bool store = true)
{
if (get_bson_cstr_bulk(result, std::integral_constant<bool, bulk_scan> {}))
if (get_bson_cstr_bulk(result, store, std::integral_constant<bool, bulk_scan> {}))
{
return check_string_utf8(result, "key");
return !store || check_string_utf8(result, "key");
}
auto out = std::back_inserter(result);
@@ -18285,7 +18319,7 @@ class binary_reader
}
if (current == 0x00)
{
return check_string_utf8(result, "key");
return !store || check_string_utf8(result, "key");
}
*out++ = static_cast<typename string_t::value_type>(current);
}
@@ -18295,10 +18329,11 @@ class binary_reader
@brief read a C-style string from contiguous input in one step
@param[in,out] result the string to append to
@param[in] store whether to append the characters to @a result
@return whether the string was read; if the input has no \\x00-byte, nothing
is read, and @ref get_bson_cstr reports the end of the input
*/
bool get_bson_cstr_bulk(string_t& result, std::true_type /*bulk*/)
bool get_bson_cstr_bulk(string_t& result, const bool store, std::true_type /*bulk*/)
{
const std::size_t remaining = ia.bulk_remaining();
if (remaining == 0)
@@ -18306,18 +18341,15 @@ class binary_reader
return false;
}
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
// a plain loop rather than std::memchr: most keys are short (array
// indices are keys, too), and the call would cost more than it saves
std::size_t length = 0;
while (length < remaining && data[length] != 0x00)
{
++length;
}
const std::size_t length = find_zero_byte(data, remaining);
if (length == remaining)
{
return false;
}
result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
if (store)
{
result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
}
// consume the string and its \x00-byte, as the byte-wise path does
ia.bulk_skip(length + 1);
chars_read += length + 1;
@@ -18326,7 +18358,7 @@ class binary_reader
}
/// input that is not contiguous: C-style strings are read byte by byte
bool get_bson_cstr_bulk(string_t& /*result*/, std::false_type /*bulk*/) const noexcept
bool get_bson_cstr_bulk(string_t& /*result*/, const bool /*store*/, std::false_type /*bulk*/) const noexcept
{
return false;
}
+37 -157
View File
@@ -1311,8 +1311,7 @@ struct parse_failure
};
/// FloatType: the number_float_t of the document, whose overflow parse() rejects
/// Store: false for accept(): the grammar, number, string and UTF-8 checks run as for parse(), but no node is written
template<typename FloatType, bool Comments, bool TrailingCommas, bool NulIsEnd, bool Sentinel, bool Store>
template<typename FloatType, bool Comments, bool TrailingCommas, bool NulIsEnd, bool Sentinel>
class builder
{
public:
@@ -1488,87 +1487,11 @@ class builder
return p;
}
/// the string from p, the first backslash, control character, ill-formed UTF-8 byte, or the end, checked and not
/// decoded (accept() stores nothing, so the arena is not written): the checks, errors and error positions of
/// slow_string(), for everything up to the closing quote
NLOHMANN_VIEW_NOINLINE decoded check_string(const unsigned char* p)
{
const unsigned char* const end = m_end;
for (;;)
{
if (p == end)
{
return failed(error_code::string_missing_quote, p);
}
const unsigned char c = *p;
if (c == '"')
{
++p;
return decoded{p, 0, 0};
}
if (c != '\\')
{
return failed(c < 0x20 ? error_code::string_control_character : error_code::string_utf8, p);
}
++p;
if (p == end)
{
return failed(error_code::string_missing_quote, p);
}
const unsigned char d = *p++;
if (d == 'u')
{
const int cp = hex4(p); // (moves p past the four digits)
if (NLOHMANN_VIEW_UNLIKELY(cp < 0))
{
return failed(error_code::string_unicode_hex, p);
}
if (NLOHMANN_VIEW_UNLIKELY((cp & 0xF800) == 0xD800)) // a surrogate
{
if (cp >= 0xDC00)
{
return failed(error_code::string_surrogate_low, p);
}
if (end - p < 2 || p[0] != '\\' || p[1] != 'u')
{
return failed(error_code::string_surrogate_high, p);
}
p += 2;
const int lo = hex4(p); // (moves p past the four digits)
if (lo < 0)
{
return failed(error_code::string_unicode_hex, p);
}
if (lo < 0xDC00 || lo > 0xDFFF)
{
return failed(error_code::string_surrogate_high, p);
}
}
}
else if (!(d == '"' || d == '\\' || d == '/' || d == 'b' || d == 'f' || d == 'n' || d == 'r' || d == 't'))
{
--p;
return failed(error_code::string_escape, p);
}
if (p != end && *p == '\\')
{
continue; // (escapes in a row: no run of text between them)
}
p = scan_string_run(p, end); // the text up to the next backslash, quote, control character, or ill-formed UTF-8
}
}
/// s points to the start of the string, p to a backslash, a control
/// character, an ill-formed UTF-8 byte, or the end: decode the string into
/// the arena
NLOHMANN_VIEW_NOINLINE decoded slow_string(const unsigned char* s, const unsigned char* p)
{
if (!enabled(Store))
{
// (accept(): the string is checked, not decoded)
static_cast<void>(s);
return check_string(p);
}
// single-character escapes; 0: invalid (and 'u', handled separately)
static const std::array<char, 128> simple_escape =
{
@@ -1929,13 +1852,10 @@ run_found:
NLOHMANN_VIEW_ALWAYS_INLINE bool run()
{
if (enabled(Store))
{
cold.m_doc.reserve(estimate_nodes(reinterpret_cast<const char*>(b), static_cast<std::size_t>(e - b)));
base = cold.m_doc.nodes;
out = base;
cap = base + cold.m_doc.node_capacity;
}
cold.m_doc.reserve(estimate_nodes(reinterpret_cast<const char*>(b), static_cast<std::size_t>(e - b)));
base = cold.m_doc.nodes;
out = base;
cap = base + cold.m_doc.node_capacity;
if (e - p >= 3 && p[0] == 0xEF && p[1] == 0xBB && p[2] == 0xBF)
{
@@ -2119,10 +2039,7 @@ close_object:
// an array pays nothing for this)
if (NLOHMANN_VIEW_UNLIKELY(cur_count >= document_data::index_min_members))
{
if (enabled(Store))
{
cold.note_large_object(cur_idx);
}
cold.note_large_object(cur_idx);
}
close_container:
@@ -2282,12 +2199,6 @@ indent_done:
/// an integer's value); two stores on little-endian targets
NLOHMANN_VIEW_ALWAYS_INLINE node* push_node(node_kind k, std::uint8_t flags, std::uint16_t extra, std::size_t off, std::uint64_t second)
{
if (!enabled(Store))
{
// accept(): nothing is stored, so what computes the arguments is not run either. base and out stay
// null: their difference, which open() takes, is 0.
return out;
}
if (NLOHMANN_VIEW_UNLIKELY(out == cap))
{
const auto n = static_cast<std::size_t>(out - base);
@@ -2346,12 +2257,9 @@ indent_done:
NLOHMANN_VIEW_ALWAYS_INLINE void close()
{
if (enabled(Store))
{
node& n = base[cur_idx];
n.len = cur_count;
n.next = static_cast<std::uint32_t>(out - base) - cur_idx;
}
node& n = base[cur_idx];
n.len = cur_count;
n.next = static_cast<std::uint32_t>(out - base) - cur_idx;
if (--depth != 0)
{
if (NLOHMANN_VIEW_LIKELY(depth <= 64))
@@ -2526,44 +2434,34 @@ indent_done:
};
/// run the builder with compile-time options
template<typename FloatType, bool Comments, bool TrailingCommas, bool NulIsEnd, bool Store>
template<typename FloatType, bool Comments, bool TrailingCommas, bool NulIsEnd>
inline bool build_with(document_data& d, const char* src, std::size_t size, bool sentinel, parse_failure& failure)
{
if (sentinel)
{
builder<FloatType, Comments, TrailingCommas, NulIsEnd, true, Store> bld(d, src, size);
builder<FloatType, Comments, TrailingCommas, NulIsEnd, true> bld(d, src, size);
const bool ok = bld.run();
failure = bld.failure();
return ok;
}
builder<FloatType, Comments, TrailingCommas, NulIsEnd, false, Store> bld(d, src, size);
builder<FloatType, Comments, TrailingCommas, NulIsEnd, false> bld(d, src, size);
const bool ok = bld.run();
failure = bld.failure();
return ok;
}
/// sentinel: src[size] is readable and 0 (e.g. std::string); FloatType: the
/// number_float_t of the document; validate_only: accept(), only the result is
/// used
/// number_float_t of the document
template<typename FloatType, bool NulIsEnd>
inline bool build(document_data& d, const char* src, std::size_t size, bool comments, bool trailing_commas, bool sentinel, bool validate_only,
parse_failure& failure)
inline bool build(document_data& d, const char* src, std::size_t size, bool comments, bool trailing_commas, bool sentinel, parse_failure& failure)
{
if (validate_only && !comments && !trailing_commas)
{
// accept() of plain JSON: nothing is stored. The other option sets
// could skip the stores just as well; they keep the builder that
// stores only so that there are two more instantiations of the parse
// loop, not eight.
return build_with<FloatType, false, false, NulIsEnd, false>(d, src, size, sentinel, failure);
}
if (comments)
{
return trailing_commas ? build_with<FloatType, true, true, NulIsEnd, true>(d, src, size, sentinel, failure)
: build_with<FloatType, true, false, NulIsEnd, true>(d, src, size, sentinel, failure);
return trailing_commas ? build_with<FloatType, true, true, NulIsEnd>(d, src, size, sentinel, failure)
: build_with<FloatType, true, false, NulIsEnd>(d, src, size, sentinel, failure);
}
return trailing_commas ? build_with<FloatType, false, true, NulIsEnd, true>(d, src, size, sentinel, failure)
: build_with<FloatType, false, false, NulIsEnd, true>(d, src, size, sentinel, failure);
return trailing_commas ? build_with<FloatType, false, true, NulIsEnd>(d, src, size, sentinel, failure)
: build_with<FloatType, false, false, NulIsEnd>(d, src, size, sentinel, failure);
}
} // namespace view
@@ -10203,7 +10101,7 @@ class basic_json_document
const bool ignore_trailing_commas = false)
{
basic_json_document d;
d.read_range(first, last, error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas);
d.read_range(first, last, allow_exceptions, ignore_comments, ignore_trailing_commas);
return d;
}
@@ -10218,7 +10116,7 @@ class basic_json_document
basic_json_document d;
std::string buf;
append_input(buf, std::forward<InputType>(input));
d.build_owned(std::move(buf), error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas);
d.build_owned(std::move(buf), allow_exceptions, ignore_comments, ignore_trailing_commas);
return d;
}
@@ -10231,8 +10129,7 @@ class basic_json_document
static bool accept(InputType&& input, const bool ignore_comments = false, const bool ignore_trailing_commas = false)
{
basic_json_document d;
d.read_kind(std::forward<InputType>(input), on_error::validate, ignore_comments, ignore_trailing_commas,
std::integral_constant<detail::view::input_kind, detail::view::classify_input<InputType>::value> {});
d.read(std::forward<InputType>(input), false, ignore_comments, ignore_trailing_commas);
return !d.is_discarded();
}
@@ -10248,7 +10145,7 @@ class basic_json_document
const bool ignore_comments = false,
const bool ignore_trailing_commas = false)
{
read_kind(std::forward<InputType>(input), error_mode(allow_exceptions), ignore_comments, ignore_trailing_commas,
read_kind(std::forward<InputType>(input), allow_exceptions, ignore_comments, ignore_trailing_commas,
std::integral_constant<detail::view::input_kind, detail::view::classify_input<InputType>::value> {});
}
@@ -10607,22 +10504,6 @@ class basic_json_document
return idx;
}
/// what a failed parse does
enum class on_error : std::uint8_t
{
discard, ///< the document is discarded
raise, ///< parse_error is thrown
validate ///< accept(): the document is discarded, and no node is stored while parsing
};
static on_error error_mode(bool allow_exceptions) noexcept
{
return allow_exceptions ? on_error::raise : on_error::discard;
}
/// NulIsEnd of the builder: a null byte ends the input unless strict_nul_handling
static constexpr bool nul_is_end = !detail::abi_config::strict_nul_handling;
/// create the storage (sized for the input) on first use
void ensure_data(const char* src, std::size_t size)
{
@@ -10634,7 +10515,7 @@ class basic_json_document
/// parse a buffer the document takes ownership of (it replaces the
/// previous owned copy)
void build_owned(std::string&& buf, on_error allow_exceptions, bool comments, bool trailing_commas)
void build_owned(std::string&& buf, bool allow_exceptions, bool comments, bool trailing_commas)
{
ensure_data(buf.data(), buf.size());
m_data->owned = std::move(buf);
@@ -10644,7 +10525,7 @@ class basic_json_document
/// parse a copy of the input; the copy is appended to the cleared owned
/// buffer, so that its capacity is reused
template<typename T>
void build_copy(T&& input, on_error allow_exceptions, bool comments, bool trailing_commas)
void build_copy(T&& input, bool allow_exceptions, bool comments, bool trailing_commas)
{
if (!m_data)
{
@@ -10665,7 +10546,7 @@ class basic_json_document
}
/// sentinel: src[size] is readable and 0 (std::string, C strings)
void build(const char* src, std::size_t size, on_error allow_exceptions, bool comments, bool trailing_commas, bool owned, bool sentinel)
void build(const char* src, std::size_t size, bool allow_exceptions, bool comments, bool trailing_commas, bool owned, bool sentinel)
{
ensure_data(src, size);
document_data& d = *m_data;
@@ -10692,8 +10573,7 @@ class basic_json_document
}
else
{
ok = detail::view::build<typename BasicJsonType::number_float_t, nul_is_end>(d, src, size, comments, trailing_commas, sentinel,
allow_exceptions == on_error::validate, failure);
ok = detail::view::build < typename BasicJsonType::number_float_t, !detail::abi_config::strict_nul_handling > (d, src, size, comments, trailing_commas, sentinel, failure);
}
if (NLOHMANN_VIEW_LIKELY(ok))
{
@@ -10706,7 +10586,7 @@ class basic_json_document
return;
}
std::vector<std::uint32_t>().swap(d.large_objects);
if (allow_exceptions == on_error::raise)
if (allow_exceptions)
{
detail::view::throw_parse_failure<BasicJsonType>(failure, src, size, comments, trailing_commas);
}
@@ -10714,13 +10594,13 @@ class basic_json_document
// --- input dispatch (see detail::view::input_kind) ---
void read_kind(std::string&& s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::move_string> /*unused*/)
void read_kind(std::string&& s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::move_string> /*unused*/)
{
build_owned(std::move(s), ae, c, tc);
}
template<typename CharT>
void read_kind(CharT* s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::c_string> /*unused*/)
void read_kind(CharT* s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::c_string> /*unused*/)
{
if (s == nullptr)
{
@@ -10734,7 +10614,7 @@ class basic_json_document
}
template<typename Array>
void read_kind(Array& a, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::char_array> /*unused*/)
void read_kind(Array& a, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::char_array> /*unused*/)
{
using CharT = typename std::remove_cv<typename std::remove_extent<Array>::type>::type;
const std::size_t n = std::extent<Array>::value;
@@ -10745,7 +10625,7 @@ class basic_json_document
}
template<typename T>
void read_kind(const T& s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::borrow_range> /*unused*/)
void read_kind(const T& s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::borrow_range> /*unused*/)
{
// std::basic_string guarantees data()[size()] == 0: use it as sentinel
const auto size = static_cast<std::size_t>(s.size());
@@ -10754,13 +10634,13 @@ class basic_json_document
}
template<typename T>
void read_kind(const T& s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::copy_range> /*unused*/)
void read_kind(const T& s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::copy_range> /*unused*/)
{
build_copy(s, ae, c, tc);
}
template<typename CharT>
void read_kind(CharT* s, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::wide_c_string> /*unused*/)
void read_kind(CharT* s, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::wide_c_string> /*unused*/)
{
if (s == nullptr)
{
@@ -10772,13 +10652,13 @@ class basic_json_document
}
template<typename T>
void read_kind(T&& input, on_error ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::adapter> /*unused*/)
void read_kind(T&& input, bool ae, bool c, bool tc, std::integral_constant<input_kind, input_kind::adapter> /*unused*/)
{
build_copy(std::forward<T>(input), ae, c, tc);
}
template<typename IteratorType>
void read_range(IteratorType first, IteratorType last, on_error ae, bool c, bool tc)
void read_range(IteratorType first, IteratorType last, bool ae, bool c, bool tc)
{
using value_type = typename std::remove_cv<typename std::iterator_traits<IteratorType>::value_type>::type;
// borrow the range where the library's input adapter scans it as one
@@ -10789,14 +10669,14 @@ class basic_json_document
}
template<typename IteratorType>
void read_range_impl(IteratorType first, IteratorType last, on_error ae, bool c, bool tc, std::true_type /*contiguous bytes*/)
void read_range_impl(IteratorType first, IteratorType last, bool ae, bool c, bool tc, std::true_type /*contiguous bytes*/)
{
const auto size = static_cast<std::size_t>(std::distance(first, last));
build(size == 0 ? "" : reinterpret_cast<const char*>(&*first), size, ae, c, tc, false, size == 0); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
}
template<typename IteratorType>
void read_range_impl(IteratorType first, IteratorType last, on_error ae, bool c, bool tc, std::false_type /*other*/)
void read_range_impl(IteratorType first, IteratorType last, bool ae, bool c, bool tc, std::false_type /*other*/)
{
build_copy(detail::view::make_adapter_input(detail::input_adapter(first, last)), ae, c, tc);
}
+35 -2
View File
@@ -1164,7 +1164,13 @@ TEST_CASE("BSON keys from contiguous and stream input")
// contiguous input reads a key up to its \x00-byte in one step, a stream
// reads it byte by byte; both must give the same value or error for the
// complete document and for every truncation of it
const json j = {{"", true}, {"k", {1, 2, 3}}, {std::string(40, 'x'), {{"nested key", "value"}}}};
json j = {{"", true}, {"k", {1, 2, 3}}, {std::string(40, 'x'), {{"nested key", "value"}}}};
// keys and array element names ("0" to "11") that end at every position of
// an 8-byte word, which contiguous input scans at a time
for (std::size_t length = 1; length <= 17; ++length)
{
j[std::string(length, static_cast<char>('a' + length))] = json::array({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, {11}});
}
const std::vector<std::uint8_t> bson = json::to_bson(j);
CHECK(json::from_bson(bson) == j);
@@ -1184,7 +1190,7 @@ TEST_CASE("BSON keys from contiguous and stream input")
}
try
{
std::istringstream stream(std::string(input.begin(), input.end()));
std::istringstream stream(std::string(reinterpret_cast<const char*>(input.data()), input.size()));
from_stream = json::from_bson(stream).dump();
}
catch (const json::parse_error& e)
@@ -1194,6 +1200,33 @@ TEST_CASE("BSON keys from contiguous and stream input")
CHECK(from_vector == from_stream);
}
}
TEST_CASE("BSON array element names")
{
// an array's element names are not part of the value, so they are not
// passed on; error_handler_t::strict still rejects ill-formed UTF-8 in them
const std::vector<std::uint8_t> bson =
{
0x1D, 0x00, 0x00, 0x00, // size (little endian)
0x04, 'a', 0x00, // entry: array "a"
0x15, 0x00, 0x00, 0x00, // size (little endian)
0x08, 0xFF, 0x00, 0x01, // entry: boolean named "\xFF", true
0x08, '1', '2', '3', '4', '5', '6', '7', '8', '9', 0x00, 0x00, // entry: boolean named "123456789", false
0x00, // end of the array
0x00 // end of the document
};
const json expected = {{"a", {true, false}}};
CHECK(json::from_bson(bson) == expected);
CHECK(json::from_bson(bson, true, true, json::error_handler_t::replace) == expected);
CHECK(json::from_bson(bson, true, true, json::error_handler_t::ignore) == expected);
json _;
CHECK_THROWS_WITH_AS(_ = json::from_bson(bson, true, true, json::error_handler_t::strict), "[json.exception.parse_error.113] parse error at byte 14: syntax error while parsing BSON key: invalid string: ill-formed UTF-8 byte", json::parse_error&);
std::istringstream stream(std::string(reinterpret_cast<const char*>(bson.data()), bson.size()));
CHECK(json::from_bson(stream) == expected);
}
#endif
TEST_CASE("Negative size of binary value")
+1 -145
View File
@@ -59,8 +59,7 @@ built build(const std::string& text, bool comments, bool trailing_commas, bool s
r.copy.assign(text.begin(), text.end());
src = r.copy.data();
}
constexpr bool nul_is_end = !nlohmann::detail::abi_config::strict_nul_handling;
r.ok = nlohmann::detail::view::build<FloatType, nul_is_end>(*r.data, src, text.size(), comments, trailing_commas, sentinel, false, r.failure);
r.ok = nlohmann::detail::view::build < FloatType, !nlohmann::detail::abi_config::strict_nul_handling > (*r.data, src, text.size(), comments, trailing_commas, sentinel, r.failure);
r.data->src = src;
r.data->base[0] = src;
r.data->base[1] = r.data->arena.data();
@@ -173,22 +172,6 @@ void check_string(const std::string& content)
}
}
// accept() stores no nodes: it must accept and reject what json::accept does, for all options and for inputs with
// and without a terminating NUL
void check_accept(const std::string& text)
{
CAPTURE(text)
const std::vector<char> bytes(text.begin(), text.end());
for (int options = 0; options < 4; ++options)
{
const bool comments = (options & 1) != 0;
const bool trailing_commas = (options & 2) != 0;
const bool accepted = json::accept(text, comments, trailing_commas);
CHECK(nlohmann::json_document::accept(text, comments, trailing_commas) == accepted);
CHECK(nlohmann::json_document::accept(bytes, comments, trailing_commas) == accepted);
}
}
// a small deterministic generator of documents
struct generator
{
@@ -440,133 +423,6 @@ TEST_CASE("json_view builder")
}
}
SECTION("accept() without an index")
{
for (const char* text :
{
"", " ", "\n", "null", "nul", "true", "[", "]", "[]", "{}", "[1,2,3]", "[1,2,3", "[1,2,3,]", "{\"a\":1,}", "{\"a\"}", "{\"a\":}",
"[1] x", "[1][2]", "\xEF\xBB\xBF[1]", "[\"a\\", "[\"\\u12\"]", "[\"\\ud800\"]", "[\"\x01\"]", "[\"\xff\"]", "[01]", "[1.]",
"[.5]", "[-]", "[1e]", "// c\n[1]", "/* c */ {\"a\" /* x */ : 1}", "/* c", "[1] // c", "1e400", "-1e400", "123456789012345678901234567890",
"{\"a\":1,\"a\":2}", "[[[]]]", "\"\\u00e4\\n\\ud83d\\ude00\""
})
{
check_accept(text);
}
// a null byte inside the text, and after it
check_accept(std::string("[1,\0]", 5));
check_accept(std::string("[1]\0", 4));
check_accept(std::string("\0", 1));
// generated documents and damaged copies, as for the builder above
generator g;
for (int i = 0; i < 2000; ++i)
{
std::string text;
g.value(text, 0);
check_accept(text);
std::string damaged = text;
const auto at = static_cast<std::size_t>(g.r(static_cast<int>(damaged.size())));
static const char replacements[] = {'x', '"', '\\', ',', ':', ']', '}', '[', '{', '1', '-', '.', 'e', '\0', '\n', '/'}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
damaged[at] = replacements[g.r(16)];
check_accept(damaged);
check_accept(text.substr(0, at));
}
// nesting beyond the 64 levels that are kept on the builder's own stack, closed and not closed
for (const std::size_t depth :
{
63, 64, 65, 66, 200, 1000
})
{
check_accept(std::string(depth, '[') + std::string(depth, ']'));
check_accept(std::string(depth, '[') + std::string(depth - 1, ']'));
std::string objects;
for (std::size_t i = 0; i < depth; ++i)
{
objects += "{\"k\":";
}
objects += "1" + std::string(depth, '}');
check_accept(objects);
check_accept(objects.substr(0, objects.size() - 1));
}
// objects with enough members to get a hash index when they are indexed (and duplicate keys)
std::string large = "{";
for (int i = 0; i < 300; ++i)
{
large += (i == 0 ? "\"k" : ",\"k") + std::to_string(i % 200) + "\":" + std::to_string(i);
}
check_accept(large + "}");
check_accept(large);
check_accept("[" + large + "},{" + large.substr(1) + "}]");
// the overflow check uses the floating-point type of the document
using float_json = nlohmann::basic_json<std::map, std::vector, std::string, bool, std::int64_t, std::uint64_t, float>;
for (const char* text :
{
"1e39", "-1e39", "3.4028235e38", "3.4028236e38", "[1.5e38, 3.5e38]", "{\"a\": 1e-50, \"b\": 1e39}", "340282356779733661637539395458142568448"
})
{
CAPTURE(text)
CHECK(nlohmann::basic_json_document<float_json>::accept(text) == float_json::accept(text));
}
// strings are checked and not decoded: runs of text of every length from 0 to 40 between valid and invalid
// escapes, UTF-8 sequences and control characters, in values and in keys, cut near the end of the text
const std::array<const char*, 9> valid = {{"\\n", "\\\"", "\\\\", "\\/", "\\t", "\\u00e4", "\\u20ac", "\\ud83d\\ude00", "\\u0000"}};
const std::array<const char*, 12> invalid = {{"\\x", "\\ud800", "\\udc00", "\\u12G4", "\\u12", "\\ud83dx", "\\ud83d\\u0041", "\\ud83d\\ud83d", "\\", "\x01", "\x1f", "\x7f"}};
const std::array<const char*, 6> utf8 = {{"\xc3\xa4", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xc3", "\xff", "\x80"}};
std::mt19937 rng(2029); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible
for (std::size_t run = 0; run <= 40; ++run)
{
for (int i = 0; i < 30; ++i)
{
std::string content = std::string(rng() % 20, 'a');
const auto pieces = 1 + (rng() % 6);
for (unsigned k = 0; k < pieces; ++k)
{
content += valid[rng() % valid.size()];
content += std::string(k == 0 ? run : rng() % 41, static_cast<char>('b' + (rng() % 20)));
switch (rng() % 8)
{
case 0:
content += utf8[rng() % 3];
break;
case 1:
content += utf8[rng() % utf8.size()];
break;
case 2:
content += invalid[rng() % invalid.size()];
break;
default:
break;
}
}
for (const std::string& text :
{
"[\"" + content + "\"]", "{\"" + content + "\":1}", "[\"" + content, "[\"" + content.substr(0, content.size() - (rng() % 4)) + "\"]"
})
{
check_accept(text);
}
}
}
}
SECTION("escapes in a row")
{
for (const char* content :
{
"\\n", "\\n\\t\\\\", "\\u00e9\\u00e9\\u4e2d", "\\u00e9\\u00e", "\\u00e9\\", "\\n\\q", "\\n\\ud83d\\ude00\\n",
"\\ud83d\\ude00\\ud83d", "\\n\\n\\n abc \\n\\n", "\\n\x01", "\\n\xC3\x28", "\\n\\"
})
{
check_accept(std::string("[\"") + content + "\"]");
check_accept(std::string("[\"") + content);
check_accept(std::string("{\"") + content + "\":1}");
}
}
SECTION("test files")
{
for (const char* name :