Compare commits

..
Author SHA1 Message Date
Niels LohmannandClaude Sonnet 5.5 1c588321c5 Fix UBSan implicit-conversion in BSON stream tests
std::string(first, last) over std::uint8_t iterators converts each byte
to char implicitly, which -fsanitize=implicit-conversion reports as soon
as a byte is >= 0x80 (the document size byte of the enlarged test input,
and 0xFF in the new array element name test). Build the string from a
const char* and a size instead.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01N5NasCxh8jKXBaa6HBx9Xk
Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-31 07:28:43 +00:00
Niels Lohmann cff8548496 BSON reader: find C-string terminators 8 bytes at a time, skip array element names
get_bson_cstr_bulk() searched for the 0x00 byte one byte at a time. It now uses
find_zero_byte() (string_scan.hpp), which tests 8 bytes per step with the
SWAR zero-byte test the lexer's scanners use.

The names of an array's elements ("0", "1", ...) are not passed to the SAX
parser, but were still copied into the key string and checked against the
error handler. They are now skipped without a copy, except with
error_handler_t::strict, which still rejects ill-formed UTF-8 in them with
the same error and position as before.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-30 22:17:08 +01:00
9 changed files with 138 additions and 12005 deletions

No files matched your search

+1 -8
View File
@@ -10,13 +10,6 @@ at its root, a [Natvis](https://learn.microsoft.com/en-us/visualstudio/debugger/
file that gives `json`/`ordered_json` values a friendly, key/value debugger view instead of showing raw internal
fields, when debugging with the MSVC debug engine (`cppvsdbg`) in Visual Studio or VS Code.
The file also has visualizers for `json_document` and `json_view` (and their `ordered_json`/editable variants). They
show the value of the document or view, and its elements as an expandable list. Because a document is only a flat index
into the source text, the strings of the direct elements are shown, but the elements of nested arrays and objects are
shown as index nodes (their kind, size, and numbers; strings only by length). Arrays and objects that have been edited
in a `json_editable_document` are not expanded. The visualizers read the internal layout of the MSVC STL (`std::array`,
`std::unique_ptr`).
The checked-in visualizers cover namespace variants with zero, one, or two ABI tags. Configurations with three or more
ABI tags are not covered and may show raw internal fields instead. To generate a Natvis file for such a configuration:
@@ -36,7 +29,7 @@ ABI tags are not covered and may show raw internal fields instead. To generate a
4. Use the generated file instead of `nlohmann_json.natvis`.
`--abi-tags` can be repeated for several configurations, and `--all` generates every combination (about 60 MB for 11
`--abi-tags` can be repeated for several configurations, and `--all` generates every combination (about 60 MB for 12
ABI tags, doubling with each further tag). See
[`tools/generate_natvis`](https://github.com/nlohmann/json/tree/develop/tools/generate_natvis) for all options.
+22 -18
View File
@@ -465,15 +465,17 @@ class binary_reader
}
const std::size_t element_type_parse_position = chars_read;
// an array's elements are named "0", "1", ... in the wire format,
// and those names are not passed on: they are skipped unless
// error_handler_t::strict must reject ill-formed UTF-8 in them
const bool is_object = container_stack.back().is_object;
key.clear();
if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))
if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key, is_object || error_handler == error_handler_t::strict)))
{
return false;
}
// an array's elements are named "0", "1", ... in the wire format,
// and those names are not passed on
if (container_stack.back().is_object && !sax->key(key))
if (is_object && !sax->key(key))
{
return false;
}
@@ -489,14 +491,18 @@ class binary_reader
@brief Parses a C-style string from the BSON input.
@param[in,out] result A reference to the string variable where the read
string is to be stored.
@param[in] store whether the characters are needed; if not (the names of
an array's elements), they are skipped and @a result may
be left unchanged, and they are not checked against
@ref error_handler
@return `true` if the \\x00-byte indicating the end of the string was
encountered before the EOF; false` indicates an unexpected EOF.
*/
bool get_bson_cstr(string_t& result)
bool get_bson_cstr(string_t& result, const bool store = true)
{
if (get_bson_cstr_bulk(result, std::integral_constant<bool, bulk_scan> {}))
if (get_bson_cstr_bulk(result, store, std::integral_constant<bool, bulk_scan> {}))
{
return check_string_utf8(result, "key");
return !store || check_string_utf8(result, "key");
}
auto out = std::back_inserter(result);
@@ -509,7 +515,7 @@ class binary_reader
}
if (current == 0x00)
{
return check_string_utf8(result, "key");
return !store || check_string_utf8(result, "key");
}
*out++ = static_cast<typename string_t::value_type>(current);
}
@@ -519,10 +525,11 @@ class binary_reader
@brief read a C-style string from contiguous input in one step
@param[in,out] result the string to append to
@param[in] store whether to append the characters to @a result
@return whether the string was read; if the input has no \\x00-byte, nothing
is read, and @ref get_bson_cstr reports the end of the input
*/
bool get_bson_cstr_bulk(string_t& result, std::true_type /*bulk*/)
bool get_bson_cstr_bulk(string_t& result, const bool store, std::true_type /*bulk*/)
{
const std::size_t remaining = ia.bulk_remaining();
if (remaining == 0)
@@ -530,18 +537,15 @@ class binary_reader
return false;
}
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
// a plain loop rather than std::memchr: most keys are short (array
// indices are keys, too), and the call would cost more than it saves
std::size_t length = 0;
while (length < remaining && data[length] != 0x00)
{
++length;
}
const std::size_t length = find_zero_byte(data, remaining);
if (length == remaining)
{
return false;
}
result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
if (store)
{
result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
}
// consume the string and its \x00-byte, as the byte-wise path does
ia.bulk_skip(length + 1);
chars_read += length + 1;
@@ -550,7 +554,7 @@ class binary_reader
}
/// input that is not contiguous: C-style strings are read byte by byte
bool get_bson_cstr_bulk(string_t& /*result*/, std::false_type /*bulk*/) const noexcept
bool get_bson_cstr_bulk(string_t& /*result*/, const bool /*store*/, std::false_type /*bulk*/) const noexcept
{
return false;
}
@@ -90,6 +90,34 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t find_string_special(const unsigned char* d
return n;
}
// return the index of the first 0x00 byte in [data, data+n), or n if there is
// none; scans 8 bytes at a time (BSON's C-style strings)
JSON_HEDLEY_ALWAYS_INLINE std::size_t find_zero_byte(const unsigned char* data, std::size_t n) noexcept
{
constexpr std::uint64_t ones = 0x0101010101010101ull;
constexpr std::uint64_t high = 0x8080808080808080ull;
std::size_t i = 0;
for (; i + 8 <= n; i += 8)
{
const std::uint64_t v = read_eight_bytes(data + i);
const std::uint64_t zero = (v - ones) & ~v & high;
if (zero != 0)
{
// the lowest flagged byte is the first zero byte (see
// find_string_special())
return i + (static_cast<std::size_t>(count_trailing_zeros(zero)) / 8);
}
}
for (; i < n; ++i)
{
if (data[i] == 0x00)
{
return i;
}
}
return n;
}
// classify a byte as one the serializer must NOT copy verbatim when
// ensure_ascii is requested: the closing quote, an escape, a control character
// (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else -
-11880
View File
File diff suppressed because it is too large. Load diff
+50 -18
View File
@@ -12458,6 +12458,34 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t find_string_special(const unsigned char* d
return n;
}
// return the index of the first 0x00 byte in [data, data+n), or n if there is
// none; scans 8 bytes at a time (BSON's C-style strings)
JSON_HEDLEY_ALWAYS_INLINE std::size_t find_zero_byte(const unsigned char* data, std::size_t n) noexcept
{
constexpr std::uint64_t ones = 0x0101010101010101ull;
constexpr std::uint64_t high = 0x8080808080808080ull;
std::size_t i = 0;
for (; i + 8 <= n; i += 8)
{
const std::uint64_t v = read_eight_bytes(data + i);
const std::uint64_t zero = (v - ones) & ~v & high;
if (zero != 0)
{
// the lowest flagged byte is the first zero byte (see
// find_string_special())
return i + (static_cast<std::size_t>(count_trailing_zeros(zero)) / 8);
}
}
for (; i < n; ++i)
{
if (data[i] == 0x00)
{
return i;
}
}
return n;
}
// classify a byte as one the serializer must NOT copy verbatim when
// ensure_ascii is requested: the closing quote, an escape, a control character
// (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else -
@@ -18241,15 +18269,17 @@ class binary_reader
}
const std::size_t element_type_parse_position = chars_read;
// an array's elements are named "0", "1", ... in the wire format,
// and those names are not passed on: they are skipped unless
// error_handler_t::strict must reject ill-formed UTF-8 in them
const bool is_object = container_stack.back().is_object;
key.clear();
if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))
if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key, is_object || error_handler == error_handler_t::strict)))
{
return false;
}
// an array's elements are named "0", "1", ... in the wire format,
// and those names are not passed on
if (container_stack.back().is_object && !sax->key(key))
if (is_object && !sax->key(key))
{
return false;
}
@@ -18265,14 +18295,18 @@ class binary_reader
@brief Parses a C-style string from the BSON input.
@param[in,out] result A reference to the string variable where the read
string is to be stored.
@param[in] store whether the characters are needed; if not (the names of
an array's elements), they are skipped and @a result may
be left unchanged, and they are not checked against
@ref error_handler
@return `true` if the \\x00-byte indicating the end of the string was
encountered before the EOF; false` indicates an unexpected EOF.
*/
bool get_bson_cstr(string_t& result)
bool get_bson_cstr(string_t& result, const bool store = true)
{
if (get_bson_cstr_bulk(result, std::integral_constant<bool, bulk_scan> {}))
if (get_bson_cstr_bulk(result, store, std::integral_constant<bool, bulk_scan> {}))
{
return check_string_utf8(result, "key");
return !store || check_string_utf8(result, "key");
}
auto out = std::back_inserter(result);
@@ -18285,7 +18319,7 @@ class binary_reader
}
if (current == 0x00)
{
return check_string_utf8(result, "key");
return !store || check_string_utf8(result, "key");
}
*out++ = static_cast<typename string_t::value_type>(current);
}
@@ -18295,10 +18329,11 @@ class binary_reader
@brief read a C-style string from contiguous input in one step
@param[in,out] result the string to append to
@param[in] store whether to append the characters to @a result
@return whether the string was read; if the input has no \\x00-byte, nothing
is read, and @ref get_bson_cstr reports the end of the input
*/
bool get_bson_cstr_bulk(string_t& result, std::true_type /*bulk*/)
bool get_bson_cstr_bulk(string_t& result, const bool store, std::true_type /*bulk*/)
{
const std::size_t remaining = ia.bulk_remaining();
if (remaining == 0)
@@ -18306,18 +18341,15 @@ class binary_reader
return false;
}
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
// a plain loop rather than std::memchr: most keys are short (array
// indices are keys, too), and the call would cost more than it saves
std::size_t length = 0;
while (length < remaining && data[length] != 0x00)
{
++length;
}
const std::size_t length = find_zero_byte(data, remaining);
if (length == remaining)
{
return false;
}
result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
if (store)
{
result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
}
// consume the string and its \x00-byte, as the byte-wise path does
ia.bulk_skip(length + 1);
chars_read += length + 1;
@@ -18326,7 +18358,7 @@ class binary_reader
}
/// input that is not contiguous: C-style strings are read byte by byte
bool get_bson_cstr_bulk(string_t& /*result*/, std::false_type /*bulk*/) const noexcept
bool get_bson_cstr_bulk(string_t& /*result*/, const bool /*store*/, std::false_type /*bulk*/) const noexcept
{
return false;
}
+35 -2
View File
@@ -1164,7 +1164,13 @@ TEST_CASE("BSON keys from contiguous and stream input")
// contiguous input reads a key up to its \x00-byte in one step, a stream
// reads it byte by byte; both must give the same value or error for the
// complete document and for every truncation of it
const json j = {{"", true}, {"k", {1, 2, 3}}, {std::string(40, 'x'), {{"nested key", "value"}}}};
json j = {{"", true}, {"k", {1, 2, 3}}, {std::string(40, 'x'), {{"nested key", "value"}}}};
// keys and array element names ("0" to "11") that end at every position of
// an 8-byte word, which contiguous input scans at a time
for (std::size_t length = 1; length <= 17; ++length)
{
j[std::string(length, static_cast<char>('a' + length))] = json::array({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, {11}});
}
const std::vector<std::uint8_t> bson = json::to_bson(j);
CHECK(json::from_bson(bson) == j);
@@ -1184,7 +1190,7 @@ TEST_CASE("BSON keys from contiguous and stream input")
}
try
{
std::istringstream stream(std::string(input.begin(), input.end()));
std::istringstream stream(std::string(reinterpret_cast<const char*>(input.data()), input.size()));
from_stream = json::from_bson(stream).dump();
}
catch (const json::parse_error& e)
@@ -1194,6 +1200,33 @@ TEST_CASE("BSON keys from contiguous and stream input")
CHECK(from_vector == from_stream);
}
}
TEST_CASE("BSON array element names")
{
// an array's element names are not part of the value, so they are not
// passed on; error_handler_t::strict still rejects ill-formed UTF-8 in them
const std::vector<std::uint8_t> bson =
{
0x1D, 0x00, 0x00, 0x00, // size (little endian)
0x04, 'a', 0x00, // entry: array "a"
0x15, 0x00, 0x00, 0x00, // size (little endian)
0x08, 0xFF, 0x00, 0x01, // entry: boolean named "\xFF", true
0x08, '1', '2', '3', '4', '5', '6', '7', '8', '9', 0x00, 0x00, // entry: boolean named "123456789", false
0x00, // end of the array
0x00 // end of the document
};
const json expected = {{"a", {true, false}}};
CHECK(json::from_bson(bson) == expected);
CHECK(json::from_bson(bson, true, true, json::error_handler_t::replace) == expected);
CHECK(json::from_bson(bson, true, true, json::error_handler_t::ignore) == expected);
json _;
CHECK_THROWS_WITH_AS(_ = json::from_bson(bson, true, true, json::error_handler_t::strict), "[json.exception.parse_error.113] parse error at byte 14: syntax error while parsing BSON key: invalid string: ill-formed UTF-8 byte", json::parse_error&);
std::istringstream stream(std::string(reinterpret_cast<const char*>(bson.data()), bson.size()));
CHECK(json::from_bson(stream) == expected);
}
#endif
TEST_CASE("Negative size of binary value")
+1 -6
View File
@@ -6,11 +6,6 @@ The checked-in `nlohmann_json.natvis` covers the default namespace (no ABI tags)
of ABI tags, with both versioned and unversioned namespace variants. Configurations that enable three or more ABI tags
are not covered by this reduced file; use `--abi-tags` or `--all` (see below) to generate a file for them.
The visualizers of `basic_json_document` and `basic_json_view` live in the same inline namespaces as `basic_json`, so the
ABI tags apply to them in the same way, and the `view_mode` template argument is covered by a wildcard: it does not
multiply the namespaces. Each namespace gets three more types (`basic_json_view<*,*>`, `basic_json_document<*,*>`, and
`detail::view::node`), which roughly doubles the size of the file.
The ABI tag list and the library version are parsed from
`include/nlohmann/detail/abi_macros.hpp`, so this script must be re-run (via
`make natvis`) whenever an `NLOHMANN_JSON_ABI_TAG_*` macro is added to that
@@ -40,7 +35,7 @@ version other than the one in `abi_macros.hpp`.
|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `--list-tags` | Print every ABI tag (in namespace order) with the macro that enables it and the `NLOHMANN_JSON_ABI_TAG_*` identifier, then exit. |
| `--abi-tags TAGS` | Add the namespaces of one configuration, given as a comma-separated list of ABI tags (`_diag,_dp,_ns`; the leading underscore is optional, and the order does not matter). Can be repeated for several configurations. Unknown tags are an error. |
| `--all` | Generate the namespaces for every combination of ABI tags. The file size doubles with each tag (about 60 MB for 11 tags). Cannot be combined with `--abi-tags`. |
| `--all` | Generate the namespaces for every combination of ABI tags. The file size doubles with each tag (about 60 MB for 12 tags). Cannot be combined with `--abi-tags`. |
| `--output FILE` | Write the Natvis file to `FILE` instead of `nlohmann_json.natvis` in the repository root. |
Each configuration gets both the versioned (`json_abi_diag_dp_ns_v3_12_0`) and the unversioned (`json_abi_diag_dp_ns`,
+1 -1
View File
@@ -95,7 +95,7 @@ if __name__ == '__main__':
'optional) of one configuration to add to the default namespaces; can be repeated')
selection.add_argument('--all', action='store_true',
help='Generate the namespaces for every combination of ABI tags; the file size doubles '
'with each ABI tag (about 60 MB for 11 tags)')
'with each ABI tag (about 60 MB for 12 tags)')
parser.add_argument('--list-tags', action='store_true',
help='List the ABI tags with the macros that enable them, and exit')
parser.add_argument('--output', metavar='FILE',
@@ -5,39 +5,6 @@
<!-- * * * * * * * * AUTO-GENERATED FILE * * * * * * * * -->
<AutoVisualizer xmlns="http://schemas.microsoft.com/vstudio/debugger/natvis/2010">
{# The children of the container node `node` (a pointer expression) of the document `doc` (a document_data pointer
expression). A node is only an index into the source text: strings are read from `doc`, so the strings of the
direct children are readable here, but the children of a nested container are shown as nodes (see the node
visualizer below). Containers with edits (flag 8, "moved") are not expanded. The node kinds are the numbers of
node_kind: 1 object, 2 array, 3 string; the float token is shown only by the node visualizer. #}
{% macro children(node, doc=none) %}
<CustomListItems MaxItemsPerView="1000">
<Variable Name="nd" InitialValue="{{ node }}" />
<Variable Name="obj" InitialValue="nd != 0 &amp;&amp; (int)nd-&gt;kind == 1" />
<Variable Name="n" InitialValue="(nd != 0 &amp;&amp; ((int)nd-&gt;kind == 1 || (int)nd-&gt;kind == 2) &amp;&amp; (nd-&gt;flags &amp; 8) == 0) ? (int)nd-&gt;len : 0" />
<Variable Name="p" InitialValue="nd + 1" />
<Variable Name="i" InitialValue="0" />
<Size>obj ? 2 * n : n</Size>
<Loop Condition="i &lt; n">
{% if doc %}
<Item Name="key" Condition="obj">{{ doc }}-&gt;base._Elems[p-&gt;flags &amp; 3] + p-&gt;off,[p-&gt;len]s8</Item>
{% else %}
<Item Name="key" Condition="obj">*p</Item>
{% endif %}
<Exec Condition="obj">p = p + 1</Exec>
{% if doc %}
<Item Name="value" Condition="obj &amp;&amp; (int)p-&gt;kind == 3">{{ doc }}-&gt;base._Elems[p-&gt;flags &amp; 3] + p-&gt;off,[p-&gt;len]s8</Item>
{% endif %}
<Item Name="value" Condition="obj{% if doc %} &amp;&amp; (int)p-&gt;kind != 3{% endif %}">*p</Item>
{% if doc %}
<Item Condition="!obj &amp;&amp; (int)p-&gt;kind == 3">{{ doc }}-&gt;base._Elems[p-&gt;flags &amp; 3] + p-&gt;off,[p-&gt;len]s8</Item>
{% endif %}
<Item Condition="!obj{% if doc %} &amp;&amp; (int)p-&gt;kind != 3{% endif %}">*p</Item>
<Exec>p = p + (((int)p-&gt;kind == 1 || (int)p-&gt;kind == 2) ? p-&gt;next : 1)</Exec>
<Exec>i = i + 1</Exec>
</Loop>
</CustomListItems>
{%- endmacro %}
{% for ns in namespaces %}
<!-- Namespace {{ ns }} -->
<Type Name="{{ ns }}::basic_json&lt;*&gt;">
@@ -95,44 +62,5 @@
</Type>
{% endfor %}
<!-- json_view / json_document (include/nlohmann/json_view.hpp). They live in the same inline namespaces as
basic_json, so the ABI tags need no extra entries; the view_mode is a template argument that the wildcard
covers. A view is a pointer to the document's data and a pointer to a node of its flat index; the
strings of the source are found through document_data::base (std::array of the MSVC STL: _Elems). -->
<Type Name="{{ ns }}::detail::view::node">
<DisplayString Condition="(int)kind == 0">null</DisplayString>
<DisplayString Condition="(int)kind == 1">object, size={len}</DisplayString>
<DisplayString Condition="(int)kind == 2">array, size={len}</DisplayString>
<DisplayString Condition="(int)kind == 3">string, length={len}</DisplayString>
<DisplayString Condition="(int)kind == 4">boolean: {(flags &amp; 4) != 0}</DisplayString>
<DisplayString Condition="(int)kind == 5">int64: {(__int64)((unsigned __int64)len | ((unsigned __int64)next &lt;&lt; 32))}</DisplayString>
<DisplayString Condition="(int)kind == 6">uint64: {(unsigned __int64)len | ((unsigned __int64)next &lt;&lt; 32)}</DisplayString>
<DisplayString Condition="(int)kind == 7">float, length={len}</DisplayString>
<DisplayString>link</DisplayString>
<Expand>
{{ children('this') }}
</Expand>
</Type>
<Type Name="{{ ns }}::basic_json_view&lt;*,*&gt;">
<DisplayString Condition="m_node == 0">discarded</DisplayString>
<DisplayString Condition="(int)m_node-&gt;kind == 3">{m_doc-&gt;base._Elems[m_node-&gt;flags &amp; 3] + m_node-&gt;off,[m_node-&gt;len]s8}</DisplayString>
<DisplayString Condition="(int)m_node-&gt;kind == 7">{m_doc-&gt;base._Elems[m_node-&gt;flags &amp; 3] + m_node-&gt;off,[m_node-&gt;len]s8b}</DisplayString>
<DisplayString>{*m_node}</DisplayString>
<Expand>
{{ children('m_node', 'm_doc') }}
</Expand>
</Type>
<Type Name="{{ ns }}::basic_json_document&lt;*,*&gt;">
<DisplayString Condition="m_data._Mypair._Myval2 == 0 || m_data._Mypair._Myval2-&gt;discarded">discarded</DisplayString>
<DisplayString>{*m_data._Mypair._Myval2-&gt;nodes}</DisplayString>
<Expand>
<Item Name="[source]" Condition="m_data._Mypair._Myval2 != 0">m_data._Mypair._Myval2-&gt;src,[m_data._Mypair._Myval2-&gt;size]s8b</Item>
<Item Name="[nodes]" Condition="m_data._Mypair._Myval2 != 0">m_data._Mypair._Myval2-&gt;node_count</Item>
{{ children('m_data._Mypair._Myval2->nodes', 'm_data._Mypair._Myval2') }}
</Expand>
</Type>
{% endfor %}
</AutoVisualizer>