From 9742edc294a0ae08fac79539da02ebef8f946948 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 17:23:12 +0100 Subject: [PATCH] BSON writer: write to_bson(j) in one pass, with sizes written afterwards (#5979) Signed-off-by: Niels Lohmann --- .../nlohmann/detail/output/binary_writer.hpp | 229 +++++++++++++++- .../detail/output/output_adapters.hpp | 26 ++ single_include/nlohmann/json.hpp | 255 +++++++++++++++++- tests/src/unit-bson.cpp | 151 +++++++++++ 4 files changed, 659 insertions(+), 2 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 8732e52b4..9d21d2a7c 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -2088,7 +2088,8 @@ class binary_writer /// arrays: the first index with one more digit; only maintained by /// @ref calc_bson_sizes std::uint64_t index_boundary = 10; - /// @ref calc_bson_sizes only: where its size goes in the table + /// @ref calc_bson_sizes: where its size goes in the table; @ref + /// write_bson_document_single_pass: the position of its size field std::size_t size_slot; /// @ref calc_bson_sizes only: the size of its entries seen so far std::size_t entries_size = 0; @@ -2339,6 +2340,199 @@ class binary_writer } } + template + using sink_written_t = decltype(std::declval().written()); + + /// whether every integer fits the 64 bits BSON has, so that writing an + /// array of numbers needs no range check + static constexpr bool bson_integers_fit_64_bits = + std::numeric_limits::is_specialized + && std::numeric_limits::digits <= std::numeric_limits::digits + && std::numeric_limits::is_specialized + && std::numeric_limits::digits <= std::numeric_limits::digits; + + /*! + @brief whether BSON documents can be written in a single pass: the sink + can overwrite what it has written (output_vector_sink), and + @ref bson_integers_fit_64_bits + */ + using bson_single_pass_sink = conjunction, std::integral_constant>; + + /// write a placeholder for the size of an object or array whose entries + /// follow; @return its position, for @ref close_bson_container + std::size_t write_bson_size_placeholder() + { + const std::size_t pos = oa.written(); + // zero in any byte order + const std::array zero{}; + oa.write_characters(zero.data(), zero.size()); + return pos; + } + + /// end the object or array whose size is at @a pos and write its size + /// there: everything from that field to the terminating null byte + /// @return false (and the size is not written) if it does not fit + bool close_bson_container(const std::size_t pos) + { + oa.write_character(to_char_type(0x00)); + const std::size_t size = oa.written() - pos; + if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(size))) + { + return false; + } + const auto le_size = static_cast(size); + oa.overwrite(pos, is_little_endian ? le_size : byteswap(le_size)); + return true; + } + + /*! + @brief check the value @a j of a BSON entry as @ref calc_bson_value_size + does, for @ref write_bson_document_single_pass + @return at least the size of the value + A number, boolean, or null needs no check there (its integers have at most + 64 bits) and has at most 8 characters; a discarded value is rejected with + the same exception when it is written. + */ + std::size_t check_bson_value(const BasicJsonType& j) + { + return (j.is_string() || j.is_binary()) ? calc_bson_value_size(j) : 8; + } + + /// whether an entry of @a entry_size characters keeps the document that + /// begins at @a start, closed by @a open_containers null bytes, within + /// the 32-bit size of BSON + bool bson_entry_fits(const std::size_t start, const std::size_t entry_size, const std::size_t open_containers) const noexcept + { + const auto limit = static_cast((std::numeric_limits::max)()); + const std::size_t written = oa.written() - start; + return written <= limit && entry_size <= limit - written && open_containers <= limit - written - entry_size; + } + + /*! + @brief Serializes the JSON object @a document as a BSON document in a + single pass, writing the size of each object and array once it is + complete + + Checks every key and value as @ref calc_bson_sizes does, in the same + order, so the same exception is thrown; the output written up to then is + discarded with the vector it was written to. + + @return false as soon as the document or a string, binary value, object, + or array in it would be larger than the 32-bit BSON sizes allow; + nothing large has been written then, and the caller writes the + document in two passes, which reports that error + */ + template::value, int>::type = 0> + bool write_bson_document_single_pass(const BasicJsonType& document) + { + const std::size_t start = oa.written(); + // the object or array whose entries are being written (with the + // position of its size), and the ones it is in + bson_frame current(&document, write_bson_size_placeholder()); + std::vector parents; + + while (true) + { + const BasicJsonType* nested = nullptr; + std::size_t nested_size_pos = 0; + if (current.value->is_object()) + { + const auto& object = *current.value->m_data.m_value.object; + while (nested == nullptr && current.member != object.cend()) + { + const auto& el = *current.member; + ++current.member; + const std::size_t header_size = calc_bson_entry_header_size(el.first, el.second); + const std::size_t value_size = el.second.is_structured() ? 0 : check_bson_value(el.second); + if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, header_size + value_size, parents.size() + 1))) + { + return false; + } + if (el.second.is_structured()) + { + write_bson_entry_header(el.first, el.second.is_object() ? 0x03 : 0x04); + nested_size_pos = write_bson_size_placeholder(); + nested = &el.second; + } + else + { + write_bson_value(el.first, el.second); + } + } + } + else + { + const auto& array = *current.value->m_data.m_value.array; + while (nested == nullptr && current.index < array.size()) + { + if (is_bson_fixed_size(array[current.index])) + { + write_bson_array_fixed_entries(current); + if (current.index == array.size()) + { + break; + } + } + + // a string, binary value, object or array + const BasicJsonType& el = array[current.index]; + const string_t index_name(current.index_name.data(), current.index_name_size); + current.next_index(); + const std::size_t value_size = el.is_structured() ? 0 : check_bson_value(el); + if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, 2 + index_name.size() + value_size, parents.size() + 1))) + { + return false; + } + if (el.is_structured()) + { + write_bson_entry_header(index_name, el.is_object() ? 0x03 : 0x04); + nested_size_pos = write_bson_size_placeholder(); + nested = ⪙ + } + else + { + write_bson_value(index_name, el); + } + } + } + + if (nested != nullptr) + { + if (nested->is_array()) + { + // an array of booleans, numbers and null is written without a frame on the stack + bson_frame leaf(nested, nested_size_pos); + write_bson_array_fixed_entries(leaf); + if (leaf.index == nested->m_data.m_value.array->size()) + { + if (JSON_HEDLEY_UNLIKELY(!close_bson_container(nested_size_pos))) + { + return false; + } + continue; + } + parents.push_back(std::move(current)); + current = std::move(leaf); + continue; + } + parents.push_back(std::move(current)); + current = bson_frame(nested, nested_size_pos); + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!close_bson_container(current.size_slot))) + { + return false; + } + if (parents.empty()) + { + return true; + } + current = std::move(parents.back()); + parents.pop_back(); + } + } + /*! @brief Serializes the JSON object @a document as a BSON document @@ -2350,6 +2544,39 @@ class binary_writer @pre document.type() == value_t::object */ void write_bson_document(const BasicJsonType& document) + { + write_bson_document(document, bson_single_pass_sink()); + } + + /// the sink can overwrite: one pass, unless the error handler replaces or + /// drops ill-formed UTF-8, which changes the sizes the checks compute + void write_bson_document(const BasicJsonType& document, std::true_type /*single_pass*/) + { + const std::size_t start = oa.written(); + if ((error_handler == error_handler_t::keep || error_handler == error_handler_t::strict) + && write_bson_document_single_pass(document)) + { + return; + } + // a length does not fit: start over, so that it is reported as the + // two passes report it, before anything is written + oa.rewind(start); + write_bson_document_two_pass(document); + } + + void write_bson_document(const BasicJsonType& document, std::false_type /*single_pass*/) + { + write_bson_document_two_pass(document); + } + + /*! + @brief Serializes the JSON object @a document as a BSON document: the + sizes of all objects and arrays first, then the entries + + @param[in] document the JSON object to serialize + @pre document.type() == value_t::object + */ + void write_bson_document_two_pass(const BasicJsonType& document) { std::vector nested_sizes; const std::size_t document_size = calc_bson_sizes(document, nested_sizes); diff --git a/include/nlohmann/detail/output/output_adapters.hpp b/include/nlohmann/detail/output/output_adapters.hpp index e40d98548..e9e57ab9a 100644 --- a/include/nlohmann/detail/output/output_adapters.hpp +++ b/include/nlohmann/detail/output/output_adapters.hpp @@ -11,6 +11,7 @@ #include // max #include // array #include // size_t +#include // uint32_t #include // memcpy #include // shared_ptr, make_shared #include // basic_string @@ -227,6 +228,31 @@ class output_vector_sink void flush() noexcept {} + /// the number of characters written so far, the position of the next one + std::size_t written() const noexcept + { + return position(); + } + + /// @brief replace the 4 bytes written before at position @a pos with + /// those of @a n, in the byte order of the machine + /// + /// Lets a writer put a length in front of data it has written since, such + /// as the size of a BSON document, without computing it in advance. + void overwrite(const std::size_t pos, const std::uint32_t n) noexcept + { + static_assert(sizeof(CharType) == 1, "overwrite() writes bytes"); + JSON_ASSERT(pos + sizeof(n) <= position()); + std::memcpy(v->data() + pos, &n, sizeof(n)); + } + + /// drop the characters written from position @a pos on + void rewind(const std::size_t pos) noexcept + { + JSON_ASSERT(pos <= position()); + cursor = v->data() + pos; + } + private: /// number of characters of *v written so far std::size_t position() const noexcept diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 4bca3a3a5..3c2670039 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -27161,6 +27161,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // max #include // array #include // size_t +#include // uint32_t #include // memcpy #include // shared_ptr, make_shared #include // basic_string @@ -27378,6 +27379,31 @@ class output_vector_sink void flush() noexcept {} + /// the number of characters written so far, the position of the next one + std::size_t written() const noexcept + { + return position(); + } + + /// @brief replace the 4 bytes written before at position @a pos with + /// those of @a n, in the byte order of the machine + /// + /// Lets a writer put a length in front of data it has written since, such + /// as the size of a BSON document, without computing it in advance. + void overwrite(const std::size_t pos, const std::uint32_t n) noexcept + { + static_assert(sizeof(CharType) == 1, "overwrite() writes bytes"); + JSON_ASSERT(pos + sizeof(n) <= position()); + std::memcpy(v->data() + pos, &n, sizeof(n)); + } + + /// drop the characters written from position @a pos on + void rewind(const std::size_t pos) noexcept + { + JSON_ASSERT(pos <= position()); + cursor = v->data() + pos; + } + private: /// number of characters of *v written so far std::size_t position() const noexcept @@ -29822,7 +29848,8 @@ class binary_writer /// arrays: the first index with one more digit; only maintained by /// @ref calc_bson_sizes std::uint64_t index_boundary = 10; - /// @ref calc_bson_sizes only: where its size goes in the table + /// @ref calc_bson_sizes: where its size goes in the table; @ref + /// write_bson_document_single_pass: the position of its size field std::size_t size_slot; /// @ref calc_bson_sizes only: the size of its entries seen so far std::size_t entries_size = 0; @@ -30073,6 +30100,199 @@ class binary_writer } } + template + using sink_written_t = decltype(std::declval().written()); + + /// whether every integer fits the 64 bits BSON has, so that writing an + /// array of numbers needs no range check + static constexpr bool bson_integers_fit_64_bits = + std::numeric_limits::is_specialized + && std::numeric_limits::digits <= std::numeric_limits::digits + && std::numeric_limits::is_specialized + && std::numeric_limits::digits <= std::numeric_limits::digits; + + /*! + @brief whether BSON documents can be written in a single pass: the sink + can overwrite what it has written (output_vector_sink), and + @ref bson_integers_fit_64_bits + */ + using bson_single_pass_sink = conjunction, std::integral_constant>; + + /// write a placeholder for the size of an object or array whose entries + /// follow; @return its position, for @ref close_bson_container + std::size_t write_bson_size_placeholder() + { + const std::size_t pos = oa.written(); + // zero in any byte order + const std::array zero{}; + oa.write_characters(zero.data(), zero.size()); + return pos; + } + + /// end the object or array whose size is at @a pos and write its size + /// there: everything from that field to the terminating null byte + /// @return false (and the size is not written) if it does not fit + bool close_bson_container(const std::size_t pos) + { + oa.write_character(to_char_type(0x00)); + const std::size_t size = oa.written() - pos; + if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(size))) + { + return false; + } + const auto le_size = static_cast(size); + oa.overwrite(pos, is_little_endian ? le_size : byteswap(le_size)); + return true; + } + + /*! + @brief check the value @a j of a BSON entry as @ref calc_bson_value_size + does, for @ref write_bson_document_single_pass + @return at least the size of the value + A number, boolean, or null needs no check there (its integers have at most + 64 bits) and has at most 8 characters; a discarded value is rejected with + the same exception when it is written. + */ + std::size_t check_bson_value(const BasicJsonType& j) + { + return (j.is_string() || j.is_binary()) ? calc_bson_value_size(j) : 8; + } + + /// whether an entry of @a entry_size characters keeps the document that + /// begins at @a start, closed by @a open_containers null bytes, within + /// the 32-bit size of BSON + bool bson_entry_fits(const std::size_t start, const std::size_t entry_size, const std::size_t open_containers) const noexcept + { + const auto limit = static_cast((std::numeric_limits::max)()); + const std::size_t written = oa.written() - start; + return written <= limit && entry_size <= limit - written && open_containers <= limit - written - entry_size; + } + + /*! + @brief Serializes the JSON object @a document as a BSON document in a + single pass, writing the size of each object and array once it is + complete + + Checks every key and value as @ref calc_bson_sizes does, in the same + order, so the same exception is thrown; the output written up to then is + discarded with the vector it was written to. + + @return false as soon as the document or a string, binary value, object, + or array in it would be larger than the 32-bit BSON sizes allow; + nothing large has been written then, and the caller writes the + document in two passes, which reports that error + */ + template::value, int>::type = 0> + bool write_bson_document_single_pass(const BasicJsonType& document) + { + const std::size_t start = oa.written(); + // the object or array whose entries are being written (with the + // position of its size), and the ones it is in + bson_frame current(&document, write_bson_size_placeholder()); + std::vector parents; + + while (true) + { + const BasicJsonType* nested = nullptr; + std::size_t nested_size_pos = 0; + if (current.value->is_object()) + { + const auto& object = *current.value->m_data.m_value.object; + while (nested == nullptr && current.member != object.cend()) + { + const auto& el = *current.member; + ++current.member; + const std::size_t header_size = calc_bson_entry_header_size(el.first, el.second); + const std::size_t value_size = el.second.is_structured() ? 0 : check_bson_value(el.second); + if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, header_size + value_size, parents.size() + 1))) + { + return false; + } + if (el.second.is_structured()) + { + write_bson_entry_header(el.first, el.second.is_object() ? 0x03 : 0x04); + nested_size_pos = write_bson_size_placeholder(); + nested = &el.second; + } + else + { + write_bson_value(el.first, el.second); + } + } + } + else + { + const auto& array = *current.value->m_data.m_value.array; + while (nested == nullptr && current.index < array.size()) + { + if (is_bson_fixed_size(array[current.index])) + { + write_bson_array_fixed_entries(current); + if (current.index == array.size()) + { + break; + } + } + + // a string, binary value, object or array + const BasicJsonType& el = array[current.index]; + const string_t index_name(current.index_name.data(), current.index_name_size); + current.next_index(); + const std::size_t value_size = el.is_structured() ? 0 : check_bson_value(el); + if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, 2 + index_name.size() + value_size, parents.size() + 1))) + { + return false; + } + if (el.is_structured()) + { + write_bson_entry_header(index_name, el.is_object() ? 0x03 : 0x04); + nested_size_pos = write_bson_size_placeholder(); + nested = ⪙ + } + else + { + write_bson_value(index_name, el); + } + } + } + + if (nested != nullptr) + { + if (nested->is_array()) + { + // an array of booleans, numbers and null is written without a frame on the stack + bson_frame leaf(nested, nested_size_pos); + write_bson_array_fixed_entries(leaf); + if (leaf.index == nested->m_data.m_value.array->size()) + { + if (JSON_HEDLEY_UNLIKELY(!close_bson_container(nested_size_pos))) + { + return false; + } + continue; + } + parents.push_back(std::move(current)); + current = std::move(leaf); + continue; + } + parents.push_back(std::move(current)); + current = bson_frame(nested, nested_size_pos); + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!close_bson_container(current.size_slot))) + { + return false; + } + if (parents.empty()) + { + return true; + } + current = std::move(parents.back()); + parents.pop_back(); + } + } + /*! @brief Serializes the JSON object @a document as a BSON document @@ -30084,6 +30304,39 @@ class binary_writer @pre document.type() == value_t::object */ void write_bson_document(const BasicJsonType& document) + { + write_bson_document(document, bson_single_pass_sink()); + } + + /// the sink can overwrite: one pass, unless the error handler replaces or + /// drops ill-formed UTF-8, which changes the sizes the checks compute + void write_bson_document(const BasicJsonType& document, std::true_type /*single_pass*/) + { + const std::size_t start = oa.written(); + if ((error_handler == error_handler_t::keep || error_handler == error_handler_t::strict) + && write_bson_document_single_pass(document)) + { + return; + } + // a length does not fit: start over, so that it is reported as the + // two passes report it, before anything is written + oa.rewind(start); + write_bson_document_two_pass(document); + } + + void write_bson_document(const BasicJsonType& document, std::false_type /*single_pass*/) + { + write_bson_document_two_pass(document); + } + + /*! + @brief Serializes the JSON object @a document as a BSON document: the + sizes of all objects and arrays first, then the entries + + @param[in] document the JSON object to serialize + @pre document.type() == value_t::object + */ + void write_bson_document_two_pass(const BasicJsonType& document) { std::vector nested_sizes; const std::size_t document_size = calc_bson_sizes(document, nested_sizes); diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index f93e0a71b..f1f43ccd7 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -19,6 +19,7 @@ #include using nlohmann::json; +#include #include #include #include @@ -2300,3 +2301,153 @@ TEST_CASE("BSON large strings and binaries (chunked reader)") } } } + +TEST_CASE("BSON written in one pass and in two passes") +{ + // to_bson() into a returned vector writes each size once the object or + // array is complete; through an output adapter, all sizes are computed + // first. Both must produce the same bytes and throw the same exceptions. + const auto two_pass = [](const json & j, const json::error_handler_t handler) + { + std::vector result; + json::to_bson(j, result, handler); + return result; + }; + + const auto check_same = [&](const json & j, const json::error_handler_t handler) + { + CAPTURE(j.dump(-1, ' ', true, json::error_handler_t::replace)) + CAPTURE(static_cast(handler)) +#if !defined(JSON_NOEXCEPTION) + std::string one_pass_error; + std::string two_pass_error; + std::vector one; + std::vector two; + try + { + one = json::to_bson(j, handler); + } + catch (const json::exception& e) + { + one_pass_error = e.what(); + } + try + { + two = two_pass(j, handler); + } + catch (const json::exception& e) + { + two_pass_error = e.what(); + } + CHECK(one_pass_error == two_pass_error); + CHECK(one == two); +#else + CHECK(json::to_bson(j, handler) == two_pass(j, handler)); +#endif + }; + + const std::array handlers = {{ + json::error_handler_t::keep, json::error_handler_t::strict, + json::error_handler_t::replace, json::error_handler_t::ignore + } + }; + + SECTION("corpus") + { + for (const auto& value : utils::round_trip_corpus::values()) + { + const json document = value.is_object() ? value : json{{"value", value}}; + for (const auto handler : handlers) + { + check_same(document, handler); + } + } + } + + SECTION("nesting and arrays of every length class") + { + json document = json::object(); + document["empty object"] = json::object(); + document["empty array"] = json::array(); + for (const std::size_t n : + { + 1u, 9u, 10u, 11u, 99u, 100u, 101u, 1000u + }) + { + json numbers = json::array(); + json mixed = json::array(); + for (std::size_t i = 0; i < n; ++i) + { + numbers.push_back(i % 3 == 0 ? json(static_cast(i) / 3) : json(static_cast(i) * 1000003)); + switch (i % 4) + { + case 0: + mixed.push_back(json("s" + std::to_string(i))); + break; + case 1: + mixed.push_back(json::array({i, json::object({{"k", i}})})); + break; + case 2: + mixed.push_back(json(nullptr)); + break; + default: + mixed.push_back(json(i % 2 == 0)); + break; + } + } + document["numbers " + std::to_string(n)] = numbers; + document["mixed " + std::to_string(n)] = mixed; + document["nested " + std::to_string(n)] = json::object({{"a", numbers}, {"b", json::array({mixed, json::object()})}}); + } + document["binary"] = json::binary({1, 2, 3}, 0x80); + document["binary old"] = json::binary({1, 2, 3}, 0x02); + for (const auto handler : handlers) + { + check_same(document, handler); + } + } + +#if !defined(JSON_NOEXCEPTION) + // every entry here throws, which aborts without exceptions + SECTION("errors at every position") + { + // an entry that cannot be written, at each place among valid ones + const std::string bad_utf8 = "\xC3\x28"; + const std::array bad_entries = + { + { + json::object({{std::string("a\0b", 3), 1}}), + json::object({{"s", bad_utf8}}), + json::object({{bad_utf8, 1}}), + json::array({1, 2, bad_utf8}), + json::object({{"b", json::binary({1}, 0x100)}}), + json::object({{"d", json(json::value_t::discarded)}}), + json::array({json(json::value_t::discarded)}), + } + }; + for (const auto& bad : bad_entries) + { + for (std::size_t pos = 0; pos < 4; ++pos) + { + json document = json::object(); + json list = json::array(); + for (std::size_t i = 0; i < 4; ++i) + { + if (i == pos) + { + document["x" + std::to_string(i)] = bad; + list.push_back(bad); + } + document["n" + std::to_string(i)] = json::array({i, "str", json::object({{"k", i}})}); + list.push_back(i); + } + document["list"] = list; + for (const auto handler : handlers) + { + check_same(document, handler); + } + } + } + } +#endif +}