diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 1a0e65985..0ec6fedc6 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -281,6 +281,9 @@ class binary_reader std::int32_t declared_size = 0; /// whether to close this container with end_object() or end_array() bool is_object; + /// BSON, when recovering: the declared size was reported as wrong, so + /// the document ends at its terminator, within the enclosing document + bool size_repaired = false; }; /*! @@ -372,16 +375,18 @@ class binary_reader since at least 5 bytes are always consumed. When recovering from errors, a document whose size does not match is - accepted: its terminator was found, so everything in it has been read. + accepted: its terminator was found, so everything in it has been read. A + size that was already reported as wrong (see @ref repair_bson_document_size) + is not reported again. @param[in] document_start value of chars_read before the size prefix @param[in] document_size the declared document size @return whether the declared size matches the number of bytes read, or the mismatch is repaired */ - bool check_bson_document_size(const std::size_t document_start, const std::int32_t document_size) + bool check_bson_document_size(const std::size_t document_start, const std::int32_t document_size, const bool size_repaired) { - if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast(document_size) != chars_read - document_start)) + if (JSON_HEDLEY_UNLIKELY(!size_repaired && (document_size < 0 || static_cast(document_size) != chars_read - document_start))) { static_cast(bson_document_size_error(document_size, chars_read - document_start)); return repair_requested(); @@ -399,7 +404,7 @@ class binary_reader bool can_skip_to_bson_document_end() const noexcept { const container_frame& top = container_stack.back(); - return top.declared_size >= 5 && top.start_position + static_cast(top.declared_size) - 1 >= chars_read; + return !top.size_repaired && top.declared_size >= 5 && top.start_position + static_cast(top.declared_size) - 1 >= chars_read; } /*! @@ -531,20 +536,38 @@ class binary_reader } // the smallest document is the size prefix and the terminator + bool size_repaired = false; + std::size_t document_end = 0; if (JSON_HEDLEY_UNLIKELY(document_size < 5)) { - return bson_document_size_error("document size ", document_size, " is less than 5"); + static_cast(bson_document_size_error("document size ", document_size, " is less than 5")); + if (!repair_requested()) + { + return false; + } + size_repaired = true; } - - // the declared size bounds where the document ends: an embedded - // document must end before the terminator of the document that contains it - const auto unsigned_size = static_cast(document_size); - const std::size_t document_end = unsigned_size > (std::numeric_limits::max)() - document_start - ? (std::numeric_limits::max)() - : document_start + unsigned_size; - if (JSON_HEDLEY_UNLIKELY(!container_stack.empty() && document_end >= container_stack.back().end_position)) + else { - return bson_document_size_error("document size ", document_size, " exceeds the size of the enclosing document"); + // the declared size bounds where the document ends: an embedded + // document must end before the terminator of the document that contains it + const auto unsigned_size = static_cast(document_size); + document_end = unsigned_size > (std::numeric_limits::max)() - document_start + ? (std::numeric_limits::max)() + : document_start + unsigned_size; + if (JSON_HEDLEY_UNLIKELY(!container_stack.empty() && document_end >= container_stack.back().end_position)) + { + static_cast(bson_document_size_error("document size ", document_size, " exceeds the size of the enclosing document")); + if (!repair_requested()) + { + return false; + } + size_repaired = true; + } + } + if (size_repaired) + { + document_end = enclosing_bson_document_end(); } if (JSON_HEDLEY_UNLIKELY(!enter_container(is_object, detail::unknown_size()))) @@ -556,9 +579,24 @@ class binary_reader frame.start_position = document_start; frame.end_position = document_end; frame.declared_size = document_size; + frame.size_repaired = size_repaired; return true; } + /*! + @brief where a BSON document whose size is wrong ends at the latest + + When recovering, such a document is read up to its terminator, which must + come before the terminator of the document that contains it. + + @return the end of the innermost open document, or the largest position if + no document is open + */ + std::size_t enclosing_bson_document_end() const noexcept + { + return container_stack.empty() ? (std::numeric_limits::max)() : container_stack.back().end_position; + } + /*! @brief read a BSON document and everything nested inside it @@ -591,7 +629,7 @@ class binary_reader // the container_stack element it would otherwise alias const container_frame top = container_stack.back(); - if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(top.start_position, top.declared_size))) + if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(top.start_position, top.declared_size, top.size_repaired))) { return false; } @@ -617,7 +655,15 @@ class binary_reader // document its own terminator, within the declared size if (JSON_HEDLEY_UNLIKELY(chars_read + 2 > container_stack.back().end_position)) { - return bson_document_size_error("document size ", container_stack.back().declared_size, " is too small for its content"); + static_cast(bson_document_size_error("document size ", container_stack.back().declared_size, " is too small for its content")); + if (!repair_requested()) + { + return false; + } + // read on up to the terminator, within the enclosing document + const std::size_t depth = container_stack.size(); + container_stack.back().size_repaired = true; + container_stack.back().end_position = depth > 1 ? container_stack[depth - 2].end_position : (std::numeric_limits::max)(); } const std::size_t element_type_parse_position = chars_read; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 315b2fdb7..37c19cc79 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -18954,6 +18954,9 @@ class binary_reader std::int32_t declared_size = 0; /// whether to close this container with end_object() or end_array() bool is_object; + /// BSON, when recovering: the declared size was reported as wrong, so + /// the document ends at its terminator, within the enclosing document + bool size_repaired = false; }; /*! @@ -19045,16 +19048,18 @@ class binary_reader since at least 5 bytes are always consumed. When recovering from errors, a document whose size does not match is - accepted: its terminator was found, so everything in it has been read. + accepted: its terminator was found, so everything in it has been read. A + size that was already reported as wrong (see @ref repair_bson_document_size) + is not reported again. @param[in] document_start value of chars_read before the size prefix @param[in] document_size the declared document size @return whether the declared size matches the number of bytes read, or the mismatch is repaired */ - bool check_bson_document_size(const std::size_t document_start, const std::int32_t document_size) + bool check_bson_document_size(const std::size_t document_start, const std::int32_t document_size, const bool size_repaired) { - if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast(document_size) != chars_read - document_start)) + if (JSON_HEDLEY_UNLIKELY(!size_repaired && (document_size < 0 || static_cast(document_size) != chars_read - document_start))) { static_cast(bson_document_size_error(document_size, chars_read - document_start)); return repair_requested(); @@ -19072,7 +19077,7 @@ class binary_reader bool can_skip_to_bson_document_end() const noexcept { const container_frame& top = container_stack.back(); - return top.declared_size >= 5 && top.start_position + static_cast(top.declared_size) - 1 >= chars_read; + return !top.size_repaired && top.declared_size >= 5 && top.start_position + static_cast(top.declared_size) - 1 >= chars_read; } /*! @@ -19204,20 +19209,38 @@ class binary_reader } // the smallest document is the size prefix and the terminator + bool size_repaired = false; + std::size_t document_end = 0; if (JSON_HEDLEY_UNLIKELY(document_size < 5)) { - return bson_document_size_error("document size ", document_size, " is less than 5"); + static_cast(bson_document_size_error("document size ", document_size, " is less than 5")); + if (!repair_requested()) + { + return false; + } + size_repaired = true; } - - // the declared size bounds where the document ends: an embedded - // document must end before the terminator of the document that contains it - const auto unsigned_size = static_cast(document_size); - const std::size_t document_end = unsigned_size > (std::numeric_limits::max)() - document_start - ? (std::numeric_limits::max)() - : document_start + unsigned_size; - if (JSON_HEDLEY_UNLIKELY(!container_stack.empty() && document_end >= container_stack.back().end_position)) + else { - return bson_document_size_error("document size ", document_size, " exceeds the size of the enclosing document"); + // the declared size bounds where the document ends: an embedded + // document must end before the terminator of the document that contains it + const auto unsigned_size = static_cast(document_size); + document_end = unsigned_size > (std::numeric_limits::max)() - document_start + ? (std::numeric_limits::max)() + : document_start + unsigned_size; + if (JSON_HEDLEY_UNLIKELY(!container_stack.empty() && document_end >= container_stack.back().end_position)) + { + static_cast(bson_document_size_error("document size ", document_size, " exceeds the size of the enclosing document")); + if (!repair_requested()) + { + return false; + } + size_repaired = true; + } + } + if (size_repaired) + { + document_end = enclosing_bson_document_end(); } if (JSON_HEDLEY_UNLIKELY(!enter_container(is_object, detail::unknown_size()))) @@ -19229,9 +19252,24 @@ class binary_reader frame.start_position = document_start; frame.end_position = document_end; frame.declared_size = document_size; + frame.size_repaired = size_repaired; return true; } + /*! + @brief where a BSON document whose size is wrong ends at the latest + + When recovering, such a document is read up to its terminator, which must + come before the terminator of the document that contains it. + + @return the end of the innermost open document, or the largest position if + no document is open + */ + std::size_t enclosing_bson_document_end() const noexcept + { + return container_stack.empty() ? (std::numeric_limits::max)() : container_stack.back().end_position; + } + /*! @brief read a BSON document and everything nested inside it @@ -19264,7 +19302,7 @@ class binary_reader // the container_stack element it would otherwise alias const container_frame top = container_stack.back(); - if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(top.start_position, top.declared_size))) + if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(top.start_position, top.declared_size, top.size_repaired))) { return false; } @@ -19290,7 +19328,15 @@ class binary_reader // document its own terminator, within the declared size if (JSON_HEDLEY_UNLIKELY(chars_read + 2 > container_stack.back().end_position)) { - return bson_document_size_error("document size ", container_stack.back().declared_size, " is too small for its content"); + static_cast(bson_document_size_error("document size ", container_stack.back().declared_size, " is too small for its content")); + if (!repair_requested()) + { + return false; + } + // read on up to the terminator, within the enclosing document + const std::size_t depth = container_stack.size(); + container_stack.back().size_repaired = true; + container_stack.back().end_position = depth > 1 ? container_stack[depth - 2].end_position : (std::numeric_limits::max)(); } const std::size_t element_type_parse_position = chars_read; diff --git a/tests/src/unit-sax_parse.cpp b/tests/src/unit-sax_parse.cpp index 6ca485ff5..daf22d5ce 100644 --- a/tests/src/unit-sax_parse.cpp +++ b/tests/src/unit-sax_parse.cpp @@ -483,6 +483,17 @@ TEST_CASE("regression test - #3989 SAX parse_error() returning true") {{"s", "a"}, {"inner", {{"a", 1}}}}, 2 }, + // BSON: so is a document whose size is too small for its content, + // which is read up to its terminator + { + json::input_format_t::bson, bson_document( + { + bson_element(0x03, "inner", bson_document({bson_element(0x10, "a", bson_int32(1)), bson_element(0x10, "b", bson_int32(2))}, -7)), + bson_element(0x10, "after", bson_int32(3)), + }), + {{"inner", {{"a", 1}, {"b", 2}}}, {"after", 3}}, + 1 + }, }; for (const auto& repair : repairs) @@ -548,12 +559,12 @@ TEST_CASE("regression test - #3989 SAX parse_error() returning true") CHECK(nested.errors == 1); CHECK(nested.value == json({1})); - // a BJData ndarray whose element type has no name: the object that - // holds the ndarray was already begun + // a BJData ndarray whose element type is not valid: the type is + // checked before the object that would hold the ndarray is begun const auto ndarray = parse_binary_recovering({'[', '$', 0x01, '#', '[', '$', 'i', '#', 'i', 2, 2, 3}, json::input_format_t::bjdata); CHECK(ndarray.errors == 1); CHECK(ndarray.balanced); - CHECK(ndarray.value == json::object()); + CHECK(ndarray.value == nullptr); // a skipped member that the input ends in const auto truncated = parse_binary_recovering({0xA2, 0x01, 0x82, 0x01}, json::input_format_t::cbor); @@ -561,9 +572,11 @@ TEST_CASE("regression test - #3989 SAX parse_error() returning true") CHECK(truncated.balanced); CHECK(truncated.value == json::object()); - // a BSON element of an unknown type in a document whose size cannot be right + // a BSON element of an unknown type in a document whose size cannot be + // right: the size is reported when it is read, and the element ends + // the document, as its end is not known const auto bson = parse_binary_recovering(bson_document({bson_element(0x10, "a", bson_int32(1)), bson_element(0x42, "x", bytes(3))}, -10), json::input_format_t::bson); - CHECK(bson.errors == 1); + CHECK(bson.errors == 2); CHECK(bson.value == json({{"a", 1}, {"x", nullptr}})); }