Files
json/tests/src/unit-ubjson_valueless.cpp
T
Niels Lohmann 934b12a9e1 Build the tests with C++17/C++20-only sections in those standards
cmake/test.cmake only adds the _cpp17/_cpp20 variants of a unit test if the
file mentions JSON_HAS_CPP_17/JSON_HAS_CPP_20. Several files have sections
that need C++17 or C++20 (std::filesystem::path, std::string_view, char8_t,
std::u8string, operator<=>, ranges) but never mention the macros, so by
default these sections were only compiled away in the C++11 build and only
ran in the CI jobs that force a standard.

Mention the macros next to the guarded sections, as the other tests do:
unit-to_json-rvalue.cpp (C++17 and C++20), unit-json_view_builder.cpp
(C++17), unit-class_parser.cpp, unit-recursion_depth_limit.cpp,
unit-ubjson_valueless.cpp, unit-custom-number-types.cpp and
unit-allocator.cpp (C++20).

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-11-16 12:40:54 +01:00

333 lines
16 KiB
C++

// __ _____ _____ _____
// __| | __| | | | JSON for Modern C++ (supporting code)
// | | |__ | | | | | | version 3.12.0
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
//
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
// SPDX-License-Identifier: MIT
// cmake/test.cmake selects the C++ standard versions with which to build a
// unit test based on the presence of JSON_HAS_CPP_<VERSION> macros.
// When using macros that are only defined for particular versions of the standard
// (e.g., JSON_HAS_FILESYSTEM for C++17 and up), please mention the corresponding
// version macro in a comment close by, like this:
// JSON_HAS_CPP_<VERSION> (do not remove; see note at top of file)
// The budget for the elements of the UBJSON containers of a valueless type
// ('Z', 'T', 'F') of a whole input: 1048576 plus
// JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE for every byte read (see
// docs/mkdocs/docs/api/macros/json_ubjson_valueless_elements_per_byte.md).
//
// The file is built several times, with the macro defined to other values (see
// tests/CMakeLists.txt), and every boundary below is computed from the macro.
// What it can test depends on the value: with a rate of at most 10000 the
// budget can be reached with the few bytes of a test, so the tests exceed it
// at the exact byte; with a rate of 131072 or more the budget cannot be reached
// by an input of any size (see the macro's page: 2^20 / 8 = 131072 elements per
// byte is what the shortest header of a container of the largest size allows), and
// the tests check that nothing is refused, in particular that the limit does
// not wrap around for a rate near the largest integer. The input is read with
// a SAX handler that counts the values, because a DOM of a million values is
// what makes such a test slow under Valgrind.
#include "doctest_compatibility.h"
#include <nlohmann/json.hpp>
using nlohmann::json;
#include <cstddef> // size_t
#include <cstdio> // FILE, tmpfile, fwrite, fseek, fclose
#include <cstdint> // uint8_t, uint32_t
#include <list>
#include <sstream>
#include <vector>
#include "ubjson_valueless.hpp"
using utils::ValuelessEventCounter;
using utils::valueless_arrays;
#ifndef JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE
#error "this test needs the macro: compile it with JSON_TEST_KEEP_MACROS (see tests/CMakeLists.txt)"
#endif
namespace
{
/// the base of the budget and the limit of a single container: both 2^20
constexpr std::uint32_t budget_base = 1048576;
#if JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE <= 10000 || JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE >= 131072
/// what version 3.12.0 wrote for a table of @a rows (at most 32767) rows of
/// @a width nulls with to_ubjson(table, true, true): `[$[#I` and the count of
/// the rows (7 bytes), then a row as `$Z#` and its count (`U` and 1 byte up to
/// 255, `I` and 2 bytes up to 32767, `l` and 4 bytes above): 5, 6 or 8 bytes
std::vector<std::uint8_t> rows_3_12_0(const std::uint32_t rows, const std::uint32_t width)
{
std::vector<std::uint8_t> v = {'[', '$', '[', '#', 'I', static_cast<std::uint8_t>(rows >> 8), static_cast<std::uint8_t>(rows)};
for (std::uint32_t row = 0; row < rows; ++row)
{
v.push_back('$');
v.push_back('Z');
v.push_back('#');
if (width <= 255)
{
v.push_back('U');
}
else if (width <= 32767)
{
v.push_back('I');
v.push_back(static_cast<std::uint8_t>(width >> 8));
}
else
{
v.push_back('l');
v.push_back(static_cast<std::uint8_t>(width >> 24));
v.push_back(static_cast<std::uint8_t>(width >> 16));
v.push_back(static_cast<std::uint8_t>(width >> 8));
}
v.push_back(static_cast<std::uint8_t>(width));
}
return v;
}
#endif
} // namespace
// the macro must not be a character or bool: every character type is refused
static_assert(nlohmann::detail::valueless_elements_rate_is_char_or_bool<bool>::value, "bool is refused");
static_assert(nlohmann::detail::valueless_elements_rate_is_char_or_bool<char>::value, "char is refused");
static_assert(nlohmann::detail::valueless_elements_rate_is_char_or_bool<signed char>::value, "signed char is refused");
static_assert(nlohmann::detail::valueless_elements_rate_is_char_or_bool<unsigned char>::value, "unsigned char is refused");
static_assert(nlohmann::detail::valueless_elements_rate_is_char_or_bool<wchar_t>::value, "wchar_t is refused");
static_assert(nlohmann::detail::valueless_elements_rate_is_char_or_bool<char16_t>::value, "char16_t is refused");
static_assert(nlohmann::detail::valueless_elements_rate_is_char_or_bool<char32_t>::value, "char32_t is refused");
// JSON_HAS_CPP_20 (do not remove; see note at top of file)
#if defined(__cpp_char8_t)
static_assert(nlohmann::detail::valueless_elements_rate_is_char_or_bool<char8_t>::value, "char8_t is refused");
#endif
static_assert(!nlohmann::detail::valueless_elements_rate_is_char_or_bool<int>::value, "int is accepted");
static_assert(!nlohmann::detail::valueless_elements_rate_is_char_or_bool<unsigned long long>::value, "unsigned long long is accepted");
TEST_CASE("UBJSON the limit of a single container does not depend on the macro")
{
#if !defined(JSON_NOEXCEPTION)
json _;
SECTION("a container of more than 1048576 elements is refused")
{
// the first check of every container; its message differs from that of
// the budget, so that it is clear which limit was exceeded
for (const char marker :
{'Z', 'T', 'F'
})
{
const auto input = valueless_arrays(marker, {budget_base + 1});
CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive array size", json::out_of_range&);
}
}
#endif
SECTION("a container of 1048576 elements is accepted at the start of an input")
{
// whatever the rate: the base is the per-container limit. The handler
// ends the parse at the first value, so that the million values are
// not read
const std::vector<std::uint8_t> single = {'[', '$', 'Z', '#', 'l', 0x00, 0x10, 0x00, 0x00};
ValuelessEventCounter sax;
sax.stop_after = 1;
CHECK(!json::sax_parse(single, &sax, json::input_format_t::ubjson));
CHECK(sax.error_id == 0);
CHECK(sax.values == 1);
}
}
#if JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE <= 10000
namespace
{
/// the rate of the build; at most 10000 here, which the boundaries rely on
constexpr std::uint32_t rate = JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE;
/// elements of the first of two containers; the second one is then refused at
/// its header, which is read at byte 1 + 9 + 9 = 19
constexpr std::uint32_t first_count = 524288;
/// the largest count the second container may have at byte 19: base plus rate
/// for each of the 19 bytes, less what the first one used. For a rate of at
/// most 27594, the largest that keeps this within the limit of one container
constexpr std::uint32_t second_allowed = budget_base - first_count + (19 * rate);
} // namespace
TEST_CASE("UBJSON the valueless elements of a whole input are bounded")
{
// The per-container limit alone let nine bytes (`[$Z#l` and a count of
// 1048576) announce a million values, and every further nine bytes another
// million. The elements of all containers of type 'Z', 'T' or 'F' in one
// input are added up: they may not exceed 1048576 plus
// JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE for every byte read when the
// container's header has been read.
#if !defined(JSON_NOEXCEPTION)
json _;
const char* const budget_message = "[json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive number of elements without payload";
#endif
SECTION("the input of the original report is rejected")
{
// the second array is the first one that is refused, by one element
#if !defined(JSON_NOEXCEPTION)
CHECK_THROWS_WITH_AS(_ = json::from_ubjson(valueless_arrays('Z', {first_count, second_allowed + 1})), budget_message, json::out_of_range&);
#endif
// and is accepted with one element less
ValuelessEventCounter sax;
CHECK(json::sax_parse(valueless_arrays('Z', {first_count, second_allowed}), &sax, json::input_format_t::ubjson));
CHECK(sax.values == first_count + second_allowed);
}
SECTION("every input adapter applies the budget")
{
const auto input = valueless_arrays('Z', {first_count, second_allowed + 1});
// stream; not strict, which does not matter
std::stringstream stream;
stream.write(reinterpret_cast<const char*>(input.data()), static_cast<std::streamsize>(input.size()));
ValuelessEventCounter from_stream;
CHECK(!json::sax_parse(stream, &from_stream, json::input_format_t::ubjson, false));
CHECK(from_stream.error_id == 408);
// a container that is not contiguous; the counts of arrays of
// different types are added up
std::vector<std::uint8_t> mixed = {'['};
utils::append_valueless_array(mixed, 'T', first_count);
utils::append_valueless_array(mixed, 'F', second_allowed + 1);
mixed.push_back(']');
const std::list<std::uint8_t> as_list(mixed.begin(), mixed.end());
ValuelessEventCounter from_list;
CHECK(!json::sax_parse(as_list.begin(), as_list.end(), &from_list, json::input_format_t::ubjson));
CHECK(from_list.error_id == 408);
CHECK(from_list.values == first_count);
// FILE*
FILE* const file = std::tmpfile(); // NOLINT(cppcoreguidelines-owning-memory)
REQUIRE(file != nullptr);
CHECK(std::fwrite(input.data(), 1, input.size(), file) == input.size());
CHECK(std::fseek(file, 0, SEEK_SET) == 0);
ValuelessEventCounter from_file;
CHECK(!json::sax_parse(file, &from_file, json::input_format_t::ubjson));
CHECK(from_file.error_id == 408);
std::fclose(file); // NOLINT(cert-err33-c,cppcoreguidelines-owning-memory)
}
SECTION("the limit is exact and the bytes read raise it")
{
// [ [$Z#l first_count] [$U#l payload + payload bytes] [$Z#l count] ]:
// chars_read is 1 + 9 + (9 + payload) + 9 = 28 + payload when the last
// header has been read, so the last array may hold
// rate * (28 + payload) values on top of the base less the
// first_count of the first array. The payload is as large as the
// limit of one container allows, but at most 4096
constexpr std::uint32_t payload_limit = (524288 / rate) - 40;
constexpr std::uint32_t payload = payload_limit < 4096 ? payload_limit : 4096;
constexpr std::uint32_t last_allowed = budget_base - first_count + (rate * (28 + payload));
static_assert(last_allowed < budget_base, "the count must be accepted by the limit of one container");
for (const std::uint32_t count :
{
last_allowed, last_allowed + 1
})
{
CAPTURE(count)
std::vector<std::uint8_t> input = {'['};
utils::append_valueless_array(input, 'Z', first_count);
input.insert(input.end(), {'[', '$', 'U', '#', 'l',
static_cast<std::uint8_t>(payload >> 24), static_cast<std::uint8_t>(payload >> 16),
static_cast<std::uint8_t>(payload >> 8), static_cast<std::uint8_t>(payload)
});
input.insert(input.end(), payload, 0x01);
utils::append_valueless_array(input, 'Z', count);
input.push_back(']');
ValuelessEventCounter sax;
const bool accepted = count == last_allowed;
CHECK(json::sax_parse(input, &sax, json::input_format_t::ubjson) == accepted);
CHECK(sax.error_id == (accepted ? 0 : 408));
CHECK(sax.values == first_count + payload + (accepted ? count : 0u));
}
}
SECTION("a table written by 3.12.0")
{
// Version 3.12.0 wrote a table of rows of nulls with one `$Z#` header
// per row (see the macro's page). A row of `width` elements has a
// header of 6 bytes up to 32767 elements (5 up to 255), so the budget
// grows by 6 * rate per row, and the row brings `width`. The 7 bytes
// of the outer header are read first. Row r is accepted while
// r * width <= 1048576 + rate * (7 + 6 * r),
// which is r <= (1048576 + 7 * rate) / (width - 6 * rate) for a width
// above 6 * rate, and always true otherwise.
constexpr std::uint32_t width = 1000;
constexpr std::uint32_t rows = 1200;
constexpr std::uint32_t credit = 6 * rate;
constexpr std::uint32_t accepted_rows = width > credit ? (budget_base + (7 * rate)) / (width - credit) : rows;
constexpr std::uint32_t read_rows = accepted_rows < rows ? accepted_rows : rows;
ValuelessEventCounter sax;
CHECK(json::sax_parse(rows_3_12_0(rows, width), &sax, json::input_format_t::ubjson) == (accepted_rows >= rows));
CHECK(sax.error_id == (accepted_rows >= rows ? 0 : 408));
CHECK(sax.values == static_cast<std::size_t>(read_rows) * width);
}
}
#elif JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE >= 131072
TEST_CASE("UBJSON a rate that cannot be reached switches the budget off")
{
// The rate is so large (2^20 / 8 = 131072 elements per byte or more)
// that no input can exceed the budget: the shortest header of a container
// of the largest size is 8 bytes (`$Z#l` and a count of 4 bytes, in an
// array of arrays). The limit is base + rate * bytes read, which wraps
// around for a rate near the largest integer in 64 and in 32
// bits already at the first byte, unless it saturates: a wrapped limit
// would refuse the second container below (at byte 19 or at byte 10, where
// a limit of 2^20 + 10 * rate wraps for a 32-bit rate of 0x7FFFFFFF).
#if !defined(JSON_NOEXCEPTION)
json _;
#endif
SECTION("containers of the largest size are accepted one after the other")
{
// the first container is accepted by the base alone, the second one is
// not: it needs the rate. The handler ends the parse at its first
// value, so that the second million values are not read
const auto input = valueless_arrays('Z', {budget_base, budget_base});
ValuelessEventCounter sax;
sax.stop_after = static_cast<std::size_t>(budget_base) + 1;
CHECK(!json::sax_parse(input, &sax, json::input_format_t::ubjson));
CHECK(sax.error_id == 0);
CHECK(sax.values == static_cast<std::size_t>(budget_base) + 1);
}
SECTION("the limit of a single container still applies")
{
const auto input = valueless_arrays('T', {budget_base, budget_base + 1});
ValuelessEventCounter sax;
CHECK(!json::sax_parse(input, &sax, json::input_format_t::ubjson));
CHECK(sax.error_id == 408);
CHECK(sax.values == budget_base);
#if !defined(JSON_NOEXCEPTION)
CHECK_THROWS_WITH_AS(_ = json::from_ubjson(valueless_arrays('F', {budget_base + 1})), "[json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive array size", json::out_of_range&);
#endif
}
SECTION("a table written by 3.12.0 is read completely")
{
// 1200 rows of 1000 values are refused after 1074 rows at the default
// rate and are accepted with every rate of 167 and more
ValuelessEventCounter sax;
CHECK(json::sax_parse(rows_3_12_0(1200, 1000), &sax, json::input_format_t::ubjson));
CHECK(sax.error_id == 0);
CHECK(sax.values == 1200u * 1000u);
}
}
// (rates from 10001 to 131071 can be reached, but not with the few bytes of a
// test: there is nothing to test for them here)
#endif