Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Niels Lohmann
|
b2c20c01b1 | ||
|
Niels Lohmann
|
9cbdcb5ed3 |
No files matched your search
@@ -23,7 +23,6 @@
|
||||
#include <cstdlib> // _byteswap_ushort, _byteswap_ulong, _byteswap_uint64
|
||||
#endif
|
||||
|
||||
#include <nlohmann/detail/bit_ops.hpp>
|
||||
#include <nlohmann/detail/input/binary_reader.hpp>
|
||||
#include <nlohmann/detail/input/string_scan.hpp>
|
||||
#include <nlohmann/detail/macro_scope.hpp>
|
||||
@@ -205,12 +204,24 @@ class binary_writer
|
||||
}
|
||||
|
||||
case value_t::number_integer:
|
||||
{
|
||||
if (j.m_data.m_value.number_integer >= 0)
|
||||
{
|
||||
// CBOR does not differentiate between positive signed
|
||||
// integers and unsigned integers
|
||||
write_cbor_head(0x00, static_cast<std::uint64_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else
|
||||
{
|
||||
// a negative integer n is encoded as -1 - n
|
||||
write_cbor_head(0x20, static_cast<std::uint64_t>(-1 - j.m_data.m_value.number_integer));
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case value_t::number_unsigned:
|
||||
{
|
||||
// one case for both, so that an array mixing negative
|
||||
// (number_integer) and non-negative (number_unsigned) values
|
||||
// does not mispredict the switch
|
||||
write_cbor_integer(j);
|
||||
write_cbor_head(0x00, j.m_data.m_value.number_unsigned);
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -245,7 +256,13 @@ class binary_writer
|
||||
|
||||
case value_t::array:
|
||||
{
|
||||
write_cbor_array(j, depth);
|
||||
// step 1: write control byte and the array size
|
||||
write_cbor_head(0x80, j.m_data.m_value.array->size());
|
||||
|
||||
for (const auto& el : *j.m_data.m_value.array)
|
||||
{
|
||||
write_cbor(el, depth + 1);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -403,12 +420,54 @@ class binary_writer
|
||||
}
|
||||
|
||||
case value_t::number_integer:
|
||||
{
|
||||
if (j.m_data.m_value.number_integer >= 0)
|
||||
{
|
||||
// MessagePack does not differentiate between positive
|
||||
// signed integers and unsigned integers.
|
||||
write_msgpack_unsigned(static_cast<std::uint64_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else
|
||||
{
|
||||
if (j.m_data.m_value.number_integer >= -32)
|
||||
{
|
||||
// negative fixnum
|
||||
write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int8_t>::min)() &&
|
||||
j.m_data.m_value.number_integer <= (std::numeric_limits<std::int8_t>::max)())
|
||||
{
|
||||
// int 8
|
||||
oa.write_character(to_char_type(0xD0));
|
||||
write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int16_t>::min)() &&
|
||||
j.m_data.m_value.number_integer <= (std::numeric_limits<std::int16_t>::max)())
|
||||
{
|
||||
// int 16
|
||||
oa.write_character(to_char_type(0xD1));
|
||||
write_number(static_cast<std::int16_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int32_t>::min)() &&
|
||||
j.m_data.m_value.number_integer <= (std::numeric_limits<std::int32_t>::max)())
|
||||
{
|
||||
// int 32
|
||||
oa.write_character(to_char_type(0xD2));
|
||||
write_number(static_cast<std::int32_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else
|
||||
{
|
||||
// int 64
|
||||
oa.write_character(to_char_type(0xD3));
|
||||
write_number(static_cast<std::int64_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case value_t::number_unsigned:
|
||||
{
|
||||
// one case for both, so that an array mixing negative
|
||||
// (number_integer) and non-negative (number_unsigned) values
|
||||
// does not mispredict the switch
|
||||
write_msgpack_integer(j, std::integral_constant<bool, msgpack_integers_fit_64_bits()>());
|
||||
write_msgpack_unsigned(static_cast<std::uint64_t>(j.m_data.m_value.number_unsigned));
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -426,7 +485,13 @@ class binary_writer
|
||||
|
||||
case value_t::array:
|
||||
{
|
||||
write_msgpack_array(j, depth);
|
||||
// step 1: write control byte and the array size
|
||||
write_msgpack_array_prefix(j.m_data.m_value.array->size(), j);
|
||||
|
||||
for (const auto& el : *j.m_data.m_value.array)
|
||||
{
|
||||
write_msgpack(el, depth + 1);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -1886,316 +1951,6 @@ class binary_writer
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief size class of an unsigned integer from its bit length
|
||||
|
||||
1, 2, 3, or 4 if @a x fits 1, 2, 4, or 8 bytes. Looked up from the number
|
||||
of leading zero bits instead of found by branches: the sizes of a stream
|
||||
of numbers are hard to predict, and a mispredicted branch costs more than
|
||||
writing the number. (A table, as comparisons are vectorized by clang into
|
||||
slower code.)
|
||||
*/
|
||||
static unsigned integer_size_class(const std::uint64_t x) noexcept
|
||||
{
|
||||
static constexpr std::array<std::uint8_t, 64> size_class_by_leading_zeros =
|
||||
{
|
||||
{
|
||||
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
2, 2, 2, 2, 2, 2, 2, 2,
|
||||
1, 1, 1, 1, 1, 1, 1, 1
|
||||
}
|
||||
};
|
||||
// x | 1 keeps count_leading_zeros() away from 0
|
||||
return size_class_by_leading_zeros[static_cast<std::size_t>(count_leading_zeros(x | 1u))];
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief encode @a first_byte and the low-order bytes of @a value that
|
||||
@a size_class calls for (none, or 1, 2, 4, or 8) in network order
|
||||
to @a out
|
||||
@return the number of characters of the encoding; up to
|
||||
@ref integer_encoding_space characters at @a out are overwritten
|
||||
|
||||
The bytes are written as one byte and one 8-byte word whose bytes after
|
||||
the value are scratch that the caller drops, so that writing them does not
|
||||
depend on the size class. For class 0 the word is all scratch; it is
|
||||
shifted by 60 bits rather than 64, which would be undefined.
|
||||
*/
|
||||
std::size_t encode_first_byte_and_tail(CharType* out, const std::uint8_t first_byte, const unsigned size_class, const std::uint64_t value) const noexcept
|
||||
{
|
||||
out[0] = to_char_type(first_byte);
|
||||
const unsigned unused_bits = 64u - ((8u << size_class) >> 1u);
|
||||
const std::uint64_t tail = is_little_endian ? byte_swap(value) >> unused_bits : value << unused_bits;
|
||||
std::memcpy(out + 1, &tail, sizeof(tail));
|
||||
return ((std::size_t{1} << size_class) >> 1u) + 1;
|
||||
}
|
||||
|
||||
/// characters encode_first_byte_and_tail() may overwrite: the first byte
|
||||
/// and one 8-byte word
|
||||
static constexpr std::size_t integer_encoding_space = 9;
|
||||
|
||||
/*!
|
||||
@brief the 64 bits of a number_integer or number_unsigned value, and a mask
|
||||
that is all ones if it is negative, else zero
|
||||
|
||||
Selected through masks rather than branches, as the type and the sign of
|
||||
the values in an array of mixed signs alternate unpredictably (non-negative
|
||||
integers are number_unsigned after parsing). Up to 64 bits, the sign is bit
|
||||
63 of the result (a narrower integer is sign-extended); computing it with
|
||||
masks keeps GCC from branching on the type. A wider integer has to be
|
||||
compared.
|
||||
*/
|
||||
static std::uint64_t integer_bits(const BasicJsonType& j, std::uint64_t& sign_mask) noexcept
|
||||
{
|
||||
const bool is_signed = j.m_data.m_type == value_t::number_integer;
|
||||
const std::uint64_t raw = is_signed
|
||||
? static_cast<std::uint64_t>(j.m_data.m_value.number_integer)
|
||||
: static_cast<std::uint64_t>(j.m_data.m_value.number_unsigned);
|
||||
sign_mask = sizeof(typename BasicJsonType::number_integer_t) <= sizeof(std::uint64_t)
|
||||
? (std::uint64_t{0} - static_cast<std::uint64_t>(is_signed)) & (std::uint64_t{0} - (raw >> 63u))
|
||||
: std::uint64_t{0} - static_cast<std::uint64_t>(is_signed && j.m_data.m_value.number_integer < 0);
|
||||
return raw;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief encode the head of a CBOR integer to @a out without branching on its
|
||||
size
|
||||
@return the number of characters of the head
|
||||
|
||||
Same output as @ref write_cbor_head. The lengths of strings and containers
|
||||
are mostly predictable, so their heads keep the branches; the sizes of a
|
||||
run of integers are not, so an integer's head is assembled without them.
|
||||
*/
|
||||
std::size_t encode_cbor_integer_head(CharType* out, const std::uint8_t major_type, const std::uint64_t argument) const noexcept
|
||||
{
|
||||
// an argument of at most 23 is in size class 1 (one byte) and is moved
|
||||
// to class 0, the first byte, by subtracting 1
|
||||
const auto is_immediate = static_cast<unsigned>(argument <= 0x17u);
|
||||
const unsigned size_class = integer_size_class(argument) - is_immediate;
|
||||
|
||||
// the first byte: the argument itself if it is immediate, else
|
||||
// 0x17 + size_class, selected through a mask
|
||||
const std::uint64_t immediate_mask = std::uint64_t{0} - is_immediate;
|
||||
const auto first_byte = static_cast<std::uint8_t>(major_type + ((argument & immediate_mask) | ((0x17u + size_class) & ~immediate_mask)));
|
||||
return encode_first_byte_and_tail(out, first_byte, size_class, argument);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief write an array value (head and elements) as CBOR
|
||||
|
||||
Integer elements, the most common scalar, are written without a call to
|
||||
@ref write_cbor. The loop is kept out of line: inlined into write_cbor, its
|
||||
registers would be saved and restored on every call of write_cbor, which
|
||||
is one call per value nested in an object.
|
||||
*/
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
void write_cbor_array(const BasicJsonType& j, const std::size_t depth)
|
||||
{
|
||||
write_cbor_head(0x80, j.m_data.m_value.array->size());
|
||||
write_array_elements(j, [this](CharType * out, const BasicJsonType & el)
|
||||
{
|
||||
return encode_cbor_integer(out, el);
|
||||
}, [this, depth](const BasicJsonType & el)
|
||||
{
|
||||
write_cbor(el, depth + 1);
|
||||
});
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief write the elements of the array @a j, runs of integers with
|
||||
@a encode_integer and all other values with @a write_value
|
||||
|
||||
Runs of integers are encoded straight into space reserved from the sink
|
||||
for a whole run: the write pointer stays a local, where it would otherwise
|
||||
be reloaded from the sink for every value, as a store of a character may
|
||||
alias it.
|
||||
*/
|
||||
template<typename EncodeInteger, typename WriteValue>
|
||||
void write_array_elements(const BasicJsonType& j, EncodeInteger encode_integer, WriteValue write_value)
|
||||
{
|
||||
constexpr std::size_t max_run = max_reserve_size / integer_encoding_space;
|
||||
auto it = j.m_data.m_value.array->begin();
|
||||
const auto end = j.m_data.m_value.array->end();
|
||||
while (it != end)
|
||||
{
|
||||
if (it->is_number_integer())
|
||||
{
|
||||
CharType* const start = oa.reserve(max_run * integer_encoding_space);
|
||||
CharType* out = start;
|
||||
for (std::size_t n = 0; n < max_run && it != end && it->is_number_integer(); ++n, ++it)
|
||||
{
|
||||
out += encode_integer(out, *it);
|
||||
}
|
||||
oa.commit(static_cast<std::size_t>(out - start));
|
||||
}
|
||||
else
|
||||
{
|
||||
write_value(*it);
|
||||
++it;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief write a number_integer or number_unsigned value as a CBOR integer
|
||||
|
||||
A negative integer n has major type 1 and the argument -1 - n, which is ~n.
|
||||
Kept out of line, like the other per-value paths, so that write_cbor stays
|
||||
small (runs of integers in an array are encoded inline instead).
|
||||
*/
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
void write_cbor_integer(const BasicJsonType& j)
|
||||
{
|
||||
oa.commit(encode_cbor_integer(oa.reserve(integer_encoding_space), j));
|
||||
}
|
||||
|
||||
/// encode a number_integer or number_unsigned value to @a out (see
|
||||
/// @ref write_cbor_integer) and return the number of characters
|
||||
std::size_t encode_cbor_integer(CharType* out, const BasicJsonType& j) const noexcept
|
||||
{
|
||||
std::uint64_t sign_mask = 0;
|
||||
const std::uint64_t raw = integer_bits(j, sign_mask);
|
||||
// ~raw is -1 - n in the 64 bits that are written
|
||||
return encode_cbor_integer_head(out, static_cast<std::uint8_t>(sign_mask & 0x20u), raw ^ sign_mask);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief whether the MessagePack integers of BasicJsonType are written by
|
||||
@ref encode_msgpack_integer
|
||||
|
||||
It picks the format from the low 64 bits; a wider number type keeps the
|
||||
range checks of @ref write_msgpack on the whole value.
|
||||
*/
|
||||
static constexpr bool msgpack_integers_fit_64_bits() noexcept
|
||||
{
|
||||
return sizeof(typename BasicJsonType::number_integer_t) <= sizeof(std::uint64_t)
|
||||
&& sizeof(typename BasicJsonType::number_unsigned_t) <= sizeof(std::uint64_t);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief write a number_integer or number_unsigned value as a MessagePack
|
||||
integer without branching on its sign or size
|
||||
|
||||
Same output as @ref write_msgpack: a non-negative integer as positive fixint
|
||||
(< 128) or uint 8/16/32/64, a negative one as negative fixint (>= -32) or
|
||||
int 8/16/32/64. Kept out of line (see @ref write_cbor_integer).
|
||||
*/
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
void write_msgpack_integer(const BasicJsonType& j, std::true_type /*fits_64_bits*/)
|
||||
{
|
||||
oa.commit(encode_msgpack_integer(oa.reserve(integer_encoding_space), j));
|
||||
}
|
||||
|
||||
/// encode a number_integer or number_unsigned value to @a out (see
|
||||
/// @ref write_msgpack_integer) and return the number of characters
|
||||
std::size_t encode_msgpack_integer(CharType* out, const BasicJsonType& j) const noexcept
|
||||
{
|
||||
std::uint64_t sign_mask = 0;
|
||||
const std::uint64_t raw = integer_bits(j, sign_mask);
|
||||
|
||||
// A negative n fits int 8/16/32 exactly if 2 * (-1 - n) + 1 fits
|
||||
// uint 8/16/32, so x gives the size class of either sign.
|
||||
const std::uint64_t is_negative = sign_mask & 1u;
|
||||
const std::uint64_t x = ((raw ^ sign_mask) << is_negative) | is_negative;
|
||||
|
||||
// A fixint is at most 127, or at least -32 (x at most 63). It is in
|
||||
// size class 1 and is moved to class 0, the first byte, by subtracting 1.
|
||||
const auto is_fixint = static_cast<unsigned>(x <= (127u ^ (sign_mask & 64u)));
|
||||
const unsigned size_class = integer_size_class(x) - is_fixint;
|
||||
|
||||
// the first byte: the low byte of the integer for a fixint, else the
|
||||
// marker 0xCC..0xCF (uint) or 0xD0..0xD3 (int), selected through a mask
|
||||
const std::uint64_t fixint_mask = std::uint64_t{0} - is_fixint;
|
||||
const std::uint64_t marker = 0xCBu + size_class + (sign_mask & 4u);
|
||||
const auto first_byte = static_cast<std::uint8_t>((raw & fixint_mask) | (marker & ~fixint_mask));
|
||||
return encode_first_byte_and_tail(out, first_byte, size_class, raw);
|
||||
}
|
||||
|
||||
/// write a number_integer or number_unsigned value of a number type wider
|
||||
/// than 64 bits as a MessagePack integer, with range checks on the whole value
|
||||
void write_msgpack_integer(const BasicJsonType& j, std::false_type /*fits_64_bits*/)
|
||||
{
|
||||
if (j.m_data.m_type == value_t::number_unsigned)
|
||||
{
|
||||
write_msgpack_unsigned(static_cast<std::uint64_t>(j.m_data.m_value.number_unsigned));
|
||||
return;
|
||||
}
|
||||
|
||||
if (j.m_data.m_value.number_integer >= 0)
|
||||
{
|
||||
// MessagePack does not differentiate between positive
|
||||
// signed integers and unsigned integers.
|
||||
write_msgpack_unsigned(static_cast<std::uint64_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else
|
||||
{
|
||||
if (j.m_data.m_value.number_integer >= -32)
|
||||
{
|
||||
// negative fixnum
|
||||
write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int8_t>::min)() &&
|
||||
j.m_data.m_value.number_integer <= (std::numeric_limits<std::int8_t>::max)())
|
||||
{
|
||||
// int 8
|
||||
oa.write_character(to_char_type(0xD0));
|
||||
write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int16_t>::min)() &&
|
||||
j.m_data.m_value.number_integer <= (std::numeric_limits<std::int16_t>::max)())
|
||||
{
|
||||
// int 16
|
||||
oa.write_character(to_char_type(0xD1));
|
||||
write_number(static_cast<std::int16_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int32_t>::min)() &&
|
||||
j.m_data.m_value.number_integer <= (std::numeric_limits<std::int32_t>::max)())
|
||||
{
|
||||
// int 32
|
||||
oa.write_character(to_char_type(0xD2));
|
||||
write_number(static_cast<std::int32_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else
|
||||
{
|
||||
// int 64
|
||||
oa.write_character(to_char_type(0xD3));
|
||||
write_number(static_cast<std::int64_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief write an array value (head and elements) as MessagePack
|
||||
|
||||
See @ref write_cbor_array.
|
||||
*/
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
void write_msgpack_array(const BasicJsonType& j, const std::size_t depth)
|
||||
{
|
||||
write_msgpack_array_prefix(j.m_data.m_value.array->size(), j);
|
||||
write_msgpack_array_elements(j, depth, std::integral_constant<bool, msgpack_integers_fit_64_bits()>());
|
||||
}
|
||||
|
||||
void write_msgpack_array_elements(const BasicJsonType& j, const std::size_t depth, std::true_type /*fits_64_bits*/)
|
||||
{
|
||||
write_array_elements(j, [this](CharType * out, const BasicJsonType & el)
|
||||
{
|
||||
return encode_msgpack_integer(out, el);
|
||||
}, [this, depth](const BasicJsonType & el)
|
||||
{
|
||||
write_msgpack(el, depth + 1);
|
||||
});
|
||||
}
|
||||
|
||||
void write_msgpack_array_elements(const BasicJsonType& j, const std::size_t depth, std::false_type /*fits_64_bits*/)
|
||||
{
|
||||
for (const auto& el : *j.m_data.m_value.array)
|
||||
{
|
||||
write_msgpack(el, depth + 1);
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief write an object key as a CBOR text string
|
||||
|
||||
|
||||
@@ -8,10 +8,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm> // max
|
||||
#include <array> // array
|
||||
#include <cstddef> // size_t
|
||||
#include <cstring> // memcpy
|
||||
#include <memory> // shared_ptr, make_shared
|
||||
#include <string> // basic_string
|
||||
#include <type_traits> // conditional, integral_constant, is_same
|
||||
@@ -51,170 +48,32 @@ template<typename CharType> struct output_adapter_protocol
|
||||
template<typename CharType>
|
||||
using output_adapter_t = std::shared_ptr<output_adapter_protocol<CharType>>;
|
||||
|
||||
/// the largest number of characters a writer may reserve() from a sink at once;
|
||||
/// output_vector_sink also appends writes of at least this size with insert()
|
||||
constexpr std::size_t max_reserve_size = 256;
|
||||
|
||||
/// @brief non-virtual output sink writing into a std::vector
|
||||
///
|
||||
/// This sink is not part of the virtual output_adapter_protocol hierarchy: it is
|
||||
/// passed to binary_writer by value as a template parameter, so
|
||||
/// write_character()/write_characters() are ordinary (inlinable) calls with no
|
||||
/// vtable lookup and no shared_ptr. It is used for the common
|
||||
/// `to_cbor`/`to_msgpack`/... into a std::vector.
|
||||
///
|
||||
/// Most writes are a few bytes - a type marker, or a number of 1, 2, 4, or 8
|
||||
/// bytes - and appending those with push_back() or insert() cost more than
|
||||
/// encoding them: insert() is an out-of-line call into a routine that can insert
|
||||
/// anywhere, and resize() is one as well with libstdc++. The sink therefore keeps
|
||||
/// its own write position: it grows the vector geometrically ahead of the
|
||||
/// position and copies each write to it, so a write is a capacity check and a
|
||||
/// short copy. The destructor shrinks the vector to the bytes actually written,
|
||||
/// also when a write throws; shrinking a vector never allocates. The sink owns
|
||||
/// that trim, so it can be moved but not copied, and the vector must not be used
|
||||
/// while the sink is alive.
|
||||
/// `to_cbor`/`to_msgpack`/... into a std::vector. output_vector_adapter below
|
||||
/// wraps this same sink to provide the virtual interface.
|
||||
template<typename CharType, typename AllocatorType = std::allocator<CharType>>
|
||||
class output_vector_sink
|
||||
{
|
||||
public:
|
||||
explicit output_vector_sink(std::vector<CharType, AllocatorType>& vec) noexcept
|
||||
: v(&vec), position(vec.size())
|
||||
{}
|
||||
|
||||
output_vector_sink(const output_vector_sink&) = delete;
|
||||
output_vector_sink& operator=(const output_vector_sink&) = delete;
|
||||
output_vector_sink& operator=(output_vector_sink&&) = delete;
|
||||
|
||||
output_vector_sink(output_vector_sink&& other) noexcept
|
||||
: v(other.v), position(other.position)
|
||||
{
|
||||
other.v = nullptr;
|
||||
}
|
||||
|
||||
~output_vector_sink()
|
||||
{
|
||||
if (v != nullptr)
|
||||
{
|
||||
v->resize(position);
|
||||
}
|
||||
}
|
||||
|
||||
// always inlined: GCC otherwise keeps these out of line in a large
|
||||
// translation unit, and a call per byte or number costs more than the write
|
||||
JSON_HEDLEY_ALWAYS_INLINE
|
||||
void write_character(CharType c)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(position == v->size()))
|
||||
{
|
||||
grow(1);
|
||||
}
|
||||
(*v)[position++] = c;
|
||||
}
|
||||
|
||||
// no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null
|
||||
// pointer with length 0 for empty strings/binary values; memcpy must not
|
||||
// see it.
|
||||
JSON_HEDLEY_ALWAYS_INLINE
|
||||
void write_characters(const CharType* s, std::size_t length)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(v->size() - position < length))
|
||||
{
|
||||
if (length >= max_reserve_size)
|
||||
{
|
||||
append(s, length);
|
||||
return;
|
||||
}
|
||||
grow(length);
|
||||
}
|
||||
if (length != 0)
|
||||
{
|
||||
std::memcpy(v->data() + position, s, length * sizeof(CharType));
|
||||
position += length;
|
||||
}
|
||||
}
|
||||
|
||||
/// @brief space for writing up to @a length characters directly
|
||||
///
|
||||
/// Returns a pointer to at least @a length (at most @ref max_reserve_size)
|
||||
/// writable characters at the write position; @ref commit then keeps the
|
||||
/// first ones of them. A writer can store whole words there and keep only
|
||||
/// the bytes it needs, and keep its write pointer in a register across many
|
||||
/// values instead of updating the sink after each.
|
||||
JSON_HEDLEY_ALWAYS_INLINE
|
||||
CharType* reserve(const std::size_t length)
|
||||
{
|
||||
JSON_ASSERT(length <= max_reserve_size);
|
||||
if (JSON_HEDLEY_UNLIKELY(v->size() - position < length))
|
||||
{
|
||||
grow(length);
|
||||
}
|
||||
return v->data() + position;
|
||||
}
|
||||
|
||||
/// keep the first @a length characters written to the space from reserve()
|
||||
JSON_HEDLEY_ALWAYS_INLINE
|
||||
void commit(const std::size_t length) noexcept
|
||||
{
|
||||
position += length;
|
||||
}
|
||||
|
||||
private:
|
||||
/// make room for at least @a length more characters after the position;
|
||||
/// kept out of line so that the write functions stay small enough for
|
||||
/// GCC to inline them into the writer
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
void grow(const std::size_t length)
|
||||
{
|
||||
// use the capacity the vector already has, else at least double it;
|
||||
// never less than 64 characters, so that a small value does not
|
||||
// reallocate for every few bytes
|
||||
const std::size_t needed = position + length;
|
||||
v->resize((std::max)(std::size_t{64}, needed <= v->capacity()
|
||||
? v->capacity()
|
||||
: (std::max)(needed, v->capacity() * 2)));
|
||||
}
|
||||
|
||||
/// append @a length (at least max_reserve_size) characters that do not fit
|
||||
/// the space ahead of the position: insert() copies them once into the
|
||||
/// grown vector, where grow() would first zero the new space and then copy
|
||||
/// over it - noticeable for a large string or binary value
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
void append(const CharType* s, const std::size_t length)
|
||||
{
|
||||
v->resize(position);
|
||||
if (v->capacity() - position < length)
|
||||
{
|
||||
// grow geometrically, with room for the writes that follow, so
|
||||
// that the next grow() need not reallocate right away
|
||||
v->reserve((std::max)(v->capacity() * 2, position + length + max_reserve_size));
|
||||
}
|
||||
v->insert(v->end(), s, s + length);
|
||||
position = v->size();
|
||||
}
|
||||
|
||||
std::vector<CharType, AllocatorType>* v;
|
||||
/// number of characters of *v written so far
|
||||
std::size_t position;
|
||||
};
|
||||
|
||||
/// output adapter for byte vectors
|
||||
template<typename CharType, typename AllocatorType = std::allocator<CharType>>
|
||||
class output_vector_adapter : public output_adapter_protocol<CharType>
|
||||
{
|
||||
public:
|
||||
explicit output_vector_adapter(std::vector<CharType, AllocatorType>& vec) noexcept
|
||||
: v(vec)
|
||||
{}
|
||||
|
||||
void write_character(CharType c) override
|
||||
void write_character(CharType c)
|
||||
{
|
||||
v.push_back(c);
|
||||
}
|
||||
|
||||
// no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null
|
||||
// pointer with length 0 for empty strings/binary values; appending an empty
|
||||
// range is a no-op
|
||||
void write_characters(const CharType* s, std::size_t length) override
|
||||
// pointer with length 0 for empty strings/binary values. Appending an empty
|
||||
// range is a no-op; the type-erased path tolerates this via the (unattributed)
|
||||
// virtual base, and the concrete sink must do the same.
|
||||
void write_characters(const CharType* s, std::size_t length)
|
||||
{
|
||||
v.insert(v.end(), s, s + length);
|
||||
}
|
||||
@@ -223,6 +82,33 @@ class output_vector_adapter : public output_adapter_protocol<CharType>
|
||||
std::vector<CharType, AllocatorType>& v;
|
||||
};
|
||||
|
||||
/// output adapter for byte vectors
|
||||
///
|
||||
/// The appending itself lives in output_vector_sink; this class only adds the
|
||||
/// virtual output_adapter_protocol interface on top of it, so both the
|
||||
/// type-erased and the templated path share one implementation.
|
||||
template<typename CharType, typename AllocatorType = std::allocator<CharType>>
|
||||
class output_vector_adapter : public output_adapter_protocol<CharType>
|
||||
{
|
||||
public:
|
||||
explicit output_vector_adapter(std::vector<CharType, AllocatorType>& vec) noexcept
|
||||
: sink(vec)
|
||||
{}
|
||||
|
||||
void write_character(CharType c) override
|
||||
{
|
||||
sink.write_character(c);
|
||||
}
|
||||
|
||||
void write_characters(const CharType* s, std::size_t length) override
|
||||
{
|
||||
sink.write_characters(s, length);
|
||||
}
|
||||
|
||||
private:
|
||||
output_vector_sink<CharType, AllocatorType> sink;
|
||||
};
|
||||
|
||||
#ifndef JSON_NO_IO
|
||||
/// output adapter for output streams
|
||||
template<typename CharType>
|
||||
@@ -302,24 +188,8 @@ class output_adapter_sink
|
||||
oa->write_characters(s, length);
|
||||
}
|
||||
|
||||
/// space for writing up to @a length characters (see
|
||||
/// output_vector_sink::reserve)
|
||||
CharType* reserve(const std::size_t length) noexcept
|
||||
{
|
||||
JSON_ASSERT(length <= max_reserve_size);
|
||||
static_cast<void>(length);
|
||||
return reserved.data();
|
||||
}
|
||||
|
||||
/// write the first @a length characters written to the space from reserve()
|
||||
void commit(const std::size_t length)
|
||||
{
|
||||
oa->write_characters(reserved.data(), length);
|
||||
}
|
||||
|
||||
private:
|
||||
output_adapter_t<CharType> oa;
|
||||
std::array<CharType, max_reserve_size> reserved{};
|
||||
};
|
||||
|
||||
/// @brief whether std::basic_string<CharType> has a non-deprecated std::char_traits
|
||||
|
||||
Generated
+110
-486
@@ -21619,8 +21619,6 @@ NLOHMANN_JSON_NAMESPACE_END
|
||||
#include <cstdlib> // _byteswap_ushort, _byteswap_ulong, _byteswap_uint64
|
||||
#endif
|
||||
|
||||
// #include <nlohmann/detail/bit_ops.hpp>
|
||||
|
||||
// #include <nlohmann/detail/input/binary_reader.hpp>
|
||||
|
||||
// #include <nlohmann/detail/input/string_scan.hpp>
|
||||
@@ -21640,10 +21638,7 @@ NLOHMANN_JSON_NAMESPACE_END
|
||||
|
||||
|
||||
|
||||
#include <algorithm> // max
|
||||
#include <array> // array
|
||||
#include <cstddef> // size_t
|
||||
#include <cstring> // memcpy
|
||||
#include <memory> // shared_ptr, make_shared
|
||||
#include <string> // basic_string
|
||||
#include <type_traits> // conditional, integral_constant, is_same
|
||||
@@ -21684,170 +21679,32 @@ template<typename CharType> struct output_adapter_protocol
|
||||
template<typename CharType>
|
||||
using output_adapter_t = std::shared_ptr<output_adapter_protocol<CharType>>;
|
||||
|
||||
/// the largest number of characters a writer may reserve() from a sink at once;
|
||||
/// output_vector_sink also appends writes of at least this size with insert()
|
||||
constexpr std::size_t max_reserve_size = 256;
|
||||
|
||||
/// @brief non-virtual output sink writing into a std::vector
|
||||
///
|
||||
/// This sink is not part of the virtual output_adapter_protocol hierarchy: it is
|
||||
/// passed to binary_writer by value as a template parameter, so
|
||||
/// write_character()/write_characters() are ordinary (inlinable) calls with no
|
||||
/// vtable lookup and no shared_ptr. It is used for the common
|
||||
/// `to_cbor`/`to_msgpack`/... into a std::vector.
|
||||
///
|
||||
/// Most writes are a few bytes - a type marker, or a number of 1, 2, 4, or 8
|
||||
/// bytes - and appending those with push_back() or insert() cost more than
|
||||
/// encoding them: insert() is an out-of-line call into a routine that can insert
|
||||
/// anywhere, and resize() is one as well with libstdc++. The sink therefore keeps
|
||||
/// its own write position: it grows the vector geometrically ahead of the
|
||||
/// position and copies each write to it, so a write is a capacity check and a
|
||||
/// short copy. The destructor shrinks the vector to the bytes actually written,
|
||||
/// also when a write throws; shrinking a vector never allocates. The sink owns
|
||||
/// that trim, so it can be moved but not copied, and the vector must not be used
|
||||
/// while the sink is alive.
|
||||
/// `to_cbor`/`to_msgpack`/... into a std::vector. output_vector_adapter below
|
||||
/// wraps this same sink to provide the virtual interface.
|
||||
template<typename CharType, typename AllocatorType = std::allocator<CharType>>
|
||||
class output_vector_sink
|
||||
{
|
||||
public:
|
||||
explicit output_vector_sink(std::vector<CharType, AllocatorType>& vec) noexcept
|
||||
: v(&vec), position(vec.size())
|
||||
{}
|
||||
|
||||
output_vector_sink(const output_vector_sink&) = delete;
|
||||
output_vector_sink& operator=(const output_vector_sink&) = delete;
|
||||
output_vector_sink& operator=(output_vector_sink&&) = delete;
|
||||
|
||||
output_vector_sink(output_vector_sink&& other) noexcept
|
||||
: v(other.v), position(other.position)
|
||||
{
|
||||
other.v = nullptr;
|
||||
}
|
||||
|
||||
~output_vector_sink()
|
||||
{
|
||||
if (v != nullptr)
|
||||
{
|
||||
v->resize(position);
|
||||
}
|
||||
}
|
||||
|
||||
// always inlined: GCC otherwise keeps these out of line in a large
|
||||
// translation unit, and a call per byte or number costs more than the write
|
||||
JSON_HEDLEY_ALWAYS_INLINE
|
||||
void write_character(CharType c)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(position == v->size()))
|
||||
{
|
||||
grow(1);
|
||||
}
|
||||
(*v)[position++] = c;
|
||||
}
|
||||
|
||||
// no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null
|
||||
// pointer with length 0 for empty strings/binary values; memcpy must not
|
||||
// see it.
|
||||
JSON_HEDLEY_ALWAYS_INLINE
|
||||
void write_characters(const CharType* s, std::size_t length)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(v->size() - position < length))
|
||||
{
|
||||
if (length >= max_reserve_size)
|
||||
{
|
||||
append(s, length);
|
||||
return;
|
||||
}
|
||||
grow(length);
|
||||
}
|
||||
if (length != 0)
|
||||
{
|
||||
std::memcpy(v->data() + position, s, length * sizeof(CharType));
|
||||
position += length;
|
||||
}
|
||||
}
|
||||
|
||||
/// @brief space for writing up to @a length characters directly
|
||||
///
|
||||
/// Returns a pointer to at least @a length (at most @ref max_reserve_size)
|
||||
/// writable characters at the write position; @ref commit then keeps the
|
||||
/// first ones of them. A writer can store whole words there and keep only
|
||||
/// the bytes it needs, and keep its write pointer in a register across many
|
||||
/// values instead of updating the sink after each.
|
||||
JSON_HEDLEY_ALWAYS_INLINE
|
||||
CharType* reserve(const std::size_t length)
|
||||
{
|
||||
JSON_ASSERT(length <= max_reserve_size);
|
||||
if (JSON_HEDLEY_UNLIKELY(v->size() - position < length))
|
||||
{
|
||||
grow(length);
|
||||
}
|
||||
return v->data() + position;
|
||||
}
|
||||
|
||||
/// keep the first @a length characters written to the space from reserve()
|
||||
JSON_HEDLEY_ALWAYS_INLINE
|
||||
void commit(const std::size_t length) noexcept
|
||||
{
|
||||
position += length;
|
||||
}
|
||||
|
||||
private:
|
||||
/// make room for at least @a length more characters after the position;
|
||||
/// kept out of line so that the write functions stay small enough for
|
||||
/// GCC to inline them into the writer
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
void grow(const std::size_t length)
|
||||
{
|
||||
// use the capacity the vector already has, else at least double it;
|
||||
// never less than 64 characters, so that a small value does not
|
||||
// reallocate for every few bytes
|
||||
const std::size_t needed = position + length;
|
||||
v->resize((std::max)(std::size_t{64}, needed <= v->capacity()
|
||||
? v->capacity()
|
||||
: (std::max)(needed, v->capacity() * 2)));
|
||||
}
|
||||
|
||||
/// append @a length (at least max_reserve_size) characters that do not fit
|
||||
/// the space ahead of the position: insert() copies them once into the
|
||||
/// grown vector, where grow() would first zero the new space and then copy
|
||||
/// over it - noticeable for a large string or binary value
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
void append(const CharType* s, const std::size_t length)
|
||||
{
|
||||
v->resize(position);
|
||||
if (v->capacity() - position < length)
|
||||
{
|
||||
// grow geometrically, with room for the writes that follow, so
|
||||
// that the next grow() need not reallocate right away
|
||||
v->reserve((std::max)(v->capacity() * 2, position + length + max_reserve_size));
|
||||
}
|
||||
v->insert(v->end(), s, s + length);
|
||||
position = v->size();
|
||||
}
|
||||
|
||||
std::vector<CharType, AllocatorType>* v;
|
||||
/// number of characters of *v written so far
|
||||
std::size_t position;
|
||||
};
|
||||
|
||||
/// output adapter for byte vectors
|
||||
template<typename CharType, typename AllocatorType = std::allocator<CharType>>
|
||||
class output_vector_adapter : public output_adapter_protocol<CharType>
|
||||
{
|
||||
public:
|
||||
explicit output_vector_adapter(std::vector<CharType, AllocatorType>& vec) noexcept
|
||||
: v(vec)
|
||||
{}
|
||||
|
||||
void write_character(CharType c) override
|
||||
void write_character(CharType c)
|
||||
{
|
||||
v.push_back(c);
|
||||
}
|
||||
|
||||
// no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null
|
||||
// pointer with length 0 for empty strings/binary values; appending an empty
|
||||
// range is a no-op
|
||||
void write_characters(const CharType* s, std::size_t length) override
|
||||
// pointer with length 0 for empty strings/binary values. Appending an empty
|
||||
// range is a no-op; the type-erased path tolerates this via the (unattributed)
|
||||
// virtual base, and the concrete sink must do the same.
|
||||
void write_characters(const CharType* s, std::size_t length)
|
||||
{
|
||||
v.insert(v.end(), s, s + length);
|
||||
}
|
||||
@@ -21856,6 +21713,33 @@ class output_vector_adapter : public output_adapter_protocol<CharType>
|
||||
std::vector<CharType, AllocatorType>& v;
|
||||
};
|
||||
|
||||
/// output adapter for byte vectors
|
||||
///
|
||||
/// The appending itself lives in output_vector_sink; this class only adds the
|
||||
/// virtual output_adapter_protocol interface on top of it, so both the
|
||||
/// type-erased and the templated path share one implementation.
|
||||
template<typename CharType, typename AllocatorType = std::allocator<CharType>>
|
||||
class output_vector_adapter : public output_adapter_protocol<CharType>
|
||||
{
|
||||
public:
|
||||
explicit output_vector_adapter(std::vector<CharType, AllocatorType>& vec) noexcept
|
||||
: sink(vec)
|
||||
{}
|
||||
|
||||
void write_character(CharType c) override
|
||||
{
|
||||
sink.write_character(c);
|
||||
}
|
||||
|
||||
void write_characters(const CharType* s, std::size_t length) override
|
||||
{
|
||||
sink.write_characters(s, length);
|
||||
}
|
||||
|
||||
private:
|
||||
output_vector_sink<CharType, AllocatorType> sink;
|
||||
};
|
||||
|
||||
#ifndef JSON_NO_IO
|
||||
/// output adapter for output streams
|
||||
template<typename CharType>
|
||||
@@ -21935,24 +21819,8 @@ class output_adapter_sink
|
||||
oa->write_characters(s, length);
|
||||
}
|
||||
|
||||
/// space for writing up to @a length characters (see
|
||||
/// output_vector_sink::reserve)
|
||||
CharType* reserve(const std::size_t length) noexcept
|
||||
{
|
||||
JSON_ASSERT(length <= max_reserve_size);
|
||||
static_cast<void>(length);
|
||||
return reserved.data();
|
||||
}
|
||||
|
||||
/// write the first @a length characters written to the space from reserve()
|
||||
void commit(const std::size_t length)
|
||||
{
|
||||
oa->write_characters(reserved.data(), length);
|
||||
}
|
||||
|
||||
private:
|
||||
output_adapter_t<CharType> oa;
|
||||
std::array<CharType, max_reserve_size> reserved{};
|
||||
};
|
||||
|
||||
/// @brief whether std::basic_string<CharType> has a non-deprecated std::char_traits
|
||||
@@ -22237,12 +22105,24 @@ class binary_writer
|
||||
}
|
||||
|
||||
case value_t::number_integer:
|
||||
{
|
||||
if (j.m_data.m_value.number_integer >= 0)
|
||||
{
|
||||
// CBOR does not differentiate between positive signed
|
||||
// integers and unsigned integers
|
||||
write_cbor_head(0x00, static_cast<std::uint64_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else
|
||||
{
|
||||
// a negative integer n is encoded as -1 - n
|
||||
write_cbor_head(0x20, static_cast<std::uint64_t>(-1 - j.m_data.m_value.number_integer));
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case value_t::number_unsigned:
|
||||
{
|
||||
// one case for both, so that an array mixing negative
|
||||
// (number_integer) and non-negative (number_unsigned) values
|
||||
// does not mispredict the switch
|
||||
write_cbor_integer(j);
|
||||
write_cbor_head(0x00, j.m_data.m_value.number_unsigned);
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -22277,7 +22157,13 @@ class binary_writer
|
||||
|
||||
case value_t::array:
|
||||
{
|
||||
write_cbor_array(j, depth);
|
||||
// step 1: write control byte and the array size
|
||||
write_cbor_head(0x80, j.m_data.m_value.array->size());
|
||||
|
||||
for (const auto& el : *j.m_data.m_value.array)
|
||||
{
|
||||
write_cbor(el, depth + 1);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -22435,12 +22321,54 @@ class binary_writer
|
||||
}
|
||||
|
||||
case value_t::number_integer:
|
||||
{
|
||||
if (j.m_data.m_value.number_integer >= 0)
|
||||
{
|
||||
// MessagePack does not differentiate between positive
|
||||
// signed integers and unsigned integers.
|
||||
write_msgpack_unsigned(static_cast<std::uint64_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else
|
||||
{
|
||||
if (j.m_data.m_value.number_integer >= -32)
|
||||
{
|
||||
// negative fixnum
|
||||
write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int8_t>::min)() &&
|
||||
j.m_data.m_value.number_integer <= (std::numeric_limits<std::int8_t>::max)())
|
||||
{
|
||||
// int 8
|
||||
oa.write_character(to_char_type(0xD0));
|
||||
write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int16_t>::min)() &&
|
||||
j.m_data.m_value.number_integer <= (std::numeric_limits<std::int16_t>::max)())
|
||||
{
|
||||
// int 16
|
||||
oa.write_character(to_char_type(0xD1));
|
||||
write_number(static_cast<std::int16_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int32_t>::min)() &&
|
||||
j.m_data.m_value.number_integer <= (std::numeric_limits<std::int32_t>::max)())
|
||||
{
|
||||
// int 32
|
||||
oa.write_character(to_char_type(0xD2));
|
||||
write_number(static_cast<std::int32_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else
|
||||
{
|
||||
// int 64
|
||||
oa.write_character(to_char_type(0xD3));
|
||||
write_number(static_cast<std::int64_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case value_t::number_unsigned:
|
||||
{
|
||||
// one case for both, so that an array mixing negative
|
||||
// (number_integer) and non-negative (number_unsigned) values
|
||||
// does not mispredict the switch
|
||||
write_msgpack_integer(j, std::integral_constant<bool, msgpack_integers_fit_64_bits()>());
|
||||
write_msgpack_unsigned(static_cast<std::uint64_t>(j.m_data.m_value.number_unsigned));
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -22458,7 +22386,13 @@ class binary_writer
|
||||
|
||||
case value_t::array:
|
||||
{
|
||||
write_msgpack_array(j, depth);
|
||||
// step 1: write control byte and the array size
|
||||
write_msgpack_array_prefix(j.m_data.m_value.array->size(), j);
|
||||
|
||||
for (const auto& el : *j.m_data.m_value.array)
|
||||
{
|
||||
write_msgpack(el, depth + 1);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -23918,316 +23852,6 @@ class binary_writer
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief size class of an unsigned integer from its bit length
|
||||
|
||||
1, 2, 3, or 4 if @a x fits 1, 2, 4, or 8 bytes. Looked up from the number
|
||||
of leading zero bits instead of found by branches: the sizes of a stream
|
||||
of numbers are hard to predict, and a mispredicted branch costs more than
|
||||
writing the number. (A table, as comparisons are vectorized by clang into
|
||||
slower code.)
|
||||
*/
|
||||
static unsigned integer_size_class(const std::uint64_t x) noexcept
|
||||
{
|
||||
static constexpr std::array<std::uint8_t, 64> size_class_by_leading_zeros =
|
||||
{
|
||||
{
|
||||
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
2, 2, 2, 2, 2, 2, 2, 2,
|
||||
1, 1, 1, 1, 1, 1, 1, 1
|
||||
}
|
||||
};
|
||||
// x | 1 keeps count_leading_zeros() away from 0
|
||||
return size_class_by_leading_zeros[static_cast<std::size_t>(count_leading_zeros(x | 1u))];
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief encode @a first_byte and the low-order bytes of @a value that
|
||||
@a size_class calls for (none, or 1, 2, 4, or 8) in network order
|
||||
to @a out
|
||||
@return the number of characters of the encoding; up to
|
||||
@ref integer_encoding_space characters at @a out are overwritten
|
||||
|
||||
The bytes are written as one byte and one 8-byte word whose bytes after
|
||||
the value are scratch that the caller drops, so that writing them does not
|
||||
depend on the size class. For class 0 the word is all scratch; it is
|
||||
shifted by 60 bits rather than 64, which would be undefined.
|
||||
*/
|
||||
std::size_t encode_first_byte_and_tail(CharType* out, const std::uint8_t first_byte, const unsigned size_class, const std::uint64_t value) const noexcept
|
||||
{
|
||||
out[0] = to_char_type(first_byte);
|
||||
const unsigned unused_bits = 64u - ((8u << size_class) >> 1u);
|
||||
const std::uint64_t tail = is_little_endian ? byte_swap(value) >> unused_bits : value << unused_bits;
|
||||
std::memcpy(out + 1, &tail, sizeof(tail));
|
||||
return ((std::size_t{1} << size_class) >> 1u) + 1;
|
||||
}
|
||||
|
||||
/// characters encode_first_byte_and_tail() may overwrite: the first byte
|
||||
/// and one 8-byte word
|
||||
static constexpr std::size_t integer_encoding_space = 9;
|
||||
|
||||
/*!
|
||||
@brief the 64 bits of a number_integer or number_unsigned value, and a mask
|
||||
that is all ones if it is negative, else zero
|
||||
|
||||
Selected through masks rather than branches, as the type and the sign of
|
||||
the values in an array of mixed signs alternate unpredictably (non-negative
|
||||
integers are number_unsigned after parsing). Up to 64 bits, the sign is bit
|
||||
63 of the result (a narrower integer is sign-extended); computing it with
|
||||
masks keeps GCC from branching on the type. A wider integer has to be
|
||||
compared.
|
||||
*/
|
||||
static std::uint64_t integer_bits(const BasicJsonType& j, std::uint64_t& sign_mask) noexcept
|
||||
{
|
||||
const bool is_signed = j.m_data.m_type == value_t::number_integer;
|
||||
const std::uint64_t raw = is_signed
|
||||
? static_cast<std::uint64_t>(j.m_data.m_value.number_integer)
|
||||
: static_cast<std::uint64_t>(j.m_data.m_value.number_unsigned);
|
||||
sign_mask = sizeof(typename BasicJsonType::number_integer_t) <= sizeof(std::uint64_t)
|
||||
? (std::uint64_t{0} - static_cast<std::uint64_t>(is_signed)) & (std::uint64_t{0} - (raw >> 63u))
|
||||
: std::uint64_t{0} - static_cast<std::uint64_t>(is_signed && j.m_data.m_value.number_integer < 0);
|
||||
return raw;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief encode the head of a CBOR integer to @a out without branching on its
|
||||
size
|
||||
@return the number of characters of the head
|
||||
|
||||
Same output as @ref write_cbor_head. The lengths of strings and containers
|
||||
are mostly predictable, so their heads keep the branches; the sizes of a
|
||||
run of integers are not, so an integer's head is assembled without them.
|
||||
*/
|
||||
std::size_t encode_cbor_integer_head(CharType* out, const std::uint8_t major_type, const std::uint64_t argument) const noexcept
|
||||
{
|
||||
// an argument of at most 23 is in size class 1 (one byte) and is moved
|
||||
// to class 0, the first byte, by subtracting 1
|
||||
const auto is_immediate = static_cast<unsigned>(argument <= 0x17u);
|
||||
const unsigned size_class = integer_size_class(argument) - is_immediate;
|
||||
|
||||
// the first byte: the argument itself if it is immediate, else
|
||||
// 0x17 + size_class, selected through a mask
|
||||
const std::uint64_t immediate_mask = std::uint64_t{0} - is_immediate;
|
||||
const auto first_byte = static_cast<std::uint8_t>(major_type + ((argument & immediate_mask) | ((0x17u + size_class) & ~immediate_mask)));
|
||||
return encode_first_byte_and_tail(out, first_byte, size_class, argument);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief write an array value (head and elements) as CBOR
|
||||
|
||||
Integer elements, the most common scalar, are written without a call to
|
||||
@ref write_cbor. The loop is kept out of line: inlined into write_cbor, its
|
||||
registers would be saved and restored on every call of write_cbor, which
|
||||
is one call per value nested in an object.
|
||||
*/
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
void write_cbor_array(const BasicJsonType& j, const std::size_t depth)
|
||||
{
|
||||
write_cbor_head(0x80, j.m_data.m_value.array->size());
|
||||
write_array_elements(j, [this](CharType * out, const BasicJsonType & el)
|
||||
{
|
||||
return encode_cbor_integer(out, el);
|
||||
}, [this, depth](const BasicJsonType & el)
|
||||
{
|
||||
write_cbor(el, depth + 1);
|
||||
});
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief write the elements of the array @a j, runs of integers with
|
||||
@a encode_integer and all other values with @a write_value
|
||||
|
||||
Runs of integers are encoded straight into space reserved from the sink
|
||||
for a whole run: the write pointer stays a local, where it would otherwise
|
||||
be reloaded from the sink for every value, as a store of a character may
|
||||
alias it.
|
||||
*/
|
||||
template<typename EncodeInteger, typename WriteValue>
|
||||
void write_array_elements(const BasicJsonType& j, EncodeInteger encode_integer, WriteValue write_value)
|
||||
{
|
||||
constexpr std::size_t max_run = max_reserve_size / integer_encoding_space;
|
||||
auto it = j.m_data.m_value.array->begin();
|
||||
const auto end = j.m_data.m_value.array->end();
|
||||
while (it != end)
|
||||
{
|
||||
if (it->is_number_integer())
|
||||
{
|
||||
CharType* const start = oa.reserve(max_run * integer_encoding_space);
|
||||
CharType* out = start;
|
||||
for (std::size_t n = 0; n < max_run && it != end && it->is_number_integer(); ++n, ++it)
|
||||
{
|
||||
out += encode_integer(out, *it);
|
||||
}
|
||||
oa.commit(static_cast<std::size_t>(out - start));
|
||||
}
|
||||
else
|
||||
{
|
||||
write_value(*it);
|
||||
++it;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief write a number_integer or number_unsigned value as a CBOR integer
|
||||
|
||||
A negative integer n has major type 1 and the argument -1 - n, which is ~n.
|
||||
Kept out of line, like the other per-value paths, so that write_cbor stays
|
||||
small (runs of integers in an array are encoded inline instead).
|
||||
*/
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
void write_cbor_integer(const BasicJsonType& j)
|
||||
{
|
||||
oa.commit(encode_cbor_integer(oa.reserve(integer_encoding_space), j));
|
||||
}
|
||||
|
||||
/// encode a number_integer or number_unsigned value to @a out (see
|
||||
/// @ref write_cbor_integer) and return the number of characters
|
||||
std::size_t encode_cbor_integer(CharType* out, const BasicJsonType& j) const noexcept
|
||||
{
|
||||
std::uint64_t sign_mask = 0;
|
||||
const std::uint64_t raw = integer_bits(j, sign_mask);
|
||||
// ~raw is -1 - n in the 64 bits that are written
|
||||
return encode_cbor_integer_head(out, static_cast<std::uint8_t>(sign_mask & 0x20u), raw ^ sign_mask);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief whether the MessagePack integers of BasicJsonType are written by
|
||||
@ref encode_msgpack_integer
|
||||
|
||||
It picks the format from the low 64 bits; a wider number type keeps the
|
||||
range checks of @ref write_msgpack on the whole value.
|
||||
*/
|
||||
static constexpr bool msgpack_integers_fit_64_bits() noexcept
|
||||
{
|
||||
return sizeof(typename BasicJsonType::number_integer_t) <= sizeof(std::uint64_t)
|
||||
&& sizeof(typename BasicJsonType::number_unsigned_t) <= sizeof(std::uint64_t);
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief write a number_integer or number_unsigned value as a MessagePack
|
||||
integer without branching on its sign or size
|
||||
|
||||
Same output as @ref write_msgpack: a non-negative integer as positive fixint
|
||||
(< 128) or uint 8/16/32/64, a negative one as negative fixint (>= -32) or
|
||||
int 8/16/32/64. Kept out of line (see @ref write_cbor_integer).
|
||||
*/
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
void write_msgpack_integer(const BasicJsonType& j, std::true_type /*fits_64_bits*/)
|
||||
{
|
||||
oa.commit(encode_msgpack_integer(oa.reserve(integer_encoding_space), j));
|
||||
}
|
||||
|
||||
/// encode a number_integer or number_unsigned value to @a out (see
|
||||
/// @ref write_msgpack_integer) and return the number of characters
|
||||
std::size_t encode_msgpack_integer(CharType* out, const BasicJsonType& j) const noexcept
|
||||
{
|
||||
std::uint64_t sign_mask = 0;
|
||||
const std::uint64_t raw = integer_bits(j, sign_mask);
|
||||
|
||||
// A negative n fits int 8/16/32 exactly if 2 * (-1 - n) + 1 fits
|
||||
// uint 8/16/32, so x gives the size class of either sign.
|
||||
const std::uint64_t is_negative = sign_mask & 1u;
|
||||
const std::uint64_t x = ((raw ^ sign_mask) << is_negative) | is_negative;
|
||||
|
||||
// A fixint is at most 127, or at least -32 (x at most 63). It is in
|
||||
// size class 1 and is moved to class 0, the first byte, by subtracting 1.
|
||||
const auto is_fixint = static_cast<unsigned>(x <= (127u ^ (sign_mask & 64u)));
|
||||
const unsigned size_class = integer_size_class(x) - is_fixint;
|
||||
|
||||
// the first byte: the low byte of the integer for a fixint, else the
|
||||
// marker 0xCC..0xCF (uint) or 0xD0..0xD3 (int), selected through a mask
|
||||
const std::uint64_t fixint_mask = std::uint64_t{0} - is_fixint;
|
||||
const std::uint64_t marker = 0xCBu + size_class + (sign_mask & 4u);
|
||||
const auto first_byte = static_cast<std::uint8_t>((raw & fixint_mask) | (marker & ~fixint_mask));
|
||||
return encode_first_byte_and_tail(out, first_byte, size_class, raw);
|
||||
}
|
||||
|
||||
/// write a number_integer or number_unsigned value of a number type wider
|
||||
/// than 64 bits as a MessagePack integer, with range checks on the whole value
|
||||
void write_msgpack_integer(const BasicJsonType& j, std::false_type /*fits_64_bits*/)
|
||||
{
|
||||
if (j.m_data.m_type == value_t::number_unsigned)
|
||||
{
|
||||
write_msgpack_unsigned(static_cast<std::uint64_t>(j.m_data.m_value.number_unsigned));
|
||||
return;
|
||||
}
|
||||
|
||||
if (j.m_data.m_value.number_integer >= 0)
|
||||
{
|
||||
// MessagePack does not differentiate between positive
|
||||
// signed integers and unsigned integers.
|
||||
write_msgpack_unsigned(static_cast<std::uint64_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else
|
||||
{
|
||||
if (j.m_data.m_value.number_integer >= -32)
|
||||
{
|
||||
// negative fixnum
|
||||
write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int8_t>::min)() &&
|
||||
j.m_data.m_value.number_integer <= (std::numeric_limits<std::int8_t>::max)())
|
||||
{
|
||||
// int 8
|
||||
oa.write_character(to_char_type(0xD0));
|
||||
write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int16_t>::min)() &&
|
||||
j.m_data.m_value.number_integer <= (std::numeric_limits<std::int16_t>::max)())
|
||||
{
|
||||
// int 16
|
||||
oa.write_character(to_char_type(0xD1));
|
||||
write_number(static_cast<std::int16_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int32_t>::min)() &&
|
||||
j.m_data.m_value.number_integer <= (std::numeric_limits<std::int32_t>::max)())
|
||||
{
|
||||
// int 32
|
||||
oa.write_character(to_char_type(0xD2));
|
||||
write_number(static_cast<std::int32_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
else
|
||||
{
|
||||
// int 64
|
||||
oa.write_character(to_char_type(0xD3));
|
||||
write_number(static_cast<std::int64_t>(j.m_data.m_value.number_integer));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief write an array value (head and elements) as MessagePack
|
||||
|
||||
See @ref write_cbor_array.
|
||||
*/
|
||||
JSON_HEDLEY_NEVER_INLINE
|
||||
void write_msgpack_array(const BasicJsonType& j, const std::size_t depth)
|
||||
{
|
||||
write_msgpack_array_prefix(j.m_data.m_value.array->size(), j);
|
||||
write_msgpack_array_elements(j, depth, std::integral_constant<bool, msgpack_integers_fit_64_bits()>());
|
||||
}
|
||||
|
||||
void write_msgpack_array_elements(const BasicJsonType& j, const std::size_t depth, std::true_type /*fits_64_bits*/)
|
||||
{
|
||||
write_array_elements(j, [this](CharType * out, const BasicJsonType & el)
|
||||
{
|
||||
return encode_msgpack_integer(out, el);
|
||||
}, [this, depth](const BasicJsonType & el)
|
||||
{
|
||||
write_msgpack(el, depth + 1);
|
||||
});
|
||||
}
|
||||
|
||||
void write_msgpack_array_elements(const BasicJsonType& j, const std::size_t depth, std::false_type /*fits_64_bits*/)
|
||||
{
|
||||
for (const auto& el : *j.m_data.m_value.array)
|
||||
{
|
||||
write_msgpack(el, depth + 1);
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief write an object key as a CBOR text string
|
||||
|
||||
|
||||
+10
-1
@@ -9,7 +9,16 @@ Each fuzzer driver (`tests/src/fuzzer-parse_*.cpp`) parses its input twice: once
|
||||
once with exceptions. Both calls must agree. Where parsing with exceptions fails, the call without exceptions must
|
||||
return a discarded value (or throw the same kind of non-parse error), and it must never throw a `parse_error`. Where
|
||||
parsing succeeds, both calls must return the same value. The drivers then serialize the value, parse the result back,
|
||||
and check that nothing was lost. The drivers check all of this with `assert`, so they refuse to build with `NDEBUG`.
|
||||
and check that nothing was lost.
|
||||
|
||||
Finally, each driver reads the same input through the other input adapters: a `std::istringstream`, a non-contiguous
|
||||
iterator range (`std::list<std::uint8_t>`), and, on POSIX systems, a `FILE*` from `fmemopen`. The pointer range takes the
|
||||
fast paths for contiguous input (bulk string scanning, multi-byte loads, lazy position tracking), while the other
|
||||
adapters read byte by byte. Every adapter must give the same value, or the same exception with the same id, message,
|
||||
and byte position. The JSON driver checks this for `parse` and `accept`, each with and without `ignore_comments`. The
|
||||
shared code is `check_input_adapters()` in `tests/src/fuzzer_common.hpp`.
|
||||
|
||||
The drivers check all of this with `assert`, so they refuse to build with `NDEBUG`.
|
||||
|
||||
## Corpus creation
|
||||
|
||||
|
||||
@@ -47,12 +47,30 @@ dumps is stable under exactly the same values that break operator==.
|
||||
The unit tests run the same checks on a fixed corpus (see the "BJData round-trip
|
||||
invariants" test case), so keep both in sync.
|
||||
|
||||
Finally, it reads the data from a stream, a non-contiguous iterator range, and
|
||||
a FILE*, which read byte by byte, and checks that each gives the same value or
|
||||
error (id, message, byte position) as the pointer range, which takes the bulk
|
||||
fast paths (see check_input_adapters()).
|
||||
|
||||
The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
|
||||
drivers.
|
||||
*/
|
||||
|
||||
#include "fuzzer_common.hpp"
|
||||
|
||||
namespace
|
||||
{
|
||||
// json::from_bjdata for check_input_adapters()
|
||||
struct read_bjdata
|
||||
{
|
||||
template<typename... Args>
|
||||
json operator()(Args&& ... args) const
|
||||
{
|
||||
return json::from_bjdata(std::forward<Args>(args)...);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
// value-stable comparison for the round-trip checks below; see the note
|
||||
// above on why this compares dump()s rather than the json values directly
|
||||
static bool is_value_stable(const json& lhs, const json& rhs)
|
||||
@@ -126,6 +144,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
assert(parsed || noexcept_threw || j_noexcept.is_discarded());
|
||||
}
|
||||
|
||||
// step 3: the other input adapters must read the data alike
|
||||
check_input_adapters(data, size, read_bjdata());
|
||||
|
||||
// return 0 - non-zero return values are reserved for future use
|
||||
return 0;
|
||||
}
|
||||
@@ -17,44 +17,33 @@ array data, it performs the following steps:
|
||||
- j2 = from_bon8(vec)
|
||||
- assert(to_bon8(j2) == vec)
|
||||
|
||||
It also checks that reading the data from a stream, which reads strings byte by
|
||||
byte, gives the same value or error as reading it from contiguous memory, which
|
||||
copies strings in bulk.
|
||||
Finally, it reads the data from a stream, a non-contiguous iterator range, and
|
||||
a FILE*, which read byte by byte, and checks that each gives the same value or
|
||||
error (id, message, byte position) as the pointer range, which takes the bulk
|
||||
fast paths (see check_input_adapters()).
|
||||
|
||||
The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
|
||||
drivers.
|
||||
*/
|
||||
|
||||
#include <sstream>
|
||||
#include "fuzzer_common.hpp"
|
||||
|
||||
namespace
|
||||
{
|
||||
// the serialization of the value read from @a input, or the error message
|
||||
template<typename InputType>
|
||||
std::string read_bon8(InputType&& input)
|
||||
// json::from_bon8 for check_input_adapters()
|
||||
struct read_bon8
|
||||
{
|
||||
try
|
||||
template<typename... Args>
|
||||
json operator()(Args&& ... args) const
|
||||
{
|
||||
const auto vec = json::to_bon8(json::from_bon8(std::forward<InputType>(input)));
|
||||
return {vec.begin(), vec.end()};
|
||||
return json::from_bon8(std::forward<Args>(args)...);
|
||||
}
|
||||
catch (const json::exception& e)
|
||||
{
|
||||
return e.what();
|
||||
}
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
// see http://llvm.org/docs/LibFuzzer.html
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
{
|
||||
// contiguous and stream input must be read alike
|
||||
{
|
||||
std::istringstream stream(std::string(reinterpret_cast<const char*>(data), size));
|
||||
assert(read_bon8(std::vector<uint8_t>(data, data + size)) == read_bon8(stream));
|
||||
}
|
||||
|
||||
std::vector<uint8_t> const vec1(data, data + size);
|
||||
|
||||
// step 0: parse input without exceptions
|
||||
@@ -106,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
assert(parsed || noexcept_threw || j_noexcept.is_discarded());
|
||||
}
|
||||
|
||||
// step 3: the other input adapters must read the data alike
|
||||
check_input_adapters(data, size, read_bon8());
|
||||
|
||||
// return 0 - non-zero return values are reserved for future use
|
||||
return 0;
|
||||
}
|
||||
@@ -17,12 +17,30 @@ array data, it performs the following steps:
|
||||
- j2 = from_bson(vec)
|
||||
- assert(to_bson(j2) == vec)
|
||||
|
||||
Finally, it reads the data from a stream, a non-contiguous iterator range, and
|
||||
a FILE*, which read byte by byte, and checks that each gives the same value or
|
||||
error (id, message, byte position) as the pointer range, which takes the bulk
|
||||
fast paths (see check_input_adapters()).
|
||||
|
||||
The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
|
||||
drivers.
|
||||
*/
|
||||
|
||||
#include "fuzzer_common.hpp"
|
||||
|
||||
namespace
|
||||
{
|
||||
// json::from_bson for check_input_adapters()
|
||||
struct read_bson
|
||||
{
|
||||
template<typename... Args>
|
||||
json operator()(Args&& ... args) const
|
||||
{
|
||||
return json::from_bson(std::forward<Args>(args)...);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
// see http://llvm.org/docs/LibFuzzer.html
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
{
|
||||
@@ -77,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
assert(parsed || noexcept_threw || j_noexcept.is_discarded());
|
||||
}
|
||||
|
||||
// step 3: the other input adapters must read the data alike
|
||||
check_input_adapters(data, size, read_bson());
|
||||
|
||||
// return 0 - non-zero return values are reserved for future use
|
||||
return 0;
|
||||
}
|
||||
@@ -17,12 +17,30 @@ array data, it performs the following steps:
|
||||
- j2 = from_cbor(vec)
|
||||
- assert(to_cbor(j2) == vec)
|
||||
|
||||
Finally, it reads the data from a stream, a non-contiguous iterator range, and
|
||||
a FILE*, which read byte by byte, and checks that each gives the same value or
|
||||
error (id, message, byte position) as the pointer range, which takes the bulk
|
||||
fast paths (see check_input_adapters()).
|
||||
|
||||
The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
|
||||
drivers.
|
||||
*/
|
||||
|
||||
#include "fuzzer_common.hpp"
|
||||
|
||||
namespace
|
||||
{
|
||||
// json::from_cbor for check_input_adapters()
|
||||
struct read_cbor
|
||||
{
|
||||
template<typename... Args>
|
||||
json operator()(Args&& ... args) const
|
||||
{
|
||||
return json::from_cbor(std::forward<Args>(args)...);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
// see http://llvm.org/docs/LibFuzzer.html
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
{
|
||||
@@ -77,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
assert(parsed || noexcept_threw || j_noexcept.is_discarded());
|
||||
}
|
||||
|
||||
// step 3: the other input adapters must read the data alike
|
||||
check_input_adapters(data, size, read_cbor());
|
||||
|
||||
// return 0 - non-zero return values are reserved for future use
|
||||
return 0;
|
||||
}
|
||||
@@ -17,6 +17,12 @@ array data, it performs the following steps:
|
||||
- j2 = parse(s1)
|
||||
- s2 = serialize(j2)
|
||||
- assert(s1 == s2)
|
||||
- for ignore_comments = false and true: assert(parse(data) and accept(data)
|
||||
give the same result for a stream, a non-contiguous iterator range, and a
|
||||
FILE* as for the pointer range)
|
||||
|
||||
The pointer range takes the bulk fast paths, the other input adapters read byte
|
||||
by byte; see check_input_adapters().
|
||||
|
||||
The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
|
||||
drivers.
|
||||
@@ -24,6 +30,32 @@ drivers.
|
||||
|
||||
#include "fuzzer_common.hpp"
|
||||
|
||||
namespace
|
||||
{
|
||||
// json::parse and json::accept for check_input_adapters()
|
||||
struct parse_json
|
||||
{
|
||||
bool ignore_comments;
|
||||
|
||||
template<typename... Args>
|
||||
json operator()(Args&& ... args) const
|
||||
{
|
||||
return json::parse(std::forward<Args>(args)..., nullptr, true, ignore_comments);
|
||||
}
|
||||
};
|
||||
|
||||
struct accept_json
|
||||
{
|
||||
bool ignore_comments;
|
||||
|
||||
template<typename... Args>
|
||||
bool operator()(Args&& ... args) const
|
||||
{
|
||||
return json::accept(std::forward<Args>(args)..., ignore_comments);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
// see http://llvm.org/docs/LibFuzzer.html
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
{
|
||||
@@ -76,6 +108,16 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
assert(parsed || noexcept_threw || j_noexcept.is_discarded());
|
||||
}
|
||||
|
||||
// step 3: the other input adapters must read the data alike
|
||||
for (const bool ignore_comments :
|
||||
{
|
||||
false, true
|
||||
})
|
||||
{
|
||||
check_input_adapters(data, size, parse_json{ignore_comments});
|
||||
check_input_adapters(data, size, accept_json{ignore_comments});
|
||||
}
|
||||
|
||||
// return 0 - non-zero return values are reserved for future use
|
||||
return 0;
|
||||
}
|
||||
@@ -17,12 +17,30 @@ array data, it performs the following steps:
|
||||
- j2 = from_msgpack(vec)
|
||||
- assert(to_msgpack(j2) == vec)
|
||||
|
||||
Finally, it reads the data from a stream, a non-contiguous iterator range, and
|
||||
a FILE*, which read byte by byte, and checks that each gives the same value or
|
||||
error (id, message, byte position) as the pointer range, which takes the bulk
|
||||
fast paths (see check_input_adapters()).
|
||||
|
||||
The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
|
||||
drivers.
|
||||
*/
|
||||
|
||||
#include "fuzzer_common.hpp"
|
||||
|
||||
namespace
|
||||
{
|
||||
// json::from_msgpack for check_input_adapters()
|
||||
struct read_msgpack
|
||||
{
|
||||
template<typename... Args>
|
||||
json operator()(Args&& ... args) const
|
||||
{
|
||||
return json::from_msgpack(std::forward<Args>(args)...);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
// see http://llvm.org/docs/LibFuzzer.html
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
{
|
||||
@@ -77,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
assert(parsed || noexcept_threw || j_noexcept.is_discarded());
|
||||
}
|
||||
|
||||
// step 3: the other input adapters must read the data alike
|
||||
check_input_adapters(data, size, read_msgpack());
|
||||
|
||||
// return 0 - non-zero return values are reserved for future use
|
||||
return 0;
|
||||
}
|
||||
@@ -26,12 +26,30 @@ array data, it performs the following steps:
|
||||
The unit tests run the same checks on a fixed corpus (see the "UBJSON round-trip
|
||||
invariants" test case), so keep both in sync.
|
||||
|
||||
Finally, it reads the data from a stream, a non-contiguous iterator range, and
|
||||
a FILE*, which read byte by byte, and checks that each gives the same value or
|
||||
error (id, message, byte position) as the pointer range, which takes the bulk
|
||||
fast paths (see check_input_adapters()).
|
||||
|
||||
The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
|
||||
drivers.
|
||||
*/
|
||||
|
||||
#include "fuzzer_common.hpp"
|
||||
|
||||
namespace
|
||||
{
|
||||
// json::from_ubjson for check_input_adapters()
|
||||
struct read_ubjson
|
||||
{
|
||||
template<typename... Args>
|
||||
json operator()(Args&& ... args) const
|
||||
{
|
||||
return json::from_ubjson(std::forward<Args>(args)...);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
// see http://llvm.org/docs/LibFuzzer.html
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
{
|
||||
@@ -96,6 +114,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
assert(parsed || noexcept_threw || j_noexcept.is_discarded());
|
||||
}
|
||||
|
||||
// step 3: the other input adapters must read the data alike
|
||||
check_input_adapters(data, size, read_ubjson());
|
||||
|
||||
// return 0 - non-zero return values are reserved for future use
|
||||
return 0;
|
||||
}
|
||||
@@ -11,6 +11,14 @@
|
||||
#pragma once
|
||||
|
||||
#include <cassert> // assert
|
||||
#include <cstddef> // size_t
|
||||
#include <cstdint> // uint8_t
|
||||
#include <cstdio> // FILE, fmemopen, fclose
|
||||
#include <list> // list
|
||||
#include <sstream> // istringstream
|
||||
#include <string> // string
|
||||
#include <utility> // forward
|
||||
#include <vector> // vector
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
using nlohmann::json; // NOLINT(google-global-names-in-headers): shared by all fuzzer drivers
|
||||
@@ -49,3 +57,89 @@ json parse_without_exceptions(Parse parse, bool& threw)
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
// the outcome of reading an input with exceptions: the serialization of the
|
||||
// value, or the id, message, and (for parse errors) byte of the exception
|
||||
struct read_outcome
|
||||
{
|
||||
std::string value{};
|
||||
int id = 0;
|
||||
std::string what{};
|
||||
std::size_t byte = 0;
|
||||
|
||||
bool operator==(const read_outcome& rhs) const
|
||||
{
|
||||
return value == rhs.value && id == rhs.id && what == rhs.what && byte == rhs.byte;
|
||||
}
|
||||
};
|
||||
|
||||
inline std::string outcome_value(const json& j)
|
||||
{
|
||||
return j.dump(-1, ' ', false, json::error_handler_t::keep);
|
||||
}
|
||||
|
||||
inline std::string outcome_value(bool b)
|
||||
{
|
||||
return b ? "true" : "false";
|
||||
}
|
||||
|
||||
template<typename Read, typename... Args>
|
||||
read_outcome read_with(const Read& parse, Args&& ... args)
|
||||
{
|
||||
read_outcome result;
|
||||
try
|
||||
{
|
||||
result.value = outcome_value(parse(std::forward<Args>(args)...));
|
||||
}
|
||||
catch (const json::parse_error& e)
|
||||
{
|
||||
result.id = e.id;
|
||||
result.what = e.what();
|
||||
result.byte = e.byte;
|
||||
}
|
||||
catch (const json::exception& e)
|
||||
{
|
||||
result.id = e.id;
|
||||
result.what = e.what();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// The contiguous input (a pointer range) takes the bulk fast paths; the other
|
||||
// input adapters read byte by byte. Reads @a data through each of them and
|
||||
// checks that all give the same value, or the same exception with the same
|
||||
// id, message, and byte position.
|
||||
//
|
||||
// @a parse is called with either an iterator range or a single input (stream
|
||||
// or FILE*) and returns the parsed value (or the result of accept()).
|
||||
template<typename Read>
|
||||
void check_input_adapters(const std::uint8_t* data, std::size_t size, const Read& parse)
|
||||
{
|
||||
const read_outcome expected = read_with(parse, data, data + size);
|
||||
|
||||
// stream adapter
|
||||
{
|
||||
std::istringstream stream(std::string(reinterpret_cast<const char*>(data), size));
|
||||
assert(read_with(parse, stream) == expected);
|
||||
}
|
||||
|
||||
// iterator adapter on a non-contiguous range
|
||||
{
|
||||
const std::list<std::uint8_t> list(data, data + size);
|
||||
assert(read_with(parse, list.begin(), list.end()) == expected);
|
||||
}
|
||||
|
||||
#if defined(__unix__) || defined(__APPLE__)
|
||||
// FILE* adapter; fmemopen may refuse an empty buffer
|
||||
{
|
||||
std::vector<std::uint8_t> buffer(data, data + size);
|
||||
std::FILE* file = buffer.empty() ? nullptr : fmemopen(buffer.data(), buffer.size(), "rb");
|
||||
if (file != nullptr)
|
||||
{
|
||||
const read_outcome actual = read_with(parse, file);
|
||||
std::fclose(file);
|
||||
assert(actual == expected);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@@ -13,7 +13,6 @@ using nlohmann::json;
|
||||
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
@@ -36,17 +35,6 @@ std::vector<json> test_values()
|
||||
big_object[std::to_string(i)] = i;
|
||||
}
|
||||
|
||||
// large strings and binary values between small ones: a write too large
|
||||
// for the space the vector sink has ahead takes a different path than a
|
||||
// small one, and the small writes after it must still land behind it
|
||||
json large_payloads = json::array();
|
||||
for (int i = 0; i < 50; ++i)
|
||||
{
|
||||
large_payloads.push_back(std::string(300, static_cast<char>('a' + (i % 26))));
|
||||
large_payloads.push_back(i);
|
||||
large_payloads.push_back(json::binary(std::vector<std::uint8_t>(static_cast<std::size_t>(100 * i), static_cast<std::uint8_t>(i))));
|
||||
}
|
||||
|
||||
return
|
||||
{
|
||||
json(nullptr), json(true), json(false),
|
||||
@@ -58,8 +46,7 @@ std::vector<json> test_values()
|
||||
json::array(), json::object(),
|
||||
json::array({1, 2, 3}), json({{"a", 1}, {"b", nullptr}}),
|
||||
json({{"nested", {{"deep", json::array({1, "two", 3.0, nullptr})}}}}),
|
||||
big_array, big_object, large_payloads,
|
||||
json::binary(std::vector<std::uint8_t>(5000, 0x42))
|
||||
big_array, big_object
|
||||
};
|
||||
}
|
||||
|
||||
@@ -183,256 +170,6 @@ TEST_CASE("binary writer output sinks")
|
||||
}
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
// the CBOR head of an integer, encoded independently of binary_writer
|
||||
void reference_cbor_head(std::vector<std::uint8_t>& out, const std::uint8_t major_type, const std::uint64_t argument)
|
||||
{
|
||||
if (argument <= 23)
|
||||
{
|
||||
out.push_back(static_cast<std::uint8_t>(major_type + argument));
|
||||
return;
|
||||
}
|
||||
|
||||
int length = 8;
|
||||
std::uint8_t additional = 27;
|
||||
if (argument <= 0xFF)
|
||||
{
|
||||
length = 1;
|
||||
additional = 24;
|
||||
}
|
||||
else if (argument <= 0xFFFF)
|
||||
{
|
||||
length = 2;
|
||||
additional = 25;
|
||||
}
|
||||
else if (argument <= 0xFFFFFFFF)
|
||||
{
|
||||
length = 4;
|
||||
additional = 26;
|
||||
}
|
||||
|
||||
out.push_back(static_cast<std::uint8_t>(major_type + additional));
|
||||
for (int i = length - 1; i >= 0; --i)
|
||||
{
|
||||
out.push_back(static_cast<std::uint8_t>(argument >> (8 * i)));
|
||||
}
|
||||
}
|
||||
|
||||
// the CBOR encoding of an integer value
|
||||
void reference_cbor_integer(std::vector<std::uint8_t>& out, const json& j)
|
||||
{
|
||||
if (j.is_number_unsigned())
|
||||
{
|
||||
reference_cbor_head(out, 0x00, j.get<std::uint64_t>());
|
||||
return;
|
||||
}
|
||||
|
||||
const auto n = j.get<std::int64_t>();
|
||||
if (n >= 0)
|
||||
{
|
||||
reference_cbor_head(out, 0x00, static_cast<std::uint64_t>(n));
|
||||
}
|
||||
else
|
||||
{
|
||||
reference_cbor_head(out, 0x20, static_cast<std::uint64_t>(-1 - n));
|
||||
}
|
||||
}
|
||||
|
||||
// the MessagePack encoding of an integer value, encoded independently of
|
||||
// binary_writer
|
||||
void reference_msgpack_integer(std::vector<std::uint8_t>& out, const json& j)
|
||||
{
|
||||
const auto put = [&out](const std::uint8_t marker, const std::uint64_t bits, const int length)
|
||||
{
|
||||
out.push_back(marker);
|
||||
for (int i = length - 1; i >= 0; --i)
|
||||
{
|
||||
out.push_back(static_cast<std::uint8_t>(bits >> (8 * i)));
|
||||
}
|
||||
};
|
||||
|
||||
if (j.is_number_unsigned() || j.get<std::int64_t>() >= 0)
|
||||
{
|
||||
const auto u = j.get<std::uint64_t>();
|
||||
if (u < 128)
|
||||
{
|
||||
out.push_back(static_cast<std::uint8_t>(u));
|
||||
}
|
||||
else if (u <= 0xFF)
|
||||
{
|
||||
put(0xCC, u, 1);
|
||||
}
|
||||
else if (u <= 0xFFFF)
|
||||
{
|
||||
put(0xCD, u, 2);
|
||||
}
|
||||
else if (u <= 0xFFFFFFFF)
|
||||
{
|
||||
put(0xCE, u, 4);
|
||||
}
|
||||
else
|
||||
{
|
||||
put(0xCF, u, 8);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const auto n = j.get<std::int64_t>();
|
||||
const auto bits = static_cast<std::uint64_t>(n);
|
||||
if (n >= -32)
|
||||
{
|
||||
out.push_back(static_cast<std::uint8_t>(bits));
|
||||
}
|
||||
else if (n >= -128)
|
||||
{
|
||||
put(0xD0, bits, 1);
|
||||
}
|
||||
else if (n >= -32768)
|
||||
{
|
||||
put(0xD1, bits, 2);
|
||||
}
|
||||
else if (n >= -2147483648LL)
|
||||
{
|
||||
put(0xD2, bits, 4);
|
||||
}
|
||||
else
|
||||
{
|
||||
put(0xD3, bits, 8);
|
||||
}
|
||||
}
|
||||
|
||||
// integers at every boundary of the CBOR and MessagePack sizes, signed and
|
||||
// unsigned
|
||||
std::vector<json> boundary_integers()
|
||||
{
|
||||
std::vector<json> result;
|
||||
const std::uint64_t boundaries[] =
|
||||
{
|
||||
0, 1, 22, 23, 24, 25, 31, 32, 33, 126, 127, 128, 129, 254, 255, 256, 257,
|
||||
32766, 32767, 32768, 32769, 65534, 65535, 65536, 65537,
|
||||
2147483646ULL, 2147483647ULL, 2147483648ULL, 2147483649ULL,
|
||||
4294967294ULL, 4294967295ULL, 4294967296ULL, 4294967297ULL,
|
||||
9223372036854775806ULL, 9223372036854775807ULL, 9223372036854775808ULL,
|
||||
18446744073709551614ULL, 18446744073709551615ULL
|
||||
};
|
||||
for (const std::uint64_t u : boundaries)
|
||||
{
|
||||
result.emplace_back(u);
|
||||
if (u <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()))
|
||||
{
|
||||
const auto n = static_cast<std::int64_t>(u);
|
||||
result.emplace_back(static_cast<json::number_integer_t>(n));
|
||||
result.emplace_back(static_cast<json::number_integer_t>(-n));
|
||||
result.emplace_back(static_cast<json::number_integer_t>(-n - 1));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// Arrays of integers are written in runs straight into space reserved from the
|
||||
// sink, with branch-free encodings; check them against independent encoders for
|
||||
// runs shorter and longer than one reservation, broken up by other values, and
|
||||
// through each sink.
|
||||
TEST_CASE("CBOR and MessagePack integer arrays")
|
||||
{
|
||||
const std::vector<json> integers = boundary_integers();
|
||||
|
||||
for (const std::size_t length :
|
||||
{
|
||||
std::size_t{1}, std::size_t{27}, std::size_t{28}, std::size_t{29}, std::size_t{57}, std::size_t{1000}
|
||||
})
|
||||
{
|
||||
for (const std::size_t break_every :
|
||||
{
|
||||
std::size_t{0}, std::size_t{1}, std::size_t{7}, std::size_t{28}
|
||||
})
|
||||
{
|
||||
CAPTURE(length)
|
||||
CAPTURE(break_every)
|
||||
|
||||
json array = json::array();
|
||||
std::vector<std::uint8_t> expected_cbor;
|
||||
std::vector<std::uint8_t> expected_msgpack;
|
||||
for (std::size_t i = 0; i < length; ++i)
|
||||
{
|
||||
if (break_every != 0 && i % (break_every + 1) == break_every)
|
||||
{
|
||||
array.push_back("x");
|
||||
expected_cbor.push_back(0x61);
|
||||
expected_cbor.push_back('x');
|
||||
expected_msgpack.push_back(0xA1);
|
||||
expected_msgpack.push_back('x');
|
||||
}
|
||||
else
|
||||
{
|
||||
const json& value = integers[(i * 7) % integers.size()];
|
||||
array.push_back(value);
|
||||
reference_cbor_integer(expected_cbor, value);
|
||||
reference_msgpack_integer(expected_msgpack, value);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<std::uint8_t> head;
|
||||
reference_cbor_head(head, 0x80, length);
|
||||
expected_cbor.insert(expected_cbor.begin(), head.begin(), head.end());
|
||||
|
||||
head.clear();
|
||||
if (length <= 15)
|
||||
{
|
||||
head.push_back(static_cast<std::uint8_t>(0x90 + length));
|
||||
}
|
||||
else
|
||||
{
|
||||
head = {0xDC, static_cast<std::uint8_t>(length >> 8), static_cast<std::uint8_t>(length)};
|
||||
}
|
||||
expected_msgpack.insert(expected_msgpack.begin(), head.begin(), head.end());
|
||||
|
||||
CHECK(json::to_cbor(array) == expected_cbor);
|
||||
CHECK(json::to_msgpack(array) == expected_msgpack);
|
||||
|
||||
std::vector<std::uint8_t> through_adapter = {0xAA};
|
||||
json::to_cbor(array, through_adapter);
|
||||
CHECK(std::vector<std::uint8_t>(through_adapter.begin() + 1, through_adapter.end()) == expected_cbor);
|
||||
through_adapter = {0xAA};
|
||||
json::to_msgpack(array, through_adapter);
|
||||
CHECK(std::vector<std::uint8_t>(through_adapter.begin() + 1, through_adapter.end()) == expected_msgpack);
|
||||
|
||||
std::ostringstream stream;
|
||||
json::to_cbor(array, stream);
|
||||
std::string streamed = stream.str();
|
||||
CHECK(std::vector<std::uint8_t>(streamed.begin(), streamed.end()) == expected_cbor);
|
||||
stream.str("");
|
||||
json::to_msgpack(array, stream);
|
||||
streamed = stream.str();
|
||||
CHECK(std::vector<std::uint8_t>(streamed.begin(), streamed.end()) == expected_msgpack);
|
||||
|
||||
CHECK(json::from_cbor(expected_cbor) == array);
|
||||
CHECK(json::from_msgpack(expected_msgpack) == array);
|
||||
|
||||
// the same array as object members
|
||||
const json object = {{"a", array}, {"b", array}};
|
||||
CHECK(json::from_cbor(json::to_cbor(object)) == object);
|
||||
CHECK(json::from_msgpack(json::to_msgpack(object)) == object);
|
||||
}
|
||||
}
|
||||
|
||||
// single values, outside an array
|
||||
for (const auto& value : integers)
|
||||
{
|
||||
CAPTURE(value)
|
||||
std::vector<std::uint8_t> expected;
|
||||
reference_cbor_integer(expected, value);
|
||||
CHECK(json::to_cbor(value) == expected);
|
||||
expected.clear();
|
||||
reference_msgpack_integer(expected, value);
|
||||
CHECK(json::to_msgpack(value) == expected);
|
||||
}
|
||||
}
|
||||
|
||||
// binary_reserve_hint() is documented as a *lower* bound on the serialized size,
|
||||
// so that reserving it up front can never leave the returned vector holding
|
||||
// capacity beyond what the value actually needs.
|
||||
|
||||
Reference in new issue
Block a user