Report the same error position for bulk number runs as for byte-wise input
For contiguous input, the CBOR and MessagePack readers read the scalars of an array in one loop, and the UBJSON/BJData readers the numbers of an optimized array. These loops passed a value on before they advanced the position and the last byte. With a number_float_t narrower than the value (for example float), passing the value on reports out_of_range.406, and SAX parse_error() then got a different position and last token than for the same bytes read through a non-contiguous adapter. Advance before passing the value on in the CBOR and MessagePack loops. The last byte is the head byte, as get_number() does not change it; the MessagePack loop used the last byte of the value. In the typed UBJSON/BJData loop, update the position per value only where passing it on can report an error, which is decided at compile time and never the case for the default number types, and do not change the last byte, as reading the values one by one does not. Instruction counts of SAX and DOM parsing of arrays of one million numbers: CBOR -1 to -4 %, MessagePack -2 to -10 %, UBJSON/BJData typed arrays -0.8 to +0.6 % (clang, -O2). Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
3 files changed
+267
-86
No files matched your search
@@ -1927,6 +1927,21 @@ class binary_reader
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*!
|
||||||
|
@brief move past a value of @a size bytes, starting with @a head, that a
|
||||||
|
bulk run reads, as get() and get_number() would
|
||||||
|
|
||||||
|
Called before the value is passed on, so that an error reported while
|
||||||
|
passing it on (see @ref emit_float) has the position and the last byte
|
||||||
|
that reading the value byte by byte gives.
|
||||||
|
*/
|
||||||
|
void bulk_advance(const std::size_t size, const unsigned char head)
|
||||||
|
{
|
||||||
|
ia.bulk_skip(size);
|
||||||
|
chars_read += size;
|
||||||
|
current = head;
|
||||||
|
}
|
||||||
|
|
||||||
/*!
|
/*!
|
||||||
@brief read the integers, floats, booleans, and nulls that follow in a
|
@brief read the integers, floats, booleans, and nulls that follow in a
|
||||||
definite-length array, without the per-element bookkeeping of
|
definite-length array, without the per-element bookkeeping of
|
||||||
@@ -1952,71 +1967,70 @@ class binary_reader
|
|||||||
{
|
{
|
||||||
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
||||||
const unsigned char head = data[0];
|
const unsigned char head = data[0];
|
||||||
std::size_t size = 0;
|
|
||||||
bool accepted = false;
|
bool accepted = false;
|
||||||
switch (head)
|
switch (head)
|
||||||
{
|
{
|
||||||
case 0x18:
|
case 0x18:
|
||||||
size = 2;
|
bulk_advance(2, head);
|
||||||
accepted = emit_unsigned(data[1]);
|
accepted = emit_unsigned(data[1]);
|
||||||
break;
|
break;
|
||||||
case 0x19:
|
case 0x19:
|
||||||
size = 3;
|
bulk_advance(3, head);
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x1A:
|
case 0x1A:
|
||||||
size = 5;
|
bulk_advance(5, head);
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x1B:
|
case 0x1B:
|
||||||
size = 9;
|
bulk_advance(9, head);
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x38:
|
case 0x38:
|
||||||
size = 2;
|
bulk_advance(2, head);
|
||||||
accepted = emit_cbor_negative_integer(data[1]);
|
accepted = emit_cbor_negative_integer(data[1]);
|
||||||
break;
|
break;
|
||||||
case 0x39:
|
case 0x39:
|
||||||
size = 3;
|
bulk_advance(3, head);
|
||||||
accepted = emit_cbor_negative_integer(load_big_endian<std::uint16_t>(data + 1));
|
accepted = emit_cbor_negative_integer(load_big_endian<std::uint16_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x3A:
|
case 0x3A:
|
||||||
size = 5;
|
bulk_advance(5, head);
|
||||||
accepted = emit_cbor_negative_integer(load_big_endian<std::uint32_t>(data + 1));
|
accepted = emit_cbor_negative_integer(load_big_endian<std::uint32_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x3B:
|
case 0x3B:
|
||||||
size = 9;
|
bulk_advance(9, head);
|
||||||
accepted = emit_cbor_negative_integer(load_big_endian<std::uint64_t>(data + 1));
|
accepted = emit_cbor_negative_integer(load_big_endian<std::uint64_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xF4:
|
case 0xF4:
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->boolean(false);
|
accepted = sax->boolean(false);
|
||||||
break;
|
break;
|
||||||
case 0xF5:
|
case 0xF5:
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->boolean(true);
|
accepted = sax->boolean(true);
|
||||||
break;
|
break;
|
||||||
case 0xF6:
|
case 0xF6:
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->null();
|
accepted = sax->null();
|
||||||
break;
|
break;
|
||||||
case 0xFA:
|
case 0xFA:
|
||||||
size = 5;
|
bulk_advance(5, head);
|
||||||
accepted = emit_float(load_big_endian<float>(data + 1));
|
accepted = emit_float(load_big_endian<float>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xFB:
|
case 0xFB:
|
||||||
size = 9;
|
bulk_advance(9, head);
|
||||||
accepted = emit_float(load_big_endian<double>(data + 1));
|
accepted = emit_float(load_big_endian<double>(data + 1));
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
if (head <= 0x17)
|
if (head <= 0x17)
|
||||||
{
|
{
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
|
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
|
||||||
}
|
}
|
||||||
else if (head >= 0x20 && head <= 0x37)
|
else if (head >= 0x20 && head <= 0x37)
|
||||||
{
|
{
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - head));
|
accepted = sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - head));
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@@ -2026,10 +2040,6 @@ class binary_reader
|
|||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
// as if read by get() and get_number(): current is the head byte
|
|
||||||
ia.bulk_skip(size);
|
|
||||||
chars_read += size;
|
|
||||||
current = head;
|
|
||||||
--remaining;
|
--remaining;
|
||||||
if (JSON_HEDLEY_UNLIKELY(!accepted))
|
if (JSON_HEDLEY_UNLIKELY(!accepted))
|
||||||
{
|
{
|
||||||
@@ -2867,73 +2877,72 @@ class binary_reader
|
|||||||
{
|
{
|
||||||
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
||||||
const unsigned char head = data[0];
|
const unsigned char head = data[0];
|
||||||
std::size_t size = 0;
|
|
||||||
bool accepted = false;
|
bool accepted = false;
|
||||||
switch (head)
|
switch (head)
|
||||||
{
|
{
|
||||||
case 0xC0: // nil
|
case 0xC0: // nil
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->null();
|
accepted = sax->null();
|
||||||
break;
|
break;
|
||||||
case 0xC2: // false
|
case 0xC2: // false
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->boolean(false);
|
accepted = sax->boolean(false);
|
||||||
break;
|
break;
|
||||||
case 0xC3: // true
|
case 0xC3: // true
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->boolean(true);
|
accepted = sax->boolean(true);
|
||||||
break;
|
break;
|
||||||
case 0xCA: // float 32
|
case 0xCA: // float 32
|
||||||
size = 5;
|
bulk_advance(5, head);
|
||||||
accepted = emit_float(load_big_endian<float>(data + 1));
|
accepted = emit_float(load_big_endian<float>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xCB: // float 64
|
case 0xCB: // float 64
|
||||||
size = 9;
|
bulk_advance(9, head);
|
||||||
accepted = emit_float(load_big_endian<double>(data + 1));
|
accepted = emit_float(load_big_endian<double>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xCC: // uint 8
|
case 0xCC: // uint 8
|
||||||
size = 2;
|
bulk_advance(2, head);
|
||||||
accepted = emit_unsigned(data[1]);
|
accepted = emit_unsigned(data[1]);
|
||||||
break;
|
break;
|
||||||
case 0xCD: // uint 16
|
case 0xCD: // uint 16
|
||||||
size = 3;
|
bulk_advance(3, head);
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xCE: // uint 32
|
case 0xCE: // uint 32
|
||||||
size = 5;
|
bulk_advance(5, head);
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xCF: // uint 64
|
case 0xCF: // uint 64
|
||||||
size = 9;
|
bulk_advance(9, head);
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xD0: // int 8
|
case 0xD0: // int 8
|
||||||
size = 2;
|
bulk_advance(2, head);
|
||||||
accepted = emit_signed(static_cast<std::int8_t>(data[1]));
|
accepted = emit_signed(static_cast<std::int8_t>(data[1]));
|
||||||
break;
|
break;
|
||||||
case 0xD1: // int 16
|
case 0xD1: // int 16
|
||||||
size = 3;
|
bulk_advance(3, head);
|
||||||
accepted = emit_signed(load_big_endian<std::int16_t>(data + 1));
|
accepted = emit_signed(load_big_endian<std::int16_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xD2: // int 32
|
case 0xD2: // int 32
|
||||||
size = 5;
|
bulk_advance(5, head);
|
||||||
accepted = emit_signed(load_big_endian<std::int32_t>(data + 1));
|
accepted = emit_signed(load_big_endian<std::int32_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xD3: // int 64
|
case 0xD3: // int 64
|
||||||
size = 9;
|
bulk_advance(9, head);
|
||||||
accepted = emit_signed(load_big_endian<std::int64_t>(data + 1));
|
accepted = emit_signed(load_big_endian<std::int64_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
if (head <= 0x7F)
|
if (head <= 0x7F)
|
||||||
{
|
{
|
||||||
// positive fixint
|
// positive fixint
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
|
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
|
||||||
}
|
}
|
||||||
else if (head >= 0xE0)
|
else if (head >= 0xE0)
|
||||||
{
|
{
|
||||||
// negative fixint
|
// negative fixint
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->number_integer(static_cast<std::int8_t>(head));
|
accepted = sax->number_integer(static_cast<std::int8_t>(head));
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@@ -2943,10 +2952,6 @@ class binary_reader
|
|||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
// as if read by get() and get_number(): current is the last byte
|
|
||||||
ia.bulk_skip(size);
|
|
||||||
chars_read += size;
|
|
||||||
current = data[size - 1];
|
|
||||||
--remaining;
|
--remaining;
|
||||||
if (JSON_HEDLEY_UNLIKELY(!accepted))
|
if (JSON_HEDLEY_UNLIKELY(!accepted))
|
||||||
{
|
{
|
||||||
@@ -3850,6 +3855,7 @@ class binary_reader
|
|||||||
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
||||||
// UBJSON stores numbers big-endian, BJData little-endian
|
// UBJSON stores numbers big-endian, BJData little-endian
|
||||||
const bool swap = is_little_endian != (input_format == input_format_t::bjdata);
|
const bool swap = is_little_endian != (input_format == input_format_t::bjdata);
|
||||||
|
const std::size_t start = chars_read;
|
||||||
std::size_t read = 0;
|
std::size_t read = 0;
|
||||||
bool accepted = true;
|
bool accepted = true;
|
||||||
while (read < count && accepted)
|
while (read < count && accepted)
|
||||||
@@ -3861,16 +3867,39 @@ class binary_reader
|
|||||||
byte_swap(number);
|
byte_swap(number);
|
||||||
}
|
}
|
||||||
++read;
|
++read;
|
||||||
|
// where passing the value on can report an error (see
|
||||||
|
// emit_float), the position is that after the value, as if it
|
||||||
|
// was read by get_number()
|
||||||
|
set_chars_read(start + (read * size), std::integral_constant<bool, may_overflow_number_float<NumberType>()> {});
|
||||||
accepted = emit_ubjson_number(number);
|
accepted = emit_ubjson_number(number);
|
||||||
}
|
}
|
||||||
// as if read by get_number(): current is the last byte read
|
// get_number() does not change current
|
||||||
ia.bulk_skip(read * size);
|
ia.bulk_skip(read * size);
|
||||||
chars_read += read * size;
|
chars_read = start + (read * size);
|
||||||
current = data[(read * size) - 1];
|
|
||||||
remaining -= read;
|
remaining -= read;
|
||||||
return accepted;
|
return accepted;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*!
|
||||||
|
@brief whether a value of type @a NumberType can be finite and still
|
||||||
|
overflow number_float_t, which @ref emit_float reports as an error
|
||||||
|
|
||||||
|
False for the default number types, so that the loop in @ref
|
||||||
|
parse_ubjson_typed_run need not track the position per value.
|
||||||
|
*/
|
||||||
|
template<typename NumberType>
|
||||||
|
static constexpr bool may_overflow_number_float() noexcept
|
||||||
|
{
|
||||||
|
return static_cast<long double>((std::numeric_limits<number_float_t>::max)()) < static_cast<long double>((std::numeric_limits<NumberType>::max)());
|
||||||
|
}
|
||||||
|
|
||||||
|
void set_chars_read(const std::size_t position, std::true_type /*track*/) noexcept
|
||||||
|
{
|
||||||
|
chars_read = position;
|
||||||
|
}
|
||||||
|
|
||||||
|
void set_chars_read(const std::size_t /*position*/, std::false_type /*track*/) const noexcept {}
|
||||||
|
|
||||||
template<typename NumberType, enable_if_t<std::is_floating_point<NumberType>::value, int> = 0>
|
template<typename NumberType, enable_if_t<std::is_floating_point<NumberType>::value, int> = 0>
|
||||||
bool emit_ubjson_number(const NumberType number)
|
bool emit_ubjson_number(const NumberType number)
|
||||||
{
|
{
|
||||||
|
|||||||
Generated
+72
-43
@@ -20605,6 +20605,21 @@ class binary_reader
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*!
|
||||||
|
@brief move past a value of @a size bytes, starting with @a head, that a
|
||||||
|
bulk run reads, as get() and get_number() would
|
||||||
|
|
||||||
|
Called before the value is passed on, so that an error reported while
|
||||||
|
passing it on (see @ref emit_float) has the position and the last byte
|
||||||
|
that reading the value byte by byte gives.
|
||||||
|
*/
|
||||||
|
void bulk_advance(const std::size_t size, const unsigned char head)
|
||||||
|
{
|
||||||
|
ia.bulk_skip(size);
|
||||||
|
chars_read += size;
|
||||||
|
current = head;
|
||||||
|
}
|
||||||
|
|
||||||
/*!
|
/*!
|
||||||
@brief read the integers, floats, booleans, and nulls that follow in a
|
@brief read the integers, floats, booleans, and nulls that follow in a
|
||||||
definite-length array, without the per-element bookkeeping of
|
definite-length array, without the per-element bookkeeping of
|
||||||
@@ -20630,71 +20645,70 @@ class binary_reader
|
|||||||
{
|
{
|
||||||
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
||||||
const unsigned char head = data[0];
|
const unsigned char head = data[0];
|
||||||
std::size_t size = 0;
|
|
||||||
bool accepted = false;
|
bool accepted = false;
|
||||||
switch (head)
|
switch (head)
|
||||||
{
|
{
|
||||||
case 0x18:
|
case 0x18:
|
||||||
size = 2;
|
bulk_advance(2, head);
|
||||||
accepted = emit_unsigned(data[1]);
|
accepted = emit_unsigned(data[1]);
|
||||||
break;
|
break;
|
||||||
case 0x19:
|
case 0x19:
|
||||||
size = 3;
|
bulk_advance(3, head);
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x1A:
|
case 0x1A:
|
||||||
size = 5;
|
bulk_advance(5, head);
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x1B:
|
case 0x1B:
|
||||||
size = 9;
|
bulk_advance(9, head);
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x38:
|
case 0x38:
|
||||||
size = 2;
|
bulk_advance(2, head);
|
||||||
accepted = emit_cbor_negative_integer(data[1]);
|
accepted = emit_cbor_negative_integer(data[1]);
|
||||||
break;
|
break;
|
||||||
case 0x39:
|
case 0x39:
|
||||||
size = 3;
|
bulk_advance(3, head);
|
||||||
accepted = emit_cbor_negative_integer(load_big_endian<std::uint16_t>(data + 1));
|
accepted = emit_cbor_negative_integer(load_big_endian<std::uint16_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x3A:
|
case 0x3A:
|
||||||
size = 5;
|
bulk_advance(5, head);
|
||||||
accepted = emit_cbor_negative_integer(load_big_endian<std::uint32_t>(data + 1));
|
accepted = emit_cbor_negative_integer(load_big_endian<std::uint32_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x3B:
|
case 0x3B:
|
||||||
size = 9;
|
bulk_advance(9, head);
|
||||||
accepted = emit_cbor_negative_integer(load_big_endian<std::uint64_t>(data + 1));
|
accepted = emit_cbor_negative_integer(load_big_endian<std::uint64_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xF4:
|
case 0xF4:
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->boolean(false);
|
accepted = sax->boolean(false);
|
||||||
break;
|
break;
|
||||||
case 0xF5:
|
case 0xF5:
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->boolean(true);
|
accepted = sax->boolean(true);
|
||||||
break;
|
break;
|
||||||
case 0xF6:
|
case 0xF6:
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->null();
|
accepted = sax->null();
|
||||||
break;
|
break;
|
||||||
case 0xFA:
|
case 0xFA:
|
||||||
size = 5;
|
bulk_advance(5, head);
|
||||||
accepted = emit_float(load_big_endian<float>(data + 1));
|
accepted = emit_float(load_big_endian<float>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xFB:
|
case 0xFB:
|
||||||
size = 9;
|
bulk_advance(9, head);
|
||||||
accepted = emit_float(load_big_endian<double>(data + 1));
|
accepted = emit_float(load_big_endian<double>(data + 1));
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
if (head <= 0x17)
|
if (head <= 0x17)
|
||||||
{
|
{
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
|
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
|
||||||
}
|
}
|
||||||
else if (head >= 0x20 && head <= 0x37)
|
else if (head >= 0x20 && head <= 0x37)
|
||||||
{
|
{
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - head));
|
accepted = sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - head));
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@@ -20704,10 +20718,6 @@ class binary_reader
|
|||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
// as if read by get() and get_number(): current is the head byte
|
|
||||||
ia.bulk_skip(size);
|
|
||||||
chars_read += size;
|
|
||||||
current = head;
|
|
||||||
--remaining;
|
--remaining;
|
||||||
if (JSON_HEDLEY_UNLIKELY(!accepted))
|
if (JSON_HEDLEY_UNLIKELY(!accepted))
|
||||||
{
|
{
|
||||||
@@ -21545,73 +21555,72 @@ class binary_reader
|
|||||||
{
|
{
|
||||||
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
||||||
const unsigned char head = data[0];
|
const unsigned char head = data[0];
|
||||||
std::size_t size = 0;
|
|
||||||
bool accepted = false;
|
bool accepted = false;
|
||||||
switch (head)
|
switch (head)
|
||||||
{
|
{
|
||||||
case 0xC0: // nil
|
case 0xC0: // nil
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->null();
|
accepted = sax->null();
|
||||||
break;
|
break;
|
||||||
case 0xC2: // false
|
case 0xC2: // false
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->boolean(false);
|
accepted = sax->boolean(false);
|
||||||
break;
|
break;
|
||||||
case 0xC3: // true
|
case 0xC3: // true
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->boolean(true);
|
accepted = sax->boolean(true);
|
||||||
break;
|
break;
|
||||||
case 0xCA: // float 32
|
case 0xCA: // float 32
|
||||||
size = 5;
|
bulk_advance(5, head);
|
||||||
accepted = emit_float(load_big_endian<float>(data + 1));
|
accepted = emit_float(load_big_endian<float>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xCB: // float 64
|
case 0xCB: // float 64
|
||||||
size = 9;
|
bulk_advance(9, head);
|
||||||
accepted = emit_float(load_big_endian<double>(data + 1));
|
accepted = emit_float(load_big_endian<double>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xCC: // uint 8
|
case 0xCC: // uint 8
|
||||||
size = 2;
|
bulk_advance(2, head);
|
||||||
accepted = emit_unsigned(data[1]);
|
accepted = emit_unsigned(data[1]);
|
||||||
break;
|
break;
|
||||||
case 0xCD: // uint 16
|
case 0xCD: // uint 16
|
||||||
size = 3;
|
bulk_advance(3, head);
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xCE: // uint 32
|
case 0xCE: // uint 32
|
||||||
size = 5;
|
bulk_advance(5, head);
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xCF: // uint 64
|
case 0xCF: // uint 64
|
||||||
size = 9;
|
bulk_advance(9, head);
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xD0: // int 8
|
case 0xD0: // int 8
|
||||||
size = 2;
|
bulk_advance(2, head);
|
||||||
accepted = emit_signed(static_cast<std::int8_t>(data[1]));
|
accepted = emit_signed(static_cast<std::int8_t>(data[1]));
|
||||||
break;
|
break;
|
||||||
case 0xD1: // int 16
|
case 0xD1: // int 16
|
||||||
size = 3;
|
bulk_advance(3, head);
|
||||||
accepted = emit_signed(load_big_endian<std::int16_t>(data + 1));
|
accepted = emit_signed(load_big_endian<std::int16_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xD2: // int 32
|
case 0xD2: // int 32
|
||||||
size = 5;
|
bulk_advance(5, head);
|
||||||
accepted = emit_signed(load_big_endian<std::int32_t>(data + 1));
|
accepted = emit_signed(load_big_endian<std::int32_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xD3: // int 64
|
case 0xD3: // int 64
|
||||||
size = 9;
|
bulk_advance(9, head);
|
||||||
accepted = emit_signed(load_big_endian<std::int64_t>(data + 1));
|
accepted = emit_signed(load_big_endian<std::int64_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
if (head <= 0x7F)
|
if (head <= 0x7F)
|
||||||
{
|
{
|
||||||
// positive fixint
|
// positive fixint
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
|
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
|
||||||
}
|
}
|
||||||
else if (head >= 0xE0)
|
else if (head >= 0xE0)
|
||||||
{
|
{
|
||||||
// negative fixint
|
// negative fixint
|
||||||
size = 1;
|
bulk_advance(1, head);
|
||||||
accepted = sax->number_integer(static_cast<std::int8_t>(head));
|
accepted = sax->number_integer(static_cast<std::int8_t>(head));
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@@ -21621,10 +21630,6 @@ class binary_reader
|
|||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
// as if read by get() and get_number(): current is the last byte
|
|
||||||
ia.bulk_skip(size);
|
|
||||||
chars_read += size;
|
|
||||||
current = data[size - 1];
|
|
||||||
--remaining;
|
--remaining;
|
||||||
if (JSON_HEDLEY_UNLIKELY(!accepted))
|
if (JSON_HEDLEY_UNLIKELY(!accepted))
|
||||||
{
|
{
|
||||||
@@ -22528,6 +22533,7 @@ class binary_reader
|
|||||||
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
||||||
// UBJSON stores numbers big-endian, BJData little-endian
|
// UBJSON stores numbers big-endian, BJData little-endian
|
||||||
const bool swap = is_little_endian != (input_format == input_format_t::bjdata);
|
const bool swap = is_little_endian != (input_format == input_format_t::bjdata);
|
||||||
|
const std::size_t start = chars_read;
|
||||||
std::size_t read = 0;
|
std::size_t read = 0;
|
||||||
bool accepted = true;
|
bool accepted = true;
|
||||||
while (read < count && accepted)
|
while (read < count && accepted)
|
||||||
@@ -22539,16 +22545,39 @@ class binary_reader
|
|||||||
byte_swap(number);
|
byte_swap(number);
|
||||||
}
|
}
|
||||||
++read;
|
++read;
|
||||||
|
// where passing the value on can report an error (see
|
||||||
|
// emit_float), the position is that after the value, as if it
|
||||||
|
// was read by get_number()
|
||||||
|
set_chars_read(start + (read * size), std::integral_constant<bool, may_overflow_number_float<NumberType>()> {});
|
||||||
accepted = emit_ubjson_number(number);
|
accepted = emit_ubjson_number(number);
|
||||||
}
|
}
|
||||||
// as if read by get_number(): current is the last byte read
|
// get_number() does not change current
|
||||||
ia.bulk_skip(read * size);
|
ia.bulk_skip(read * size);
|
||||||
chars_read += read * size;
|
chars_read = start + (read * size);
|
||||||
current = data[(read * size) - 1];
|
|
||||||
remaining -= read;
|
remaining -= read;
|
||||||
return accepted;
|
return accepted;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*!
|
||||||
|
@brief whether a value of type @a NumberType can be finite and still
|
||||||
|
overflow number_float_t, which @ref emit_float reports as an error
|
||||||
|
|
||||||
|
False for the default number types, so that the loop in @ref
|
||||||
|
parse_ubjson_typed_run need not track the position per value.
|
||||||
|
*/
|
||||||
|
template<typename NumberType>
|
||||||
|
static constexpr bool may_overflow_number_float() noexcept
|
||||||
|
{
|
||||||
|
return static_cast<long double>((std::numeric_limits<number_float_t>::max)()) < static_cast<long double>((std::numeric_limits<NumberType>::max)());
|
||||||
|
}
|
||||||
|
|
||||||
|
void set_chars_read(const std::size_t position, std::true_type /*track*/) noexcept
|
||||||
|
{
|
||||||
|
chars_read = position;
|
||||||
|
}
|
||||||
|
|
||||||
|
void set_chars_read(const std::size_t /*position*/, std::false_type /*track*/) const noexcept {}
|
||||||
|
|
||||||
template<typename NumberType, enable_if_t<std::is_floating_point<NumberType>::value, int> = 0>
|
template<typename NumberType, enable_if_t<std::is_floating_point<NumberType>::value, int> = 0>
|
||||||
bool emit_ubjson_number(const NumberType number)
|
bool emit_ubjson_number(const NumberType number)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ using nlohmann::json;
|
|||||||
#include <cmath>
|
#include <cmath>
|
||||||
#include <fstream>
|
#include <fstream>
|
||||||
#include <limits>
|
#include <limits>
|
||||||
|
#include <list>
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
@@ -367,3 +368,125 @@ TEST_CASE("Binary formats with narrow number types")
|
|||||||
CHECK(format.decode(format.encode(json(1e300)), false).is_discarded());
|
CHECK(format.decode(format.encode(json(1e300)), false).is_discarded());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
namespace
|
||||||
|
{
|
||||||
|
// records the SAX events of narrow_json, including the arguments of parse_error
|
||||||
|
struct narrow_event_recorder
|
||||||
|
{
|
||||||
|
using number_integer_t = narrow_json::number_integer_t;
|
||||||
|
using number_unsigned_t = narrow_json::number_unsigned_t;
|
||||||
|
using number_float_t = narrow_json::number_float_t;
|
||||||
|
using string_t = narrow_json::string_t;
|
||||||
|
using binary_t = narrow_json::binary_t;
|
||||||
|
|
||||||
|
bool null()
|
||||||
|
{
|
||||||
|
events.emplace_back("null");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
bool boolean(bool val)
|
||||||
|
{
|
||||||
|
events.emplace_back(val ? "true" : "false");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
bool number_integer(number_integer_t val)
|
||||||
|
{
|
||||||
|
events.push_back(std::to_string(val));
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
bool number_unsigned(number_unsigned_t val)
|
||||||
|
{
|
||||||
|
events.push_back(std::to_string(val));
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
bool number_float(number_float_t val, const string_t& /*unused*/)
|
||||||
|
{
|
||||||
|
events.push_back(std::to_string(val));
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
bool string(string_t& val)
|
||||||
|
{
|
||||||
|
events.push_back(val);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
bool binary(binary_t& /*unused*/)
|
||||||
|
{
|
||||||
|
events.emplace_back("binary");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
bool start_object(std::size_t /*unused*/)
|
||||||
|
{
|
||||||
|
events.emplace_back("{");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
bool key(string_t& val)
|
||||||
|
{
|
||||||
|
events.push_back(val);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
bool end_object()
|
||||||
|
{
|
||||||
|
events.emplace_back("}");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
bool start_array(std::size_t /*unused*/)
|
||||||
|
{
|
||||||
|
events.emplace_back("[");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
bool end_array()
|
||||||
|
{
|
||||||
|
events.emplace_back("]");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
bool parse_error(std::size_t position, const std::string& last_token, const nlohmann::detail::exception& ex)
|
||||||
|
{
|
||||||
|
events.push_back("parse_error(" + std::to_string(position) + ", " + last_token + ", " + ex.what() + ")");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<std::string> events {}; // NOLINT(readability-redundant-member-init)
|
||||||
|
};
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
TEST_CASE("Binary formats with narrow number types from contiguous and non-contiguous input")
|
||||||
|
{
|
||||||
|
// contiguous input reads runs of numbers in one loop; a number that
|
||||||
|
// overflows number_float_t must be reported with the same position and
|
||||||
|
// last byte as when the input is read byte by byte
|
||||||
|
const json values = {1, 70000, -70000, 1.5, 2.5, 1e300, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5};
|
||||||
|
const json doubles = {1.5, 2.5, 3.5, 1e300, 4.5, 5.5, 6.5, 7.5, 8.5};
|
||||||
|
|
||||||
|
struct binary_input
|
||||||
|
{
|
||||||
|
const char* name;
|
||||||
|
json::input_format_t format;
|
||||||
|
bytes data;
|
||||||
|
};
|
||||||
|
const std::vector<binary_input> inputs =
|
||||||
|
{
|
||||||
|
{"CBOR", json::input_format_t::cbor, json::to_cbor(values)},
|
||||||
|
{"CBOR doubles", json::input_format_t::cbor, json::to_cbor(doubles)},
|
||||||
|
{"MessagePack", json::input_format_t::msgpack, json::to_msgpack(values)},
|
||||||
|
{"MessagePack doubles", json::input_format_t::msgpack, json::to_msgpack(doubles)},
|
||||||
|
{"UBJSON", json::input_format_t::ubjson, json::to_ubjson(values, true)},
|
||||||
|
{"UBJSON typed doubles", json::input_format_t::ubjson, json::to_ubjson(doubles, true, true)},
|
||||||
|
{"BJData", json::input_format_t::bjdata, json::to_bjdata(values, true)},
|
||||||
|
{"BJData typed doubles", json::input_format_t::bjdata, json::to_bjdata(doubles, true, true)},
|
||||||
|
};
|
||||||
|
|
||||||
|
for (const auto& input : inputs)
|
||||||
|
{
|
||||||
|
const std::string name = input.name;
|
||||||
|
CAPTURE(name)
|
||||||
|
const std::list<std::uint8_t> bytewise(input.data.begin(), input.data.end());
|
||||||
|
narrow_event_recorder from_contiguous;
|
||||||
|
narrow_event_recorder from_bytewise;
|
||||||
|
CHECK_FALSE(narrow_json::sax_parse(input.data, &from_contiguous, input.format));
|
||||||
|
CHECK_FALSE(narrow_json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, input.format));
|
||||||
|
CHECK(from_contiguous.events == from_bytewise.events);
|
||||||
|
REQUIRE(!from_contiguous.events.empty());
|
||||||
|
CHECK(from_contiguous.events.back().find("out_of_range.406") != std::string::npos);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user