Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
04fbced73d |
No files matched your search
@@ -140,7 +140,7 @@ Exceptions have ids 1xx.
|
|||||||
|
|
||||||
This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member `byte` indicates the error position.
|
This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member `byte` indicates the error position.
|
||||||
|
|
||||||
The message ends with the last read token (`last read: '...'`), with control characters written as `<U+XXXX>`. A token of more than 8192 characters is shown by its first and its last 4096 characters, separated by `...`.
|
The message ends with the last read token (`last read: '...'`), with control characters written as `<U+XXXX>`. A token of more than 8192 bytes is shown by its first and its last 4096 bytes, separated by `...`; each part is shortened by up to three bytes so that it does not end or begin within a UTF-8 character.
|
||||||
|
|
||||||
!!! failure "Example message"
|
!!! failure "Example message"
|
||||||
|
|
||||||
|
|||||||
@@ -1927,21 +1927,6 @@ class binary_reader
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*!
|
|
||||||
@brief move past a value of @a size bytes, starting with @a head, that a
|
|
||||||
bulk run reads, as get() and get_number() would
|
|
||||||
|
|
||||||
Called before the value is passed on, so that an error reported while
|
|
||||||
passing it on (see @ref emit_float) has the position and the last byte
|
|
||||||
that reading the value byte by byte gives.
|
|
||||||
*/
|
|
||||||
void bulk_advance(const std::size_t size, const unsigned char head)
|
|
||||||
{
|
|
||||||
ia.bulk_skip(size);
|
|
||||||
chars_read += size;
|
|
||||||
current = head;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*!
|
/*!
|
||||||
@brief read the integers, floats, booleans, and nulls that follow in a
|
@brief read the integers, floats, booleans, and nulls that follow in a
|
||||||
definite-length array, without the per-element bookkeeping of
|
definite-length array, without the per-element bookkeeping of
|
||||||
@@ -1967,70 +1952,71 @@ class binary_reader
|
|||||||
{
|
{
|
||||||
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
||||||
const unsigned char head = data[0];
|
const unsigned char head = data[0];
|
||||||
|
std::size_t size = 0;
|
||||||
bool accepted = false;
|
bool accepted = false;
|
||||||
switch (head)
|
switch (head)
|
||||||
{
|
{
|
||||||
case 0x18:
|
case 0x18:
|
||||||
bulk_advance(2, head);
|
size = 2;
|
||||||
accepted = emit_unsigned(data[1]);
|
accepted = emit_unsigned(data[1]);
|
||||||
break;
|
break;
|
||||||
case 0x19:
|
case 0x19:
|
||||||
bulk_advance(3, head);
|
size = 3;
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x1A:
|
case 0x1A:
|
||||||
bulk_advance(5, head);
|
size = 5;
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x1B:
|
case 0x1B:
|
||||||
bulk_advance(9, head);
|
size = 9;
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x38:
|
case 0x38:
|
||||||
bulk_advance(2, head);
|
size = 2;
|
||||||
accepted = emit_cbor_negative_integer(data[1]);
|
accepted = emit_cbor_negative_integer(data[1]);
|
||||||
break;
|
break;
|
||||||
case 0x39:
|
case 0x39:
|
||||||
bulk_advance(3, head);
|
size = 3;
|
||||||
accepted = emit_cbor_negative_integer(load_big_endian<std::uint16_t>(data + 1));
|
accepted = emit_cbor_negative_integer(load_big_endian<std::uint16_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x3A:
|
case 0x3A:
|
||||||
bulk_advance(5, head);
|
size = 5;
|
||||||
accepted = emit_cbor_negative_integer(load_big_endian<std::uint32_t>(data + 1));
|
accepted = emit_cbor_negative_integer(load_big_endian<std::uint32_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x3B:
|
case 0x3B:
|
||||||
bulk_advance(9, head);
|
size = 9;
|
||||||
accepted = emit_cbor_negative_integer(load_big_endian<std::uint64_t>(data + 1));
|
accepted = emit_cbor_negative_integer(load_big_endian<std::uint64_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xF4:
|
case 0xF4:
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->boolean(false);
|
accepted = sax->boolean(false);
|
||||||
break;
|
break;
|
||||||
case 0xF5:
|
case 0xF5:
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->boolean(true);
|
accepted = sax->boolean(true);
|
||||||
break;
|
break;
|
||||||
case 0xF6:
|
case 0xF6:
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->null();
|
accepted = sax->null();
|
||||||
break;
|
break;
|
||||||
case 0xFA:
|
case 0xFA:
|
||||||
bulk_advance(5, head);
|
size = 5;
|
||||||
accepted = emit_float(load_big_endian<float>(data + 1));
|
accepted = emit_float(load_big_endian<float>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xFB:
|
case 0xFB:
|
||||||
bulk_advance(9, head);
|
size = 9;
|
||||||
accepted = emit_float(load_big_endian<double>(data + 1));
|
accepted = emit_float(load_big_endian<double>(data + 1));
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
if (head <= 0x17)
|
if (head <= 0x17)
|
||||||
{
|
{
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
|
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
|
||||||
}
|
}
|
||||||
else if (head >= 0x20 && head <= 0x37)
|
else if (head >= 0x20 && head <= 0x37)
|
||||||
{
|
{
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - head));
|
accepted = sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - head));
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@@ -2040,6 +2026,10 @@ class binary_reader
|
|||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
// as if read by get() and get_number(): current is the head byte
|
||||||
|
ia.bulk_skip(size);
|
||||||
|
chars_read += size;
|
||||||
|
current = head;
|
||||||
--remaining;
|
--remaining;
|
||||||
if (JSON_HEDLEY_UNLIKELY(!accepted))
|
if (JSON_HEDLEY_UNLIKELY(!accepted))
|
||||||
{
|
{
|
||||||
@@ -2877,72 +2867,73 @@ class binary_reader
|
|||||||
{
|
{
|
||||||
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
||||||
const unsigned char head = data[0];
|
const unsigned char head = data[0];
|
||||||
|
std::size_t size = 0;
|
||||||
bool accepted = false;
|
bool accepted = false;
|
||||||
switch (head)
|
switch (head)
|
||||||
{
|
{
|
||||||
case 0xC0: // nil
|
case 0xC0: // nil
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->null();
|
accepted = sax->null();
|
||||||
break;
|
break;
|
||||||
case 0xC2: // false
|
case 0xC2: // false
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->boolean(false);
|
accepted = sax->boolean(false);
|
||||||
break;
|
break;
|
||||||
case 0xC3: // true
|
case 0xC3: // true
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->boolean(true);
|
accepted = sax->boolean(true);
|
||||||
break;
|
break;
|
||||||
case 0xCA: // float 32
|
case 0xCA: // float 32
|
||||||
bulk_advance(5, head);
|
size = 5;
|
||||||
accepted = emit_float(load_big_endian<float>(data + 1));
|
accepted = emit_float(load_big_endian<float>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xCB: // float 64
|
case 0xCB: // float 64
|
||||||
bulk_advance(9, head);
|
size = 9;
|
||||||
accepted = emit_float(load_big_endian<double>(data + 1));
|
accepted = emit_float(load_big_endian<double>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xCC: // uint 8
|
case 0xCC: // uint 8
|
||||||
bulk_advance(2, head);
|
size = 2;
|
||||||
accepted = emit_unsigned(data[1]);
|
accepted = emit_unsigned(data[1]);
|
||||||
break;
|
break;
|
||||||
case 0xCD: // uint 16
|
case 0xCD: // uint 16
|
||||||
bulk_advance(3, head);
|
size = 3;
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xCE: // uint 32
|
case 0xCE: // uint 32
|
||||||
bulk_advance(5, head);
|
size = 5;
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xCF: // uint 64
|
case 0xCF: // uint 64
|
||||||
bulk_advance(9, head);
|
size = 9;
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xD0: // int 8
|
case 0xD0: // int 8
|
||||||
bulk_advance(2, head);
|
size = 2;
|
||||||
accepted = emit_signed(static_cast<std::int8_t>(data[1]));
|
accepted = emit_signed(static_cast<std::int8_t>(data[1]));
|
||||||
break;
|
break;
|
||||||
case 0xD1: // int 16
|
case 0xD1: // int 16
|
||||||
bulk_advance(3, head);
|
size = 3;
|
||||||
accepted = emit_signed(load_big_endian<std::int16_t>(data + 1));
|
accepted = emit_signed(load_big_endian<std::int16_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xD2: // int 32
|
case 0xD2: // int 32
|
||||||
bulk_advance(5, head);
|
size = 5;
|
||||||
accepted = emit_signed(load_big_endian<std::int32_t>(data + 1));
|
accepted = emit_signed(load_big_endian<std::int32_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xD3: // int 64
|
case 0xD3: // int 64
|
||||||
bulk_advance(9, head);
|
size = 9;
|
||||||
accepted = emit_signed(load_big_endian<std::int64_t>(data + 1));
|
accepted = emit_signed(load_big_endian<std::int64_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
if (head <= 0x7F)
|
if (head <= 0x7F)
|
||||||
{
|
{
|
||||||
// positive fixint
|
// positive fixint
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
|
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
|
||||||
}
|
}
|
||||||
else if (head >= 0xE0)
|
else if (head >= 0xE0)
|
||||||
{
|
{
|
||||||
// negative fixint
|
// negative fixint
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->number_integer(static_cast<std::int8_t>(head));
|
accepted = sax->number_integer(static_cast<std::int8_t>(head));
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@@ -2952,6 +2943,10 @@ class binary_reader
|
|||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
// as if read by get() and get_number(): current is the last byte
|
||||||
|
ia.bulk_skip(size);
|
||||||
|
chars_read += size;
|
||||||
|
current = data[size - 1];
|
||||||
--remaining;
|
--remaining;
|
||||||
if (JSON_HEDLEY_UNLIKELY(!accepted))
|
if (JSON_HEDLEY_UNLIKELY(!accepted))
|
||||||
{
|
{
|
||||||
@@ -3855,7 +3850,6 @@ class binary_reader
|
|||||||
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
||||||
// UBJSON stores numbers big-endian, BJData little-endian
|
// UBJSON stores numbers big-endian, BJData little-endian
|
||||||
const bool swap = is_little_endian != (input_format == input_format_t::bjdata);
|
const bool swap = is_little_endian != (input_format == input_format_t::bjdata);
|
||||||
const std::size_t start = chars_read;
|
|
||||||
std::size_t read = 0;
|
std::size_t read = 0;
|
||||||
bool accepted = true;
|
bool accepted = true;
|
||||||
while (read < count && accepted)
|
while (read < count && accepted)
|
||||||
@@ -3867,39 +3861,16 @@ class binary_reader
|
|||||||
byte_swap(number);
|
byte_swap(number);
|
||||||
}
|
}
|
||||||
++read;
|
++read;
|
||||||
// where passing the value on can report an error (see
|
|
||||||
// emit_float), the position is that after the value, as if it
|
|
||||||
// was read by get_number()
|
|
||||||
set_chars_read(start + (read * size), std::integral_constant<bool, may_overflow_number_float<NumberType>()> {});
|
|
||||||
accepted = emit_ubjson_number(number);
|
accepted = emit_ubjson_number(number);
|
||||||
}
|
}
|
||||||
// get_number() does not change current
|
// as if read by get_number(): current is the last byte read
|
||||||
ia.bulk_skip(read * size);
|
ia.bulk_skip(read * size);
|
||||||
chars_read = start + (read * size);
|
chars_read += read * size;
|
||||||
|
current = data[(read * size) - 1];
|
||||||
remaining -= read;
|
remaining -= read;
|
||||||
return accepted;
|
return accepted;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*!
|
|
||||||
@brief whether a value of type @a NumberType can be finite and still
|
|
||||||
overflow number_float_t, which @ref emit_float reports as an error
|
|
||||||
|
|
||||||
False for the default number types, so that the loop in @ref
|
|
||||||
parse_ubjson_typed_run need not track the position per value.
|
|
||||||
*/
|
|
||||||
template<typename NumberType>
|
|
||||||
static constexpr bool may_overflow_number_float() noexcept
|
|
||||||
{
|
|
||||||
return static_cast<long double>((std::numeric_limits<number_float_t>::max)()) < static_cast<long double>((std::numeric_limits<NumberType>::max)());
|
|
||||||
}
|
|
||||||
|
|
||||||
void set_chars_read(const std::size_t position, std::true_type /*track*/) noexcept
|
|
||||||
{
|
|
||||||
chars_read = position;
|
|
||||||
}
|
|
||||||
|
|
||||||
void set_chars_read(const std::size_t /*position*/, std::false_type /*track*/) const noexcept {}
|
|
||||||
|
|
||||||
template<typename NumberType, enable_if_t<std::is_floating_point<NumberType>::value, int> = 0>
|
template<typename NumberType, enable_if_t<std::is_floating_point<NumberType>::value, int> = 0>
|
||||||
bool emit_ubjson_number(const NumberType number)
|
bool emit_ubjson_number(const NumberType number)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -2279,9 +2279,14 @@ scan_number_done:
|
|||||||
std::string result;
|
std::string result;
|
||||||
if (chars.size() > 2 * shown)
|
if (chars.size() > 2 * shown)
|
||||||
{
|
{
|
||||||
append_escaped(result, chars.begin(), chars.begin() + static_cast<std::ptrdiff_t>(shown));
|
// the lexer reads bytes (wide characters arrive as UTF-8), so
|
||||||
|
// move both cuts to character boundaries to keep the message
|
||||||
|
// valid UTF-8; this only shortens the two parts
|
||||||
|
const auto head_end = chars.begin() + static_cast<std::ptrdiff_t>(utf8_head_cut(chars.data(), shown));
|
||||||
|
const auto tail_begin = chars.end() - static_cast<std::ptrdiff_t>(shown);
|
||||||
|
append_escaped(result, chars.begin(), head_end);
|
||||||
result += "...";
|
result += "...";
|
||||||
append_escaped(result, chars.end() - static_cast<std::ptrdiff_t>(shown), chars.end());
|
append_escaped(result, tail_begin + static_cast<std::ptrdiff_t>(utf8_tail_skip(&*tail_begin)), chars.end());
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@@ -2291,6 +2296,55 @@ scan_number_done:
|
|||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
/// whether @a c is a UTF-8 continuation byte
|
||||||
|
static bool is_utf8_continuation(const char_type c) noexcept
|
||||||
|
{
|
||||||
|
return (static_cast<unsigned char>(c) & 0xC0u) == 0x80u;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*!
|
||||||
|
@brief the length of the first @a n characters at @a data without a UTF-8
|
||||||
|
character that begins there but does not end there
|
||||||
|
|
||||||
|
Looks only at the last 4 of these characters: in valid UTF-8, at most 3
|
||||||
|
bytes of a character can be before the cut.
|
||||||
|
*/
|
||||||
|
static std::size_t utf8_head_cut(const char_type* data, const std::size_t n) noexcept
|
||||||
|
{
|
||||||
|
std::size_t lead = n - 1;
|
||||||
|
while (lead > n - 4 && is_utf8_continuation(data[lead]))
|
||||||
|
{
|
||||||
|
--lead;
|
||||||
|
}
|
||||||
|
const auto c = static_cast<unsigned char>(data[lead]);
|
||||||
|
std::size_t length = 1;
|
||||||
|
if (c >= 0xF0u)
|
||||||
|
{
|
||||||
|
length = 4;
|
||||||
|
}
|
||||||
|
else if (c >= 0xE0u)
|
||||||
|
{
|
||||||
|
length = 3;
|
||||||
|
}
|
||||||
|
else if (c >= 0xC0u)
|
||||||
|
{
|
||||||
|
length = 2;
|
||||||
|
}
|
||||||
|
return lead + length > n ? lead : n;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// the number of UTF-8 continuation bytes (at most 3) at @a data, which
|
||||||
|
/// belong to a character that begins before it
|
||||||
|
static std::size_t utf8_tail_skip(const char_type* data) noexcept
|
||||||
|
{
|
||||||
|
std::size_t skip = 0;
|
||||||
|
while (skip < 3 && is_utf8_continuation(data[skip]))
|
||||||
|
{
|
||||||
|
++skip;
|
||||||
|
}
|
||||||
|
return skip;
|
||||||
|
}
|
||||||
|
|
||||||
/// append the characters [first, last) to @a result, escaping control characters
|
/// append the characters [first, last) to @a result, escaping control characters
|
||||||
template<typename Iterator>
|
template<typename Iterator>
|
||||||
static void append_escaped(std::string& result, Iterator first, Iterator last)
|
static void append_escaped(std::string& result, Iterator first, Iterator last)
|
||||||
|
|||||||
Generated
+99
-74
@@ -15782,9 +15782,14 @@ scan_number_done:
|
|||||||
std::string result;
|
std::string result;
|
||||||
if (chars.size() > 2 * shown)
|
if (chars.size() > 2 * shown)
|
||||||
{
|
{
|
||||||
append_escaped(result, chars.begin(), chars.begin() + static_cast<std::ptrdiff_t>(shown));
|
// the lexer reads bytes (wide characters arrive as UTF-8), so
|
||||||
|
// move both cuts to character boundaries to keep the message
|
||||||
|
// valid UTF-8; this only shortens the two parts
|
||||||
|
const auto head_end = chars.begin() + static_cast<std::ptrdiff_t>(utf8_head_cut(chars.data(), shown));
|
||||||
|
const auto tail_begin = chars.end() - static_cast<std::ptrdiff_t>(shown);
|
||||||
|
append_escaped(result, chars.begin(), head_end);
|
||||||
result += "...";
|
result += "...";
|
||||||
append_escaped(result, chars.end() - static_cast<std::ptrdiff_t>(shown), chars.end());
|
append_escaped(result, tail_begin + static_cast<std::ptrdiff_t>(utf8_tail_skip(&*tail_begin)), chars.end());
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@@ -15794,6 +15799,55 @@ scan_number_done:
|
|||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
/// whether @a c is a UTF-8 continuation byte
|
||||||
|
static bool is_utf8_continuation(const char_type c) noexcept
|
||||||
|
{
|
||||||
|
return (static_cast<unsigned char>(c) & 0xC0u) == 0x80u;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*!
|
||||||
|
@brief the length of the first @a n characters at @a data without a UTF-8
|
||||||
|
character that begins there but does not end there
|
||||||
|
|
||||||
|
Looks only at the last 4 of these characters: in valid UTF-8, at most 3
|
||||||
|
bytes of a character can be before the cut.
|
||||||
|
*/
|
||||||
|
static std::size_t utf8_head_cut(const char_type* data, const std::size_t n) noexcept
|
||||||
|
{
|
||||||
|
std::size_t lead = n - 1;
|
||||||
|
while (lead > n - 4 && is_utf8_continuation(data[lead]))
|
||||||
|
{
|
||||||
|
--lead;
|
||||||
|
}
|
||||||
|
const auto c = static_cast<unsigned char>(data[lead]);
|
||||||
|
std::size_t length = 1;
|
||||||
|
if (c >= 0xF0u)
|
||||||
|
{
|
||||||
|
length = 4;
|
||||||
|
}
|
||||||
|
else if (c >= 0xE0u)
|
||||||
|
{
|
||||||
|
length = 3;
|
||||||
|
}
|
||||||
|
else if (c >= 0xC0u)
|
||||||
|
{
|
||||||
|
length = 2;
|
||||||
|
}
|
||||||
|
return lead + length > n ? lead : n;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// the number of UTF-8 continuation bytes (at most 3) at @a data, which
|
||||||
|
/// belong to a character that begins before it
|
||||||
|
static std::size_t utf8_tail_skip(const char_type* data) noexcept
|
||||||
|
{
|
||||||
|
std::size_t skip = 0;
|
||||||
|
while (skip < 3 && is_utf8_continuation(data[skip]))
|
||||||
|
{
|
||||||
|
++skip;
|
||||||
|
}
|
||||||
|
return skip;
|
||||||
|
}
|
||||||
|
|
||||||
/// append the characters [first, last) to @a result, escaping control characters
|
/// append the characters [first, last) to @a result, escaping control characters
|
||||||
template<typename Iterator>
|
template<typename Iterator>
|
||||||
static void append_escaped(std::string& result, Iterator first, Iterator last)
|
static void append_escaped(std::string& result, Iterator first, Iterator last)
|
||||||
@@ -20605,21 +20659,6 @@ class binary_reader
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*!
|
|
||||||
@brief move past a value of @a size bytes, starting with @a head, that a
|
|
||||||
bulk run reads, as get() and get_number() would
|
|
||||||
|
|
||||||
Called before the value is passed on, so that an error reported while
|
|
||||||
passing it on (see @ref emit_float) has the position and the last byte
|
|
||||||
that reading the value byte by byte gives.
|
|
||||||
*/
|
|
||||||
void bulk_advance(const std::size_t size, const unsigned char head)
|
|
||||||
{
|
|
||||||
ia.bulk_skip(size);
|
|
||||||
chars_read += size;
|
|
||||||
current = head;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*!
|
/*!
|
||||||
@brief read the integers, floats, booleans, and nulls that follow in a
|
@brief read the integers, floats, booleans, and nulls that follow in a
|
||||||
definite-length array, without the per-element bookkeeping of
|
definite-length array, without the per-element bookkeeping of
|
||||||
@@ -20645,70 +20684,71 @@ class binary_reader
|
|||||||
{
|
{
|
||||||
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
||||||
const unsigned char head = data[0];
|
const unsigned char head = data[0];
|
||||||
|
std::size_t size = 0;
|
||||||
bool accepted = false;
|
bool accepted = false;
|
||||||
switch (head)
|
switch (head)
|
||||||
{
|
{
|
||||||
case 0x18:
|
case 0x18:
|
||||||
bulk_advance(2, head);
|
size = 2;
|
||||||
accepted = emit_unsigned(data[1]);
|
accepted = emit_unsigned(data[1]);
|
||||||
break;
|
break;
|
||||||
case 0x19:
|
case 0x19:
|
||||||
bulk_advance(3, head);
|
size = 3;
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x1A:
|
case 0x1A:
|
||||||
bulk_advance(5, head);
|
size = 5;
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x1B:
|
case 0x1B:
|
||||||
bulk_advance(9, head);
|
size = 9;
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x38:
|
case 0x38:
|
||||||
bulk_advance(2, head);
|
size = 2;
|
||||||
accepted = emit_cbor_negative_integer(data[1]);
|
accepted = emit_cbor_negative_integer(data[1]);
|
||||||
break;
|
break;
|
||||||
case 0x39:
|
case 0x39:
|
||||||
bulk_advance(3, head);
|
size = 3;
|
||||||
accepted = emit_cbor_negative_integer(load_big_endian<std::uint16_t>(data + 1));
|
accepted = emit_cbor_negative_integer(load_big_endian<std::uint16_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x3A:
|
case 0x3A:
|
||||||
bulk_advance(5, head);
|
size = 5;
|
||||||
accepted = emit_cbor_negative_integer(load_big_endian<std::uint32_t>(data + 1));
|
accepted = emit_cbor_negative_integer(load_big_endian<std::uint32_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0x3B:
|
case 0x3B:
|
||||||
bulk_advance(9, head);
|
size = 9;
|
||||||
accepted = emit_cbor_negative_integer(load_big_endian<std::uint64_t>(data + 1));
|
accepted = emit_cbor_negative_integer(load_big_endian<std::uint64_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xF4:
|
case 0xF4:
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->boolean(false);
|
accepted = sax->boolean(false);
|
||||||
break;
|
break;
|
||||||
case 0xF5:
|
case 0xF5:
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->boolean(true);
|
accepted = sax->boolean(true);
|
||||||
break;
|
break;
|
||||||
case 0xF6:
|
case 0xF6:
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->null();
|
accepted = sax->null();
|
||||||
break;
|
break;
|
||||||
case 0xFA:
|
case 0xFA:
|
||||||
bulk_advance(5, head);
|
size = 5;
|
||||||
accepted = emit_float(load_big_endian<float>(data + 1));
|
accepted = emit_float(load_big_endian<float>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xFB:
|
case 0xFB:
|
||||||
bulk_advance(9, head);
|
size = 9;
|
||||||
accepted = emit_float(load_big_endian<double>(data + 1));
|
accepted = emit_float(load_big_endian<double>(data + 1));
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
if (head <= 0x17)
|
if (head <= 0x17)
|
||||||
{
|
{
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
|
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
|
||||||
}
|
}
|
||||||
else if (head >= 0x20 && head <= 0x37)
|
else if (head >= 0x20 && head <= 0x37)
|
||||||
{
|
{
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - head));
|
accepted = sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - head));
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@@ -20718,6 +20758,10 @@ class binary_reader
|
|||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
// as if read by get() and get_number(): current is the head byte
|
||||||
|
ia.bulk_skip(size);
|
||||||
|
chars_read += size;
|
||||||
|
current = head;
|
||||||
--remaining;
|
--remaining;
|
||||||
if (JSON_HEDLEY_UNLIKELY(!accepted))
|
if (JSON_HEDLEY_UNLIKELY(!accepted))
|
||||||
{
|
{
|
||||||
@@ -21555,72 +21599,73 @@ class binary_reader
|
|||||||
{
|
{
|
||||||
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
||||||
const unsigned char head = data[0];
|
const unsigned char head = data[0];
|
||||||
|
std::size_t size = 0;
|
||||||
bool accepted = false;
|
bool accepted = false;
|
||||||
switch (head)
|
switch (head)
|
||||||
{
|
{
|
||||||
case 0xC0: // nil
|
case 0xC0: // nil
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->null();
|
accepted = sax->null();
|
||||||
break;
|
break;
|
||||||
case 0xC2: // false
|
case 0xC2: // false
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->boolean(false);
|
accepted = sax->boolean(false);
|
||||||
break;
|
break;
|
||||||
case 0xC3: // true
|
case 0xC3: // true
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->boolean(true);
|
accepted = sax->boolean(true);
|
||||||
break;
|
break;
|
||||||
case 0xCA: // float 32
|
case 0xCA: // float 32
|
||||||
bulk_advance(5, head);
|
size = 5;
|
||||||
accepted = emit_float(load_big_endian<float>(data + 1));
|
accepted = emit_float(load_big_endian<float>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xCB: // float 64
|
case 0xCB: // float 64
|
||||||
bulk_advance(9, head);
|
size = 9;
|
||||||
accepted = emit_float(load_big_endian<double>(data + 1));
|
accepted = emit_float(load_big_endian<double>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xCC: // uint 8
|
case 0xCC: // uint 8
|
||||||
bulk_advance(2, head);
|
size = 2;
|
||||||
accepted = emit_unsigned(data[1]);
|
accepted = emit_unsigned(data[1]);
|
||||||
break;
|
break;
|
||||||
case 0xCD: // uint 16
|
case 0xCD: // uint 16
|
||||||
bulk_advance(3, head);
|
size = 3;
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xCE: // uint 32
|
case 0xCE: // uint 32
|
||||||
bulk_advance(5, head);
|
size = 5;
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xCF: // uint 64
|
case 0xCF: // uint 64
|
||||||
bulk_advance(9, head);
|
size = 9;
|
||||||
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
|
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xD0: // int 8
|
case 0xD0: // int 8
|
||||||
bulk_advance(2, head);
|
size = 2;
|
||||||
accepted = emit_signed(static_cast<std::int8_t>(data[1]));
|
accepted = emit_signed(static_cast<std::int8_t>(data[1]));
|
||||||
break;
|
break;
|
||||||
case 0xD1: // int 16
|
case 0xD1: // int 16
|
||||||
bulk_advance(3, head);
|
size = 3;
|
||||||
accepted = emit_signed(load_big_endian<std::int16_t>(data + 1));
|
accepted = emit_signed(load_big_endian<std::int16_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xD2: // int 32
|
case 0xD2: // int 32
|
||||||
bulk_advance(5, head);
|
size = 5;
|
||||||
accepted = emit_signed(load_big_endian<std::int32_t>(data + 1));
|
accepted = emit_signed(load_big_endian<std::int32_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
case 0xD3: // int 64
|
case 0xD3: // int 64
|
||||||
bulk_advance(9, head);
|
size = 9;
|
||||||
accepted = emit_signed(load_big_endian<std::int64_t>(data + 1));
|
accepted = emit_signed(load_big_endian<std::int64_t>(data + 1));
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
if (head <= 0x7F)
|
if (head <= 0x7F)
|
||||||
{
|
{
|
||||||
// positive fixint
|
// positive fixint
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
|
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
|
||||||
}
|
}
|
||||||
else if (head >= 0xE0)
|
else if (head >= 0xE0)
|
||||||
{
|
{
|
||||||
// negative fixint
|
// negative fixint
|
||||||
bulk_advance(1, head);
|
size = 1;
|
||||||
accepted = sax->number_integer(static_cast<std::int8_t>(head));
|
accepted = sax->number_integer(static_cast<std::int8_t>(head));
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@@ -21630,6 +21675,10 @@ class binary_reader
|
|||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
// as if read by get() and get_number(): current is the last byte
|
||||||
|
ia.bulk_skip(size);
|
||||||
|
chars_read += size;
|
||||||
|
current = data[size - 1];
|
||||||
--remaining;
|
--remaining;
|
||||||
if (JSON_HEDLEY_UNLIKELY(!accepted))
|
if (JSON_HEDLEY_UNLIKELY(!accepted))
|
||||||
{
|
{
|
||||||
@@ -22533,7 +22582,6 @@ class binary_reader
|
|||||||
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
||||||
// UBJSON stores numbers big-endian, BJData little-endian
|
// UBJSON stores numbers big-endian, BJData little-endian
|
||||||
const bool swap = is_little_endian != (input_format == input_format_t::bjdata);
|
const bool swap = is_little_endian != (input_format == input_format_t::bjdata);
|
||||||
const std::size_t start = chars_read;
|
|
||||||
std::size_t read = 0;
|
std::size_t read = 0;
|
||||||
bool accepted = true;
|
bool accepted = true;
|
||||||
while (read < count && accepted)
|
while (read < count && accepted)
|
||||||
@@ -22545,39 +22593,16 @@ class binary_reader
|
|||||||
byte_swap(number);
|
byte_swap(number);
|
||||||
}
|
}
|
||||||
++read;
|
++read;
|
||||||
// where passing the value on can report an error (see
|
|
||||||
// emit_float), the position is that after the value, as if it
|
|
||||||
// was read by get_number()
|
|
||||||
set_chars_read(start + (read * size), std::integral_constant<bool, may_overflow_number_float<NumberType>()> {});
|
|
||||||
accepted = emit_ubjson_number(number);
|
accepted = emit_ubjson_number(number);
|
||||||
}
|
}
|
||||||
// get_number() does not change current
|
// as if read by get_number(): current is the last byte read
|
||||||
ia.bulk_skip(read * size);
|
ia.bulk_skip(read * size);
|
||||||
chars_read = start + (read * size);
|
chars_read += read * size;
|
||||||
|
current = data[(read * size) - 1];
|
||||||
remaining -= read;
|
remaining -= read;
|
||||||
return accepted;
|
return accepted;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*!
|
|
||||||
@brief whether a value of type @a NumberType can be finite and still
|
|
||||||
overflow number_float_t, which @ref emit_float reports as an error
|
|
||||||
|
|
||||||
False for the default number types, so that the loop in @ref
|
|
||||||
parse_ubjson_typed_run need not track the position per value.
|
|
||||||
*/
|
|
||||||
template<typename NumberType>
|
|
||||||
static constexpr bool may_overflow_number_float() noexcept
|
|
||||||
{
|
|
||||||
return static_cast<long double>((std::numeric_limits<number_float_t>::max)()) < static_cast<long double>((std::numeric_limits<NumberType>::max)());
|
|
||||||
}
|
|
||||||
|
|
||||||
void set_chars_read(const std::size_t position, std::true_type /*track*/) noexcept
|
|
||||||
{
|
|
||||||
chars_read = position;
|
|
||||||
}
|
|
||||||
|
|
||||||
void set_chars_read(const std::size_t /*position*/, std::false_type /*track*/) const noexcept {}
|
|
||||||
|
|
||||||
template<typename NumberType, enable_if_t<std::is_floating_point<NumberType>::value, int> = 0>
|
template<typename NumberType, enable_if_t<std::is_floating_point<NumberType>::value, int> = 0>
|
||||||
bool emit_ubjson_number(const NumberType number)
|
bool emit_ubjson_number(const NumberType number)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ using nlohmann::json;
|
|||||||
#include <cmath>
|
#include <cmath>
|
||||||
#include <fstream>
|
#include <fstream>
|
||||||
#include <limits>
|
#include <limits>
|
||||||
#include <list>
|
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
@@ -368,125 +367,3 @@ TEST_CASE("Binary formats with narrow number types")
|
|||||||
CHECK(format.decode(format.encode(json(1e300)), false).is_discarded());
|
CHECK(format.decode(format.encode(json(1e300)), false).is_discarded());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace
|
|
||||||
{
|
|
||||||
// records the SAX events of narrow_json, including the arguments of parse_error
|
|
||||||
struct narrow_event_recorder
|
|
||||||
{
|
|
||||||
using number_integer_t = narrow_json::number_integer_t;
|
|
||||||
using number_unsigned_t = narrow_json::number_unsigned_t;
|
|
||||||
using number_float_t = narrow_json::number_float_t;
|
|
||||||
using string_t = narrow_json::string_t;
|
|
||||||
using binary_t = narrow_json::binary_t;
|
|
||||||
|
|
||||||
bool null()
|
|
||||||
{
|
|
||||||
events.emplace_back("null");
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
bool boolean(bool val)
|
|
||||||
{
|
|
||||||
events.emplace_back(val ? "true" : "false");
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
bool number_integer(number_integer_t val)
|
|
||||||
{
|
|
||||||
events.push_back(std::to_string(val));
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
bool number_unsigned(number_unsigned_t val)
|
|
||||||
{
|
|
||||||
events.push_back(std::to_string(val));
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
bool number_float(number_float_t val, const string_t& /*unused*/)
|
|
||||||
{
|
|
||||||
events.push_back(std::to_string(val));
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
bool string(string_t& val)
|
|
||||||
{
|
|
||||||
events.push_back(val);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
bool binary(binary_t& /*unused*/)
|
|
||||||
{
|
|
||||||
events.emplace_back("binary");
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
bool start_object(std::size_t /*unused*/)
|
|
||||||
{
|
|
||||||
events.emplace_back("{");
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
bool key(string_t& val)
|
|
||||||
{
|
|
||||||
events.push_back(val);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
bool end_object()
|
|
||||||
{
|
|
||||||
events.emplace_back("}");
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
bool start_array(std::size_t /*unused*/)
|
|
||||||
{
|
|
||||||
events.emplace_back("[");
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
bool end_array()
|
|
||||||
{
|
|
||||||
events.emplace_back("]");
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
bool parse_error(std::size_t position, const std::string& last_token, const nlohmann::detail::exception& ex)
|
|
||||||
{
|
|
||||||
events.push_back("parse_error(" + std::to_string(position) + ", " + last_token + ", " + ex.what() + ")");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::vector<std::string> events {}; // NOLINT(readability-redundant-member-init)
|
|
||||||
};
|
|
||||||
} // namespace
|
|
||||||
|
|
||||||
TEST_CASE("Binary formats with narrow number types from contiguous and non-contiguous input")
|
|
||||||
{
|
|
||||||
// contiguous input reads runs of numbers in one loop; a number that
|
|
||||||
// overflows number_float_t must be reported with the same position and
|
|
||||||
// last byte as when the input is read byte by byte
|
|
||||||
const json values = {1, 70000, -70000, 1.5, 2.5, 1e300, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5};
|
|
||||||
const json doubles = {1.5, 2.5, 3.5, 1e300, 4.5, 5.5, 6.5, 7.5, 8.5};
|
|
||||||
|
|
||||||
struct binary_input
|
|
||||||
{
|
|
||||||
const char* name;
|
|
||||||
json::input_format_t format;
|
|
||||||
bytes data;
|
|
||||||
};
|
|
||||||
const std::vector<binary_input> inputs =
|
|
||||||
{
|
|
||||||
{"CBOR", json::input_format_t::cbor, json::to_cbor(values)},
|
|
||||||
{"CBOR doubles", json::input_format_t::cbor, json::to_cbor(doubles)},
|
|
||||||
{"MessagePack", json::input_format_t::msgpack, json::to_msgpack(values)},
|
|
||||||
{"MessagePack doubles", json::input_format_t::msgpack, json::to_msgpack(doubles)},
|
|
||||||
{"UBJSON", json::input_format_t::ubjson, json::to_ubjson(values, true)},
|
|
||||||
{"UBJSON typed doubles", json::input_format_t::ubjson, json::to_ubjson(doubles, true, true)},
|
|
||||||
{"BJData", json::input_format_t::bjdata, json::to_bjdata(values, true)},
|
|
||||||
{"BJData typed doubles", json::input_format_t::bjdata, json::to_bjdata(doubles, true, true)},
|
|
||||||
};
|
|
||||||
|
|
||||||
for (const auto& input : inputs)
|
|
||||||
{
|
|
||||||
const std::string name = input.name;
|
|
||||||
CAPTURE(name)
|
|
||||||
const std::list<std::uint8_t> bytewise(input.data.begin(), input.data.end());
|
|
||||||
narrow_event_recorder from_contiguous;
|
|
||||||
narrow_event_recorder from_bytewise;
|
|
||||||
CHECK_FALSE(narrow_json::sax_parse(input.data, &from_contiguous, input.format));
|
|
||||||
CHECK_FALSE(narrow_json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, input.format));
|
|
||||||
CHECK(from_contiguous.events == from_bytewise.events);
|
|
||||||
REQUIRE(!from_contiguous.events.empty());
|
|
||||||
CHECK(from_contiguous.events.back().find("out_of_range.406") != std::string::npos);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -3380,5 +3380,108 @@ TEST_CASE("parse error message of a long token")
|
|||||||
CHECK(message(text, kind) == "no error");
|
CHECK(message(text, kind) == "no error");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
SECTION("a long string of multi-byte characters")
|
||||||
|
{
|
||||||
|
// the parts of a long token are cut at character boundaries, so the
|
||||||
|
// message of valid UTF-8 input is valid UTF-8, also for wide strings
|
||||||
|
const auto last_read = [](const std::string & m) -> std::string
|
||||||
|
{
|
||||||
|
const auto pos = m.find("last read: '");
|
||||||
|
return pos == std::string::npos ? std::string() : m.substr(pos + 12, m.size() - pos - 13);
|
||||||
|
};
|
||||||
|
const auto utf8 = [](const char32_t c) -> std::string
|
||||||
|
{
|
||||||
|
std::string result;
|
||||||
|
if (c < 0x800)
|
||||||
|
{
|
||||||
|
result.push_back(static_cast<char>(0xC0 | (c >> 6)));
|
||||||
|
}
|
||||||
|
else if (c < 0x10000)
|
||||||
|
{
|
||||||
|
result.push_back(static_cast<char>(0xE0 | (c >> 12)));
|
||||||
|
result.push_back(static_cast<char>(0x80 | ((c >> 6) & 0x3F)));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
result.push_back(static_cast<char>(0xF0 | (c >> 18)));
|
||||||
|
result.push_back(static_cast<char>(0x80 | ((c >> 12) & 0x3F)));
|
||||||
|
result.push_back(static_cast<char>(0x80 | ((c >> 6) & 0x3F)));
|
||||||
|
}
|
||||||
|
result.push_back(static_cast<char>(0x80 | (c & 0x3F)));
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
// characters of 2, 3, and 4 bytes
|
||||||
|
const std::vector<char32_t> characters = {0xE9, 0x20AC, 0x1F600};
|
||||||
|
for (const char32_t c : characters)
|
||||||
|
{
|
||||||
|
for (std::size_t offset = 0; offset < 4; ++offset)
|
||||||
|
{
|
||||||
|
CAPTURE(static_cast<std::uint32_t>(c))
|
||||||
|
CAPTURE(offset)
|
||||||
|
// an unterminated string: the token is the whole input
|
||||||
|
std::string text = "\"" + std::string(offset, 'a');
|
||||||
|
std::u32string text32 = U"\"" + std::u32string(offset, U'a');
|
||||||
|
while (text.size() < 9000)
|
||||||
|
{
|
||||||
|
text += utf8(c);
|
||||||
|
text32.push_back(c);
|
||||||
|
}
|
||||||
|
std::u16string text16;
|
||||||
|
for (const char32_t c32 : text32)
|
||||||
|
{
|
||||||
|
if (c32 < 0x10000)
|
||||||
|
{
|
||||||
|
text16.push_back(static_cast<char16_t>(c32));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
text16.push_back(static_cast<char16_t>(0xD800 + ((c32 - 0x10000) >> 10)));
|
||||||
|
text16.push_back(static_cast<char16_t>(0xDC00 + ((c32 - 0x10000) & 0x3FF)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::string shown = last_read(message(text, 0));
|
||||||
|
const auto dots = shown.find("...");
|
||||||
|
REQUIRE(dots != std::string::npos);
|
||||||
|
// the first and the last at most 4096 bytes, without parts of characters
|
||||||
|
CHECK(dots <= 4096);
|
||||||
|
CHECK(dots > 4092);
|
||||||
|
CHECK(text.compare(0, dots, shown, 0, dots) == 0);
|
||||||
|
const std::size_t tail = shown.size() - dots - 3;
|
||||||
|
CHECK(tail <= 4096);
|
||||||
|
CHECK(tail > 4092);
|
||||||
|
CHECK(text.compare(text.size() - tail, tail, shown, dots + 3, tail) == 0);
|
||||||
|
CHECK_NOTHROW(json(shown).dump());
|
||||||
|
|
||||||
|
for (int kind = 1; kind < 4; ++kind)
|
||||||
|
{
|
||||||
|
CAPTURE(kind)
|
||||||
|
CHECK(last_read(message(text, kind)) == shown);
|
||||||
|
}
|
||||||
|
json _;
|
||||||
|
std::string m32;
|
||||||
|
std::string m16;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_ = json::parse(text32);
|
||||||
|
}
|
||||||
|
catch (const json::parse_error& e)
|
||||||
|
{
|
||||||
|
m32 = e.what();
|
||||||
|
}
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_ = json::parse(text16);
|
||||||
|
}
|
||||||
|
catch (const json::parse_error& e)
|
||||||
|
{
|
||||||
|
m16 = e.what();
|
||||||
|
}
|
||||||
|
CHECK(last_read(m32) == shown);
|
||||||
|
CHECK(last_read(m16) == shown);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
Reference in new issue
Block a user