Compare commits

..
Author SHA1 Message Date
Niels Lohmann 04fbced73d Cut long tokens in parse error messages at UTF-8 character boundaries
A parse error message shows a token of more than 8192 bytes by its
first and its last 4096 bytes around "...". The cuts were at fixed byte
offsets, so they could split a UTF-8 character: for valid UTF-8 input,
what() and the last token passed to SAX parse_error() were not valid
UTF-8, and storing the message in a json value and dumping it threw
type_error.316.

Move the end of the first part back and the start of the last part
forward to a character boundary, by at most three bytes each. The
lexer always reads bytes (wide strings are converted to UTF-8 first),
and the decision depends only on bytes inside the two parts, so the
message stays the same for every kind of input. Both parts only get
shorter, so the buffers that keep the token need no more room.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-11-16 08:12:02 +01:00
6 changed files with 302 additions and 272 deletions

No files matched your search

+1 -1
View File
@@ -140,7 +140,7 @@ Exceptions have ids 1xx.
This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member `byte` indicates the error position.
The message ends with the last read token (`last read: '...'`), with control characters written as `<U+XXXX>`. A token of more than 8192 characters is shown by its first and its last 4096 characters, separated by `...`.
The message ends with the last read token (`last read: '...'`), with control characters written as `<U+XXXX>`. A token of more than 8192 bytes is shown by its first and its last 4096 bytes, separated by `...`; each part is shortened by up to three bytes so that it does not end or begin within a UTF-8 character.
!!! failure "Example message"
+43 -72
View File
@@ -1927,21 +1927,6 @@ class binary_reader
return result;
}
/*!
@brief move past a value of @a size bytes, starting with @a head, that a
bulk run reads, as get() and get_number() would
Called before the value is passed on, so that an error reported while
passing it on (see @ref emit_float) has the position and the last byte
that reading the value byte by byte gives.
*/
void bulk_advance(const std::size_t size, const unsigned char head)
{
ia.bulk_skip(size);
chars_read += size;
current = head;
}
/*!
@brief read the integers, floats, booleans, and nulls that follow in a
definite-length array, without the per-element bookkeeping of
@@ -1967,70 +1952,71 @@ class binary_reader
{
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
const unsigned char head = data[0];
std::size_t size = 0;
bool accepted = false;
switch (head)
{
case 0x18:
bulk_advance(2, head);
size = 2;
accepted = emit_unsigned(data[1]);
break;
case 0x19:
bulk_advance(3, head);
size = 3;
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
break;
case 0x1A:
bulk_advance(5, head);
size = 5;
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
break;
case 0x1B:
bulk_advance(9, head);
size = 9;
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
break;
case 0x38:
bulk_advance(2, head);
size = 2;
accepted = emit_cbor_negative_integer(data[1]);
break;
case 0x39:
bulk_advance(3, head);
size = 3;
accepted = emit_cbor_negative_integer(load_big_endian<std::uint16_t>(data + 1));
break;
case 0x3A:
bulk_advance(5, head);
size = 5;
accepted = emit_cbor_negative_integer(load_big_endian<std::uint32_t>(data + 1));
break;
case 0x3B:
bulk_advance(9, head);
size = 9;
accepted = emit_cbor_negative_integer(load_big_endian<std::uint64_t>(data + 1));
break;
case 0xF4:
bulk_advance(1, head);
size = 1;
accepted = sax->boolean(false);
break;
case 0xF5:
bulk_advance(1, head);
size = 1;
accepted = sax->boolean(true);
break;
case 0xF6:
bulk_advance(1, head);
size = 1;
accepted = sax->null();
break;
case 0xFA:
bulk_advance(5, head);
size = 5;
accepted = emit_float(load_big_endian<float>(data + 1));
break;
case 0xFB:
bulk_advance(9, head);
size = 9;
accepted = emit_float(load_big_endian<double>(data + 1));
break;
default:
if (head <= 0x17)
{
bulk_advance(1, head);
size = 1;
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
}
else if (head >= 0x20 && head <= 0x37)
{
bulk_advance(1, head);
size = 1;
accepted = sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - head));
}
else
@@ -2040,6 +2026,10 @@ class binary_reader
}
break;
}
// as if read by get() and get_number(): current is the head byte
ia.bulk_skip(size);
chars_read += size;
current = head;
--remaining;
if (JSON_HEDLEY_UNLIKELY(!accepted))
{
@@ -2877,72 +2867,73 @@ class binary_reader
{
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
const unsigned char head = data[0];
std::size_t size = 0;
bool accepted = false;
switch (head)
{
case 0xC0: // nil
bulk_advance(1, head);
size = 1;
accepted = sax->null();
break;
case 0xC2: // false
bulk_advance(1, head);
size = 1;
accepted = sax->boolean(false);
break;
case 0xC3: // true
bulk_advance(1, head);
size = 1;
accepted = sax->boolean(true);
break;
case 0xCA: // float 32
bulk_advance(5, head);
size = 5;
accepted = emit_float(load_big_endian<float>(data + 1));
break;
case 0xCB: // float 64
bulk_advance(9, head);
size = 9;
accepted = emit_float(load_big_endian<double>(data + 1));
break;
case 0xCC: // uint 8
bulk_advance(2, head);
size = 2;
accepted = emit_unsigned(data[1]);
break;
case 0xCD: // uint 16
bulk_advance(3, head);
size = 3;
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
break;
case 0xCE: // uint 32
bulk_advance(5, head);
size = 5;
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
break;
case 0xCF: // uint 64
bulk_advance(9, head);
size = 9;
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
break;
case 0xD0: // int 8
bulk_advance(2, head);
size = 2;
accepted = emit_signed(static_cast<std::int8_t>(data[1]));
break;
case 0xD1: // int 16
bulk_advance(3, head);
size = 3;
accepted = emit_signed(load_big_endian<std::int16_t>(data + 1));
break;
case 0xD2: // int 32
bulk_advance(5, head);
size = 5;
accepted = emit_signed(load_big_endian<std::int32_t>(data + 1));
break;
case 0xD3: // int 64
bulk_advance(9, head);
size = 9;
accepted = emit_signed(load_big_endian<std::int64_t>(data + 1));
break;
default:
if (head <= 0x7F)
{
// positive fixint
bulk_advance(1, head);
size = 1;
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
}
else if (head >= 0xE0)
{
// negative fixint
bulk_advance(1, head);
size = 1;
accepted = sax->number_integer(static_cast<std::int8_t>(head));
}
else
@@ -2952,6 +2943,10 @@ class binary_reader
}
break;
}
// as if read by get() and get_number(): current is the last byte
ia.bulk_skip(size);
chars_read += size;
current = data[size - 1];
--remaining;
if (JSON_HEDLEY_UNLIKELY(!accepted))
{
@@ -3855,7 +3850,6 @@ class binary_reader
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
// UBJSON stores numbers big-endian, BJData little-endian
const bool swap = is_little_endian != (input_format == input_format_t::bjdata);
const std::size_t start = chars_read;
std::size_t read = 0;
bool accepted = true;
while (read < count && accepted)
@@ -3867,39 +3861,16 @@ class binary_reader
byte_swap(number);
}
++read;
// where passing the value on can report an error (see
// emit_float), the position is that after the value, as if it
// was read by get_number()
set_chars_read(start + (read * size), std::integral_constant<bool, may_overflow_number_float<NumberType>()> {});
accepted = emit_ubjson_number(number);
}
// get_number() does not change current
// as if read by get_number(): current is the last byte read
ia.bulk_skip(read * size);
chars_read = start + (read * size);
chars_read += read * size;
current = data[(read * size) - 1];
remaining -= read;
return accepted;
}
/*!
@brief whether a value of type @a NumberType can be finite and still
overflow number_float_t, which @ref emit_float reports as an error
False for the default number types, so that the loop in @ref
parse_ubjson_typed_run need not track the position per value.
*/
template<typename NumberType>
static constexpr bool may_overflow_number_float() noexcept
{
return static_cast<long double>((std::numeric_limits<number_float_t>::max)()) < static_cast<long double>((std::numeric_limits<NumberType>::max)());
}
void set_chars_read(const std::size_t position, std::true_type /*track*/) noexcept
{
chars_read = position;
}
void set_chars_read(const std::size_t /*position*/, std::false_type /*track*/) const noexcept {}
template<typename NumberType, enable_if_t<std::is_floating_point<NumberType>::value, int> = 0>
bool emit_ubjson_number(const NumberType number)
{
+56 -2
View File
@@ -2279,9 +2279,14 @@ scan_number_done:
std::string result;
if (chars.size() > 2 * shown)
{
append_escaped(result, chars.begin(), chars.begin() + static_cast<std::ptrdiff_t>(shown));
// the lexer reads bytes (wide characters arrive as UTF-8), so
// move both cuts to character boundaries to keep the message
// valid UTF-8; this only shortens the two parts
const auto head_end = chars.begin() + static_cast<std::ptrdiff_t>(utf8_head_cut(chars.data(), shown));
const auto tail_begin = chars.end() - static_cast<std::ptrdiff_t>(shown);
append_escaped(result, chars.begin(), head_end);
result += "...";
append_escaped(result, chars.end() - static_cast<std::ptrdiff_t>(shown), chars.end());
append_escaped(result, tail_begin + static_cast<std::ptrdiff_t>(utf8_tail_skip(&*tail_begin)), chars.end());
}
else
{
@@ -2291,6 +2296,55 @@ scan_number_done:
}
private:
/// whether @a c is a UTF-8 continuation byte
static bool is_utf8_continuation(const char_type c) noexcept
{
return (static_cast<unsigned char>(c) & 0xC0u) == 0x80u;
}
/*!
@brief the length of the first @a n characters at @a data without a UTF-8
character that begins there but does not end there
Looks only at the last 4 of these characters: in valid UTF-8, at most 3
bytes of a character can be before the cut.
*/
static std::size_t utf8_head_cut(const char_type* data, const std::size_t n) noexcept
{
std::size_t lead = n - 1;
while (lead > n - 4 && is_utf8_continuation(data[lead]))
{
--lead;
}
const auto c = static_cast<unsigned char>(data[lead]);
std::size_t length = 1;
if (c >= 0xF0u)
{
length = 4;
}
else if (c >= 0xE0u)
{
length = 3;
}
else if (c >= 0xC0u)
{
length = 2;
}
return lead + length > n ? lead : n;
}
/// the number of UTF-8 continuation bytes (at most 3) at @a data, which
/// belong to a character that begins before it
static std::size_t utf8_tail_skip(const char_type* data) noexcept
{
std::size_t skip = 0;
while (skip < 3 && is_utf8_continuation(data[skip]))
{
++skip;
}
return skip;
}
/// append the characters [first, last) to @a result, escaping control characters
template<typename Iterator>
static void append_escaped(std::string& result, Iterator first, Iterator last)
+99 -74
View File
@@ -15782,9 +15782,14 @@ scan_number_done:
std::string result;
if (chars.size() > 2 * shown)
{
append_escaped(result, chars.begin(), chars.begin() + static_cast<std::ptrdiff_t>(shown));
// the lexer reads bytes (wide characters arrive as UTF-8), so
// move both cuts to character boundaries to keep the message
// valid UTF-8; this only shortens the two parts
const auto head_end = chars.begin() + static_cast<std::ptrdiff_t>(utf8_head_cut(chars.data(), shown));
const auto tail_begin = chars.end() - static_cast<std::ptrdiff_t>(shown);
append_escaped(result, chars.begin(), head_end);
result += "...";
append_escaped(result, chars.end() - static_cast<std::ptrdiff_t>(shown), chars.end());
append_escaped(result, tail_begin + static_cast<std::ptrdiff_t>(utf8_tail_skip(&*tail_begin)), chars.end());
}
else
{
@@ -15794,6 +15799,55 @@ scan_number_done:
}
private:
/// whether @a c is a UTF-8 continuation byte
static bool is_utf8_continuation(const char_type c) noexcept
{
return (static_cast<unsigned char>(c) & 0xC0u) == 0x80u;
}
/*!
@brief the length of the first @a n characters at @a data without a UTF-8
character that begins there but does not end there
Looks only at the last 4 of these characters: in valid UTF-8, at most 3
bytes of a character can be before the cut.
*/
static std::size_t utf8_head_cut(const char_type* data, const std::size_t n) noexcept
{
std::size_t lead = n - 1;
while (lead > n - 4 && is_utf8_continuation(data[lead]))
{
--lead;
}
const auto c = static_cast<unsigned char>(data[lead]);
std::size_t length = 1;
if (c >= 0xF0u)
{
length = 4;
}
else if (c >= 0xE0u)
{
length = 3;
}
else if (c >= 0xC0u)
{
length = 2;
}
return lead + length > n ? lead : n;
}
/// the number of UTF-8 continuation bytes (at most 3) at @a data, which
/// belong to a character that begins before it
static std::size_t utf8_tail_skip(const char_type* data) noexcept
{
std::size_t skip = 0;
while (skip < 3 && is_utf8_continuation(data[skip]))
{
++skip;
}
return skip;
}
/// append the characters [first, last) to @a result, escaping control characters
template<typename Iterator>
static void append_escaped(std::string& result, Iterator first, Iterator last)
@@ -20605,21 +20659,6 @@ class binary_reader
return result;
}
/*!
@brief move past a value of @a size bytes, starting with @a head, that a
bulk run reads, as get() and get_number() would
Called before the value is passed on, so that an error reported while
passing it on (see @ref emit_float) has the position and the last byte
that reading the value byte by byte gives.
*/
void bulk_advance(const std::size_t size, const unsigned char head)
{
ia.bulk_skip(size);
chars_read += size;
current = head;
}
/*!
@brief read the integers, floats, booleans, and nulls that follow in a
definite-length array, without the per-element bookkeeping of
@@ -20645,70 +20684,71 @@ class binary_reader
{
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
const unsigned char head = data[0];
std::size_t size = 0;
bool accepted = false;
switch (head)
{
case 0x18:
bulk_advance(2, head);
size = 2;
accepted = emit_unsigned(data[1]);
break;
case 0x19:
bulk_advance(3, head);
size = 3;
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
break;
case 0x1A:
bulk_advance(5, head);
size = 5;
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
break;
case 0x1B:
bulk_advance(9, head);
size = 9;
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
break;
case 0x38:
bulk_advance(2, head);
size = 2;
accepted = emit_cbor_negative_integer(data[1]);
break;
case 0x39:
bulk_advance(3, head);
size = 3;
accepted = emit_cbor_negative_integer(load_big_endian<std::uint16_t>(data + 1));
break;
case 0x3A:
bulk_advance(5, head);
size = 5;
accepted = emit_cbor_negative_integer(load_big_endian<std::uint32_t>(data + 1));
break;
case 0x3B:
bulk_advance(9, head);
size = 9;
accepted = emit_cbor_negative_integer(load_big_endian<std::uint64_t>(data + 1));
break;
case 0xF4:
bulk_advance(1, head);
size = 1;
accepted = sax->boolean(false);
break;
case 0xF5:
bulk_advance(1, head);
size = 1;
accepted = sax->boolean(true);
break;
case 0xF6:
bulk_advance(1, head);
size = 1;
accepted = sax->null();
break;
case 0xFA:
bulk_advance(5, head);
size = 5;
accepted = emit_float(load_big_endian<float>(data + 1));
break;
case 0xFB:
bulk_advance(9, head);
size = 9;
accepted = emit_float(load_big_endian<double>(data + 1));
break;
default:
if (head <= 0x17)
{
bulk_advance(1, head);
size = 1;
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
}
else if (head >= 0x20 && head <= 0x37)
{
bulk_advance(1, head);
size = 1;
accepted = sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - head));
}
else
@@ -20718,6 +20758,10 @@ class binary_reader
}
break;
}
// as if read by get() and get_number(): current is the head byte
ia.bulk_skip(size);
chars_read += size;
current = head;
--remaining;
if (JSON_HEDLEY_UNLIKELY(!accepted))
{
@@ -21555,72 +21599,73 @@ class binary_reader
{
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
const unsigned char head = data[0];
std::size_t size = 0;
bool accepted = false;
switch (head)
{
case 0xC0: // nil
bulk_advance(1, head);
size = 1;
accepted = sax->null();
break;
case 0xC2: // false
bulk_advance(1, head);
size = 1;
accepted = sax->boolean(false);
break;
case 0xC3: // true
bulk_advance(1, head);
size = 1;
accepted = sax->boolean(true);
break;
case 0xCA: // float 32
bulk_advance(5, head);
size = 5;
accepted = emit_float(load_big_endian<float>(data + 1));
break;
case 0xCB: // float 64
bulk_advance(9, head);
size = 9;
accepted = emit_float(load_big_endian<double>(data + 1));
break;
case 0xCC: // uint 8
bulk_advance(2, head);
size = 2;
accepted = emit_unsigned(data[1]);
break;
case 0xCD: // uint 16
bulk_advance(3, head);
size = 3;
accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
break;
case 0xCE: // uint 32
bulk_advance(5, head);
size = 5;
accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
break;
case 0xCF: // uint 64
bulk_advance(9, head);
size = 9;
accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
break;
case 0xD0: // int 8
bulk_advance(2, head);
size = 2;
accepted = emit_signed(static_cast<std::int8_t>(data[1]));
break;
case 0xD1: // int 16
bulk_advance(3, head);
size = 3;
accepted = emit_signed(load_big_endian<std::int16_t>(data + 1));
break;
case 0xD2: // int 32
bulk_advance(5, head);
size = 5;
accepted = emit_signed(load_big_endian<std::int32_t>(data + 1));
break;
case 0xD3: // int 64
bulk_advance(9, head);
size = 9;
accepted = emit_signed(load_big_endian<std::int64_t>(data + 1));
break;
default:
if (head <= 0x7F)
{
// positive fixint
bulk_advance(1, head);
size = 1;
accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
}
else if (head >= 0xE0)
{
// negative fixint
bulk_advance(1, head);
size = 1;
accepted = sax->number_integer(static_cast<std::int8_t>(head));
}
else
@@ -21630,6 +21675,10 @@ class binary_reader
}
break;
}
// as if read by get() and get_number(): current is the last byte
ia.bulk_skip(size);
chars_read += size;
current = data[size - 1];
--remaining;
if (JSON_HEDLEY_UNLIKELY(!accepted))
{
@@ -22533,7 +22582,6 @@ class binary_reader
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
// UBJSON stores numbers big-endian, BJData little-endian
const bool swap = is_little_endian != (input_format == input_format_t::bjdata);
const std::size_t start = chars_read;
std::size_t read = 0;
bool accepted = true;
while (read < count && accepted)
@@ -22545,39 +22593,16 @@ class binary_reader
byte_swap(number);
}
++read;
// where passing the value on can report an error (see
// emit_float), the position is that after the value, as if it
// was read by get_number()
set_chars_read(start + (read * size), std::integral_constant<bool, may_overflow_number_float<NumberType>()> {});
accepted = emit_ubjson_number(number);
}
// get_number() does not change current
// as if read by get_number(): current is the last byte read
ia.bulk_skip(read * size);
chars_read = start + (read * size);
chars_read += read * size;
current = data[(read * size) - 1];
remaining -= read;
return accepted;
}
/*!
@brief whether a value of type @a NumberType can be finite and still
overflow number_float_t, which @ref emit_float reports as an error
False for the default number types, so that the loop in @ref
parse_ubjson_typed_run need not track the position per value.
*/
template<typename NumberType>
static constexpr bool may_overflow_number_float() noexcept
{
return static_cast<long double>((std::numeric_limits<number_float_t>::max)()) < static_cast<long double>((std::numeric_limits<NumberType>::max)());
}
void set_chars_read(const std::size_t position, std::true_type /*track*/) noexcept
{
chars_read = position;
}
void set_chars_read(const std::size_t /*position*/, std::false_type /*track*/) const noexcept {}
template<typename NumberType, enable_if_t<std::is_floating_point<NumberType>::value, int> = 0>
bool emit_ubjson_number(const NumberType number)
{
-123
View File
@@ -14,7 +14,6 @@ using nlohmann::json;
#include <cmath>
#include <fstream>
#include <limits>
#include <list>
#include <map>
#include <string>
#include <vector>
@@ -368,125 +367,3 @@ TEST_CASE("Binary formats with narrow number types")
CHECK(format.decode(format.encode(json(1e300)), false).is_discarded());
}
}
namespace
{
// records the SAX events of narrow_json, including the arguments of parse_error
struct narrow_event_recorder
{
using number_integer_t = narrow_json::number_integer_t;
using number_unsigned_t = narrow_json::number_unsigned_t;
using number_float_t = narrow_json::number_float_t;
using string_t = narrow_json::string_t;
using binary_t = narrow_json::binary_t;
bool null()
{
events.emplace_back("null");
return true;
}
bool boolean(bool val)
{
events.emplace_back(val ? "true" : "false");
return true;
}
bool number_integer(number_integer_t val)
{
events.push_back(std::to_string(val));
return true;
}
bool number_unsigned(number_unsigned_t val)
{
events.push_back(std::to_string(val));
return true;
}
bool number_float(number_float_t val, const string_t& /*unused*/)
{
events.push_back(std::to_string(val));
return true;
}
bool string(string_t& val)
{
events.push_back(val);
return true;
}
bool binary(binary_t& /*unused*/)
{
events.emplace_back("binary");
return true;
}
bool start_object(std::size_t /*unused*/)
{
events.emplace_back("{");
return true;
}
bool key(string_t& val)
{
events.push_back(val);
return true;
}
bool end_object()
{
events.emplace_back("}");
return true;
}
bool start_array(std::size_t /*unused*/)
{
events.emplace_back("[");
return true;
}
bool end_array()
{
events.emplace_back("]");
return true;
}
bool parse_error(std::size_t position, const std::string& last_token, const nlohmann::detail::exception& ex)
{
events.push_back("parse_error(" + std::to_string(position) + ", " + last_token + ", " + ex.what() + ")");
return false;
}
std::vector<std::string> events {}; // NOLINT(readability-redundant-member-init)
};
} // namespace
TEST_CASE("Binary formats with narrow number types from contiguous and non-contiguous input")
{
// contiguous input reads runs of numbers in one loop; a number that
// overflows number_float_t must be reported with the same position and
// last byte as when the input is read byte by byte
const json values = {1, 70000, -70000, 1.5, 2.5, 1e300, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5};
const json doubles = {1.5, 2.5, 3.5, 1e300, 4.5, 5.5, 6.5, 7.5, 8.5};
struct binary_input
{
const char* name;
json::input_format_t format;
bytes data;
};
const std::vector<binary_input> inputs =
{
{"CBOR", json::input_format_t::cbor, json::to_cbor(values)},
{"CBOR doubles", json::input_format_t::cbor, json::to_cbor(doubles)},
{"MessagePack", json::input_format_t::msgpack, json::to_msgpack(values)},
{"MessagePack doubles", json::input_format_t::msgpack, json::to_msgpack(doubles)},
{"UBJSON", json::input_format_t::ubjson, json::to_ubjson(values, true)},
{"UBJSON typed doubles", json::input_format_t::ubjson, json::to_ubjson(doubles, true, true)},
{"BJData", json::input_format_t::bjdata, json::to_bjdata(values, true)},
{"BJData typed doubles", json::input_format_t::bjdata, json::to_bjdata(doubles, true, true)},
};
for (const auto& input : inputs)
{
const std::string name = input.name;
CAPTURE(name)
const std::list<std::uint8_t> bytewise(input.data.begin(), input.data.end());
narrow_event_recorder from_contiguous;
narrow_event_recorder from_bytewise;
CHECK_FALSE(narrow_json::sax_parse(input.data, &from_contiguous, input.format));
CHECK_FALSE(narrow_json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, input.format));
CHECK(from_contiguous.events == from_bytewise.events);
REQUIRE(!from_contiguous.events.empty());
CHECK(from_contiguous.events.back().find("out_of_range.406") != std::string::npos);
}
}
+103
View File
@@ -3380,5 +3380,108 @@ TEST_CASE("parse error message of a long token")
CHECK(message(text, kind) == "no error");
}
}
SECTION("a long string of multi-byte characters")
{
// the parts of a long token are cut at character boundaries, so the
// message of valid UTF-8 input is valid UTF-8, also for wide strings
const auto last_read = [](const std::string & m) -> std::string
{
const auto pos = m.find("last read: '");
return pos == std::string::npos ? std::string() : m.substr(pos + 12, m.size() - pos - 13);
};
const auto utf8 = [](const char32_t c) -> std::string
{
std::string result;
if (c < 0x800)
{
result.push_back(static_cast<char>(0xC0 | (c >> 6)));
}
else if (c < 0x10000)
{
result.push_back(static_cast<char>(0xE0 | (c >> 12)));
result.push_back(static_cast<char>(0x80 | ((c >> 6) & 0x3F)));
}
else
{
result.push_back(static_cast<char>(0xF0 | (c >> 18)));
result.push_back(static_cast<char>(0x80 | ((c >> 12) & 0x3F)));
result.push_back(static_cast<char>(0x80 | ((c >> 6) & 0x3F)));
}
result.push_back(static_cast<char>(0x80 | (c & 0x3F)));
return result;
};
// characters of 2, 3, and 4 bytes
const std::vector<char32_t> characters = {0xE9, 0x20AC, 0x1F600};
for (const char32_t c : characters)
{
for (std::size_t offset = 0; offset < 4; ++offset)
{
CAPTURE(static_cast<std::uint32_t>(c))
CAPTURE(offset)
// an unterminated string: the token is the whole input
std::string text = "\"" + std::string(offset, 'a');
std::u32string text32 = U"\"" + std::u32string(offset, U'a');
while (text.size() < 9000)
{
text += utf8(c);
text32.push_back(c);
}
std::u16string text16;
for (const char32_t c32 : text32)
{
if (c32 < 0x10000)
{
text16.push_back(static_cast<char16_t>(c32));
}
else
{
text16.push_back(static_cast<char16_t>(0xD800 + ((c32 - 0x10000) >> 10)));
text16.push_back(static_cast<char16_t>(0xDC00 + ((c32 - 0x10000) & 0x3FF)));
}
}
const std::string shown = last_read(message(text, 0));
const auto dots = shown.find("...");
REQUIRE(dots != std::string::npos);
// the first and the last at most 4096 bytes, without parts of characters
CHECK(dots <= 4096);
CHECK(dots > 4092);
CHECK(text.compare(0, dots, shown, 0, dots) == 0);
const std::size_t tail = shown.size() - dots - 3;
CHECK(tail <= 4096);
CHECK(tail > 4092);
CHECK(text.compare(text.size() - tail, tail, shown, dots + 3, tail) == 0);
CHECK_NOTHROW(json(shown).dump());
for (int kind = 1; kind < 4; ++kind)
{
CAPTURE(kind)
CHECK(last_read(message(text, kind)) == shown);
}
json _;
std::string m32;
std::string m16;
try
{
_ = json::parse(text32);
}
catch (const json::parse_error& e)
{
m32 = e.what();
}
try
{
_ = json::parse(text16);
}
catch (const json::parse_error& e)
{
m16 = e.what();
}
CHECK(last_read(m32) == shown);
CHECK(last_read(m16) == shown);
}
}
}
}
#endif