Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Niels Lohmann
|
b2c20c01b1 | ||
|
Niels Lohmann
|
9cbdcb5ed3 |
No files matched your search
@@ -772,33 +772,6 @@ class serializer
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// A byte below 0x60 is a complete code point and needs no
|
||||
// UTF-8 decoding. The scanner stops only at a control
|
||||
// character, a quote, or a backslash in that range, so look up
|
||||
// its escape sequence and write all 8 bytes of the table entry
|
||||
// straight into the write buffer; only the length stored in the
|
||||
// last byte of the entry is kept.
|
||||
if (data[i] < 0x60)
|
||||
{
|
||||
// emit any bytes still pending in string_buffer first to
|
||||
// preserve output order
|
||||
if (bytes != 0)
|
||||
{
|
||||
put_buffer(string_buffer, bytes);
|
||||
bytes = 0;
|
||||
}
|
||||
const auto& escape = ascii_escapes()[data[i]];
|
||||
if (JSON_HEDLEY_UNLIKELY(write_buffer.size() - write_buffer_pos < escape.size()))
|
||||
{
|
||||
flush();
|
||||
}
|
||||
std::memcpy(write_buffer.data() + write_buffer_pos, escape.data(), escape.size());
|
||||
write_buffer_pos += static_cast<std::size_t>(escape[7]);
|
||||
bytes_after_last_accept = 0;
|
||||
undumped_chars = 0;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const auto byte = static_cast<std::uint8_t>(s[i]);
|
||||
@@ -807,38 +780,91 @@ class serializer
|
||||
{
|
||||
case UTF8_ACCEPT: // decode found a new code point
|
||||
{
|
||||
// the code point is at least 0x60 (smaller ones are handled
|
||||
// above) and is not printable ASCII (which the scanner
|
||||
// copies): it is DEL (0x7F) under EnsureAscii or a non-ASCII
|
||||
// character
|
||||
if (EnsureAscii)
|
||||
switch (codepoint)
|
||||
{
|
||||
JSON_ASSERT(codepoint >= 0x7F);
|
||||
if (error_handler == error_handler_t::keep)
|
||||
case 0x08: // backspace
|
||||
{
|
||||
// this character was buffered as raw bytes below
|
||||
// in case it turned out to be part of an
|
||||
// ill-formed sequence (which is kept as is); now
|
||||
// that it decoded to a well-formed code point,
|
||||
// undo that and \u-escape it like any other
|
||||
// character instead
|
||||
bytes = bytes_after_last_accept;
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 'b';
|
||||
break;
|
||||
}
|
||||
if (codepoint <= 0xFFFF)
|
||||
|
||||
case 0x09: // horizontal tab
|
||||
{
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(codepoint));
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 't';
|
||||
break;
|
||||
}
|
||||
else
|
||||
|
||||
case 0x0A: // newline
|
||||
{
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(0xD7C0u + (codepoint >> 10u)));
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(0xDC00u + (codepoint & 0x3FFu)));
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 'n';
|
||||
break;
|
||||
}
|
||||
|
||||
case 0x0C: // formfeed
|
||||
{
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 'f';
|
||||
break;
|
||||
}
|
||||
|
||||
case 0x0D: // carriage return
|
||||
{
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 'r';
|
||||
break;
|
||||
}
|
||||
|
||||
case 0x22: // quotation mark
|
||||
{
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = '"';
|
||||
break;
|
||||
}
|
||||
|
||||
case 0x5C: // reverse solidus
|
||||
{
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = '\\';
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
{
|
||||
// escape control characters (0x00..0x1F) or, if
|
||||
// EnsureAscii parameter is used, non-ASCII characters
|
||||
if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F)))
|
||||
{
|
||||
if (EnsureAscii && error_handler == error_handler_t::keep)
|
||||
{
|
||||
// this character was buffered as raw bytes
|
||||
// below in case it turned out to be part of
|
||||
// an ill-formed sequence (which is kept as
|
||||
// is); now that it decoded to a well-formed
|
||||
// code point, undo that and \u-escape it
|
||||
// like any other character instead
|
||||
bytes = bytes_after_last_accept;
|
||||
}
|
||||
if (codepoint <= 0xFFFF)
|
||||
{
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(codepoint));
|
||||
}
|
||||
else
|
||||
{
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(0xD7C0u + (codepoint >> 10u)));
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(0xDC00u + (codepoint & 0x3FFu)));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// copy byte to buffer (all previous bytes
|
||||
// been copied have in default case above)
|
||||
string_buffer[bytes++] = s[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// copy byte to buffer (all previous bytes of the
|
||||
// sequence have been copied in the default case below)
|
||||
string_buffer[bytes++] = s[i];
|
||||
}
|
||||
|
||||
// write buffer and reset index; there must be 13 bytes
|
||||
@@ -1227,121 +1253,6 @@ class serializer
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief the JSON form of each character below 0x60
|
||||
|
||||
Entry c holds the escape sequence of character c - the short form for
|
||||
\\b, \\t, \\n, \\f, \\r, \\" and \\\\, \\u00xx for the other control characters -
|
||||
or c itself if it needs no escaping, padded with zeros to 7 bytes. The last
|
||||
byte is the length of the sequence. The 8 bytes of an entry are copied at
|
||||
once, so writing an escape needs neither a branch nor a byte-wise loop.
|
||||
*/
|
||||
static const std::array<std::array<char, 8>, 0x60>& ascii_escapes() noexcept
|
||||
{
|
||||
static const std::array<std::array<char, 8>, 0x60> escapes =
|
||||
{
|
||||
{
|
||||
{{'\\', 'u', '0', '0', '0', '0', 0, 6}}, // 0x00
|
||||
{{'\\', 'u', '0', '0', '0', '1', 0, 6}}, // 0x01
|
||||
{{'\\', 'u', '0', '0', '0', '2', 0, 6}}, // 0x02
|
||||
{{'\\', 'u', '0', '0', '0', '3', 0, 6}}, // 0x03
|
||||
{{'\\', 'u', '0', '0', '0', '4', 0, 6}}, // 0x04
|
||||
{{'\\', 'u', '0', '0', '0', '5', 0, 6}}, // 0x05
|
||||
{{'\\', 'u', '0', '0', '0', '6', 0, 6}}, // 0x06
|
||||
{{'\\', 'u', '0', '0', '0', '7', 0, 6}}, // 0x07
|
||||
{{'\\', 'b', 0, 0, 0, 0, 0, 2}}, // 0x08
|
||||
{{'\\', 't', 0, 0, 0, 0, 0, 2}}, // 0x09
|
||||
{{'\\', 'n', 0, 0, 0, 0, 0, 2}}, // 0x0A
|
||||
{{'\\', 'u', '0', '0', '0', 'b', 0, 6}}, // 0x0B
|
||||
{{'\\', 'f', 0, 0, 0, 0, 0, 2}}, // 0x0C
|
||||
{{'\\', 'r', 0, 0, 0, 0, 0, 2}}, // 0x0D
|
||||
{{'\\', 'u', '0', '0', '0', 'e', 0, 6}}, // 0x0E
|
||||
{{'\\', 'u', '0', '0', '0', 'f', 0, 6}}, // 0x0F
|
||||
{{'\\', 'u', '0', '0', '1', '0', 0, 6}}, // 0x10
|
||||
{{'\\', 'u', '0', '0', '1', '1', 0, 6}}, // 0x11
|
||||
{{'\\', 'u', '0', '0', '1', '2', 0, 6}}, // 0x12
|
||||
{{'\\', 'u', '0', '0', '1', '3', 0, 6}}, // 0x13
|
||||
{{'\\', 'u', '0', '0', '1', '4', 0, 6}}, // 0x14
|
||||
{{'\\', 'u', '0', '0', '1', '5', 0, 6}}, // 0x15
|
||||
{{'\\', 'u', '0', '0', '1', '6', 0, 6}}, // 0x16
|
||||
{{'\\', 'u', '0', '0', '1', '7', 0, 6}}, // 0x17
|
||||
{{'\\', 'u', '0', '0', '1', '8', 0, 6}}, // 0x18
|
||||
{{'\\', 'u', '0', '0', '1', '9', 0, 6}}, // 0x19
|
||||
{{'\\', 'u', '0', '0', '1', 'a', 0, 6}}, // 0x1A
|
||||
{{'\\', 'u', '0', '0', '1', 'b', 0, 6}}, // 0x1B
|
||||
{{'\\', 'u', '0', '0', '1', 'c', 0, 6}}, // 0x1C
|
||||
{{'\\', 'u', '0', '0', '1', 'd', 0, 6}}, // 0x1D
|
||||
{{'\\', 'u', '0', '0', '1', 'e', 0, 6}}, // 0x1E
|
||||
{{'\\', 'u', '0', '0', '1', 'f', 0, 6}}, // 0x1F
|
||||
{{' ', 0, 0, 0, 0, 0, 0, 1}}, // 0x20
|
||||
{{'!', 0, 0, 0, 0, 0, 0, 1}}, // 0x21
|
||||
{{'\\', '"', 0, 0, 0, 0, 0, 2}}, // 0x22
|
||||
{{'#', 0, 0, 0, 0, 0, 0, 1}}, // 0x23
|
||||
{{'$', 0, 0, 0, 0, 0, 0, 1}}, // 0x24
|
||||
{{'%', 0, 0, 0, 0, 0, 0, 1}}, // 0x25
|
||||
{{'&', 0, 0, 0, 0, 0, 0, 1}}, // 0x26
|
||||
{{'\'', 0, 0, 0, 0, 0, 0, 1}}, // 0x27
|
||||
{{'(', 0, 0, 0, 0, 0, 0, 1}}, // 0x28
|
||||
{{')', 0, 0, 0, 0, 0, 0, 1}}, // 0x29
|
||||
{{'*', 0, 0, 0, 0, 0, 0, 1}}, // 0x2A
|
||||
{{'+', 0, 0, 0, 0, 0, 0, 1}}, // 0x2B
|
||||
{{',', 0, 0, 0, 0, 0, 0, 1}}, // 0x2C
|
||||
{{'-', 0, 0, 0, 0, 0, 0, 1}}, // 0x2D
|
||||
{{'.', 0, 0, 0, 0, 0, 0, 1}}, // 0x2E
|
||||
{{'/', 0, 0, 0, 0, 0, 0, 1}}, // 0x2F
|
||||
{{'0', 0, 0, 0, 0, 0, 0, 1}}, // 0x30
|
||||
{{'1', 0, 0, 0, 0, 0, 0, 1}}, // 0x31
|
||||
{{'2', 0, 0, 0, 0, 0, 0, 1}}, // 0x32
|
||||
{{'3', 0, 0, 0, 0, 0, 0, 1}}, // 0x33
|
||||
{{'4', 0, 0, 0, 0, 0, 0, 1}}, // 0x34
|
||||
{{'5', 0, 0, 0, 0, 0, 0, 1}}, // 0x35
|
||||
{{'6', 0, 0, 0, 0, 0, 0, 1}}, // 0x36
|
||||
{{'7', 0, 0, 0, 0, 0, 0, 1}}, // 0x37
|
||||
{{'8', 0, 0, 0, 0, 0, 0, 1}}, // 0x38
|
||||
{{'9', 0, 0, 0, 0, 0, 0, 1}}, // 0x39
|
||||
{{':', 0, 0, 0, 0, 0, 0, 1}}, // 0x3A
|
||||
{{';', 0, 0, 0, 0, 0, 0, 1}}, // 0x3B
|
||||
{{'<', 0, 0, 0, 0, 0, 0, 1}}, // 0x3C
|
||||
{{'=', 0, 0, 0, 0, 0, 0, 1}}, // 0x3D
|
||||
{{'>', 0, 0, 0, 0, 0, 0, 1}}, // 0x3E
|
||||
{{'?', 0, 0, 0, 0, 0, 0, 1}}, // 0x3F
|
||||
{{'@', 0, 0, 0, 0, 0, 0, 1}}, // 0x40
|
||||
{{'A', 0, 0, 0, 0, 0, 0, 1}}, // 0x41
|
||||
{{'B', 0, 0, 0, 0, 0, 0, 1}}, // 0x42
|
||||
{{'C', 0, 0, 0, 0, 0, 0, 1}}, // 0x43
|
||||
{{'D', 0, 0, 0, 0, 0, 0, 1}}, // 0x44
|
||||
{{'E', 0, 0, 0, 0, 0, 0, 1}}, // 0x45
|
||||
{{'F', 0, 0, 0, 0, 0, 0, 1}}, // 0x46
|
||||
{{'G', 0, 0, 0, 0, 0, 0, 1}}, // 0x47
|
||||
{{'H', 0, 0, 0, 0, 0, 0, 1}}, // 0x48
|
||||
{{'I', 0, 0, 0, 0, 0, 0, 1}}, // 0x49
|
||||
{{'J', 0, 0, 0, 0, 0, 0, 1}}, // 0x4A
|
||||
{{'K', 0, 0, 0, 0, 0, 0, 1}}, // 0x4B
|
||||
{{'L', 0, 0, 0, 0, 0, 0, 1}}, // 0x4C
|
||||
{{'M', 0, 0, 0, 0, 0, 0, 1}}, // 0x4D
|
||||
{{'N', 0, 0, 0, 0, 0, 0, 1}}, // 0x4E
|
||||
{{'O', 0, 0, 0, 0, 0, 0, 1}}, // 0x4F
|
||||
{{'P', 0, 0, 0, 0, 0, 0, 1}}, // 0x50
|
||||
{{'Q', 0, 0, 0, 0, 0, 0, 1}}, // 0x51
|
||||
{{'R', 0, 0, 0, 0, 0, 0, 1}}, // 0x52
|
||||
{{'S', 0, 0, 0, 0, 0, 0, 1}}, // 0x53
|
||||
{{'T', 0, 0, 0, 0, 0, 0, 1}}, // 0x54
|
||||
{{'U', 0, 0, 0, 0, 0, 0, 1}}, // 0x55
|
||||
{{'V', 0, 0, 0, 0, 0, 0, 1}}, // 0x56
|
||||
{{'W', 0, 0, 0, 0, 0, 0, 1}}, // 0x57
|
||||
{{'X', 0, 0, 0, 0, 0, 0, 1}}, // 0x58
|
||||
{{'Y', 0, 0, 0, 0, 0, 0, 1}}, // 0x59
|
||||
{{'Z', 0, 0, 0, 0, 0, 0, 1}}, // 0x5A
|
||||
{{'[', 0, 0, 0, 0, 0, 0, 1}}, // 0x5B
|
||||
{{'\\', '\\', 0, 0, 0, 0, 0, 2}}, // 0x5C
|
||||
{{']', 0, 0, 0, 0, 0, 0, 1}}, // 0x5D
|
||||
{{'^', 0, 0, 0, 0, 0, 0, 1}}, // 0x5E
|
||||
{{'_', 0, 0, 0, 0, 0, 0, 1}}, // 0x5F
|
||||
}
|
||||
};
|
||||
return escapes;
|
||||
}
|
||||
|
||||
/*!
|
||||
* @brief write a lowercase "\\uXXXX" escape sequence into @a string_buffer
|
||||
*
|
||||
|
||||
Generated
+78
-167
@@ -27087,33 +27087,6 @@ class serializer
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// A byte below 0x60 is a complete code point and needs no
|
||||
// UTF-8 decoding. The scanner stops only at a control
|
||||
// character, a quote, or a backslash in that range, so look up
|
||||
// its escape sequence and write all 8 bytes of the table entry
|
||||
// straight into the write buffer; only the length stored in the
|
||||
// last byte of the entry is kept.
|
||||
if (data[i] < 0x60)
|
||||
{
|
||||
// emit any bytes still pending in string_buffer first to
|
||||
// preserve output order
|
||||
if (bytes != 0)
|
||||
{
|
||||
put_buffer(string_buffer, bytes);
|
||||
bytes = 0;
|
||||
}
|
||||
const auto& escape = ascii_escapes()[data[i]];
|
||||
if (JSON_HEDLEY_UNLIKELY(write_buffer.size() - write_buffer_pos < escape.size()))
|
||||
{
|
||||
flush();
|
||||
}
|
||||
std::memcpy(write_buffer.data() + write_buffer_pos, escape.data(), escape.size());
|
||||
write_buffer_pos += static_cast<std::size_t>(escape[7]);
|
||||
bytes_after_last_accept = 0;
|
||||
undumped_chars = 0;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const auto byte = static_cast<std::uint8_t>(s[i]);
|
||||
@@ -27122,38 +27095,91 @@ class serializer
|
||||
{
|
||||
case UTF8_ACCEPT: // decode found a new code point
|
||||
{
|
||||
// the code point is at least 0x60 (smaller ones are handled
|
||||
// above) and is not printable ASCII (which the scanner
|
||||
// copies): it is DEL (0x7F) under EnsureAscii or a non-ASCII
|
||||
// character
|
||||
if (EnsureAscii)
|
||||
switch (codepoint)
|
||||
{
|
||||
JSON_ASSERT(codepoint >= 0x7F);
|
||||
if (error_handler == error_handler_t::keep)
|
||||
case 0x08: // backspace
|
||||
{
|
||||
// this character was buffered as raw bytes below
|
||||
// in case it turned out to be part of an
|
||||
// ill-formed sequence (which is kept as is); now
|
||||
// that it decoded to a well-formed code point,
|
||||
// undo that and \u-escape it like any other
|
||||
// character instead
|
||||
bytes = bytes_after_last_accept;
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 'b';
|
||||
break;
|
||||
}
|
||||
if (codepoint <= 0xFFFF)
|
||||
|
||||
case 0x09: // horizontal tab
|
||||
{
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(codepoint));
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 't';
|
||||
break;
|
||||
}
|
||||
else
|
||||
|
||||
case 0x0A: // newline
|
||||
{
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(0xD7C0u + (codepoint >> 10u)));
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(0xDC00u + (codepoint & 0x3FFu)));
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 'n';
|
||||
break;
|
||||
}
|
||||
|
||||
case 0x0C: // formfeed
|
||||
{
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 'f';
|
||||
break;
|
||||
}
|
||||
|
||||
case 0x0D: // carriage return
|
||||
{
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 'r';
|
||||
break;
|
||||
}
|
||||
|
||||
case 0x22: // quotation mark
|
||||
{
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = '"';
|
||||
break;
|
||||
}
|
||||
|
||||
case 0x5C: // reverse solidus
|
||||
{
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = '\\';
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
{
|
||||
// escape control characters (0x00..0x1F) or, if
|
||||
// EnsureAscii parameter is used, non-ASCII characters
|
||||
if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F)))
|
||||
{
|
||||
if (EnsureAscii && error_handler == error_handler_t::keep)
|
||||
{
|
||||
// this character was buffered as raw bytes
|
||||
// below in case it turned out to be part of
|
||||
// an ill-formed sequence (which is kept as
|
||||
// is); now that it decoded to a well-formed
|
||||
// code point, undo that and \u-escape it
|
||||
// like any other character instead
|
||||
bytes = bytes_after_last_accept;
|
||||
}
|
||||
if (codepoint <= 0xFFFF)
|
||||
{
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(codepoint));
|
||||
}
|
||||
else
|
||||
{
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(0xD7C0u + (codepoint >> 10u)));
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(0xDC00u + (codepoint & 0x3FFu)));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// copy byte to buffer (all previous bytes
|
||||
// been copied have in default case above)
|
||||
string_buffer[bytes++] = s[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// copy byte to buffer (all previous bytes of the
|
||||
// sequence have been copied in the default case below)
|
||||
string_buffer[bytes++] = s[i];
|
||||
}
|
||||
|
||||
// write buffer and reset index; there must be 13 bytes
|
||||
@@ -27542,121 +27568,6 @@ class serializer
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief the JSON form of each character below 0x60
|
||||
|
||||
Entry c holds the escape sequence of character c - the short form for
|
||||
\\b, \\t, \\n, \\f, \\r, \\" and \\\\, \\u00xx for the other control characters -
|
||||
or c itself if it needs no escaping, padded with zeros to 7 bytes. The last
|
||||
byte is the length of the sequence. The 8 bytes of an entry are copied at
|
||||
once, so writing an escape needs neither a branch nor a byte-wise loop.
|
||||
*/
|
||||
static const std::array<std::array<char, 8>, 0x60>& ascii_escapes() noexcept
|
||||
{
|
||||
static const std::array<std::array<char, 8>, 0x60> escapes =
|
||||
{
|
||||
{
|
||||
{{'\\', 'u', '0', '0', '0', '0', 0, 6}}, // 0x00
|
||||
{{'\\', 'u', '0', '0', '0', '1', 0, 6}}, // 0x01
|
||||
{{'\\', 'u', '0', '0', '0', '2', 0, 6}}, // 0x02
|
||||
{{'\\', 'u', '0', '0', '0', '3', 0, 6}}, // 0x03
|
||||
{{'\\', 'u', '0', '0', '0', '4', 0, 6}}, // 0x04
|
||||
{{'\\', 'u', '0', '0', '0', '5', 0, 6}}, // 0x05
|
||||
{{'\\', 'u', '0', '0', '0', '6', 0, 6}}, // 0x06
|
||||
{{'\\', 'u', '0', '0', '0', '7', 0, 6}}, // 0x07
|
||||
{{'\\', 'b', 0, 0, 0, 0, 0, 2}}, // 0x08
|
||||
{{'\\', 't', 0, 0, 0, 0, 0, 2}}, // 0x09
|
||||
{{'\\', 'n', 0, 0, 0, 0, 0, 2}}, // 0x0A
|
||||
{{'\\', 'u', '0', '0', '0', 'b', 0, 6}}, // 0x0B
|
||||
{{'\\', 'f', 0, 0, 0, 0, 0, 2}}, // 0x0C
|
||||
{{'\\', 'r', 0, 0, 0, 0, 0, 2}}, // 0x0D
|
||||
{{'\\', 'u', '0', '0', '0', 'e', 0, 6}}, // 0x0E
|
||||
{{'\\', 'u', '0', '0', '0', 'f', 0, 6}}, // 0x0F
|
||||
{{'\\', 'u', '0', '0', '1', '0', 0, 6}}, // 0x10
|
||||
{{'\\', 'u', '0', '0', '1', '1', 0, 6}}, // 0x11
|
||||
{{'\\', 'u', '0', '0', '1', '2', 0, 6}}, // 0x12
|
||||
{{'\\', 'u', '0', '0', '1', '3', 0, 6}}, // 0x13
|
||||
{{'\\', 'u', '0', '0', '1', '4', 0, 6}}, // 0x14
|
||||
{{'\\', 'u', '0', '0', '1', '5', 0, 6}}, // 0x15
|
||||
{{'\\', 'u', '0', '0', '1', '6', 0, 6}}, // 0x16
|
||||
{{'\\', 'u', '0', '0', '1', '7', 0, 6}}, // 0x17
|
||||
{{'\\', 'u', '0', '0', '1', '8', 0, 6}}, // 0x18
|
||||
{{'\\', 'u', '0', '0', '1', '9', 0, 6}}, // 0x19
|
||||
{{'\\', 'u', '0', '0', '1', 'a', 0, 6}}, // 0x1A
|
||||
{{'\\', 'u', '0', '0', '1', 'b', 0, 6}}, // 0x1B
|
||||
{{'\\', 'u', '0', '0', '1', 'c', 0, 6}}, // 0x1C
|
||||
{{'\\', 'u', '0', '0', '1', 'd', 0, 6}}, // 0x1D
|
||||
{{'\\', 'u', '0', '0', '1', 'e', 0, 6}}, // 0x1E
|
||||
{{'\\', 'u', '0', '0', '1', 'f', 0, 6}}, // 0x1F
|
||||
{{' ', 0, 0, 0, 0, 0, 0, 1}}, // 0x20
|
||||
{{'!', 0, 0, 0, 0, 0, 0, 1}}, // 0x21
|
||||
{{'\\', '"', 0, 0, 0, 0, 0, 2}}, // 0x22
|
||||
{{'#', 0, 0, 0, 0, 0, 0, 1}}, // 0x23
|
||||
{{'$', 0, 0, 0, 0, 0, 0, 1}}, // 0x24
|
||||
{{'%', 0, 0, 0, 0, 0, 0, 1}}, // 0x25
|
||||
{{'&', 0, 0, 0, 0, 0, 0, 1}}, // 0x26
|
||||
{{'\'', 0, 0, 0, 0, 0, 0, 1}}, // 0x27
|
||||
{{'(', 0, 0, 0, 0, 0, 0, 1}}, // 0x28
|
||||
{{')', 0, 0, 0, 0, 0, 0, 1}}, // 0x29
|
||||
{{'*', 0, 0, 0, 0, 0, 0, 1}}, // 0x2A
|
||||
{{'+', 0, 0, 0, 0, 0, 0, 1}}, // 0x2B
|
||||
{{',', 0, 0, 0, 0, 0, 0, 1}}, // 0x2C
|
||||
{{'-', 0, 0, 0, 0, 0, 0, 1}}, // 0x2D
|
||||
{{'.', 0, 0, 0, 0, 0, 0, 1}}, // 0x2E
|
||||
{{'/', 0, 0, 0, 0, 0, 0, 1}}, // 0x2F
|
||||
{{'0', 0, 0, 0, 0, 0, 0, 1}}, // 0x30
|
||||
{{'1', 0, 0, 0, 0, 0, 0, 1}}, // 0x31
|
||||
{{'2', 0, 0, 0, 0, 0, 0, 1}}, // 0x32
|
||||
{{'3', 0, 0, 0, 0, 0, 0, 1}}, // 0x33
|
||||
{{'4', 0, 0, 0, 0, 0, 0, 1}}, // 0x34
|
||||
{{'5', 0, 0, 0, 0, 0, 0, 1}}, // 0x35
|
||||
{{'6', 0, 0, 0, 0, 0, 0, 1}}, // 0x36
|
||||
{{'7', 0, 0, 0, 0, 0, 0, 1}}, // 0x37
|
||||
{{'8', 0, 0, 0, 0, 0, 0, 1}}, // 0x38
|
||||
{{'9', 0, 0, 0, 0, 0, 0, 1}}, // 0x39
|
||||
{{':', 0, 0, 0, 0, 0, 0, 1}}, // 0x3A
|
||||
{{';', 0, 0, 0, 0, 0, 0, 1}}, // 0x3B
|
||||
{{'<', 0, 0, 0, 0, 0, 0, 1}}, // 0x3C
|
||||
{{'=', 0, 0, 0, 0, 0, 0, 1}}, // 0x3D
|
||||
{{'>', 0, 0, 0, 0, 0, 0, 1}}, // 0x3E
|
||||
{{'?', 0, 0, 0, 0, 0, 0, 1}}, // 0x3F
|
||||
{{'@', 0, 0, 0, 0, 0, 0, 1}}, // 0x40
|
||||
{{'A', 0, 0, 0, 0, 0, 0, 1}}, // 0x41
|
||||
{{'B', 0, 0, 0, 0, 0, 0, 1}}, // 0x42
|
||||
{{'C', 0, 0, 0, 0, 0, 0, 1}}, // 0x43
|
||||
{{'D', 0, 0, 0, 0, 0, 0, 1}}, // 0x44
|
||||
{{'E', 0, 0, 0, 0, 0, 0, 1}}, // 0x45
|
||||
{{'F', 0, 0, 0, 0, 0, 0, 1}}, // 0x46
|
||||
{{'G', 0, 0, 0, 0, 0, 0, 1}}, // 0x47
|
||||
{{'H', 0, 0, 0, 0, 0, 0, 1}}, // 0x48
|
||||
{{'I', 0, 0, 0, 0, 0, 0, 1}}, // 0x49
|
||||
{{'J', 0, 0, 0, 0, 0, 0, 1}}, // 0x4A
|
||||
{{'K', 0, 0, 0, 0, 0, 0, 1}}, // 0x4B
|
||||
{{'L', 0, 0, 0, 0, 0, 0, 1}}, // 0x4C
|
||||
{{'M', 0, 0, 0, 0, 0, 0, 1}}, // 0x4D
|
||||
{{'N', 0, 0, 0, 0, 0, 0, 1}}, // 0x4E
|
||||
{{'O', 0, 0, 0, 0, 0, 0, 1}}, // 0x4F
|
||||
{{'P', 0, 0, 0, 0, 0, 0, 1}}, // 0x50
|
||||
{{'Q', 0, 0, 0, 0, 0, 0, 1}}, // 0x51
|
||||
{{'R', 0, 0, 0, 0, 0, 0, 1}}, // 0x52
|
||||
{{'S', 0, 0, 0, 0, 0, 0, 1}}, // 0x53
|
||||
{{'T', 0, 0, 0, 0, 0, 0, 1}}, // 0x54
|
||||
{{'U', 0, 0, 0, 0, 0, 0, 1}}, // 0x55
|
||||
{{'V', 0, 0, 0, 0, 0, 0, 1}}, // 0x56
|
||||
{{'W', 0, 0, 0, 0, 0, 0, 1}}, // 0x57
|
||||
{{'X', 0, 0, 0, 0, 0, 0, 1}}, // 0x58
|
||||
{{'Y', 0, 0, 0, 0, 0, 0, 1}}, // 0x59
|
||||
{{'Z', 0, 0, 0, 0, 0, 0, 1}}, // 0x5A
|
||||
{{'[', 0, 0, 0, 0, 0, 0, 1}}, // 0x5B
|
||||
{{'\\', '\\', 0, 0, 0, 0, 0, 2}}, // 0x5C
|
||||
{{']', 0, 0, 0, 0, 0, 0, 1}}, // 0x5D
|
||||
{{'^', 0, 0, 0, 0, 0, 0, 1}}, // 0x5E
|
||||
{{'_', 0, 0, 0, 0, 0, 0, 1}}, // 0x5F
|
||||
}
|
||||
};
|
||||
return escapes;
|
||||
}
|
||||
|
||||
/*!
|
||||
* @brief write a lowercase "\\uXXXX" escape sequence into @a string_buffer
|
||||
*
|
||||
|
||||
+10
-1
@@ -9,7 +9,16 @@ Each fuzzer driver (`tests/src/fuzzer-parse_*.cpp`) parses its input twice: once
|
||||
once with exceptions. Both calls must agree. Where parsing with exceptions fails, the call without exceptions must
|
||||
return a discarded value (or throw the same kind of non-parse error), and it must never throw a `parse_error`. Where
|
||||
parsing succeeds, both calls must return the same value. The drivers then serialize the value, parse the result back,
|
||||
and check that nothing was lost. The drivers check all of this with `assert`, so they refuse to build with `NDEBUG`.
|
||||
and check that nothing was lost.
|
||||
|
||||
Finally, each driver reads the same input through the other input adapters: a `std::istringstream`, a non-contiguous
|
||||
iterator range (`std::list<std::uint8_t>`), and, on POSIX systems, a `FILE*` from `fmemopen`. The pointer range takes the
|
||||
fast paths for contiguous input (bulk string scanning, multi-byte loads, lazy position tracking), while the other
|
||||
adapters read byte by byte. Every adapter must give the same value, or the same exception with the same id, message,
|
||||
and byte position. The JSON driver checks this for `parse` and `accept`, each with and without `ignore_comments`. The
|
||||
shared code is `check_input_adapters()` in `tests/src/fuzzer_common.hpp`.
|
||||
|
||||
The drivers check all of this with `assert`, so they refuse to build with `NDEBUG`.
|
||||
|
||||
## Corpus creation
|
||||
|
||||
|
||||
@@ -47,12 +47,30 @@ dumps is stable under exactly the same values that break operator==.
|
||||
The unit tests run the same checks on a fixed corpus (see the "BJData round-trip
|
||||
invariants" test case), so keep both in sync.
|
||||
|
||||
Finally, it reads the data from a stream, a non-contiguous iterator range, and
|
||||
a FILE*, which read byte by byte, and checks that each gives the same value or
|
||||
error (id, message, byte position) as the pointer range, which takes the bulk
|
||||
fast paths (see check_input_adapters()).
|
||||
|
||||
The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
|
||||
drivers.
|
||||
*/
|
||||
|
||||
#include "fuzzer_common.hpp"
|
||||
|
||||
namespace
|
||||
{
|
||||
// json::from_bjdata for check_input_adapters()
|
||||
struct read_bjdata
|
||||
{
|
||||
template<typename... Args>
|
||||
json operator()(Args&& ... args) const
|
||||
{
|
||||
return json::from_bjdata(std::forward<Args>(args)...);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
// value-stable comparison for the round-trip checks below; see the note
|
||||
// above on why this compares dump()s rather than the json values directly
|
||||
static bool is_value_stable(const json& lhs, const json& rhs)
|
||||
@@ -126,6 +144,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
assert(parsed || noexcept_threw || j_noexcept.is_discarded());
|
||||
}
|
||||
|
||||
// step 3: the other input adapters must read the data alike
|
||||
check_input_adapters(data, size, read_bjdata());
|
||||
|
||||
// return 0 - non-zero return values are reserved for future use
|
||||
return 0;
|
||||
}
|
||||
@@ -17,44 +17,33 @@ array data, it performs the following steps:
|
||||
- j2 = from_bon8(vec)
|
||||
- assert(to_bon8(j2) == vec)
|
||||
|
||||
It also checks that reading the data from a stream, which reads strings byte by
|
||||
byte, gives the same value or error as reading it from contiguous memory, which
|
||||
copies strings in bulk.
|
||||
Finally, it reads the data from a stream, a non-contiguous iterator range, and
|
||||
a FILE*, which read byte by byte, and checks that each gives the same value or
|
||||
error (id, message, byte position) as the pointer range, which takes the bulk
|
||||
fast paths (see check_input_adapters()).
|
||||
|
||||
The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
|
||||
drivers.
|
||||
*/
|
||||
|
||||
#include <sstream>
|
||||
#include "fuzzer_common.hpp"
|
||||
|
||||
namespace
|
||||
{
|
||||
// the serialization of the value read from @a input, or the error message
|
||||
template<typename InputType>
|
||||
std::string read_bon8(InputType&& input)
|
||||
// json::from_bon8 for check_input_adapters()
|
||||
struct read_bon8
|
||||
{
|
||||
try
|
||||
template<typename... Args>
|
||||
json operator()(Args&& ... args) const
|
||||
{
|
||||
const auto vec = json::to_bon8(json::from_bon8(std::forward<InputType>(input)));
|
||||
return {vec.begin(), vec.end()};
|
||||
return json::from_bon8(std::forward<Args>(args)...);
|
||||
}
|
||||
catch (const json::exception& e)
|
||||
{
|
||||
return e.what();
|
||||
}
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
// see http://llvm.org/docs/LibFuzzer.html
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
{
|
||||
// contiguous and stream input must be read alike
|
||||
{
|
||||
std::istringstream stream(std::string(reinterpret_cast<const char*>(data), size));
|
||||
assert(read_bon8(std::vector<uint8_t>(data, data + size)) == read_bon8(stream));
|
||||
}
|
||||
|
||||
std::vector<uint8_t> const vec1(data, data + size);
|
||||
|
||||
// step 0: parse input without exceptions
|
||||
@@ -106,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
assert(parsed || noexcept_threw || j_noexcept.is_discarded());
|
||||
}
|
||||
|
||||
// step 3: the other input adapters must read the data alike
|
||||
check_input_adapters(data, size, read_bon8());
|
||||
|
||||
// return 0 - non-zero return values are reserved for future use
|
||||
return 0;
|
||||
}
|
||||
@@ -17,12 +17,30 @@ array data, it performs the following steps:
|
||||
- j2 = from_bson(vec)
|
||||
- assert(to_bson(j2) == vec)
|
||||
|
||||
Finally, it reads the data from a stream, a non-contiguous iterator range, and
|
||||
a FILE*, which read byte by byte, and checks that each gives the same value or
|
||||
error (id, message, byte position) as the pointer range, which takes the bulk
|
||||
fast paths (see check_input_adapters()).
|
||||
|
||||
The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
|
||||
drivers.
|
||||
*/
|
||||
|
||||
#include "fuzzer_common.hpp"
|
||||
|
||||
namespace
|
||||
{
|
||||
// json::from_bson for check_input_adapters()
|
||||
struct read_bson
|
||||
{
|
||||
template<typename... Args>
|
||||
json operator()(Args&& ... args) const
|
||||
{
|
||||
return json::from_bson(std::forward<Args>(args)...);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
// see http://llvm.org/docs/LibFuzzer.html
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
{
|
||||
@@ -77,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
assert(parsed || noexcept_threw || j_noexcept.is_discarded());
|
||||
}
|
||||
|
||||
// step 3: the other input adapters must read the data alike
|
||||
check_input_adapters(data, size, read_bson());
|
||||
|
||||
// return 0 - non-zero return values are reserved for future use
|
||||
return 0;
|
||||
}
|
||||
@@ -17,12 +17,30 @@ array data, it performs the following steps:
|
||||
- j2 = from_cbor(vec)
|
||||
- assert(to_cbor(j2) == vec)
|
||||
|
||||
Finally, it reads the data from a stream, a non-contiguous iterator range, and
|
||||
a FILE*, which read byte by byte, and checks that each gives the same value or
|
||||
error (id, message, byte position) as the pointer range, which takes the bulk
|
||||
fast paths (see check_input_adapters()).
|
||||
|
||||
The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
|
||||
drivers.
|
||||
*/
|
||||
|
||||
#include "fuzzer_common.hpp"
|
||||
|
||||
namespace
|
||||
{
|
||||
// json::from_cbor for check_input_adapters()
|
||||
struct read_cbor
|
||||
{
|
||||
template<typename... Args>
|
||||
json operator()(Args&& ... args) const
|
||||
{
|
||||
return json::from_cbor(std::forward<Args>(args)...);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
// see http://llvm.org/docs/LibFuzzer.html
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
{
|
||||
@@ -77,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
assert(parsed || noexcept_threw || j_noexcept.is_discarded());
|
||||
}
|
||||
|
||||
// step 3: the other input adapters must read the data alike
|
||||
check_input_adapters(data, size, read_cbor());
|
||||
|
||||
// return 0 - non-zero return values are reserved for future use
|
||||
return 0;
|
||||
}
|
||||
@@ -17,6 +17,12 @@ array data, it performs the following steps:
|
||||
- j2 = parse(s1)
|
||||
- s2 = serialize(j2)
|
||||
- assert(s1 == s2)
|
||||
- for ignore_comments = false and true: assert(parse(data) and accept(data)
|
||||
give the same result for a stream, a non-contiguous iterator range, and a
|
||||
FILE* as for the pointer range)
|
||||
|
||||
The pointer range takes the bulk fast paths, the other input adapters read byte
|
||||
by byte; see check_input_adapters().
|
||||
|
||||
The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
|
||||
drivers.
|
||||
@@ -24,6 +30,32 @@ drivers.
|
||||
|
||||
#include "fuzzer_common.hpp"
|
||||
|
||||
namespace
|
||||
{
|
||||
// json::parse and json::accept for check_input_adapters()
|
||||
struct parse_json
|
||||
{
|
||||
bool ignore_comments;
|
||||
|
||||
template<typename... Args>
|
||||
json operator()(Args&& ... args) const
|
||||
{
|
||||
return json::parse(std::forward<Args>(args)..., nullptr, true, ignore_comments);
|
||||
}
|
||||
};
|
||||
|
||||
struct accept_json
|
||||
{
|
||||
bool ignore_comments;
|
||||
|
||||
template<typename... Args>
|
||||
bool operator()(Args&& ... args) const
|
||||
{
|
||||
return json::accept(std::forward<Args>(args)..., ignore_comments);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
// see http://llvm.org/docs/LibFuzzer.html
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
{
|
||||
@@ -76,6 +108,16 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
assert(parsed || noexcept_threw || j_noexcept.is_discarded());
|
||||
}
|
||||
|
||||
// step 3: the other input adapters must read the data alike
|
||||
for (const bool ignore_comments :
|
||||
{
|
||||
false, true
|
||||
})
|
||||
{
|
||||
check_input_adapters(data, size, parse_json{ignore_comments});
|
||||
check_input_adapters(data, size, accept_json{ignore_comments});
|
||||
}
|
||||
|
||||
// return 0 - non-zero return values are reserved for future use
|
||||
return 0;
|
||||
}
|
||||
@@ -17,12 +17,30 @@ array data, it performs the following steps:
|
||||
- j2 = from_msgpack(vec)
|
||||
- assert(to_msgpack(j2) == vec)
|
||||
|
||||
Finally, it reads the data from a stream, a non-contiguous iterator range, and
|
||||
a FILE*, which read byte by byte, and checks that each gives the same value or
|
||||
error (id, message, byte position) as the pointer range, which takes the bulk
|
||||
fast paths (see check_input_adapters()).
|
||||
|
||||
The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
|
||||
drivers.
|
||||
*/
|
||||
|
||||
#include "fuzzer_common.hpp"
|
||||
|
||||
namespace
|
||||
{
|
||||
// json::from_msgpack for check_input_adapters()
|
||||
struct read_msgpack
|
||||
{
|
||||
template<typename... Args>
|
||||
json operator()(Args&& ... args) const
|
||||
{
|
||||
return json::from_msgpack(std::forward<Args>(args)...);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
// see http://llvm.org/docs/LibFuzzer.html
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
{
|
||||
@@ -77,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
assert(parsed || noexcept_threw || j_noexcept.is_discarded());
|
||||
}
|
||||
|
||||
// step 3: the other input adapters must read the data alike
|
||||
check_input_adapters(data, size, read_msgpack());
|
||||
|
||||
// return 0 - non-zero return values are reserved for future use
|
||||
return 0;
|
||||
}
|
||||
@@ -26,12 +26,30 @@ array data, it performs the following steps:
|
||||
The unit tests run the same checks on a fixed corpus (see the "UBJSON round-trip
|
||||
invariants" test case), so keep both in sync.
|
||||
|
||||
Finally, it reads the data from a stream, a non-contiguous iterator range, and
|
||||
a FILE*, which read byte by byte, and checks that each gives the same value or
|
||||
error (id, message, byte position) as the pointer range, which takes the bulk
|
||||
fast paths (see check_input_adapters()).
|
||||
|
||||
The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
|
||||
drivers.
|
||||
*/
|
||||
|
||||
#include "fuzzer_common.hpp"
|
||||
|
||||
namespace
|
||||
{
|
||||
// json::from_ubjson for check_input_adapters()
|
||||
struct read_ubjson
|
||||
{
|
||||
template<typename... Args>
|
||||
json operator()(Args&& ... args) const
|
||||
{
|
||||
return json::from_ubjson(std::forward<Args>(args)...);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
// see http://llvm.org/docs/LibFuzzer.html
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
{
|
||||
@@ -96,6 +114,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
|
||||
assert(parsed || noexcept_threw || j_noexcept.is_discarded());
|
||||
}
|
||||
|
||||
// step 3: the other input adapters must read the data alike
|
||||
check_input_adapters(data, size, read_ubjson());
|
||||
|
||||
// return 0 - non-zero return values are reserved for future use
|
||||
return 0;
|
||||
}
|
||||
@@ -11,6 +11,14 @@
|
||||
#pragma once
|
||||
|
||||
#include <cassert> // assert
|
||||
#include <cstddef> // size_t
|
||||
#include <cstdint> // uint8_t
|
||||
#include <cstdio> // FILE, fmemopen, fclose
|
||||
#include <list> // list
|
||||
#include <sstream> // istringstream
|
||||
#include <string> // string
|
||||
#include <utility> // forward
|
||||
#include <vector> // vector
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
using nlohmann::json; // NOLINT(google-global-names-in-headers): shared by all fuzzer drivers
|
||||
@@ -49,3 +57,89 @@ json parse_without_exceptions(Parse parse, bool& threw)
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
// the outcome of reading an input with exceptions: the serialization of the
|
||||
// value, or the id, message, and (for parse errors) byte of the exception
|
||||
struct read_outcome
|
||||
{
|
||||
std::string value{};
|
||||
int id = 0;
|
||||
std::string what{};
|
||||
std::size_t byte = 0;
|
||||
|
||||
bool operator==(const read_outcome& rhs) const
|
||||
{
|
||||
return value == rhs.value && id == rhs.id && what == rhs.what && byte == rhs.byte;
|
||||
}
|
||||
};
|
||||
|
||||
inline std::string outcome_value(const json& j)
|
||||
{
|
||||
return j.dump(-1, ' ', false, json::error_handler_t::keep);
|
||||
}
|
||||
|
||||
inline std::string outcome_value(bool b)
|
||||
{
|
||||
return b ? "true" : "false";
|
||||
}
|
||||
|
||||
template<typename Read, typename... Args>
|
||||
read_outcome read_with(const Read& parse, Args&& ... args)
|
||||
{
|
||||
read_outcome result;
|
||||
try
|
||||
{
|
||||
result.value = outcome_value(parse(std::forward<Args>(args)...));
|
||||
}
|
||||
catch (const json::parse_error& e)
|
||||
{
|
||||
result.id = e.id;
|
||||
result.what = e.what();
|
||||
result.byte = e.byte;
|
||||
}
|
||||
catch (const json::exception& e)
|
||||
{
|
||||
result.id = e.id;
|
||||
result.what = e.what();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// The contiguous input (a pointer range) takes the bulk fast paths; the other
|
||||
// input adapters read byte by byte. Reads @a data through each of them and
|
||||
// checks that all give the same value, or the same exception with the same
|
||||
// id, message, and byte position.
|
||||
//
|
||||
// @a parse is called with either an iterator range or a single input (stream
|
||||
// or FILE*) and returns the parsed value (or the result of accept()).
|
||||
template<typename Read>
|
||||
void check_input_adapters(const std::uint8_t* data, std::size_t size, const Read& parse)
|
||||
{
|
||||
const read_outcome expected = read_with(parse, data, data + size);
|
||||
|
||||
// stream adapter
|
||||
{
|
||||
std::istringstream stream(std::string(reinterpret_cast<const char*>(data), size));
|
||||
assert(read_with(parse, stream) == expected);
|
||||
}
|
||||
|
||||
// iterator adapter on a non-contiguous range
|
||||
{
|
||||
const std::list<std::uint8_t> list(data, data + size);
|
||||
assert(read_with(parse, list.begin(), list.end()) == expected);
|
||||
}
|
||||
|
||||
#if defined(__unix__) || defined(__APPLE__)
|
||||
// FILE* adapter; fmemopen may refuse an empty buffer
|
||||
{
|
||||
std::vector<std::uint8_t> buffer(data, data + size);
|
||||
std::FILE* file = buffer.empty() ? nullptr : fmemopen(buffer.data(), buffer.size(), "rb");
|
||||
if (file != nullptr)
|
||||
{
|
||||
const read_outcome actual = read_with(parse, file);
|
||||
std::fclose(file);
|
||||
assert(actual == expected);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@@ -20,11 +20,8 @@
|
||||
using nlohmann::json;
|
||||
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <sstream>
|
||||
#include <iomanip>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "test_utils.hpp"
|
||||
|
||||
@@ -926,68 +923,66 @@ TEST_CASE("serialization boundary values for the string buffer")
|
||||
// mid-string once fewer than 13 bytes remain (`string_buffer.size() - bytes
|
||||
// < 13`), 13 being one more than the most a single code point can ever
|
||||
// write at once (a surrogate pair: two back-to-back "\uXXXX" escapes, 12
|
||||
// bytes). Characters below 0x60 (control characters, '"' and '\\') are
|
||||
// escaped straight into the write buffer and never reach string_buffer, so
|
||||
// under ensure_ascii it fills in steps of 6 (one "\uXXXX" unit for a
|
||||
// character below U+10000) or 12 (a surrogate pair). With 2-byte characters
|
||||
// such as U+00E4, `bytes` takes multiples of 6 at the point the check runs;
|
||||
// the tightest values reachable are therefore 498 (512 - 498 == 14, after
|
||||
// 83 characters) and 504 (512 - 504 == 8, after 84 characters, where the
|
||||
// flush fires immediately and resets bytes to 0).
|
||||
// bytes). Every write into string_buffer that this check protects happens
|
||||
// in steps of 2 (a simple "\\x" escape) or 6 (one "\uXXXX" unit), so
|
||||
// `bytes` only ever takes even values at the point the check runs - the
|
||||
// tightest values actually reachable are therefore 498 (512 - 498 == 14,
|
||||
// one simple escape away from the threshold) and 500 (512 - 500 == 12,
|
||||
// where the flush fires immediately and resets bytes to 0).
|
||||
|
||||
SECTION("a run of 6-byte escapes lands bytes on, and one step past, the flush threshold")
|
||||
SECTION("a run of 2-byte escapes lands bytes on, and one step past, the flush threshold")
|
||||
{
|
||||
for (const int count :
|
||||
{
|
||||
83, 84, 85
|
||||
249, 250, 251
|
||||
})
|
||||
{
|
||||
CAPTURE(count)
|
||||
std::string umlauts;
|
||||
const json j = std::string(static_cast<std::size_t>(count), '\n');
|
||||
std::string expected = "\"";
|
||||
for (int i = 0; i < count; ++i)
|
||||
{
|
||||
umlauts += "\xC3\xA4";
|
||||
expected += "\\u00e4";
|
||||
expected += "\\n";
|
||||
}
|
||||
expected += '"';
|
||||
CHECK(json(umlauts).dump(-1, ' ', true) == expected);
|
||||
CHECK(j.dump() == expected);
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("a prefix leaves the tightest reachable margin before a 12-byte surrogate pair")
|
||||
SECTION("an ASCII prefix leaves the tightest reachable margin before a 12-byte surrogate pair")
|
||||
{
|
||||
// U+1F600 (the "\xF0\x9F\x98\x80" UTF-8 bytes) is dumped under
|
||||
// ensure_ascii as the 12-byte surrogate pair "\ud83d\ude00"; that
|
||||
// write happens in a single step with no intermediate flush check, so
|
||||
// it is the write most exposed by an off-by-one in the "< 13" guard.
|
||||
// A prefix of 83 U+00E4 characters leaves exactly 14 bytes of headroom
|
||||
// A prefix of 249 newlines leaves exactly 14 bytes of headroom
|
||||
// (512 - 498), the smallest margin the guard ever actually allows
|
||||
// into a new code point; 84 characters instead trigger the guard's own
|
||||
// into a new code point; 250 newlines instead trigger the guard's own
|
||||
// flush first, so the emoji starts from a freshly emptied (512-byte)
|
||||
// buffer, and 85 repeats that with one more escape already past the
|
||||
// buffer, and 251 repeats that with one more escape already past the
|
||||
// reset. Together they cover the margin the guard allows landing on,
|
||||
// one step before, and one step after - all must still produce the
|
||||
// identical, correct escapes.
|
||||
for (const int prefix_count :
|
||||
{
|
||||
83, 84, 85
|
||||
249, 250, 251
|
||||
})
|
||||
{
|
||||
CAPTURE(prefix_count)
|
||||
std::string prefix;
|
||||
std::string expected_prefix;
|
||||
for (int i = 0; i < prefix_count; ++i)
|
||||
{
|
||||
prefix += "\xC3\xA4";
|
||||
expected_prefix += "\\u00e4";
|
||||
}
|
||||
const std::string prefix(static_cast<std::size_t>(prefix_count), '\n');
|
||||
const std::string emoji = "\xF0\x9F\x98\x80";
|
||||
const json j = prefix + emoji;
|
||||
|
||||
// without ensure_ascii, the whole string is copied unchanged
|
||||
std::string expected_prefix;
|
||||
for (int i = 0; i < prefix_count; ++i)
|
||||
{
|
||||
expected_prefix += "\\n";
|
||||
}
|
||||
|
||||
// newline escaping does not depend on ensure_ascii: only the
|
||||
// emoji differs (raw UTF-8 bytes vs. a \u-escaped surrogate pair)
|
||||
std::string expected_raw = "\"";
|
||||
expected_raw += prefix;
|
||||
expected_raw += expected_prefix;
|
||||
expected_raw += emoji;
|
||||
expected_raw += '"';
|
||||
std::string expected_ascii = "\"";
|
||||
@@ -1032,524 +1027,3 @@ TEST_CASE("serialization boundary values for the string buffer")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
// appends the escaped form of the ASCII character c to out; written
|
||||
// independently of the table in serializer::ascii_escapes()
|
||||
void append_escaped_ascii(std::string& out, const char c, const bool ensure_ascii)
|
||||
{
|
||||
const auto byte = static_cast<unsigned char>(c);
|
||||
switch (byte)
|
||||
{
|
||||
case '\b':
|
||||
out += "\\b";
|
||||
break;
|
||||
case '\t':
|
||||
out += "\\t";
|
||||
break;
|
||||
case '\n':
|
||||
out += "\\n";
|
||||
break;
|
||||
case '\f':
|
||||
out += "\\f";
|
||||
break;
|
||||
case '\r':
|
||||
out += "\\r";
|
||||
break;
|
||||
case '"':
|
||||
out += "\\\"";
|
||||
break;
|
||||
case '\\':
|
||||
out += "\\\\";
|
||||
break;
|
||||
default:
|
||||
if (byte < 0x20 || (ensure_ascii && byte == 0x7F))
|
||||
{
|
||||
// byte < 0x80, so the first two hex digits are 0
|
||||
const char* const hex = "0123456789abcdef";
|
||||
out += "\\u00";
|
||||
out += hex[byte >> 4U];
|
||||
out += hex[byte & 0x0FU];
|
||||
}
|
||||
else
|
||||
{
|
||||
out += c;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// reference escaper for ASCII strings
|
||||
std::string reference_escape_ascii(const std::string& s, const bool ensure_ascii)
|
||||
{
|
||||
std::string result = "\"";
|
||||
for (const char c : s)
|
||||
{
|
||||
append_escaped_ascii(result, c, ensure_ascii);
|
||||
}
|
||||
result += '"';
|
||||
return result;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("characters below 0x60 are escaped from a table straight into the write buffer")
|
||||
{
|
||||
SECTION("every ASCII character at every position")
|
||||
{
|
||||
// lengths up to 17 cover the 8-byte strides of the bulk scanner on
|
||||
// both sides of the character, so the scanner stops at it from every
|
||||
// offset
|
||||
for (std::size_t length = 1; length <= 17; ++length)
|
||||
{
|
||||
for (std::size_t pos = 0; pos < length; ++pos)
|
||||
{
|
||||
for (unsigned int c = 0; c < 0x80; ++c)
|
||||
{
|
||||
std::string s(length, 'a');
|
||||
s[pos] = static_cast<char>(c);
|
||||
CAPTURE(length)
|
||||
CAPTURE(pos)
|
||||
CAPTURE(c)
|
||||
const json j = s;
|
||||
CHECK(j.dump(-1, ' ', false) == reference_escape_ascii(s, false));
|
||||
CHECK(j.dump(-1, ' ', true) == reference_escape_ascii(s, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("runs of escaped characters")
|
||||
{
|
||||
std::string s;
|
||||
for (unsigned int c = 0; c < 0x20; ++c)
|
||||
{
|
||||
s += static_cast<char>(c);
|
||||
}
|
||||
s += R"("\"\)";
|
||||
CHECK(json(s).dump() == reference_escape_ascii(s, false));
|
||||
CHECK(json(s).dump(-1, ' ', true) == reference_escape_ascii(s, true));
|
||||
}
|
||||
|
||||
SECTION("escapes straddle the end of the write buffer")
|
||||
{
|
||||
// the 8-byte store of a table entry needs room for all 8 bytes, even
|
||||
// when the escape itself is shorter; a prefix of k plain characters
|
||||
// shifts the 6-byte escapes across every offset of the 1024-byte
|
||||
// write buffer
|
||||
for (std::size_t k = 0; k < 8; ++k)
|
||||
{
|
||||
CAPTURE(k)
|
||||
const std::string s = std::string(k, 'a') + std::string(400, '\x01');
|
||||
CHECK(json(s).dump() == reference_escape_ascii(s, false));
|
||||
CHECK(json::array({s, s}).dump() == "[" + reference_escape_ascii(s, false) + "," + reference_escape_ascii(s, false) + "]");
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("pending \\u escapes of non-ASCII characters are written first")
|
||||
{
|
||||
// under ensure_ascii, U+00E4 is \u-escaped into the string buffer;
|
||||
// a following escape from the table must not overtake it
|
||||
const json j = "\xC3\xA4\n\xC3\xA4\"\xF0\x9F\x98\x80\\";
|
||||
CHECK(j.dump(-1, ' ', true) == "\"\\u00e4\\n\\u00e4\\\"\\ud83d\\ude00\\\\\"");
|
||||
CHECK(j.dump(-1, ' ', false) == "\"\xC3\xA4\\n\xC3\xA4\\\"\xF0\x9F\x98\x80\\\\\"");
|
||||
}
|
||||
|
||||
SECTION("escapes after invalid UTF-8 for every error handler")
|
||||
{
|
||||
const json j = "a\xFF\n\xC3\"";
|
||||
CHECK_THROWS_WITH_AS(utils::ignore_return_value(j.dump()), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0xFF", json::type_error&);
|
||||
CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"a\\n\\\"\"");
|
||||
CHECK(j.dump(-1, ' ', true, json::error_handler_t::ignore) == "\"a\\n\\\"\"");
|
||||
CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"a\xEF\xBF\xBD\\n\xEF\xBF\xBD\\\"\"");
|
||||
CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"a\\ufffd\\n\\ufffd\\\"\"");
|
||||
CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"a\xFF\\n\xC3\\\"\"");
|
||||
CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"a\xFF\\n\xC3\\\"\"");
|
||||
}
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
// expected result of dumping a string: the output, or (with
|
||||
// error_handler_t::strict) the message of the type_error.316 exception
|
||||
struct expected_dump
|
||||
{
|
||||
bool throws = false;
|
||||
std::string text{}; // NOLINT(readability-redundant-member-init)
|
||||
};
|
||||
|
||||
std::string upper_hex_byte(const unsigned char byte)
|
||||
{
|
||||
const char* const hex = "0123456789ABCDEF";
|
||||
return std::string(1, hex[byte >> 4U]) + hex[byte & 0x0FU];
|
||||
}
|
||||
|
||||
void append_u_escape(std::string& out, const std::uint32_t code_unit)
|
||||
{
|
||||
const char* const hex = "0123456789abcdef";
|
||||
out += "\\u";
|
||||
for (int shift = 12; shift >= 0; shift -= 4)
|
||||
{
|
||||
out += hex[(code_unit >> static_cast<unsigned int>(shift)) & 0x0FU];
|
||||
}
|
||||
}
|
||||
|
||||
void append_replacement_character(std::string& out, const bool ensure_ascii)
|
||||
{
|
||||
out += ensure_ascii ? "\\ufffd" : "\xEF\xBF\xBD";
|
||||
}
|
||||
|
||||
// reference serializer for arbitrary byte strings, written from the documented
|
||||
// behavior of dump() and independently of serializer::dump_escaped(): UTF-8 is
|
||||
// decoded strictly (well-formed sequences as in Table 3-7 of the Unicode
|
||||
// Standard), and each ill-formed sequence is handled as the error handler says
|
||||
expected_dump reference_dump_string(const std::string& s, const bool ensure_ascii, const json::error_handler_t error_handler)
|
||||
{
|
||||
expected_dump result;
|
||||
std::string& out = result.text;
|
||||
out = "\"";
|
||||
|
||||
// handle the ill-formed bytes s[begin, end), of which s[index] is the byte
|
||||
// that could not be decoded; returns false if an exception is expected
|
||||
const auto ill_formed = [&](const std::size_t begin, const std::size_t end, const std::size_t index)
|
||||
{
|
||||
switch (error_handler)
|
||||
{
|
||||
case json::error_handler_t::strict:
|
||||
result.throws = true;
|
||||
out = "[json.exception.type_error.316] invalid UTF-8 byte at index " + std::to_string(index) + ": 0x" + upper_hex_byte(static_cast<unsigned char>(s[index]));
|
||||
return false;
|
||||
case json::error_handler_t::ignore:
|
||||
break;
|
||||
case json::error_handler_t::replace:
|
||||
append_replacement_character(out, ensure_ascii);
|
||||
break;
|
||||
case json::error_handler_t::keep:
|
||||
default:
|
||||
// kept as is, even with ensure_ascii
|
||||
out.append(s, begin, end - begin);
|
||||
break;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
std::size_t i = 0;
|
||||
while (i < s.size())
|
||||
{
|
||||
const auto lead = static_cast<unsigned char>(s[i]);
|
||||
if (lead < 0x80)
|
||||
{
|
||||
append_escaped_ascii(out, s[i], ensure_ascii);
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
// number of bytes of the sequence and the range of its second byte
|
||||
std::size_t length = 0;
|
||||
std::uint32_t code_point = 0;
|
||||
unsigned char lower = 0x80;
|
||||
unsigned char upper = 0xBF;
|
||||
if (lead >= 0xC2 && lead <= 0xDF)
|
||||
{
|
||||
length = 2;
|
||||
code_point = lead & 0x1FU;
|
||||
}
|
||||
else if (lead >= 0xE0 && lead <= 0xEF)
|
||||
{
|
||||
length = 3;
|
||||
code_point = lead & 0x0FU;
|
||||
lower = (lead == 0xE0) ? 0xA0 : 0x80; // no overlong encodings
|
||||
upper = (lead == 0xED) ? 0x9F : 0xBF; // no surrogates
|
||||
}
|
||||
else if (lead >= 0xF0 && lead <= 0xF4)
|
||||
{
|
||||
length = 4;
|
||||
code_point = lead & 0x07U;
|
||||
lower = (lead == 0xF0) ? 0x90 : 0x80; // no overlong encodings
|
||||
upper = (lead == 0xF4) ? 0x8F : 0xBF; // nothing above U+10FFFF
|
||||
}
|
||||
else
|
||||
{
|
||||
// a byte that cannot start a sequence is consumed
|
||||
if (!ill_formed(i, i + 1, i))
|
||||
{
|
||||
return result;
|
||||
}
|
||||
++i;
|
||||
continue;
|
||||
}
|
||||
|
||||
std::size_t j = i + 1;
|
||||
for (; j < i + length; ++j)
|
||||
{
|
||||
if (j == s.size())
|
||||
{
|
||||
// the string ends in the middle of the sequence
|
||||
switch (error_handler)
|
||||
{
|
||||
case json::error_handler_t::strict:
|
||||
result.throws = true;
|
||||
out = "[json.exception.type_error.316] incomplete UTF-8 string; last byte: 0x" + upper_hex_byte(static_cast<unsigned char>(s.back()));
|
||||
return result;
|
||||
case json::error_handler_t::ignore:
|
||||
break;
|
||||
case json::error_handler_t::replace:
|
||||
append_replacement_character(out, ensure_ascii);
|
||||
break;
|
||||
case json::error_handler_t::keep:
|
||||
default:
|
||||
out.append(s, i, s.size() - i);
|
||||
break;
|
||||
}
|
||||
out += '"';
|
||||
return result;
|
||||
}
|
||||
|
||||
const auto byte = static_cast<unsigned char>(s[j]);
|
||||
if (byte < lower || byte > upper)
|
||||
{
|
||||
break;
|
||||
}
|
||||
code_point = (code_point << 6U) | (byte & 0x3FU);
|
||||
lower = 0x80;
|
||||
upper = 0xBF;
|
||||
}
|
||||
|
||||
if (j < i + length)
|
||||
{
|
||||
// the sequence was abandoned at s[j]; that byte is read again as
|
||||
// the start of a new character
|
||||
if (!ill_formed(i, j, j))
|
||||
{
|
||||
return result;
|
||||
}
|
||||
i = j;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!ensure_ascii)
|
||||
{
|
||||
out.append(s, i, length);
|
||||
}
|
||||
else if (code_point <= 0xFFFF)
|
||||
{
|
||||
append_u_escape(out, code_point);
|
||||
}
|
||||
else
|
||||
{
|
||||
append_u_escape(out, 0xD800U + ((code_point - 0x10000U) >> 10U));
|
||||
append_u_escape(out, 0xDC00U + ((code_point - 0x10000U) & 0x3FFU));
|
||||
}
|
||||
i = j;
|
||||
}
|
||||
|
||||
out += '"';
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string hex_dump(const std::string& s)
|
||||
{
|
||||
std::string result;
|
||||
for (const char c : s)
|
||||
{
|
||||
result += upper_hex_byte(static_cast<unsigned char>(c));
|
||||
result += ' ';
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// compares dump() of s with the reference for every error handler, with
|
||||
// ensure_ascii on and off; CHECKs only on a mismatch to keep the sweep fast;
|
||||
// error_handler_t::strict can be skipped where the position of the string
|
||||
// buffer does not matter, as throwing is slow on some platforms
|
||||
void check_dump_against_reference(const std::string& s, const bool with_strict = true)
|
||||
{
|
||||
const json j = s;
|
||||
const std::array<json::error_handler_t, 4> error_handlers =
|
||||
{
|
||||
{
|
||||
json::error_handler_t::strict, json::error_handler_t::ignore,
|
||||
json::error_handler_t::replace, json::error_handler_t::keep
|
||||
}
|
||||
};
|
||||
for (const auto error_handler : error_handlers)
|
||||
{
|
||||
if (!with_strict && error_handler == json::error_handler_t::strict)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
for (const bool ensure_ascii :
|
||||
{
|
||||
false, true
|
||||
})
|
||||
{
|
||||
const expected_dump expected = reference_dump_string(s, ensure_ascii, error_handler);
|
||||
expected_dump actual;
|
||||
int id = 0;
|
||||
try
|
||||
{
|
||||
actual.text = j.dump(-1, ' ', ensure_ascii, error_handler);
|
||||
}
|
||||
catch (const json::exception& e)
|
||||
{
|
||||
actual.throws = true;
|
||||
actual.text = e.what();
|
||||
id = e.id;
|
||||
}
|
||||
if (actual.throws != expected.throws || actual.text != expected.text || (actual.throws && id != 316))
|
||||
{
|
||||
const std::string input = hex_dump(s);
|
||||
CAPTURE(input)
|
||||
CAPTURE(ensure_ascii)
|
||||
CAPTURE(static_cast<int>(error_handler))
|
||||
CHECK(actual.throws == expected.throws);
|
||||
CHECK(actual.text == expected.text);
|
||||
CHECK((!actual.throws || id == 316));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("invalid UTF-8 at every position is handled like the reference")
|
||||
{
|
||||
// ill-formed byte sequences; those that end in a truncated sequence are
|
||||
// also cut short by the filler that follows, or by the end of the string
|
||||
const std::vector<std::string> patterns =
|
||||
{
|
||||
// stray continuation bytes
|
||||
"\x80", "\xBF",
|
||||
// bytes that cannot start a sequence
|
||||
"\xC0", "\xC1", "\xF5", "\xF6", "\xF7", "\xF8", "\xF9", "\xFA", "\xFB", "\xFC", "\xFD", "\xFE", "\xFF",
|
||||
// overlong encodings
|
||||
"\xC0\xAF", "\xE0\x80\x80", "\xF0\x80\x80\x80",
|
||||
// UTF-16 surrogates
|
||||
"\xED\xA0\x80", "\xED\xBF\xBF",
|
||||
// a code point above U+10FFFF
|
||||
"\xF4\x90\x80\x80",
|
||||
// sequences cut short by ASCII, a control character, a quote, a
|
||||
// backslash, DEL, or the lead byte of a well-formed character
|
||||
"\xC3" "a", "\xE2\x82" "\n", "\xE2\x82\"", "\xF0\x9F\x98\\", "\xC3\x7F", "\xC3\xC3\xA4", "\xE2\xF0\x9F\x98\x80",
|
||||
// truncated sequences
|
||||
"\xC3", "\xE2\x82", "\xF0\x9F\x98"
|
||||
};
|
||||
|
||||
// well-formed characters of 1, 2, 3, and 4 bytes
|
||||
const std::vector<std::string> fillers = {"a", "\xC3\xA4", "\xE2\x82\xAC", "\xF0\x9F\x98\x80"};
|
||||
|
||||
const auto make_string = [](const std::string & filler, const std::size_t before, const std::string & pattern, const std::size_t after)
|
||||
{
|
||||
std::string s;
|
||||
for (std::size_t k = 0; k < before; ++k)
|
||||
{
|
||||
s += filler;
|
||||
}
|
||||
s += pattern;
|
||||
for (std::size_t k = 0; k < after; ++k)
|
||||
{
|
||||
s += filler;
|
||||
}
|
||||
return s;
|
||||
};
|
||||
|
||||
SECTION("the reference")
|
||||
{
|
||||
// the reference itself, against hand-written results
|
||||
const std::string s = "a\xFF\n\xC3\"\xE0\x80\xF0\x9F\x98\x80\xE2\x82";
|
||||
CHECK(reference_dump_string(s, false, json::error_handler_t::strict).text == "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0xFF");
|
||||
CHECK(reference_dump_string(s.substr(2), false, json::error_handler_t::strict).text == "[json.exception.type_error.316] invalid UTF-8 byte at index 2: 0x22");
|
||||
CHECK(reference_dump_string("\xF0\x9F\x98\x80\xE2\x82", true, json::error_handler_t::strict).text == "[json.exception.type_error.316] incomplete UTF-8 string; last byte: 0x82");
|
||||
CHECK(reference_dump_string(s, false, json::error_handler_t::ignore).text == "\"a\\n\\\"\xF0\x9F\x98\x80\"");
|
||||
CHECK(reference_dump_string(s, true, json::error_handler_t::ignore).text == "\"a\\n\\\"\\ud83d\\ude00\"");
|
||||
CHECK(reference_dump_string(s, false, json::error_handler_t::replace).text == "\"a\xEF\xBF\xBD\\n\xEF\xBF\xBD\\\"\xEF\xBF\xBD\xEF\xBF\xBD\xF0\x9F\x98\x80\xEF\xBF\xBD\"");
|
||||
CHECK(reference_dump_string(s, true, json::error_handler_t::replace).text == "\"a\\ufffd\\n\\ufffd\\\"\\ufffd\\ufffd\\ud83d\\ude00\\ufffd\"");
|
||||
CHECK(reference_dump_string(s, false, json::error_handler_t::keep).text == "\"" + std::string("a\xFF\\n\xC3\\\"\xE0\x80\xF0\x9F\x98\x80\xE2\x82") + "\"");
|
||||
CHECK(reference_dump_string(s, true, json::error_handler_t::keep).text == "\"" + std::string("a\xFF\\n\xC3\\\"\xE0\x80\\ud83d\\ude00\xE2\x82") + "\"");
|
||||
check_dump_against_reference(s);
|
||||
}
|
||||
|
||||
SECTION("short strings")
|
||||
{
|
||||
// strings of up to 17 bytes put the pattern at every offset of an
|
||||
// 8-byte stride of the bulk scanners, with whole strides on both
|
||||
// sides; lengths around 24, 32, and 40 bytes cross the 16- and 32-byte
|
||||
// strides; error_handler_t::strict is checked for up to 17 bytes of
|
||||
// ASCII and 8 bytes of other characters only, as throwing is slow on
|
||||
// some platforms
|
||||
for (const auto& filler : fillers)
|
||||
{
|
||||
for (std::size_t length = 0; length * filler.size() <= 40; ++length)
|
||||
{
|
||||
const std::size_t bytes = length * filler.size();
|
||||
if (filler.size() == 1 && bytes > 17 && (bytes + 1) % 8 > 2)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
for (std::size_t pos = 0; pos <= length; ++pos)
|
||||
{
|
||||
for (const auto& pattern : patterns)
|
||||
{
|
||||
check_dump_against_reference(make_string(filler, pos, pattern, length - pos), bytes <= (filler.size() == 1 ? 17 : 8));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("strings crossing the end of the string buffer")
|
||||
{
|
||||
// characters that are not copied in bulk go through a 512-byte string
|
||||
// buffer that is written out once fewer than 13 bytes are left; with
|
||||
// ensure_ascii, each 2-byte character takes 6 bytes of it and each
|
||||
// 4-byte character 12, so the pattern meets the end of the buffer after
|
||||
// about 84 or 42 characters (3-byte characters behave like 2-byte
|
||||
// ones); plain ASCII, copied in bulk, is checked around 512 bytes;
|
||||
// positions further away are sampled; error_handler_t::strict throws
|
||||
// at the first ill-formed byte, regardless of the string buffer, and is
|
||||
// covered above
|
||||
for (const auto& filler : fillers)
|
||||
{
|
||||
if (filler.size() == 3)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
const std::size_t escaped_size = filler.size() == 1 ? 1 : filler.size() * 3;
|
||||
const std::size_t middle = 506 / escaped_size;
|
||||
const std::size_t window = filler.size() == 1 ? 8 : 16;
|
||||
for (std::size_t pos = 0; pos <= middle + window; pos += (pos + window >= middle) ? 1 : 29)
|
||||
{
|
||||
for (const std::size_t after :
|
||||
{
|
||||
std::size_t{0}, std::size_t{12}
|
||||
})
|
||||
{
|
||||
for (const auto& pattern : patterns)
|
||||
{
|
||||
check_dump_against_reference(make_string(filler, pos, pattern, after), false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("repeated invalid sequences at every offset of the string buffer")
|
||||
{
|
||||
// ill-formed sequences that are kept or replaced fill the string
|
||||
// buffer without a bulk copy in between; a prefix of 0 to 12 bytes
|
||||
// 0xFF (kept as 1 byte, replaced by 3 or 6) shifts them across every
|
||||
// offset of the end of the buffer
|
||||
for (const auto& pattern : patterns)
|
||||
{
|
||||
for (std::size_t shift = 0; shift <= 12; ++shift)
|
||||
{
|
||||
std::string s(shift, '\xFF');
|
||||
while (s.size() < 600)
|
||||
{
|
||||
s += pattern;
|
||||
}
|
||||
check_dump_against_reference(s, shift == 0);
|
||||
check_dump_against_reference(s + "\xF0\x9F\x98\x80", false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user