diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index f290120b2..fc170278e 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -140,7 +140,7 @@ Exceptions have ids 1xx. This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member `byte` indicates the error position. -The message ends with the last read token (`last read: '...'`), with control characters written as ``. A token of more than 8192 characters is shown by its first and its last 4096 characters, separated by `...`. +The message ends with the last read token (`last read: '...'`), with control characters written as ``. A token of more than 8192 bytes is shown by its first and its last 4096 bytes, separated by `...`; each part is shortened by up to three bytes so that it does not end or begin within a UTF-8 character. !!! failure "Example message" diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 54420867b..a67af2712 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -2279,9 +2279,14 @@ scan_number_done: std::string result; if (chars.size() > 2 * shown) { - append_escaped(result, chars.begin(), chars.begin() + static_cast(shown)); + // the lexer reads bytes (wide characters arrive as UTF-8), so + // move both cuts to character boundaries to keep the message + // valid UTF-8; this only shortens the two parts + const auto head_end = chars.begin() + static_cast(utf8_head_cut(chars.data(), shown)); + const auto tail_begin = chars.end() - static_cast(shown); + append_escaped(result, chars.begin(), head_end); result += "..."; - append_escaped(result, chars.end() - static_cast(shown), chars.end()); + append_escaped(result, tail_begin + static_cast(utf8_tail_skip(&*tail_begin)), chars.end()); } else { @@ -2291,6 +2296,55 @@ scan_number_done: } private: + /// whether @a c is a UTF-8 continuation byte + static bool is_utf8_continuation(const char_type c) noexcept + { + return (static_cast(c) & 0xC0u) == 0x80u; + } + + /*! + @brief the length of the first @a n characters at @a data without a UTF-8 + character that begins there but does not end there + + Looks only at the last 4 of these characters: in valid UTF-8, at most 3 + bytes of a character can be before the cut. + */ + static std::size_t utf8_head_cut(const char_type* data, const std::size_t n) noexcept + { + std::size_t lead = n - 1; + while (lead > n - 4 && is_utf8_continuation(data[lead])) + { + --lead; + } + const auto c = static_cast(data[lead]); + std::size_t length = 1; + if (c >= 0xF0u) + { + length = 4; + } + else if (c >= 0xE0u) + { + length = 3; + } + else if (c >= 0xC0u) + { + length = 2; + } + return lead + length > n ? lead : n; + } + + /// the number of UTF-8 continuation bytes (at most 3) at @a data, which + /// belong to a character that begins before it + static std::size_t utf8_tail_skip(const char_type* data) noexcept + { + std::size_t skip = 0; + while (skip < 3 && is_utf8_continuation(data[skip])) + { + ++skip; + } + return skip; + } + /// append the characters [first, last) to @a result, escaping control characters template static void append_escaped(std::string& result, Iterator first, Iterator last) diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 473bfa894..5a18ec9e3 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -15782,9 +15782,14 @@ scan_number_done: std::string result; if (chars.size() > 2 * shown) { - append_escaped(result, chars.begin(), chars.begin() + static_cast(shown)); + // the lexer reads bytes (wide characters arrive as UTF-8), so + // move both cuts to character boundaries to keep the message + // valid UTF-8; this only shortens the two parts + const auto head_end = chars.begin() + static_cast(utf8_head_cut(chars.data(), shown)); + const auto tail_begin = chars.end() - static_cast(shown); + append_escaped(result, chars.begin(), head_end); result += "..."; - append_escaped(result, chars.end() - static_cast(shown), chars.end()); + append_escaped(result, tail_begin + static_cast(utf8_tail_skip(&*tail_begin)), chars.end()); } else { @@ -15794,6 +15799,55 @@ scan_number_done: } private: + /// whether @a c is a UTF-8 continuation byte + static bool is_utf8_continuation(const char_type c) noexcept + { + return (static_cast(c) & 0xC0u) == 0x80u; + } + + /*! + @brief the length of the first @a n characters at @a data without a UTF-8 + character that begins there but does not end there + + Looks only at the last 4 of these characters: in valid UTF-8, at most 3 + bytes of a character can be before the cut. + */ + static std::size_t utf8_head_cut(const char_type* data, const std::size_t n) noexcept + { + std::size_t lead = n - 1; + while (lead > n - 4 && is_utf8_continuation(data[lead])) + { + --lead; + } + const auto c = static_cast(data[lead]); + std::size_t length = 1; + if (c >= 0xF0u) + { + length = 4; + } + else if (c >= 0xE0u) + { + length = 3; + } + else if (c >= 0xC0u) + { + length = 2; + } + return lead + length > n ? lead : n; + } + + /// the number of UTF-8 continuation bytes (at most 3) at @a data, which + /// belong to a character that begins before it + static std::size_t utf8_tail_skip(const char_type* data) noexcept + { + std::size_t skip = 0; + while (skip < 3 && is_utf8_continuation(data[skip])) + { + ++skip; + } + return skip; + } + /// append the characters [first, last) to @a result, escaping control characters template static void append_escaped(std::string& result, Iterator first, Iterator last) diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index b015a87d0..74fcc6cac 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -3380,5 +3380,108 @@ TEST_CASE("parse error message of a long token") CHECK(message(text, kind) == "no error"); } } + + SECTION("a long string of multi-byte characters") + { + // the parts of a long token are cut at character boundaries, so the + // message of valid UTF-8 input is valid UTF-8, also for wide strings + const auto last_read = [](const std::string & m) -> std::string + { + const auto pos = m.find("last read: '"); + return pos == std::string::npos ? std::string() : m.substr(pos + 12, m.size() - pos - 13); + }; + const auto utf8 = [](const char32_t c) -> std::string + { + std::string result; + if (c < 0x800) + { + result.push_back(static_cast(0xC0 | (c >> 6))); + } + else if (c < 0x10000) + { + result.push_back(static_cast(0xE0 | (c >> 12))); + result.push_back(static_cast(0x80 | ((c >> 6) & 0x3F))); + } + else + { + result.push_back(static_cast(0xF0 | (c >> 18))); + result.push_back(static_cast(0x80 | ((c >> 12) & 0x3F))); + result.push_back(static_cast(0x80 | ((c >> 6) & 0x3F))); + } + result.push_back(static_cast(0x80 | (c & 0x3F))); + return result; + }; + // characters of 2, 3, and 4 bytes + const std::vector characters = {0xE9, 0x20AC, 0x1F600}; + for (const char32_t c : characters) + { + for (std::size_t offset = 0; offset < 4; ++offset) + { + CAPTURE(static_cast(c)) + CAPTURE(offset) + // an unterminated string: the token is the whole input + std::string text = "\"" + std::string(offset, 'a'); + std::u32string text32 = U"\"" + std::u32string(offset, U'a'); + while (text.size() < 9000) + { + text += utf8(c); + text32.push_back(c); + } + std::u16string text16; + for (const char32_t c32 : text32) + { + if (c32 < 0x10000) + { + text16.push_back(static_cast(c32)); + } + else + { + text16.push_back(static_cast(0xD800 + ((c32 - 0x10000) >> 10))); + text16.push_back(static_cast(0xDC00 + ((c32 - 0x10000) & 0x3FF))); + } + } + + const std::string shown = last_read(message(text, 0)); + const auto dots = shown.find("..."); + REQUIRE(dots != std::string::npos); + // the first and the last at most 4096 bytes, without parts of characters + CHECK(dots <= 4096); + CHECK(dots > 4092); + CHECK(text.compare(0, dots, shown, 0, dots) == 0); + const std::size_t tail = shown.size() - dots - 3; + CHECK(tail <= 4096); + CHECK(tail > 4092); + CHECK(text.compare(text.size() - tail, tail, shown, dots + 3, tail) == 0); + CHECK_NOTHROW(json(shown).dump()); + + for (int kind = 1; kind < 4; ++kind) + { + CAPTURE(kind) + CHECK(last_read(message(text, kind)) == shown); + } + json _; + std::string m32; + std::string m16; + try + { + _ = json::parse(text32); + } + catch (const json::parse_error& e) + { + m32 = e.what(); + } + try + { + _ = json::parse(text16); + } + catch (const json::parse_error& e) + { + m16 = e.what(); + } + CHECK(last_read(m32) == shown); + CHECK(last_read(m16) == shown); + } + } + } } #endif