Stop compiling unit-wstring.cpp out entirely on classic ICC
tests/src/unit-wstring.cpp wrapped the whole file in #ifndef __INTEL_COMPILER, with the comment "ICPC errors out on multibyte character sequences in source files". The ci_icpc job (intel/oneapi-hpckit:2023.2.1) still exists, so that job ran none of the wstring/u16string/u32string input adapter tests, including the malformed-input checks #5704 (open) extends. Only 9 lines contained non-ASCII bytes: the three *_is_utf16()/ *_is_utf32() probe functions, and three std::wstring/u16string/ u32string literals plus their narrow-string dump() expectations. Rewrite all of them with \u/\U escapes in the wide/u16/u32 literals and \x escapes (split into separate string-literal tokens so a following byte is never read as part of the same hex escape, e.g. "\xE1\x83\x85" "a") in the narrow ones. Remove the #ifndef __INTEL_COMPILER/#endif guard along with it. The *_is_utf16()/*_is_utf32() probes compared a raw multibyte literal against an escape-based one to detect a compiler that misreads the source file's encoding; with no raw literals left to misread, the comparison is now tautological, so drop the probes and the "if" guards around each SECTION's body instead of leaving them in as dead checks. The same non-ASCII-in-source-and-in-a-narrow-comparison pattern existed once more in unit-deserialization.cpp's "Using _json with char8_t literals #4945" test: a raw emoji character in a u8R"(...)" literal, guarded by a check_utf8() that returned false for ICC (same reason) and for Windows without the active UTF-8 code page. Rewrite the literal with a \U escape and compare it against a \x-escaped expectation instead of a second raw literal, and drop check_utf8() and the now-unused <windows.h> include along with the guard. Verified locally (clang, -std=c++11 and -std=c++20, -fsanitize=address,undefined, and a plain build): test-wstring keeps 18/18 assertions and unit-deserialization keeps 466/466 (c++11) and 477/477 (c++20) assertions, matching this branch before the change exactly - no coverage was gained or lost, only the source-encoding dependency was removed. ci_icpc has to confirm classic ICC actually builds and passes test-wstring now; if it does not, that is a real finding, not a reason to restore the guard. Overlaps #5704 (open), which edits unit-wstring.cpp inside the previously-guarded region (an include near the top, checks in the invalid-string sections, and a new section at the end). Closes #5713 item 5. Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
2 files changed
+18
-78
No files matched your search
@@ -28,11 +28,6 @@ using nlohmann::json;
|
|||||||
#include <string>
|
#include <string>
|
||||||
#include <valarray>
|
#include <valarray>
|
||||||
|
|
||||||
#if defined(_WIN32)
|
|
||||||
#define NOMINMAX
|
|
||||||
#include <windows.h> // for GetACP()
|
|
||||||
#endif
|
|
||||||
|
|
||||||
namespace
|
namespace
|
||||||
{
|
{
|
||||||
struct SaxEventLogger : public nlohmann::json_sax<json>
|
struct SaxEventLogger : public nlohmann::json_sax<json>
|
||||||
@@ -228,24 +223,6 @@ class proxy_iterator
|
|||||||
iterator* m_it = nullptr;
|
iterator* m_it = nullptr;
|
||||||
};
|
};
|
||||||
|
|
||||||
// JSON_HAS_CPP_20
|
|
||||||
#if defined(__cpp_char8_t)
|
|
||||||
bool check_utf8()
|
|
||||||
{
|
|
||||||
#if defined(_WIN32)
|
|
||||||
// Runtime check of the active ANSI code page
|
|
||||||
// 65001 == UTF-8
|
|
||||||
return GetACP() == 65001;
|
|
||||||
#elif defined(__ICC) || defined(__INTEL_COMPILER)
|
|
||||||
// classic Intel ICC does not encode narrow string literals containing
|
|
||||||
// non-ASCII source characters as UTF-8, so comparing a decoded u8 literal
|
|
||||||
// against a narrow string literal containing the same characters fails
|
|
||||||
return false;
|
|
||||||
#else
|
|
||||||
return true;
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
TEST_CASE("deserialization")
|
TEST_CASE("deserialization")
|
||||||
@@ -1297,14 +1274,15 @@ TEST_CASE("deserialization")
|
|||||||
CHECK(j1["key"] == "value");
|
CHECK(j1["key"] == "value");
|
||||||
CHECK(j1["num"] == 42);
|
CHECK(j1["num"] == 42);
|
||||||
|
|
||||||
// UTF-8 prefixed literal (C++20 and later);
|
// UTF-8 prefixed literal (C++20 and later); the emoji is written as a
|
||||||
// MSVC may not set /utf-8, so we need to check
|
// \U escape rather than a raw multibyte character so this does not
|
||||||
if (check_utf8())
|
// depend on the compiler's source-file encoding (e.g., MSVC without
|
||||||
{
|
// /utf-8, or classic ICC, which does not encode non-ASCII narrow
|
||||||
const auto j2 = u8R"({"emoji": "😀", "msg": "hello"})"_json;
|
// string literals as UTF-8 - compare against a \x-escaped expectation
|
||||||
CHECK(j2["emoji"] == "😀");
|
// for the same reason)
|
||||||
|
const auto j2 = u8"{\"emoji\": \"\U0001F600\", \"msg\": \"hello\"}"_json;
|
||||||
|
CHECK(j2["emoji"] == "\xF0\x9F\x98\x80");
|
||||||
CHECK(j2["msg"] == "hello");
|
CHECK(j2["msg"] == "hello");
|
||||||
}
|
|
||||||
|
|
||||||
const auto j3 = u8R"({"key": "value", "num": 42})"_json;
|
const auto j3 = u8R"({"key": "value", "num": 42})"_json;
|
||||||
CHECK(j3["key"] == "value");
|
CHECK(j3["key"] == "value");
|
||||||
|
|||||||
+10
-48
@@ -11,44 +11,20 @@
|
|||||||
#include <nlohmann/json.hpp>
|
#include <nlohmann/json.hpp>
|
||||||
using nlohmann::json;
|
using nlohmann::json;
|
||||||
|
|
||||||
// ICPC errors out on multibyte character sequences in source files
|
|
||||||
#ifndef __INTEL_COMPILER
|
|
||||||
namespace
|
|
||||||
{
|
|
||||||
bool wstring_is_utf16();
|
|
||||||
bool wstring_is_utf16()
|
|
||||||
{
|
|
||||||
return (std::wstring(L"💩") == std::wstring(L"\U0001F4A9"));
|
|
||||||
}
|
|
||||||
|
|
||||||
bool u16string_is_utf16();
|
|
||||||
bool u16string_is_utf16()
|
|
||||||
{
|
|
||||||
return (std::u16string(u"💩") == std::u16string(u"\U0001F4A9"));
|
|
||||||
}
|
|
||||||
|
|
||||||
bool u32string_is_utf32();
|
|
||||||
bool u32string_is_utf32()
|
|
||||||
{
|
|
||||||
return (std::u32string(U"💩") == std::u32string(U"\U0001F4A9"));
|
|
||||||
}
|
|
||||||
} // namespace
|
|
||||||
|
|
||||||
TEST_CASE("wide strings")
|
TEST_CASE("wide strings")
|
||||||
{
|
{
|
||||||
SECTION("std::wstring")
|
SECTION("std::wstring")
|
||||||
{
|
{
|
||||||
if (wstring_is_utf16())
|
// U+10C5 U+0061(a) U+00E4 U+00F6 U+1F4A4 U+1F9E2, written with \u/\U
|
||||||
{
|
// escapes rather than as raw multibyte characters so this file
|
||||||
std::wstring const w = L"[12.2,\"Ⴥaäö💤🧢\"]";
|
// compiles on toolchains (e.g. classic ICC) that error out on
|
||||||
|
// multibyte character sequences in source files
|
||||||
|
std::wstring const w = L"[12.2,\"\u10C5a\u00E4\u00F6\U0001F4A4\U0001F9E2\"]";
|
||||||
json const j = json::parse(w);
|
json const j = json::parse(w);
|
||||||
CHECK(j.dump() == "[12.2,\"Ⴥaäö💤🧢\"]");
|
CHECK(j.dump() == "[12.2,\"" "\xE1\x83\x85" "a" "\xC3\xA4" "\xC3\xB6" "\xF0\x9F\x92\xA4" "\xF0\x9F\xA7\xA2" "\"]");
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
SECTION("invalid std::wstring")
|
SECTION("invalid std::wstring")
|
||||||
{
|
|
||||||
if (wstring_is_utf16())
|
|
||||||
{
|
{
|
||||||
std::wstring const w = L"\"\xDBFF";
|
std::wstring const w = L"\"\xDBFF";
|
||||||
json _;
|
json _;
|
||||||
@@ -77,21 +53,15 @@ TEST_CASE("wide strings")
|
|||||||
// has to report an ill-formed byte at the surrogate's own position
|
// has to report an ill-formed byte at the surrogate's own position
|
||||||
CHECK_THROWS_WITH_AS(_ = json::parse(std::wstring{L'"', static_cast<wchar_t>(0xDC00), L'a', L'"'}), error_low_surrogate, json::parse_error&);
|
CHECK_THROWS_WITH_AS(_ = json::parse(std::wstring{L'"', static_cast<wchar_t>(0xDC00), L'a', L'"'}), error_low_surrogate, json::parse_error&);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
SECTION("std::u16string")
|
SECTION("std::u16string")
|
||||||
{
|
{
|
||||||
if (u16string_is_utf16())
|
std::u16string const w = u"[12.2,\"\u10C5a\u00E4\u00F6\U0001F4A4\U0001F9E2\"]";
|
||||||
{
|
|
||||||
std::u16string const w = u"[12.2,\"Ⴥaäö💤🧢\"]";
|
|
||||||
json const j = json::parse(w);
|
json const j = json::parse(w);
|
||||||
CHECK(j.dump() == "[12.2,\"Ⴥaäö💤🧢\"]");
|
CHECK(j.dump() == "[12.2,\"" "\xE1\x83\x85" "a" "\xC3\xA4" "\xC3\xB6" "\xF0\x9F\x92\xA4" "\xF0\x9F\xA7\xA2" "\"]");
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
SECTION("invalid std::u16string")
|
SECTION("invalid std::u16string")
|
||||||
{
|
|
||||||
if (u16string_is_utf16())
|
|
||||||
{
|
{
|
||||||
std::u16string const w = u"\"\xDBFF";
|
std::u16string const w = u"\"\xDBFF";
|
||||||
json _;
|
json _;
|
||||||
@@ -110,21 +80,15 @@ TEST_CASE("wide strings")
|
|||||||
// a valid surrogate pair is still decoded (U+1F600)
|
// a valid surrogate pair is still decoded (U+1F600)
|
||||||
CHECK(json::parse(std::u16string{u'"', 0xD83D, 0xDE00, u'"'}).get<std::string>() == "\xF0\x9F\x98\x80");
|
CHECK(json::parse(std::u16string{u'"', 0xD83D, 0xDE00, u'"'}).get<std::string>() == "\xF0\x9F\x98\x80");
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
SECTION("std::u32string")
|
SECTION("std::u32string")
|
||||||
{
|
{
|
||||||
if (u32string_is_utf32())
|
std::u32string const w = U"[12.2,\"\u10C5a\u00E4\u00F6\U0001F4A4\U0001F9E2\"]";
|
||||||
{
|
|
||||||
std::u32string const w = U"[12.2,\"Ⴥaäö💤🧢\"]";
|
|
||||||
json const j = json::parse(w);
|
json const j = json::parse(w);
|
||||||
CHECK(j.dump() == "[12.2,\"Ⴥaäö💤🧢\"]");
|
CHECK(j.dump() == "[12.2,\"" "\xE1\x83\x85" "a" "\xC3\xA4" "\xC3\xB6" "\xF0\x9F\x92\xA4" "\xF0\x9F\xA7\xA2" "\"]");
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
SECTION("invalid std::u32string")
|
SECTION("invalid std::u32string")
|
||||||
{
|
|
||||||
if (u32string_is_utf32())
|
|
||||||
{
|
{
|
||||||
std::u32string const w = U"\"\x110000";
|
std::u32string const w = U"\"\x110000";
|
||||||
json _;
|
json _;
|
||||||
@@ -140,6 +104,4 @@ TEST_CASE("wide strings")
|
|||||||
// the same unit inside a string is reported as an ill-formed byte
|
// the same unit inside a string is reported as an ill-formed byte
|
||||||
CHECK_THROWS_WITH_AS(_ = json::parse(std::u32string{U'"', static_cast<char32_t>(0xFFFFFFFF), U'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xFF'", json::parse_error&);
|
CHECK_THROWS_WITH_AS(_ = json::parse(std::u32string{U'"', static_cast<char32_t>(0xFFFFFFFF), U'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xFF'", json::parse_error&);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
#endif
|
|
||||||
Reference in new issue
Block a user