From e894077e173fe95d2812f0d97cde6f0ebe635633 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 11 Nov 2026 08:48:06 +0100 Subject: [PATCH] Read a std::u8string from a JSON string and store it as one in every configuration In C++20, json(std::u8string) was stored as a JSON string only when std::filesystem::path support was available (the to_json overload sat inside the JSON_HAS_FILESYSTEM guard), and there was no matching from_json, so get() threw type_error.302 ("type must be array, but is string"). Without std::filesystem, the same expression still produced an array of code units (the 3.12.0 behavior). Move the to_json overload out of the filesystem guard so that a std::u8string is a string in every configuration, build the value directly from the code units (this also works for a custom string_t such as the tests' alt_string), and add a from_json overload that reads a JSON string. The array of numbers written by 3.12.0 is still accepted; other types throw type_error.302 ("type must be string or array"). std::u8string_view and char8_t arrays are unchanged. The tests now run in C++20/23 (the file mentions JSON_HAS_CPP_20/23), also without std::filesystem (test-conversions2_no_filesystem), and with ordered_json and a custom string type. The new behavior is documented. Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/api/basic_json/basic_json.md | 6 +- docs/mkdocs/docs/api/basic_json/get.md | 6 + .../features/types/template_parameters.md | 2 +- docs/mkdocs/docs/home/releases/unreleased.md | 10 ++ .../nlohmann/detail/conversions/from_json.hpp | 31 +++++ .../nlohmann/detail/conversions/to_json.hpp | 10 +- single_include/nlohmann/json.hpp | 41 ++++++- tests/CMakeLists.txt | 10 ++ tests/src/unit-alt-string.cpp | 16 +++ tests/src/unit-conversions2.cpp | 108 +++++++++++++++--- 10 files changed, 217 insertions(+), 23 deletions(-) diff --git a/docs/mkdocs/docs/api/basic_json/basic_json.md b/docs/mkdocs/docs/api/basic_json/basic_json.md index dadb2d887..4d46ea281 100644 --- a/docs/mkdocs/docs/api/basic_json/basic_json.md +++ b/docs/mkdocs/docs/api/basic_json/basic_json.md @@ -67,7 +67,11 @@ basic_json(basic_json&& other) noexcept; - **objects**: [`object_t`](object_t.md) and all kinds of compatible associative containers such as `std::map`, `std::unordered_map`, `std::multimap`, and `std::unordered_multimap` with a `key_type` compatible to `string_t` and a `value_type` from which a `basic_json` value can be constructed. - - **strings**: `string_t`, string literals, and all compatible string containers can be used. + - **strings**: `string_t`, string literals, and all compatible string containers can be used. Since version 3.13.0, + this includes `#!cpp std::u8string` (C++20): its code units are stored as a JSON string and + [`get()`](get.md) reads it back (in version 3.12.0, a `#!cpp std::u8string` was stored as an array + of numbers; `get()` still accepts such an array). `#!cpp std::u8string_view` and `#!cpp char8_t` + arrays are not strings; they are stored as arrays of code units. !!! note "Rvalue containers are moved from" diff --git a/docs/mkdocs/docs/api/basic_json/get.md b/docs/mkdocs/docs/api/basic_json/get.md index 06e34218a..a590ae03a 100644 --- a/docs/mkdocs/docs/api/basic_json/get.md +++ b/docs/mkdocs/docs/api/basic_json/get.md @@ -130,6 +130,12 @@ overload (3). `#!cpp json(true).get()` throw. This is the behavior of all 3.x versions; [`basic_json_view::get()`](../basic_json_view/get.md) behaves the same. +!!! note "`std::u8string` conversions" + + Since version 3.13.0, `#!cpp get()` (C++20) reads a JSON string; it also accepts an array of numbers, + the form in which version 3.12.0 stored a `#!cpp std::u8string`. Any other JSON type throws + [`type_error.302`](../../home/exceptions.md#jsonexceptiontype_error302) ("type must be string or array"). + !!! note "`std::optional` conversions" Prior to version 3.13.0, `#!cpp get>()` (and other conversions to `std::optional`) failed to diff --git a/docs/mkdocs/docs/features/types/template_parameters.md b/docs/mkdocs/docs/features/types/template_parameters.md index 6ae6820f5..8679bf6f9 100644 --- a/docs/mkdocs/docs/features/types/template_parameters.md +++ b/docs/mkdocs/docs/features/types/template_parameters.md @@ -425,7 +425,7 @@ using array_t = ArrayType>; | Type | Reason | |----------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------| | `#!cpp std::wstring`, `#!cpp std::u16string`, `#!cpp std::u32string` | the character type is not one byte wide | -| `#!cpp std::u8string` | one byte wide, but `#!cpp char8_t` is not `#!cpp char`-compatible | +| `#!cpp std::u8string` | one byte wide, but `#!cpp char8_t` is not `#!cpp char`-compatible (as a *value*, it converts to and from a JSON string) | | `absl::Cord` | no `value_type`, and the storage is not contiguous | | `QString` | no `append(const char*, size_type)`; its `QChar` is also two bytes wide, though that is never diagnosed | diff --git a/docs/mkdocs/docs/home/releases/unreleased.md b/docs/mkdocs/docs/home/releases/unreleased.md index b27da9748..01b334247 100644 --- a/docs/mkdocs/docs/home/releases/unreleased.md +++ b/docs/mkdocs/docs/home/releases/unreleased.md @@ -8,6 +8,16 @@ These changes are part of the `develop` branch, but not of a release yet. exception is thrown during the conversion, the source may be partially moved-from, so the strong exception guarantee no longer holds for the source of an rvalue conversion (the JSON value itself is unaffected). Lvalues and `const` objects are still copied. See [`basic_json`](../../api/basic_json/basic_json.md). +- In C++20 (library support for `char8_t`, `__cpp_lib_char8_t`), a `#!cpp std::u8string` is a JSON **string**: + `json j = std::u8string(u8"abc");` stores `"abc"`, and `#!cpp j.get()` reads it back. Version 3.12.0 + stored a `#!cpp std::u8string` as an array of code units (`[97,98,99]`); on the 3.13.0 development branch it was + already stored as a string if `std::filesystem` is available, but could not be read back. Now the result no longer + depends on whether `std::filesystem::path` is supported, and `#!cpp get()` accepts both a JSON string + and the array of numbers written by 3.12.0. **Consequence:** JSON documents produced from a `#!cpp std::u8string` by + 3.12.0 (arrays) are still read, but code that checks `is_array()` on such a value, or that reads it as + `#!cpp std::vector` / `#!cpp std::vector`, now sees a string; to keep the array form, convert + explicitly, e.g., `#!cpp json(std::vector(s.begin(), s.end()))`. `#!cpp std::u8string_view` and + `#!cpp char8_t` arrays are unchanged (arrays of code units). See [`basic_json`](../../api/basic_json/basic_json.md). ## API changes diff --git a/include/nlohmann/detail/conversions/from_json.hpp b/include/nlohmann/detail/conversions/from_json.hpp index 09f0713ea..eb1f3d189 100644 --- a/include/nlohmann/detail/conversions/from_json.hpp +++ b/include/nlohmann/detail/conversions/from_json.hpp @@ -785,6 +785,37 @@ void from_json(const BasicJsonType& j, std::unordered_map::string_t::value_type, char>::value, int > = 0 > +inline void from_json(const BasicJsonType& j, std::basic_string& s) +{ + if (j.is_string()) + { + const auto& str = string_value(j); + const char* const data = str.data(); + const std::size_t size = str.size(); + std::basic_string ret(s.get_allocator()); + ret.reserve(size); + for (std::size_t i = 0; i < size; ++i) + { + ret.push_back(static_cast(static_cast(data[i]))); + } + s = std::move(ret); + } + else if (j.is_array()) + { + from_json_array_impl(j, s, priority_tag<3> {}); + } + else + { + throw_type_must_be("string or array", j); + } +} +#endif + #if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM // Workaround for MSVC 19.51 (and possibly later): in large cpp files, the compiler may fail to resolve with generic has_from_json (issue #4996) diff --git a/include/nlohmann/detail/conversions/to_json.hpp b/include/nlohmann/detail/conversions/to_json.hpp index 05ae82370..3d95370d6 100644 --- a/include/nlohmann/detail/conversions/to_json.hpp +++ b/include/nlohmann/detail/conversions/to_json.hpp @@ -625,16 +625,20 @@ inline void to_json(BasicJsonType& j, const T& t) to_json_tuple_impl(j, t, make_index_sequence::value> {}); } -#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM +// a std::u8string is a JSON string (it was an array of code units before 3.13.0); +// defined outside the std::filesystem guard so that the result does not depend +// on whether std::filesystem::path is supported #if defined(__cpp_lib_char8_t) template inline void to_json(BasicJsonType& j, const std::basic_string& s) { - using OtherAllocator = typename std::allocator_traits::template rebind_alloc; - j = std::basic_string, OtherAllocator>(s.begin(), s.end(), s.get_allocator()); + using string_t = typename BasicJsonType::string_t; + // char may alias any object; the code units are copied unchanged + j = string_t(reinterpret_cast(s.data()), s.size()); } #endif +#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM // Workaround for MSVC 19.51 (and possibly later): in large cpp files, the compiler may fail to resolve with generic has_to_json (issue #4996) template struct has_to_json : std::true_type {}; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 0db1793ac..45c9b92a6 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -6928,6 +6928,37 @@ void from_json(const BasicJsonType& j, std::unordered_map::string_t::value_type, char>::value, int > = 0 > +inline void from_json(const BasicJsonType& j, std::basic_string& s) +{ + if (j.is_string()) + { + const auto& str = string_value(j); + const char* const data = str.data(); + const std::size_t size = str.size(); + std::basic_string ret(s.get_allocator()); + ret.reserve(size); + for (std::size_t i = 0; i < size; ++i) + { + ret.push_back(static_cast(static_cast(data[i]))); + } + s = std::move(ret); + } + else if (j.is_array()) + { + from_json_array_impl(j, s, priority_tag<3> {}); + } + else + { + throw_type_must_be("string or array", j); + } +} +#endif + #if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM // Workaround for MSVC 19.51 (and possibly later): in large cpp files, the compiler may fail to resolve with generic has_from_json (issue #4996) @@ -8329,16 +8360,20 @@ inline void to_json(BasicJsonType& j, const T& t) to_json_tuple_impl(j, t, make_index_sequence::value> {}); } -#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM +// a std::u8string is a JSON string (it was an array of code units before 3.13.0); +// defined outside the std::filesystem guard so that the result does not depend +// on whether std::filesystem::path is supported #if defined(__cpp_lib_char8_t) template inline void to_json(BasicJsonType& j, const std::basic_string& s) { - using OtherAllocator = typename std::allocator_traits::template rebind_alloc; - j = std::basic_string, OtherAllocator>(s.begin(), s.end(), s.get_allocator()); + using string_t = typename BasicJsonType::string_t; + // char may alias any object; the code units are copied unchanged + j = string_t(reinterpret_cast(s.data()), s.size()); } #endif +#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM // Workaround for MSVC 19.51 (and possibly later): in large cpp files, the compiler may fail to resolve with generic has_to_json (issue #4996) template struct has_to_json : std::true_type {}; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f4e574fc1..a1aca2bfd 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -364,6 +364,16 @@ json_test_add_test_for(src/unit-class_parser.cpp MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force} ) +# test the conversions again without std::filesystem::path support: a +# std::u8string must be a JSON string in this configuration, too +json_test_set_test_options(test-conversions2_no_filesystem + COMPILE_DEFINITIONS JSON_HAS_FILESYSTEM=0 JSON_HAS_EXPERIMENTAL_FILESYSTEM=0 +) +json_test_add_test_for(src/unit-conversions2.cpp + NAME test-conversions2_no_filesystem + MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force} +) + # test diagnostic positions again without regular diagnostics (JSON pointer paths) json_test_set_test_options(test-diagnostic-positions_only COMPILE_DEFINITIONS JSON_DIAGNOSTICS=0 diff --git a/tests/src/unit-alt-string.cpp b/tests/src/unit-alt-string.cpp index 4a0260a10..dc13cbd53 100644 --- a/tests/src/unit-alt-string.cpp +++ b/tests/src/unit-alt-string.cpp @@ -536,4 +536,20 @@ TEST_CASE("alternative string type with json_view") } } +// a std::u8string is a JSON string, whatever the string type of the JSON value +// JSON_HAS_CPP_20 JSON_HAS_CPP_23 (do not remove; see note at top of file) +#if defined(__cpp_lib_char8_t) +TEST_CASE("alternative string type and std::u8string") +{ + const std::u8string s = u8"abc"; + const alt_json j = s; + CHECK(j.is_string()); + CHECK(j.get() == alt_string("abc")); + CHECK(j.get() == s); + + // the array of code units written by versions before 3.13.0 + CHECK(alt_json::array({0x61, 0x62, 0x63}).get() == s); +} +#endif + DOCTEST_CLANG_SUPPRESS_WARNING_POP diff --git a/tests/src/unit-conversions2.cpp b/tests/src/unit-conversions2.cpp index c543e4ff8..8cb44a231 100644 --- a/tests/src/unit-conversions2.cpp +++ b/tests/src/unit-conversions2.cpp @@ -27,10 +27,12 @@ using nlohmann::json; #include #include #include +#include #include #include #include #include +#include // NLOHMANN_JSON_SERIALIZE_ENUM uses a static std::pair DOCTEST_CLANG_SUPPRESS_WARNING_PUSH @@ -707,38 +709,114 @@ TEST_CASE("std::filesystem::path") } #endif -// the ADL to_json overload for std::u8string only exists under the same guard -// as std::filesystem::path support (it is otherwise only reached indirectly, -// via std::filesystem::path::u8string()) -- mirror both #if conditions from -// include/nlohmann/detail/conversions/to_json.hpp exactly -#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM +// a std::u8string is a JSON string since 3.13.0 (it was an array of code units +// before); independent of whether std::filesystem::path is supported, see the +// test-conversions2_no_filesystem target +// JSON_HAS_CPP_20 JSON_HAS_CPP_23 (do not remove; see note at top of file) #if defined(__cpp_lib_char8_t) TEST_CASE("std::u8string") { + // use \u universal-character-names (rather than raw \x byte escapes + // or literal non-ASCII source bytes) to compose the multi-byte UTF-8 + // encoding -- MSVC treats \x escapes used that way inside a u8 + // literal as a nonstandard extension (warning C5321), which some of + // our CI configs promote to an error; \u is portable and produces + // the exact same encoded bytes without depending on the source + // file's encoding + const std::u8string utf8 = u8"P\u011B\u0161ina"; + SECTION("ascii") { const std::u8string s = u8"Path"; json const j = s; + CHECK(j.is_string()); CHECK(j.template get() == "Path"); } SECTION("utf-8") { - // use \u universal-character-names (rather than raw \x byte escapes - // or literal non-ASCII source bytes) to compose the multi-byte UTF-8 - // encoding -- MSVC treats \x escapes used that way inside a u8 - // literal as a nonstandard extension (warning C5321), which some of - // our CI configs promote to an error; \u is portable and produces - // the exact same encoded bytes without depending on the source - // file's encoding - const std::u8string s = u8"P\u011B\u0161ina"; - json const j = s; + json const j = utf8; + CHECK(j.is_string()); CHECK(j.template get() == "P\xc4\x9b\xc5\xa1ina"); } -} + + SECTION("round trip") + { + json const j = utf8; + CHECK(j.template get() == utf8); + + std::u8string back; + j.get_to(back); + CHECK(back == utf8); + + // from a string + json const j_string = "P\xc4\x9b\xc5\xa1ina"; + CHECK(j_string.template get() == utf8); + + // empty, and with an embedded NUL + CHECK(json(std::u8string()).template get().empty()); + const std::u8string with_nul(u8"a\0b", 3); + CHECK(json(with_nul).template get() == with_nul); + + // the lvalue and rvalue paths, and in containers + std::u8string lvalue = utf8; + CHECK(json(lvalue).template get() == utf8); + CHECK(json(std::move(lvalue)).template get() == utf8); + const std::vector vec = {utf8, u8"x"}; + json const j_vec = vec; + CHECK(j_vec.is_array()); + CHECK(j_vec[0].is_string()); + CHECK(j_vec.template get>() == vec); + const std::map map = {{"k", utf8}}; + CHECK(json(map).template get>() == map); + } + + SECTION("array of code units (written by versions before 3.13.0)") + { + json const j = json::array({0x50, 0xC4, 0x9B, 0xC5, 0xA1, 0x69, 0x6E, 0x61}); + CHECK(j.template get() == utf8); + CHECK(json::array().template get().empty()); + CHECK(json::parse("[97, 98]").template get() == u8"ab"); + + std::u8string s = u8"overwritten"; + j.get_to(s); + CHECK(s == utf8); + } + + SECTION("other types") + { +#if !defined(JSON_NOEXCEPTION) + CHECK_THROWS_WITH_AS(json(1).template get(), "[json.exception.type_error.302] type must be string or array, but is number", json::type_error); + CHECK_THROWS_WITH_AS(json(nullptr).template get(), "[json.exception.type_error.302] type must be string or array, but is null", json::type_error); + CHECK_THROWS_AS(json::object().template get(), json::type_error); + // an element that is not a number + CHECK_THROWS_AS(json::array({"a"}).template get(), json::type_error); #endif + } + + SECTION("ordered_json") + { + nlohmann::ordered_json const j = utf8; + CHECK(j.is_string()); + CHECK(j.template get() == "P\xc4\x9b\xc5\xa1ina"); + CHECK(j.template get() == utf8); + CHECK(nlohmann::ordered_json::array({0x61, 0x62}).template get() == u8"ab"); + } + + SECTION("std::u8string_view and char8_t arrays are unchanged (arrays of code units)") + { + const std::u8string_view view = utf8; + json const j_view = view; + CHECK(j_view.is_array()); + CHECK(j_view.size() == 8); + + const char8_t arr[] = u8"ab"; + json const j_arr = arr; + CHECK(j_arr == json::array({0x61, 0x62, 0})); + } +} #endif #if !defined(JSON_NOEXCEPTION)