Read a std::u8string from a JSON string and store it as one in every configuration

In C++20, json(std::u8string) was stored as a JSON string only when
std::filesystem::path support was available (the to_json overload sat
inside the JSON_HAS_FILESYSTEM guard), and there was no matching
from_json, so get<std::u8string>() threw type_error.302 ("type must be
array, but is string"). Without std::filesystem, the same expression
still produced an array of code units (the 3.12.0 behavior).

Move the to_json overload out of the filesystem guard so that a
std::u8string is a string in every configuration, build the value
directly from the code units (this also works for a custom string_t
such as the tests' alt_string), and add a from_json overload that reads
a JSON string. The array of numbers written by 3.12.0 is still accepted;
other types throw type_error.302 ("type must be string or array").
std::u8string_view and char8_t arrays are unchanged.

The tests now run in C++20/23 (the file mentions JSON_HAS_CPP_20/23),
also without std::filesystem (test-conversions2_no_filesystem), and with
ordered_json and a custom string type. The new behavior is documented.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

.github/dependabot.yml

@@ -67,4 +67,3 @@ updates:
       - dependency-name: mcp
       - dependency-name: boltons
       - dependency-name: "opentelemetry-*"
-      - dependency-name: importlib-metadata

.github/workflows/codeql-analysis.yml

@@ -38,7 +38,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@33d7b85da1a04171e211befe2c739303b9a4c066 # v4.39.3
+      uses: github/codeql-action/init@1fc58d04a4d4374f921bdac573c3811c240f9c26 # v4.39.2
       with:
         languages: c-cpp
         # Analyze the sources without compiling them. Autobuild compiled the
@@ -47,4 +47,4 @@ jobs:
         build-mode: none
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@33d7b85da1a04171e211befe2c739303b9a4c066 # v4.39.3
+      uses: github/codeql-action/analyze@1fc58d04a4d4374f921bdac573c3811c240f9c26 # v4.39.2

.github/workflows/flawfinder.yml

@@ -48,6 +48,6 @@ jobs:
           output: 'flawfinder_results.sarif'
 
       - name: Upload analysis results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@33d7b85da1a04171e211befe2c739303b9a4c066 # v4.39.3
+        uses: github/codeql-action/upload-sarif@1fc58d04a4d4374f921bdac573c3811c240f9c26 # v4.39.2
         with:
           sarif_file: ${{github.workspace}}/flawfinder_results.sarif

.github/workflows/scorecards.yml

@@ -80,6 +80,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@33d7b85da1a04171e211befe2c739303b9a4c066 # v4.39.3
+        uses: github/codeql-action/upload-sarif@1fc58d04a4d4374f921bdac573c3811c240f9c26 # v4.39.2
         with:
           sarif_file: results.sarif

.github/workflows/semgrep.yml

@@ -65,7 +65,7 @@ jobs:
 
       # Upload SARIF file generated in previous step
       - name: Upload SARIF file
-        uses: github/codeql-action/upload-sarif@33d7b85da1a04171e211befe2c739303b9a4c066 # v4.39.3
+        uses: github/codeql-action/upload-sarif@1fc58d04a4d4374f921bdac573c3811c240f9c26 # v4.39.2
         with:
           sarif_file: semgrep.sarif
         if: always()

cmake/requirements/requirements-semgrep.txt

@@ -391,9 +391,9 @@ httpx-sse==0.4.3 \
 idna==3.20 \
     --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \
     --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c
-importlib-metadata==8.7.1 \
-    --hash=sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb \
-    --hash=sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151
+importlib-metadata==10.1.0 \
+    --hash=sha256:82b04047fa09792c1a8cfeddfe08bec79e4b13c29c8469b0d3949860f61017e8 \
+    --hash=sha256:cf45ce66a929c64e994185a809c04840d0501db17e663810d993d0f4829265c7
 jsonschema==4.25.1 \
     --hash=sha256:3fba0169e345c7175110351d456342c364814cfcf3b964ba4587f22915230a63 \
     --hash=sha256:e4a9655ce0da0c0b67a085847e00a3a51449e1157f4f75e9fb5aa545e122eb85

docs/mkdocs/docs/api/basic_json/basic_json.md

@@ -67,7 +67,11 @@ basic_json(basic_json&& other) noexcept;
     - **objects**: [`object_t`](object_t.md) and all kinds of compatible associative containers such as `std::map`,
      `std::unordered_map`, `std::multimap`, and `std::unordered_multimap` with a `key_type` compatible to `string_t`
      and a `value_type` from which a `basic_json` value can be constructed.
-    - **strings**: `string_t`, string literals, and all compatible string containers can be used.
+    - **strings**: `string_t`, string literals, and all compatible string containers can be used. Since version 3.13.0,
+      this includes `#!cpp std::u8string` (C++20): its code units are stored as a JSON string and
+      [`get<std::u8string>()`](get.md) reads it back (in version 3.12.0, a `#!cpp std::u8string` was stored as an array
+      of numbers; `get<std::u8string>()` still accepts such an array). `#!cpp std::u8string_view` and `#!cpp char8_t`
+      arrays are not strings; they are stored as arrays of code units.
 
     !!! note "Rvalue containers are moved from"
 

docs/mkdocs/docs/api/basic_json/get.md

@@ -130,6 +130,12 @@ overload (3).
     `#!cpp json(true).get<double>()` throw. This is the behavior of all 3.x versions;
     [`basic_json_view::get<T>()`](../basic_json_view/get.md) behaves the same.
 
+!!! note "`std::u8string` conversions"
+
+    Since version 3.13.0, `#!cpp get<std::u8string>()` (C++20) reads a JSON string; it also accepts an array of numbers,
+    the form in which version 3.12.0 stored a `#!cpp std::u8string`. Any other JSON type throws
+    [`type_error.302`](../../home/exceptions.md#jsonexceptiontype_error302) ("type must be string or array").
+
 !!! note "`std::optional` conversions"
 
     Prior to version 3.13.0, `#!cpp get<std::optional<T>>()` (and other conversions to `std::optional<T>`) failed to

docs/mkdocs/docs/features/types/template_parameters.md

@@ -425,7 +425,7 @@ using array_t = ArrayType<basic_json, AllocatorType<basic_json>>;
 | Type                                                                 | Reason                                                                                                  |
 |----------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------|
 | `#!cpp std::wstring`, `#!cpp std::u16string`, `#!cpp std::u32string` | the character type is not one byte wide                                                                 |
-| `#!cpp std::u8string`                                                | one byte wide, but `#!cpp char8_t` is not `#!cpp char`-compatible                                       |
+| `#!cpp std::u8string`                                                | one byte wide, but `#!cpp char8_t` is not `#!cpp char`-compatible (as a *value*, it converts to and from a JSON string) |
 | `absl::Cord`                                                         | no `value_type`, and the storage is not contiguous                                                      |
 | `QString`                                                            | no `append(const char*, size_type)`; its `QChar` is also two bytes wide, though that is never diagnosed |
 

docs/mkdocs/docs/home/releases/unreleased.md

@@ -8,6 +8,16 @@ These changes are part of the `develop` branch, but not of a release yet.
   exception is thrown during the conversion, the source may be partially moved-from, so the strong exception guarantee
   no longer holds for the source of an rvalue conversion (the JSON value itself is unaffected). Lvalues and `const`
   objects are still copied. See [`basic_json`](../../api/basic_json/basic_json.md).
+- In C++20 (library support for `char8_t`, `__cpp_lib_char8_t`), a `#!cpp std::u8string` is a JSON **string**:
+  `json j = std::u8string(u8"abc");` stores `"abc"`, and `#!cpp j.get<std::u8string>()` reads it back. Version 3.12.0
+  stored a `#!cpp std::u8string` as an array of code units (`[97,98,99]`); on the 3.13.0 development branch it was
+  already stored as a string if `std::filesystem` is available, but could not be read back. Now the result no longer
+  depends on whether `std::filesystem::path` is supported, and `#!cpp get<std::u8string>()` accepts both a JSON string
+  and the array of numbers written by 3.12.0. **Consequence:** JSON documents produced from a `#!cpp std::u8string` by
+  3.12.0 (arrays) are still read, but code that checks `is_array()` on such a value, or that reads it as
+  `#!cpp std::vector<char8_t>` / `#!cpp std::vector<std::uint8_t>`, now sees a string; to keep the array form, convert
+  explicitly, e.g., `#!cpp json(std::vector<char8_t>(s.begin(), s.end()))`. `#!cpp std::u8string_view` and
+  `#!cpp char8_t` arrays are unchanged (arrays of code units). See [`basic_json`](../../api/basic_json/basic_json.md).
 
 ## API changes
 

include/nlohmann/detail/conversions/from_json.hpp

@@ -785,6 +785,37 @@ void from_json(const BasicJsonType& j, std::unordered_map<Key, Value, Hash, KeyE
     from_json_pair_array_to_map(j, m);
 }
 
+#if defined(__cpp_lib_char8_t)
+// a std::u8string is read from a JSON string; the array of code units written by
+// versions before 3.13.0 is still accepted
+template < typename BasicJsonType, typename Tr, typename Allocator,
+           enable_if_t < std::is_same<typename json_type_of_t<BasicJsonType>::string_t::value_type, char>::value, int > = 0 >
+inline void from_json(const BasicJsonType& j, std::basic_string<char8_t, Tr, Allocator>& s)
+{
+    if (j.is_string())
+    {
+        const auto& str = string_value(j);
+        const char* const data = str.data();
+        const std::size_t size = str.size();
+        std::basic_string<char8_t, Tr, Allocator> ret(s.get_allocator());
+        ret.reserve(size);
+        for (std::size_t i = 0; i < size; ++i)
+        {
+            ret.push_back(static_cast<char8_t>(static_cast<unsigned char>(data[i])));
+        }
+        s = std::move(ret);
+    }
+    else if (j.is_array())
+    {
+        from_json_array_impl(j, s, priority_tag<3> {});
+    }
+    else
+    {
+        throw_type_must_be("string or array", j);
+    }
+}
+#endif
+
 #if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM
 
 // Workaround for MSVC 19.51 (and possibly later): in large cpp files, the compiler may fail to resolve with generic has_from_json (issue #4996)

include/nlohmann/detail/conversions/to_json.hpp

@@ -625,16 +625,20 @@ inline void to_json(BasicJsonType& j, const T& t)
     to_json_tuple_impl(j, t, make_index_sequence<std::tuple_size<T>::value> {});
 }
 
-#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM
+// a std::u8string is a JSON string (it was an array of code units before 3.13.0);
+// defined outside the std::filesystem guard so that the result does not depend
+// on whether std::filesystem::path is supported
 #if defined(__cpp_lib_char8_t)
 template<typename BasicJsonType, typename Tr, typename Allocator>
 inline void to_json(BasicJsonType& j, const std::basic_string<char8_t, Tr, Allocator>& s)
 {
-    using OtherAllocator = typename std::allocator_traits<Allocator>::template rebind_alloc<char>;
-    j = std::basic_string<char, std::char_traits<char>, OtherAllocator>(s.begin(), s.end(), s.get_allocator());
+    using string_t = typename BasicJsonType::string_t;
+    // char may alias any object; the code units are copied unchanged
+    j = string_t(reinterpret_cast<const char*>(s.data()), s.size());
 }
 #endif
 
+#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM
 // Workaround for MSVC 19.51 (and possibly later): in large cpp files, the compiler may fail to resolve with generic has_to_json (issue #4996)
 template<typename BasicJsonType>
 struct has_to_json<BasicJsonType, std_fs::path, void> : std::true_type {};

single_include/nlohmann/json.hpp

@@ -6928,6 +6928,37 @@ void from_json(const BasicJsonType& j, std::unordered_map<Key, Value, Hash, KeyE
     from_json_pair_array_to_map(j, m);
 }
 
+#if defined(__cpp_lib_char8_t)
+// a std::u8string is read from a JSON string; the array of code units written by
+// versions before 3.13.0 is still accepted
+template < typename BasicJsonType, typename Tr, typename Allocator,
+           enable_if_t < std::is_same<typename json_type_of_t<BasicJsonType>::string_t::value_type, char>::value, int > = 0 >
+inline void from_json(const BasicJsonType& j, std::basic_string<char8_t, Tr, Allocator>& s)
+{
+    if (j.is_string())
+    {
+        const auto& str = string_value(j);
+        const char* const data = str.data();
+        const std::size_t size = str.size();
+        std::basic_string<char8_t, Tr, Allocator> ret(s.get_allocator());
+        ret.reserve(size);
+        for (std::size_t i = 0; i < size; ++i)
+        {
+            ret.push_back(static_cast<char8_t>(static_cast<unsigned char>(data[i])));
+        }
+        s = std::move(ret);
+    }
+    else if (j.is_array())
+    {
+        from_json_array_impl(j, s, priority_tag<3> {});
+    }
+    else
+    {
+        throw_type_must_be("string or array", j);
+    }
+}
+#endif
+
 #if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM
 
 // Workaround for MSVC 19.51 (and possibly later): in large cpp files, the compiler may fail to resolve with generic has_from_json (issue #4996)
@@ -8329,16 +8360,20 @@ inline void to_json(BasicJsonType& j, const T& t)
     to_json_tuple_impl(j, t, make_index_sequence<std::tuple_size<T>::value> {});
 }
 
-#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM
+// a std::u8string is a JSON string (it was an array of code units before 3.13.0);
+// defined outside the std::filesystem guard so that the result does not depend
+// on whether std::filesystem::path is supported
 #if defined(__cpp_lib_char8_t)
 template<typename BasicJsonType, typename Tr, typename Allocator>
 inline void to_json(BasicJsonType& j, const std::basic_string<char8_t, Tr, Allocator>& s)
 {
-    using OtherAllocator = typename std::allocator_traits<Allocator>::template rebind_alloc<char>;
-    j = std::basic_string<char, std::char_traits<char>, OtherAllocator>(s.begin(), s.end(), s.get_allocator());
+    using string_t = typename BasicJsonType::string_t;
+    // char may alias any object; the code units are copied unchanged
+    j = string_t(reinterpret_cast<const char*>(s.data()), s.size());
 }
 #endif
 
+#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM
 // Workaround for MSVC 19.51 (and possibly later): in large cpp files, the compiler may fail to resolve with generic has_to_json (issue #4996)
 template<typename BasicJsonType>
 struct has_to_json<BasicJsonType, std_fs::path, void> : std::true_type {};

tests/CMakeLists.txt

@@ -364,6 +364,16 @@ json_test_add_test_for(src/unit-class_parser.cpp
     MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force}
 )
 
+# test the conversions again without std::filesystem::path support: a
+# std::u8string must be a JSON string in this configuration, too
+json_test_set_test_options(test-conversions2_no_filesystem
+    COMPILE_DEFINITIONS JSON_HAS_FILESYSTEM=0 JSON_HAS_EXPERIMENTAL_FILESYSTEM=0
+)
+json_test_add_test_for(src/unit-conversions2.cpp
+    NAME test-conversions2_no_filesystem
+    MAIN test_main CXX_STANDARDS ${test_cxx_standards} ${test_force}
+)
+
 # test diagnostic positions again without regular diagnostics (JSON pointer paths)
 json_test_set_test_options(test-diagnostic-positions_only
     COMPILE_DEFINITIONS JSON_DIAGNOSTICS=0

tests/src/unit-alt-string.cpp

@@ -536,4 +536,20 @@ TEST_CASE("alternative string type with json_view")
     }
 }
 
+// a std::u8string is a JSON string, whatever the string type of the JSON value
+// JSON_HAS_CPP_20 JSON_HAS_CPP_23 (do not remove; see note at top of file)
+#if defined(__cpp_lib_char8_t)
+TEST_CASE("alternative string type and std::u8string")
+{
+    const std::u8string s = u8"abc";
+    const alt_json j = s;
+    CHECK(j.is_string());
+    CHECK(j.get<alt_string>() == alt_string("abc"));
+    CHECK(j.get<std::u8string>() == s);
+
+    // the array of code units written by versions before 3.13.0
+    CHECK(alt_json::array({0x61, 0x62, 0x63}).get<std::u8string>() == s);
+}
+#endif
+
 DOCTEST_CLANG_SUPPRESS_WARNING_POP

tests/src/unit-conversions2.cpp

@@ -27,10 +27,12 @@ using nlohmann::json;
 #include <deque>
 #include <forward_list>
 #include <list>
+#include <map>
 #include <set>
 #include <unordered_map>
 #include <unordered_set>
 #include <valarray>
+#include <vector>
 
 // NLOHMANN_JSON_SERIALIZE_ENUM uses a static std::pair
 DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
@@ -707,23 +709,12 @@ TEST_CASE("std::filesystem::path")
 }
 #endif
 
-// the ADL to_json overload for std::u8string only exists under the same guard
-// as std::filesystem::path support (it is otherwise only reached indirectly,
-// via std::filesystem::path::u8string()) -- mirror both #if conditions from
-// include/nlohmann/detail/conversions/to_json.hpp exactly
-#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM
+// a std::u8string is a JSON string since 3.13.0 (it was an array of code units
+// before); independent of whether std::filesystem::path is supported, see the
+// test-conversions2_no_filesystem target
+// JSON_HAS_CPP_20 JSON_HAS_CPP_23 (do not remove; see note at top of file)
 #if defined(__cpp_lib_char8_t)
 TEST_CASE("std::u8string")
-{
-    SECTION("ascii")
-    {
-        const std::u8string s = u8"Path";
-        json const j = s;
-
-        CHECK(j.template get<std::string>() == "Path");
-    }
-
-    SECTION("utf-8")
 {
     // use \u universal-character-names (rather than raw \x byte escapes
     // or literal non-ASCII source bytes) to compose the multi-byte UTF-8
@@ -732,13 +723,100 @@ TEST_CASE("std::u8string")
     // our CI configs promote to an error; \u is portable and produces
     // the exact same encoded bytes without depending on the source
     // file's encoding
-        const std::u8string s = u8"P\u011B\u0161ina";
+    const std::u8string utf8 = u8"P\u011B\u0161ina";
+
+    SECTION("ascii")
+    {
+        const std::u8string s = u8"Path";
         json const j = s;
 
+        CHECK(j.is_string());
+        CHECK(j.template get<std::string>() == "Path");
+    }
+
+    SECTION("utf-8")
+    {
+        json const j = utf8;
+
+        CHECK(j.is_string());
         CHECK(j.template get<std::string>() == "P\xc4\x9b\xc5\xa1ina");
     }
+
+    SECTION("round trip")
+    {
+        json const j = utf8;
+        CHECK(j.template get<std::u8string>() == utf8);
+
+        std::u8string back;
+        j.get_to(back);
+        CHECK(back == utf8);
+
+        // from a string
+        json const j_string = "P\xc4\x9b\xc5\xa1ina";
+        CHECK(j_string.template get<std::u8string>() == utf8);
+
+        // empty, and with an embedded NUL
+        CHECK(json(std::u8string()).template get<std::u8string>().empty());
+        const std::u8string with_nul(u8"a\0b", 3);
+        CHECK(json(with_nul).template get<std::u8string>() == with_nul);
+
+        // the lvalue and rvalue paths, and in containers
+        std::u8string lvalue = utf8;
+        CHECK(json(lvalue).template get<std::u8string>() == utf8);
+        CHECK(json(std::move(lvalue)).template get<std::u8string>() == utf8);
+        const std::vector<std::u8string> vec = {utf8, u8"x"};
+        json const j_vec = vec;
+        CHECK(j_vec.is_array());
+        CHECK(j_vec[0].is_string());
+        CHECK(j_vec.template get<std::vector<std::u8string>>() == vec);
+        const std::map<std::string, std::u8string> map = {{"k", utf8}};
+        CHECK(json(map).template get<std::map<std::string, std::u8string>>() == map);
     }
+
+    SECTION("array of code units (written by versions before 3.13.0)")
+    {
+        json const j = json::array({0x50, 0xC4, 0x9B, 0xC5, 0xA1, 0x69, 0x6E, 0x61});
+        CHECK(j.template get<std::u8string>() == utf8);
+        CHECK(json::array().template get<std::u8string>().empty());
+        CHECK(json::parse("[97, 98]").template get<std::u8string>() == u8"ab");
+
+        std::u8string s = u8"overwritten";
+        j.get_to(s);
+        CHECK(s == utf8);
+    }
+
+    SECTION("other types")
+    {
+#if !defined(JSON_NOEXCEPTION)
+        CHECK_THROWS_WITH_AS(json(1).template get<std::u8string>(), "[json.exception.type_error.302] type must be string or array, but is number", json::type_error);
+        CHECK_THROWS_WITH_AS(json(nullptr).template get<std::u8string>(), "[json.exception.type_error.302] type must be string or array, but is null", json::type_error);
+        CHECK_THROWS_AS(json::object().template get<std::u8string>(), json::type_error);
+        // an element that is not a number
+        CHECK_THROWS_AS(json::array({"a"}).template get<std::u8string>(), json::type_error);
 #endif
+    }
+
+    SECTION("ordered_json")
+    {
+        nlohmann::ordered_json const j = utf8;
+        CHECK(j.is_string());
+        CHECK(j.template get<std::string>() == "P\xc4\x9b\xc5\xa1ina");
+        CHECK(j.template get<std::u8string>() == utf8);
+        CHECK(nlohmann::ordered_json::array({0x61, 0x62}).template get<std::u8string>() == u8"ab");
+    }
+
+    SECTION("std::u8string_view and char8_t arrays are unchanged (arrays of code units)")
+    {
+        const std::u8string_view view = utf8;
+        json const j_view = view;
+        CHECK(j_view.is_array());
+        CHECK(j_view.size() == 8);
+
+        const char8_t arr[] = u8"ab";
+        json const j_arr = arr;
+        CHECK(j_arr == json::array({0x61, 0x62, 0}));
+    }
+}
 #endif
 
 #if !defined(JSON_NOEXCEPTION)