json_view: accept() checks strings without decoding them

The builder that accept() runs stores no nodes, but still decoded every string
with escapes into the arena (slow_string(): memcpy of the runs between escapes
and the UTF-8 of every \u escape). It now calls check_string() instead, which
has the same checks and the same errors and error positions (control characters,
ill-formed UTF-8, bad escapes, bad hex digits, lone and unpaired surrogates, a
missing closing quote) and skips the text between them with scan_string_run(),
without writing anything. The builder that stores keeps slow_string().

The flag that passes the request from accept() to build() now sits next to
`discarded` in document_data, where it uses padding: no member of document_data
moves (the first version put it after `nodes`, which moved all members behind it
by eight bytes).

Tests: the new section of the accept test checks runs of text of every length from
0 to 40 between valid and invalid escapes, UTF-8 sequences and control characters,
in values and in keys, cut near the end of the text, against json::accept().
Checked against a checker that accepts a lone low surrogate.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/view/builder.hpp

@@ -16,6 +16,7 @@
 #include <cstring> // memcmp, memcpy
 #include <limits> // numeric_limits
 #include <string> // string
+#include <type_traits> // false_type, integral_constant, true_type
 #include <vector> // vector
 
 #include <nlohmann/json.hpp>
@@ -246,6 +247,88 @@ class builder
         return p;
     }
 
+    /// the string from p, the first backslash, control character, ill-formed UTF-8 byte, or the end, checked and not
+    /// decoded (accept() stores nothing, so the arena is not written): the checks, errors and error positions of
+    /// slow_string(), for everything up to the closing quote
+    NLOHMANN_VIEW_NOINLINE decoded check_string(const unsigned char* p)
+    {
+        const unsigned char* const end = m_end;
+        for (;;)
+        {
+            if (p == end)
+            {
+                return failed(error_code::string_missing_quote, p);
+            }
+            const unsigned char c = *p;
+            if (c == '"')
+            {
+                ++p;
+                return decoded{p, 0, 0};
+            }
+            if (c != '\\')
+            {
+                return failed(c < 0x20 ? error_code::string_control_character : error_code::string_utf8, p);
+            }
+            ++p;
+            if (p == end)
+            {
+                return failed(error_code::string_missing_quote, p);
+            }
+            const unsigned char d = *p++;
+            if (d == 'u')
+            {
+                const int cp = hex4(p); // (moves p past the four digits)
+                if (NLOHMANN_VIEW_UNLIKELY(cp < 0))
+                {
+                    return failed(error_code::string_unicode_hex, p);
+                }
+                if (NLOHMANN_VIEW_UNLIKELY((cp & 0xF800) == 0xD800)) // a surrogate
+                {
+                    if (cp >= 0xDC00)
+                    {
+                        return failed(error_code::string_surrogate_low, p);
+                    }
+                    if (end - p < 2 || p[0] != '\\' || p[1] != 'u')
+                    {
+                        return failed(error_code::string_surrogate_high, p);
+                    }
+                    p += 2;
+                    const int lo = hex4(p); // (moves p past the four digits)
+                    if (lo < 0)
+                    {
+                        return failed(error_code::string_unicode_hex, p);
+                    }
+                    if (lo < 0xDC00 || lo > 0xDFFF)
+                    {
+                        return failed(error_code::string_surrogate_high, p);
+                    }
+                }
+            }
+            else if (!(d == '"' || d == '\\' || d == '/' || d == 'b' || d == 'f' || d == 'n' || d == 'r' || d == 't'))
+            {
+                --p;
+                return failed(error_code::string_escape, p);
+            }
+            p = scan_string_run(p, end); // the text up to the next backslash, quote, control character, or ill-formed UTF-8
+        }
+    }
+
+    /// the string from p, decoded into the arena, or only checked (Store == false)
+    decoded decode_string(const unsigned char* s, const unsigned char* p)
+    {
+        return decode_string(s, p, std::integral_constant<bool, Store>());
+    }
+
+    decoded decode_string(const unsigned char* s, const unsigned char* p, std::true_type /*unused*/)
+    {
+        return slow_string(s, p);
+    }
+
+    decoded decode_string(const unsigned char* /*unused*/, const unsigned char* p, std::false_type /*unused*/)
+    {
+        return check_string(p);
+    }
+
     /// s points to the start of the string, p to a backslash, a control
     /// character, an ill-formed UTF-8 byte, or the end: decode the string into
     /// the arena
@@ -1193,7 +1276,7 @@ indent_done:
                 ++p;
                 return true;
             }
-            const decoded r = cold.slow_string(s, p);
+            const decoded r = cold.decode_string(s, p);
             if (r.p == nullptr)
             {
                 return false;

include/nlohmann/detail/view/document_data.hpp

@@ -36,7 +36,6 @@ struct document_data
     const char* src = nullptr;
     std::size_t size = 0;
     node* nodes = nullptr; ///< the node index (see node.hpp)
-    bool validate_only = false; ///< accept(): build() checks the text and stores no nodes
     std::size_t node_count = 0;
     std::size_t node_capacity = 0;
     node* inline_nodes = nullptr; ///< node array allocated together with this header
@@ -59,6 +58,7 @@ struct document_data
     std::vector<std::uint32_t> large_objects; ///< positions of the objects to index (noted while parsing)
     std::array<const char*, 4> base = {{nullptr, nullptr, nullptr, nullptr}}; ///< string bases: source, arena, edit arena, unused (indexed by flags & node_flags::storage)
     bool discarded = true;
+    bool validate_only = false; ///< accept(): build() checks the text and stores no nodes (next to `discarded`: no member moves)
 
     /// The storage of edits (editable documents only; see edit_storage.hpp).
     /// No edit moves or resizes the parsed index, and the nodes of values

single_include/nlohmann/json_view.hpp

@@ -88,6 +88,7 @@
 #include <cstring> // memcmp, memcpy
 #include <limits> // numeric_limits
 #include <string> // string
+#include <type_traits> // false_type, integral_constant, true_type
 #include <vector> // vector
 
 // #include <nlohmann/json.hpp>
@@ -366,7 +367,6 @@ struct document_data
     const char* src = nullptr;
     std::size_t size = 0;
     node* nodes = nullptr; ///< the node index (see node.hpp)
-    bool validate_only = false; ///< accept(): build() checks the text and stores no nodes
     std::size_t node_count = 0;
     std::size_t node_capacity = 0;
     node* inline_nodes = nullptr; ///< node array allocated together with this header
@@ -389,6 +389,7 @@ struct document_data
     std::vector<std::uint32_t> large_objects; ///< positions of the objects to index (noted while parsing)
     std::array<const char*, 4> base = {{nullptr, nullptr, nullptr, nullptr}}; ///< string bases: source, arena, edit arena, unused (indexed by flags & node_flags::storage)
     bool discarded = true;
+    bool validate_only = false; ///< accept(): build() checks the text and stores no nodes (next to `discarded`: no member moves)
 
     /// The storage of edits (editable documents only; see edit_storage.hpp).
     /// No edit moves or resizes the parsed index, and the nodes of values
@@ -1489,6 +1490,88 @@ class builder
         return p;
     }
 
+    /// the string from p, the first backslash, control character, ill-formed UTF-8 byte, or the end, checked and not
+    /// decoded (accept() stores nothing, so the arena is not written): the checks, errors and error positions of
+    /// slow_string(), for everything up to the closing quote
+    NLOHMANN_VIEW_NOINLINE decoded check_string(const unsigned char* p)
+    {
+        const unsigned char* const end = m_end;
+        for (;;)
+        {
+            if (p == end)
+            {
+                return failed(error_code::string_missing_quote, p);
+            }
+            const unsigned char c = *p;
+            if (c == '"')
+            {
+                ++p;
+                return decoded{p, 0, 0};
+            }
+            if (c != '\\')
+            {
+                return failed(c < 0x20 ? error_code::string_control_character : error_code::string_utf8, p);
+            }
+            ++p;
+            if (p == end)
+            {
+                return failed(error_code::string_missing_quote, p);
+            }
+            const unsigned char d = *p++;
+            if (d == 'u')
+            {
+                const int cp = hex4(p); // (moves p past the four digits)
+                if (NLOHMANN_VIEW_UNLIKELY(cp < 0))
+                {
+                    return failed(error_code::string_unicode_hex, p);
+                }
+                if (NLOHMANN_VIEW_UNLIKELY((cp & 0xF800) == 0xD800)) // a surrogate
+                {
+                    if (cp >= 0xDC00)
+                    {
+                        return failed(error_code::string_surrogate_low, p);
+                    }
+                    if (end - p < 2 || p[0] != '\\' || p[1] != 'u')
+                    {
+                        return failed(error_code::string_surrogate_high, p);
+                    }
+                    p += 2;
+                    const int lo = hex4(p); // (moves p past the four digits)
+                    if (lo < 0)
+                    {
+                        return failed(error_code::string_unicode_hex, p);
+                    }
+                    if (lo < 0xDC00 || lo > 0xDFFF)
+                    {
+                        return failed(error_code::string_surrogate_high, p);
+                    }
+                }
+            }
+            else if (!(d == '"' || d == '\\' || d == '/' || d == 'b' || d == 'f' || d == 'n' || d == 'r' || d == 't'))
+            {
+                --p;
+                return failed(error_code::string_escape, p);
+            }
+            p = scan_string_run(p, end); // the text up to the next backslash, quote, control character, or ill-formed UTF-8
+        }
+    }
+
+    /// the string from p, decoded into the arena, or only checked (Store == false)
+    decoded decode_string(const unsigned char* s, const unsigned char* p)
+    {
+        return decode_string(s, p, std::integral_constant<bool, Store>());
+    }
+
+    decoded decode_string(const unsigned char* s, const unsigned char* p, std::true_type /*unused*/)
+    {
+        return slow_string(s, p);
+    }
+
+    decoded decode_string(const unsigned char* /*unused*/, const unsigned char* p, std::false_type /*unused*/)
+    {
+        return check_string(p);
+    }
+
     /// s points to the start of the string, p to a backslash, a control
     /// character, an ill-formed UTF-8 byte, or the end: decode the string into
     /// the arena
@@ -2436,7 +2519,7 @@ indent_done:
                 ++p;
                 return true;
             }
-            const decoded r = cold.slow_string(s, p);
+            const decoded r = cold.decode_string(s, p);
             if (r.p == nullptr)
             {
                 return false;

tests/src/unit-json_view_builder.cpp

@@ -509,6 +509,47 @@ TEST_CASE("json_view builder")
             CAPTURE(text)
             CHECK(nlohmann::basic_json_document<float_json>::accept(text) == float_json::accept(text));
         }
+
+        // strings are checked and not decoded: runs of text of every length from 0 to 40 between valid and invalid
+        // escapes, UTF-8 sequences and control characters, in values and in keys, cut near the end of the text
+        const std::array<const char*, 9> valid = {{"\\n", "\\\"", "\\\\", "\\/", "\\t", "\\u00e4", "\\u20ac", "\\ud83d\\ude00", "\\u0000"}};
+        const std::array<const char*, 12> invalid = {{"\\x", "\\ud800", "\\udc00", "\\u12G4", "\\u12", "\\ud83dx", "\\ud83d\\u0041", "\\ud83d\\ud83d", "\\", "\x01", "\x1f", "\x7f"}};
+        const std::array<const char*, 6> utf8 = {{"\xc3\xa4", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xc3", "\xff", "\x80"}};
+        std::mt19937 rng(2029); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible
+        for (std::size_t run = 0; run <= 40; ++run)
+        {
+            for (int i = 0; i < 30; ++i)
+            {
+                std::string content = std::string(rng() % 20, 'a');
+                const auto pieces = 1 + (rng() % 6);
+                for (unsigned k = 0; k < pieces; ++k)
+                {
+                    content += valid[rng() % valid.size()];
+                    content += std::string(k == 0 ? run : rng() % 41, static_cast<char>('b' + (rng() % 20)));
+                    switch (rng() % 8)
+                    {
+                        case 0:
+                            content += utf8[rng() % 3];
+                            break;
+                        case 1:
+                            content += utf8[rng() % utf8.size()];
+                            break;
+                        case 2:
+                            content += invalid[rng() % invalid.size()];
+                            break;
+                        default:
+                            break;
+                    }
+                }
+                for (const std::string& text :
+                        {
+                            "[\"" + content + "\"]", "{\"" + content + "\":1}", "[\"" + content, "[\"" + content.substr(0, content.size() - (rng() % 4)) + "\"]"
+                        })
+                {
+                    check_accept(text);
+                }
+            }
+        }
     }
 
     SECTION("test files")