Scan the strings of json_view's dump() with ensure_ascii 16 bytes at a time

With ensure_ascii, a string of the source that has a byte >= 0x7F was written
by a generic path: put() per piece and a validate-and-decode step for every
character beyond ASCII. It now goes through a writer with the cursor in a
local variable. The part of the string that has nothing to escape is found
16 bytes at a time (SSE2 or NEON; the existing 8-byte scan without vector
support) and copied; strings that are ASCII throughout are still decided by
the 8-byte check of dump() with ensure_ascii (a 16-byte scan for them was
2-5 % slower for short strings). The two-byte sequences (lead C2..DF) and
the three-byte sequences whose continuation bytes may have any value (lead
E1..EF except ED) are decoded with a few compares; E0, ED, four-byte
sequences, and damaged input go through validate_one_utf8() as before, so
damaged images still throw type_error.316.

dump(-1, ' ', true) instructions per call, macOS arm64 (develop -> now);
the compact dump without ensure_ascii and dumps of ASCII text are unchanged:

  twitter 5.42M -> 3.57M    citm 3.28M -> 3.24M     jeopardy(4k) 5.78M -> 5.65M
  CJK text, raw 66.5M -> 35.3M    escaped \u 32.7M -> 17.0M
  Latin text with accents 11.5M -> 9.4M    ASCII text 4.40M -> 4.40M

Linux instructions (GCC 13 / Clang 18), dump(-1, ' ', true):

  twitter -9 % / -33 %    CJK text -14 % / -47 %    Latin text -8 % / -18 %
  citm -0.2 % / -0.4 %    ASCII text -0.1 % / -0.3 %

The compact dump without ensure_ascii is unchanged (canada 0.0 %).

The unit test covers every character of the Basic Multilingual Plane, and a
character that needs an escape at each position of strings of 0..40 bytes.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

single_include/nlohmann/json_view.hpp

@@ -6089,6 +6089,8 @@ NLOHMANN_JSON_NAMESPACE_END
 
 // #include <nlohmann/detail/view/number.hpp>
 
+// #include <nlohmann/detail/view/simd.hpp>
+
 
 NLOHMANN_JSON_NAMESPACE_BEGIN
 namespace detail
@@ -6255,6 +6257,70 @@ inline bool is_ascii_without_delete(const char* s, std::size_t n) noexcept
     return true;
 }
 
+#if NLOHMANN_VIEW_VECTOR
+/// the bytes of the 16 at p that ensure_ascii cannot copy (a quote, a
+/// backslash, a control character, DEL, or not ASCII; a signed compare with
+/// 0x20 finds control characters and non-ASCII bytes at once), one bit per
+/// byte (x86) or one nibble per byte (NEON)
+NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t ascii_stop_mask(const unsigned char* p) noexcept
+{
+#if NLOHMANN_VIEW_NEON
+    const uint8x16_t in = vld1q_u8(p);
+    const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))),
+                                        vorrq_u8(vceqq_u8(in, vdupq_n_u8(0x7F)), vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))));
+    return vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0);
+#else
+    const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(p))); // NOLINT(bugprone-casting-through-void)
+    const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))),
+                                         _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F)), _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))));
+    return static_cast<std::uint64_t>(static_cast<unsigned>(_mm_movemask_epi8(special)));
+#endif
+}
+
+/// the width in bits of a byte in the mask of ascii_stop_mask()
+constexpr unsigned ascii_stop_bits() noexcept
+{
+    return NLOHMANN_VIEW_NEON ? 4u : 1u;
+}
+#endif
+
+/// As find_ascii_copyable_run(), 16 bytes per step. end is the end of the
+/// buffer that holds the n bytes at p (a short run is read as 16 bytes if
+/// they are in the buffer).
+inline std::size_t ascii_copyable_run(const unsigned char* p, std::size_t n, const unsigned char* end) noexcept
+{
+#if NLOHMANN_VIEW_VECTOR
+    if (n >= 16)
+    {
+        std::size_t i = 0;
+        for (; i + 16 <= n; i += 16)
+        {
+            const std::uint64_t mask = ascii_stop_mask(p + i);
+            if (mask != 0)
+            {
+                return i + (static_cast<std::size_t>(count_trailing_zeros(mask)) / ascii_stop_bits());
+            }
+        }
+        if (i == n)
+        {
+            return n;
+        }
+        // the last 16 bytes: those before i are known to be copyable
+        const std::uint64_t mask = ascii_stop_mask(p + n - 16);
+        return mask != 0 ? n - 16 + (static_cast<std::size_t>(count_trailing_zeros(mask)) / ascii_stop_bits()) : n;
+    }
+    if (end - p >= 16)
+    {
+        // (the bytes after the run are masked out)
+        const std::uint64_t mask = ascii_stop_mask(p) & ((std::uint64_t{1} << (n * ascii_stop_bits())) - 1u);
+        return mask != 0 ? static_cast<std::size_t>(count_trailing_zeros(mask)) / ascii_stop_bits() : n;
+    }
+#else
+    static_cast<void>(end);
+#endif
+    return find_ascii_copyable_run(p, n);
+}
+
 /// A stack that starts in a buffer of the caller (a local array) and moves to
 /// the heap (a vector of the caller) only when that is full, so that dumps of
 /// shallow documents need no allocation. The top is a pointer, as in
@@ -6506,10 +6572,151 @@ class view_serializer
         return (n.flags & node_flags::storage) == 0 && is_ascii_without_delete(src + n.off, n.len);
     }
 
-    /// a string with ensure_ascii (dump_text()), out of line
-    NLOHMANN_VIEW_NOINLINE void write_ascii_string(const node& n)
+    /*!
+    @brief a string with ensure_ascii: the characters that are not printable ASCII as \\uXXXX
+    (a character beyond U+FFFF as a surrogate pair); strings of a document
+    are valid UTF-8, except in a damaged image loaded with image_check::bounds,
+    for which this throws what basic_json::dump() throws for the string
+    */
+    NLOHMANN_VIEW_NOINLINE void write_ascii(const node& n)
     {
-        write_string(n);
+        const auto* const s = reinterpret_cast<const unsigned char*>(m_doc.str(n)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
+        const std::size_t size = n.len;
+        char* w = m_out.cursor();
+        char* lim = m_out.limit();
+        // room for k bytes
+        const auto room = [&](std::size_t k)
+        {
+            if (NLOHMANN_VIEW_UNLIKELY(static_cast<std::size_t>(lim - w) < k))
+            {
+                m_out.set_cursor(w);
+                m_out.reserve(k);
+                w = m_out.cursor();
+                lim = m_out.limit();
+            }
+        };
+        static constexpr const char* hex = "0123456789abcdef";
+        // \\uXXXX
+        const auto u_escape = [&](std::uint32_t u)
+        {
+            w[0] = '\\';
+            w[1] = 'u';
+            w[2] = hex[(u >> 12u) & 0xFu];
+            w[3] = hex[(u >> 8u) & 0xFu];
+            w[4] = hex[(u >> 4u) & 0xFu];
+            w[5] = hex[u & 0xFu];
+            w += 6;
+        };
+        room(1);
+        *w++ = '"';
+        std::size_t i = 0;
+        // (a string of the source starts with a run that has nothing to escape, 16 bytes per step)
+        const std::size_t prefix = (n.flags & node_flags::storage) == 0 ? ascii_copyable_run(s, size, reinterpret_cast<const unsigned char*>(m_doc.src + m_doc.size)) : 0; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
+        if (prefix != 0)
+        {
+            room(prefix);
+            std::memcpy(w, s, prefix);
+            w += prefix;
+            i = prefix;
+        }
+        while (i < size)
+        {
+            // (a character that needs an escape sequence is not worth a call)
+            const std::size_t run = is_ascii_copyable(s[i]) ? find_ascii_copyable_run(s + i, size - i) : 0;
+            if (run != 0)
+            {
+                room(run);
+                std::memcpy(w, s + i, run);
+                w += run;
+                i += run;
+                continue;
+            }
+            room(12);
+            if (s[i] >= 0x80)
+            {
+                const unsigned char lead = s[i];
+                // (the two-byte sequences of Latin text and the three-byte sequences of most other
+                // text, with continuation bytes of any value: the ranges that are restricted (E0,
+                // ED, and the sequences of four bytes) go through validate_one_utf8())
+                if (lead >= 0xE1 && lead <= 0xEF && lead != 0xED && size - i >= 3 && (s[i + 1] & 0xC0u) == 0x80 && (s[i + 2] & 0xC0u) == 0x80)
+                {
+                    u_escape(((lead & 0x0Fu) << 12u) | ((s[i + 1] & 0x3Fu) << 6u) | (s[i + 2] & 0x3Fu));
+                    i += 3;
+                    continue;
+                }
+                if (lead >= 0xC2 && lead <= 0xDF && size - i >= 2 && (s[i + 1] & 0xC0u) == 0x80)
+                {
+                    u_escape(((lead & 0x1Fu) << 6u) | (s[i + 1] & 0x3Fu));
+                    i += 2;
+                    continue;
+                }
+                const std::size_t len = validate_one_utf8(s + i, size - i);
+                if (NLOHMANN_VIEW_UNLIKELY(len == 0))
+                {
+                    m_out.set_cursor(w);
+                    invalid_utf8(s, size);
+                    return; // LCOV_EXCL_LINE (invalid_utf8 throws)
+                }
+                std::uint32_t codepoint = s[i] & (0xFFu >> (len + 1));
+                for (std::size_t k = 1; k < len; ++k)
+                {
+                    codepoint = (codepoint << 6u) | (s[i + k] & 0x3Fu);
+                }
+                i += len;
+                if (codepoint <= 0xFFFF)
+                {
+                    u_escape(codepoint);
+                }
+                else
+                {
+                    u_escape(0xD7C0u + (codepoint >> 10u));
+                    u_escape(0xDC00u + (codepoint & 0x3FFu));
+                }
+                continue;
+            }
+            // a quote, a backslash, or a control character (or DEL)
+            const unsigned char c = s[i++];
+            char escape = 0;
+            switch (c)
+            {
+                case 0x08:
+                    escape = 'b';
+                    break;
+                case 0x09:
+                    escape = 't';
+                    break;
+                case 0x0A:
+                    escape = 'n';
+                    break;
+                case 0x0C:
+                    escape = 'f';
+                    break;
+                case 0x0D:
+                    escape = 'r';
+                    break;
+                case 0x22:
+                    escape = '"';
+                    break;
+                case 0x5C:
+                    escape = '\\';
+                    break;
+                default:
+                    break;
+            }
+            if (escape != 0)
+            {
+                w[0] = '\\';
+                w[1] = escape;
+                w += 2;
+            }
+            else
+            {
+                u_escape(c);
+            }
+        }
+        room(1);
+        *w++ = '"';
+        m_out.set_cursor(w);
     }
 
     /// the copies of dump_text() that are not fixed-size moves (long
@@ -6537,8 +6744,8 @@ class view_serializer
     the source copied by fixed-size moves of 32 bytes where the source has
     that many bytes left, instead of a library call per token. The buffer
     keeps 64 bytes of slack for the overshoot. With ensure_ascii (Ascii), a
-    string of the source is copied only if it has no byte >= 0x7F (see
-    is_ascii_without_delete()); any other string is written by write_string().
+    string of the source is copied only if it has nothing to escape (see
+    copied()); any other string is written by write_ascii().
     */
     template<bool SourceNumbers, bool Pretty, bool Ascii>
     void dump_text(const node* root)
@@ -6620,7 +6827,7 @@ class view_serializer
             m_out.set_cursor(w);
             if (enabled(Ascii))
             {
-                write_ascii_string(n);
+                write_ascii(n);
             }
             else
             {
@@ -6935,72 +7142,6 @@ class view_serializer
         m_out.put(s.data(), s.size());
     }
 
-    void write_string(const node& n)
-    {
-        const char* const s = m_doc.str(n);
-        m_out.put('"');
-        if ((n.flags & node_flags::storage) == 0 && (!m_style.ensure_ascii || is_ascii_without_delete(s, n.len)))
-        {
-            // a string of the source without escape sequences has nothing to
-            // escape (with ensure_ascii, if it is printable ASCII)
-            m_out.put(s, n.len);
-        }
-        else if (m_style.ensure_ascii)
-        {
-            write_escaped<true>(reinterpret_cast<const unsigned char*>(s), n.len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
-        }
-        else
-        {
-            write_escaped<false>(reinterpret_cast<const unsigned char*>(s), n.len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
-        }
-        m_out.put('"');
-    }
-
-    /// as serializer::dump_escaped(); strings of a document are valid UTF-8,
-    /// except in a damaged image loaded with image_check::bounds, for which
-    /// this throws what basic_json::dump() throws for the string
-    template<bool EnsureAscii>
-    void write_escaped(const unsigned char* s, std::size_t n)
-    {
-        std::size_t i = 0;
-        while (i < n)
-        {
-            std::size_t run = 0;
-            if (!EnsureAscii)
-            {
-                run = string_bulk_run(s + i, n - i);
-            }
-            else if (is_ascii_copyable(s[i]))
-            {
-                run = find_ascii_copyable_run(s + i, n - i);
-            }
-            if (run != 0)
-            {
-                m_out.put(reinterpret_cast<const char*>(s + i), run); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
-                i += run;
-                continue;
-            }
-            std::uint32_t codepoint = s[i];
-            std::size_t len = 1;
-            if (codepoint >= 0x80)
-            {
-                len = validate_one_utf8(s + i, n - i);
-                if (NLOHMANN_VIEW_UNLIKELY(len == 0))
-                {
-                    invalid_utf8(s, n);
-                    return;
-                }
-                codepoint &= 0xFFu >> (len + 1);
-                for (std::size_t k = 1; k < len; ++k)
-                {
-                    codepoint = (codepoint << 6u) | (s[i + k] & 0x3Fu);
-                }
-            }
-            write_codepoint<EnsureAscii>(codepoint, s + i, len);
-            i += len;
-        }
-    }
-
     /// throw what basic_json::dump() throws for a string that is not valid UTF-8
     NLOHMANN_VIEW_NOINLINE static void invalid_utf8(const unsigned char* s, std::size_t n)
     {