diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index fde164dd9..ae159a8bb 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -418,30 +418,43 @@ class lexer : public lexer_base } /// contiguous input: bulk-append the run of ordinary characters and complete - /// well-formed UTF-8 sequences starting at the current read position, leaving - /// the first byte that needs individual handling (the closing quote, an - /// escape, a control character, or an ill-formed UTF-8 byte) for get() - void scan_string_bulk(std::true_type /*bulk*/) + /// well-formed UTF-8 sequences starting at the current read position, then + /// read the first byte that needs individual handling (the closing quote, an + /// escape, a control character, or an ill-formed UTF-8 byte) like get(). + /// That byte is taken from the buffer that was just scanned. Through get(), + /// Apple Clang merged this read with the other get() calls of the scan + /// function, which load the read position again right after it was stored + /// (3% of json::accept() on jeopardy.json on Apple M1). + char_int_type get_after_string_run(std::true_type /*bulk*/) { // a pending unget must be consumed through the normal path first if (next_unget) { - return; + return get(); } const std::size_t remaining = ia.bulk_remaining(); if (remaining == 0) { - return; + return get(); } - const auto* const data = reinterpret_cast(ia.bulk_data()); + const auto* const raw = ia.bulk_data(); + const auto* const data = reinterpret_cast(raw); - const std::size_t pos = string_bulk_run(data, remaining); - if (pos == 0) + const std::size_t pos = lexer_string_bulk_run(data, remaining); + if (pos != 0) { - return; + token_buffer.append(reinterpret_cast(data), pos); + bulk_consume(reinterpret_cast(data), pos); } - token_buffer.append(reinterpret_cast(data), pos); - bulk_consume(reinterpret_cast(data), pos); + if (pos == remaining) + { + return get(); // end of the contiguous input + } + // what get() does, with the byte at hand + advance_position(); + ia.bulk_skip(1); + current = char_traits::to_int_type(raw[pos]); + return track_after_read(); } /// consume @a n characters at @a data (the adapter's bulk_data()) and do @@ -474,7 +487,10 @@ class lexer : public lexer_base } /// streaming input: no bulk fast path - void scan_string_bulk(std::false_type /*bulk*/) const noexcept {} + char_int_type get_after_string_run(std::false_type /*bulk*/) + { + return get(); + } /*! @brief scan a string literal @@ -502,11 +518,8 @@ class lexer : public lexer_base while (true) { // bulk-consume ordinary characters from contiguous input, then - // handle the next special byte through the switch below - scan_string_bulk(std::integral_constant {}); - - // get the next character - switch (get()) + // get the next character, the one that ends the run + switch (get_after_string_run(std::integral_constant())) { // end of file while parsing the string case char_traits::eof(): diff --git a/include/nlohmann/detail/input/string_scan.hpp b/include/nlohmann/detail/input/string_scan.hpp index d41005fcd..11cf405d0 100644 --- a/include/nlohmann/detail/input/string_scan.hpp +++ b/include/nlohmann/detail/input/string_scan.hpp @@ -35,7 +35,7 @@ // This file contains the byte-level string-scanning helpers used by the lexer's // contiguous fast path. They operate purely on raw bytes (no dependency on the // lexer's template parameters) so they are free functions, keeping the lexer -// itself focused on the state machine; see lexer::scan_string_bulk(). +// itself focused on the state machine; see lexer::get_after_string_run(). NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -311,12 +311,19 @@ JSON_HEDLEY_ALWAYS_INLINE const unsigned char* plain_run_end(const unsigned char } p += 8; } - p = vector_plain_run_long(p, e); - if (e - p < 16) // else the vector loop stopped at a special byte + // the vector loop only for 16 bytes or more: the serializer passes a + // string without its closing quote, so most of its runs end at e, and a + // call of the out-of-line NEON loop for a shorter rest made dump() up to + // 10% slower + if (e - p >= 16) { - p += find_string_special(p, static_cast(e - p)); + p = vector_plain_run_long(p, e); + if (e - p >= 16) + { + return p; // the vector loop stopped at a special byte + } } - return p; + return p + find_string_special(p, static_cast(e - p)); } #endif @@ -418,6 +425,42 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t string_bulk_run(const unsigned char* data, return scalar_string_bulk_run(data, n); } +#if JSON_SCAN_NEON +/// scalar_string_bulk_run() from @a start on, out of line (see +/// lexer_string_bulk_run()) +JSON_HEDLEY_NEVER_INLINE inline std::size_t string_bulk_run_from(const unsigned char* data, std::size_t start, std::size_t n) noexcept +{ + return start + scalar_string_bulk_run(data + start, n - start); +} +#endif + +// string_bulk_run() for the lexer. With NEON, only the first four words are +// checked inline (most keys and short values end there), and longer runs and +// non-ASCII text continue out of line. The lexer's scan function runs once per +// token, and the vector code inlined into it needs more registers, which every +// token pays for: 5% of json::accept() on jeopardy.json with Apple Clang on +// Apple M1, even though no string of that file reaches the vector code. The +// serializer keeps string_bulk_run() inline, which is faster there. +JSON_HEDLEY_ALWAYS_INLINE std::size_t lexer_string_bulk_run(const unsigned char* data, std::size_t n) noexcept +{ +#if JSON_SCAN_NEON && !(defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17)) + std::size_t i = 0; + for (int words = 4; words > 0 && n - i >= 8; --words) + { + const std::uint64_t special = swar_string_special(read_eight_bytes(data + i)); + if (special != 0) + { + const std::size_t pos = i + (static_cast(count_trailing_zeros(special)) / 8); + return data[pos] < 0x80u ? pos : string_bulk_run_from(data, pos, n); + } + i += 8; + } + return string_bulk_run_from(data, i, n); +#else + return string_bulk_run(data, n); +#endif +} + // Decode the 4 hex digits at [data, data+4) - the digits following a `\u` // escape - into a codepoint 0x0000..0xFFFF via one table lookup per byte // (after yyjson's read_hex_u16), or return -1 if any of the 4 bytes is not a diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 1b1193c8c..27eae4ba7 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -11494,7 +11494,7 @@ NLOHMANN_JSON_NAMESPACE_END // This file contains the byte-level string-scanning helpers used by the lexer's // contiguous fast path. They operate purely on raw bytes (no dependency on the // lexer's template parameters) so they are free functions, keeping the lexer -// itself focused on the state machine; see lexer::scan_string_bulk(). +// itself focused on the state machine; see lexer::get_after_string_run(). NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -11770,12 +11770,19 @@ JSON_HEDLEY_ALWAYS_INLINE const unsigned char* plain_run_end(const unsigned char } p += 8; } - p = vector_plain_run_long(p, e); - if (e - p < 16) // else the vector loop stopped at a special byte + // the vector loop only for 16 bytes or more: the serializer passes a + // string without its closing quote, so most of its runs end at e, and a + // call of the out-of-line NEON loop for a shorter rest made dump() up to + // 10% slower + if (e - p >= 16) { - p += find_string_special(p, static_cast(e - p)); + p = vector_plain_run_long(p, e); + if (e - p >= 16) + { + return p; // the vector loop stopped at a special byte + } } - return p; + return p + find_string_special(p, static_cast(e - p)); } #endif @@ -11877,6 +11884,42 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t string_bulk_run(const unsigned char* data, return scalar_string_bulk_run(data, n); } +#if JSON_SCAN_NEON +/// scalar_string_bulk_run() from @a start on, out of line (see +/// lexer_string_bulk_run()) +JSON_HEDLEY_NEVER_INLINE inline std::size_t string_bulk_run_from(const unsigned char* data, std::size_t start, std::size_t n) noexcept +{ + return start + scalar_string_bulk_run(data + start, n - start); +} +#endif + +// string_bulk_run() for the lexer. With NEON, only the first four words are +// checked inline (most keys and short values end there), and longer runs and +// non-ASCII text continue out of line. The lexer's scan function runs once per +// token, and the vector code inlined into it needs more registers, which every +// token pays for: 5% of json::accept() on jeopardy.json with Apple Clang on +// Apple M1, even though no string of that file reaches the vector code. The +// serializer keeps string_bulk_run() inline, which is faster there. +JSON_HEDLEY_ALWAYS_INLINE std::size_t lexer_string_bulk_run(const unsigned char* data, std::size_t n) noexcept +{ +#if JSON_SCAN_NEON && !(defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17)) + std::size_t i = 0; + for (int words = 4; words > 0 && n - i >= 8; --words) + { + const std::uint64_t special = swar_string_special(read_eight_bytes(data + i)); + if (special != 0) + { + const std::size_t pos = i + (static_cast(count_trailing_zeros(special)) / 8); + return data[pos] < 0x80u ? pos : string_bulk_run_from(data, pos, n); + } + i += 8; + } + return string_bulk_run_from(data, i, n); +#else + return string_bulk_run(data, n); +#endif +} + // Decode the 4 hex digits at [data, data+4) - the digits following a `\u` // escape - into a codepoint 0x0000..0xFFFF via one table lookup per byte // (after yyjson's read_hex_u16), or return -1 if any of the 4 bytes is not a @@ -12323,30 +12366,43 @@ class lexer : public lexer_base } /// contiguous input: bulk-append the run of ordinary characters and complete - /// well-formed UTF-8 sequences starting at the current read position, leaving - /// the first byte that needs individual handling (the closing quote, an - /// escape, a control character, or an ill-formed UTF-8 byte) for get() - void scan_string_bulk(std::true_type /*bulk*/) + /// well-formed UTF-8 sequences starting at the current read position, then + /// read the first byte that needs individual handling (the closing quote, an + /// escape, a control character, or an ill-formed UTF-8 byte) like get(). + /// That byte is taken from the buffer that was just scanned. Through get(), + /// Apple Clang merged this read with the other get() calls of the scan + /// function, which load the read position again right after it was stored + /// (3% of json::accept() on jeopardy.json on Apple M1). + char_int_type get_after_string_run(std::true_type /*bulk*/) { // a pending unget must be consumed through the normal path first if (next_unget) { - return; + return get(); } const std::size_t remaining = ia.bulk_remaining(); if (remaining == 0) { - return; + return get(); } - const auto* const data = reinterpret_cast(ia.bulk_data()); + const auto* const raw = ia.bulk_data(); + const auto* const data = reinterpret_cast(raw); - const std::size_t pos = string_bulk_run(data, remaining); - if (pos == 0) + const std::size_t pos = lexer_string_bulk_run(data, remaining); + if (pos != 0) { - return; + token_buffer.append(reinterpret_cast(data), pos); + bulk_consume(reinterpret_cast(data), pos); } - token_buffer.append(reinterpret_cast(data), pos); - bulk_consume(reinterpret_cast(data), pos); + if (pos == remaining) + { + return get(); // end of the contiguous input + } + // what get() does, with the byte at hand + advance_position(); + ia.bulk_skip(1); + current = char_traits::to_int_type(raw[pos]); + return track_after_read(); } /// consume @a n characters at @a data (the adapter's bulk_data()) and do @@ -12379,7 +12435,10 @@ class lexer : public lexer_base } /// streaming input: no bulk fast path - void scan_string_bulk(std::false_type /*bulk*/) const noexcept {} + char_int_type get_after_string_run(std::false_type /*bulk*/) + { + return get(); + } /*! @brief scan a string literal @@ -12407,11 +12466,8 @@ class lexer : public lexer_base while (true) { // bulk-consume ordinary characters from contiguous input, then - // handle the next special byte through the switch below - scan_string_bulk(std::integral_constant {}); - - // get the next character - switch (get()) + // get the next character, the one that ends the run + switch (get_after_string_run(std::integral_constant())) { // end of file while parsing the string case char_traits::eof():