wt/json
1
0 Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-11-13 17:00:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Address the clang-tidy findings of the SIMD scan

Browse Source 
Hold the UTF-8 lookup tables in std::array, compute the length of a
sequence without nested conditionals, and use std::array in the tests.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann committed 2026-09-30 21:01:56 +02:00
1 parent c3b51addbf
commit 1f0c3be6f3
3 files changed
+79 -64
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

include/nlohmann/detail/view/simd.hpp
+35 -28
View File
@@ -9,6 +9,7 @@
#pragma once #pragma once
#include <array> // array
#include <cstddef> // size_t #include <cstddef> // size_t
#include <cstdint> // uint8_t, uint64_t #include <cstdint> // uint8_t, uint64_t
@@ -36,9 +37,9 @@
#endif #endif
#if NLOHMANN_VIEW_SSE2 && defined(JSON_VIEW_USE_SSSE3) #if NLOHMANN_VIEW_SSE2 && defined(JSON_VIEW_USE_SSSE3)
#include <tmmintrin.h> #include <tmmintrin.h>
#define NLOHMANN_VIEW_SSSE3 1 #define NLOHMANN_VIEW_SSSE3 1 // NOLINT(cppcoreguidelines-macro-to-enum,modernize-macro-to-enum)
#else #else
#define NLOHMANN_VIEW_SSSE3 0 #define NLOHMANN_VIEW_SSSE3 0 // NOLINT(cppcoreguidelines-macro-to-enum,modernize-macro-to-enum)
#endif #endif
#define NLOHMANN_VIEW_VECTOR (NLOHMANN_VIEW_NEON || NLOHMANN_VIEW_SSE2) #define NLOHMANN_VIEW_VECTOR (NLOHMANN_VIEW_NEON || NLOHMANN_VIEW_SSE2)
#define NLOHMANN_VIEW_VECTOR_UTF8 (NLOHMANN_VIEW_NEON || NLOHMANN_VIEW_SSSE3) #define NLOHMANN_VIEW_VECTOR_UTF8 (NLOHMANN_VIEW_NEON || NLOHMANN_VIEW_SSSE3)
@@ -101,37 +102,43 @@ struct utf8_lookup4
static constexpr std::uint8_t too_short = 1u << 0u, too_long = 1u << 1u, overlong_3 = 1u << 2u, too_large = 1u << 3u; static constexpr std::uint8_t too_short = 1u << 0u, too_long = 1u << 1u, overlong_3 = 1u << 2u, too_large = 1u << 3u;
static constexpr std::uint8_t surrogate = 1u << 4u, overlong_2 = 1u << 5u, too_large_1000 = 1u << 6u, overlong_4 = 1u << 6u; static constexpr std::uint8_t surrogate = 1u << 4u, overlong_2 = 1u << 5u, too_large_1000 = 1u << 6u, overlong_4 = 1u << 6u;
static constexpr std::uint8_t two_conts = 1u << 7u, carry = too_short | too_long | two_conts; static constexpr std::uint8_t two_conts = 1u << 7u, carry = too_short | too_long | two_conts;
static const std::uint8_t byte_1_high[16]; static const std::array<std::uint8_t, 16> byte_1_high;
static const std::uint8_t byte_1_low[16]; static const std::array<std::uint8_t, 16> byte_1_low;
static const std::uint8_t byte_2_high[16]; static const std::array<std::uint8_t, 16> byte_2_high;
}; };
template<typename Dummy> template<typename Dummy>
const std::uint8_t utf8_lookup4<Dummy>::byte_1_high[16] = const std::array<std::uint8_t, 16> utf8_lookup4<Dummy>::byte_1_high =
{ {
too_long, too_long, too_long, too_long, too_long, too_long, too_long, too_long, {
two_conts, two_conts, two_conts, two_conts, too_long, too_long, too_long, too_long, too_long, too_long, too_long, too_long,
too_short | overlong_2, too_short, too_short | overlong_3 | surrogate, too_short | too_large | too_large_1000 | overlong_4 two_conts, two_conts, two_conts, two_conts,
too_short | overlong_2, too_short, too_short | overlong_3 | surrogate, too_short | too_large | too_large_1000 | overlong_4
}
}; };
template<typename Dummy> template<typename Dummy>
const std::uint8_t utf8_lookup4<Dummy>::byte_1_low[16] = const std::array<std::uint8_t, 16> utf8_lookup4<Dummy>::byte_1_low =
{ {
carry | overlong_3 | overlong_2 | overlong_4, carry | overlong_2, carry, carry, {
carry | too_large, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | overlong_3 | overlong_2 | overlong_4, carry | overlong_2, carry, carry,
carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000,
carry | too_large | too_large_1000, carry | too_large | too_large_1000 | surrogate, carry | too_large | too_large_1000, carry | too_large | too_large_1000 carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000,
carry | too_large | too_large_1000, carry | too_large | too_large_1000 | surrogate, carry | too_large | too_large_1000, carry | too_large | too_large_1000
}
}; };
template<typename Dummy> template<typename Dummy>
const std::uint8_t utf8_lookup4<Dummy>::byte_2_high[16] = const std::array<std::uint8_t, 16> utf8_lookup4<Dummy>::byte_2_high =
{ {
too_short, too_short, too_short, too_short, too_short, too_short, too_short, too_short, {
static_cast<std::uint8_t>(too_long | overlong_2 | two_conts | overlong_3 | too_large_1000 | overlong_4), too_short, too_short, too_short, too_short, too_short, too_short, too_short, too_short,
static_cast<std::uint8_t>(too_long | overlong_2 | two_conts | overlong_3 | too_large), static_cast<std::uint8_t>(too_long | overlong_2 | two_conts | overlong_3 | too_large_1000 | overlong_4),
static_cast<std::uint8_t>(too_long | overlong_2 | two_conts | surrogate | too_large), static_cast<std::uint8_t>(too_long | overlong_2 | two_conts | overlong_3 | too_large),
static_cast<std::uint8_t>(too_long | overlong_2 | two_conts | surrogate | too_large), static_cast<std::uint8_t>(too_long | overlong_2 | two_conts | surrogate | too_large),
too_short, too_short, too_short, too_short static_cast<std::uint8_t>(too_long | overlong_2 | two_conts | surrogate | too_large),
too_short, too_short, too_short, too_short
}
}; };
/// the end of scan_string_vector from block, where the vector loop stopped /// the end of scan_string_vector from block, where the vector loop stopped
@@ -148,7 +155,7 @@ inline const unsigned char* scan_string_finish(const unsigned char* p, const uns
} }
if (c >= 0xC0) if (c >= 0xC0)
{ {
const int len = c >= 0xF0 ? 4 : (c >= 0xE0 ? 3 : 2); const int len = 2 + static_cast<int>(c >= 0xE0) + static_cast<int>(c >= 0xF0);
if (len > i) if (len > i)
{ {
block -= i; block -= i;
@@ -191,9 +198,9 @@ NLOHMANN_VIEW_NOINLINE inline const unsigned char* scan_string_vector(const unsi
using lookup = utf8_lookup4<>; using lookup = utf8_lookup4<>;
const unsigned char* block = p; const unsigned char* block = p;
#if NLOHMANN_VIEW_NEON #if NLOHMANN_VIEW_NEON
const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high); const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high.data());
const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low); const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low.data());
const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high); const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high.data());
uint8x16_t prev = vdupq_n_u8(0); uint8x16_t prev = vdupq_n_u8(0);
while (e - block >= 16) while (e - block >= 16)
{ {
@@ -228,9 +235,9 @@ NLOHMANN_VIEW_NOINLINE inline const unsigned char* scan_string_vector(const unsi
#else #else
// the same with SSSE3 (pshufb for the table lookups; nibbles from 16-bit // the same with SSSE3 (pshufb for the table lookups; nibbles from 16-bit
// shifts, as there are no byte shifts) // shifts, as there are no byte shifts)
const __m128i t1h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_high))); const __m128i t1h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_high.data())));
const __m128i t1l = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_low))); const __m128i t1l = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_low.data())));
const __m128i t2h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_2_high))); const __m128i t2h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_2_high.data())));
const __m128i nibble = _mm_set1_epi8(0x0F); const __m128i nibble = _mm_set1_epi8(0x0F);
const __m128i zero = _mm_setzero_si128(); const __m128i zero = _mm_setzero_si128();
__m128i prev = zero; __m128i prev = zero;
single_include/nlohmann/json_view.hpp
+35 -28
View File
@@ -407,6 +407,7 @@ NLOHMANN_JSON_NAMESPACE_END
#include <array> // array
#include <cstddef> // size_t #include <cstddef> // size_t
#include <cstdint> // uint8_t, uint64_t #include <cstdint> // uint8_t, uint64_t
@@ -435,9 +436,9 @@ NLOHMANN_JSON_NAMESPACE_END
#endif #endif
#if NLOHMANN_VIEW_SSE2 && defined(JSON_VIEW_USE_SSSE3) #if NLOHMANN_VIEW_SSE2 && defined(JSON_VIEW_USE_SSSE3)
#include <tmmintrin.h> #include <tmmintrin.h>
#define NLOHMANN_VIEW_SSSE3 1 #define NLOHMANN_VIEW_SSSE3 1 // NOLINT(cppcoreguidelines-macro-to-enum,modernize-macro-to-enum)
#else #else
#define NLOHMANN_VIEW_SSSE3 0 #define NLOHMANN_VIEW_SSSE3 0 // NOLINT(cppcoreguidelines-macro-to-enum,modernize-macro-to-enum)
#endif #endif
#define NLOHMANN_VIEW_VECTOR (NLOHMANN_VIEW_NEON || NLOHMANN_VIEW_SSE2) #define NLOHMANN_VIEW_VECTOR (NLOHMANN_VIEW_NEON || NLOHMANN_VIEW_SSE2)
#define NLOHMANN_VIEW_VECTOR_UTF8 (NLOHMANN_VIEW_NEON || NLOHMANN_VIEW_SSSE3) #define NLOHMANN_VIEW_VECTOR_UTF8 (NLOHMANN_VIEW_NEON || NLOHMANN_VIEW_SSSE3)
@@ -500,37 +501,43 @@ struct utf8_lookup4
static constexpr std::uint8_t too_short = 1u << 0u, too_long = 1u << 1u, overlong_3 = 1u << 2u, too_large = 1u << 3u; static constexpr std::uint8_t too_short = 1u << 0u, too_long = 1u << 1u, overlong_3 = 1u << 2u, too_large = 1u << 3u;
static constexpr std::uint8_t surrogate = 1u << 4u, overlong_2 = 1u << 5u, too_large_1000 = 1u << 6u, overlong_4 = 1u << 6u; static constexpr std::uint8_t surrogate = 1u << 4u, overlong_2 = 1u << 5u, too_large_1000 = 1u << 6u, overlong_4 = 1u << 6u;
static constexpr std::uint8_t two_conts = 1u << 7u, carry = too_short | too_long | two_conts; static constexpr std::uint8_t two_conts = 1u << 7u, carry = too_short | too_long | two_conts;
static const std::uint8_t byte_1_high[16]; static const std::array<std::uint8_t, 16> byte_1_high;
static const std::uint8_t byte_1_low[16]; static const std::array<std::uint8_t, 16> byte_1_low;
static const std::uint8_t byte_2_high[16]; static const std::array<std::uint8_t, 16> byte_2_high;
}; };
template<typename Dummy> template<typename Dummy>
const std::uint8_t utf8_lookup4<Dummy>::byte_1_high[16] = const std::array<std::uint8_t, 16> utf8_lookup4<Dummy>::byte_1_high =
{ {
too_long, too_long, too_long, too_long, too_long, too_long, too_long, too_long, {
two_conts, two_conts, two_conts, two_conts, too_long, too_long, too_long, too_long, too_long, too_long, too_long, too_long,
too_short | overlong_2, too_short, too_short | overlong_3 | surrogate, too_short | too_large | too_large_1000 | overlong_4 two_conts, two_conts, two_conts, two_conts,
too_short | overlong_2, too_short, too_short | overlong_3 | surrogate, too_short | too_large | too_large_1000 | overlong_4
}
}; };
template<typename Dummy> template<typename Dummy>
const std::uint8_t utf8_lookup4<Dummy>::byte_1_low[16] = const std::array<std::uint8_t, 16> utf8_lookup4<Dummy>::byte_1_low =
{ {
carry | overlong_3 | overlong_2 | overlong_4, carry | overlong_2, carry, carry, {
carry | too_large, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | overlong_3 | overlong_2 | overlong_4, carry | overlong_2, carry, carry,
carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000,
carry | too_large | too_large_1000, carry | too_large | too_large_1000 | surrogate, carry | too_large | too_large_1000, carry | too_large | too_large_1000 carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000, carry | too_large | too_large_1000,
carry | too_large | too_large_1000, carry | too_large | too_large_1000 | surrogate, carry | too_large | too_large_1000, carry | too_large | too_large_1000
}
}; };
template<typename Dummy> template<typename Dummy>
const std::uint8_t utf8_lookup4<Dummy>::byte_2_high[16] = const std::array<std::uint8_t, 16> utf8_lookup4<Dummy>::byte_2_high =
{ {
too_short, too_short, too_short, too_short, too_short, too_short, too_short, too_short, {
static_cast<std::uint8_t>(too_long | overlong_2 | two_conts | overlong_3 | too_large_1000 | overlong_4), too_short, too_short, too_short, too_short, too_short, too_short, too_short, too_short,
static_cast<std::uint8_t>(too_long | overlong_2 | two_conts | overlong_3 | too_large), static_cast<std::uint8_t>(too_long | overlong_2 | two_conts | overlong_3 | too_large_1000 | overlong_4),
static_cast<std::uint8_t>(too_long | overlong_2 | two_conts | surrogate | too_large), static_cast<std::uint8_t>(too_long | overlong_2 | two_conts | overlong_3 | too_large),
static_cast<std::uint8_t>(too_long | overlong_2 | two_conts | surrogate | too_large), static_cast<std::uint8_t>(too_long | overlong_2 | two_conts | surrogate | too_large),
too_short, too_short, too_short, too_short static_cast<std::uint8_t>(too_long | overlong_2 | two_conts | surrogate | too_large),
too_short, too_short, too_short, too_short
}
}; };
/// the end of scan_string_vector from block, where the vector loop stopped /// the end of scan_string_vector from block, where the vector loop stopped
@@ -547,7 +554,7 @@ inline const unsigned char* scan_string_finish(const unsigned char* p, const uns
} }
if (c >= 0xC0) if (c >= 0xC0)
{ {
const int len = c >= 0xF0 ? 4 : (c >= 0xE0 ? 3 : 2); const int len = 2 + static_cast<int>(c >= 0xE0) + static_cast<int>(c >= 0xF0);
if (len > i) if (len > i)
{ {
block -= i; block -= i;
@@ -590,9 +597,9 @@ NLOHMANN_VIEW_NOINLINE inline const unsigned char* scan_string_vector(const unsi
using lookup = utf8_lookup4<>; using lookup = utf8_lookup4<>;
const unsigned char* block = p; const unsigned char* block = p;
#if NLOHMANN_VIEW_NEON #if NLOHMANN_VIEW_NEON
const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high); const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high.data());
const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low); const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low.data());
const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high); const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high.data());
uint8x16_t prev = vdupq_n_u8(0); uint8x16_t prev = vdupq_n_u8(0);
while (e - block >= 16) while (e - block >= 16)
{ {
@@ -627,9 +634,9 @@ NLOHMANN_VIEW_NOINLINE inline const unsigned char* scan_string_vector(const unsi
#else #else
// the same with SSSE3 (pshufb for the table lookups; nibbles from 16-bit // the same with SSSE3 (pshufb for the table lookups; nibbles from 16-bit
// shifts, as there are no byte shifts) // shifts, as there are no byte shifts)
const __m128i t1h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_high))); const __m128i t1h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_high.data())));
const __m128i t1l = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_low))); const __m128i t1l = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_low.data())));
const __m128i t2h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_2_high))); const __m128i t2h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_2_high.data())));
const __m128i nibble = _mm_set1_epi8(0x0F); const __m128i nibble = _mm_set1_epi8(0x0F);
const __m128i zero = _mm_setzero_si128(); const __m128i zero = _mm_setzero_si128();
__m128i prev = zero; __m128i prev = zero;
tests/src/unit-json_view_builder.cpp
+9 -8
View File
@@ -17,6 +17,7 @@
#endif #endif
using nlohmann::json; using nlohmann::json;
#include <array>
#include <cstdint> #include <cstdint>
#include <fstream> #include <fstream>
#include <map> #include <map>
@@ -435,8 +436,8 @@ TEST_CASE("json_view builder: strings across vector blocks")
// at a time. Sequences are placed so that they start at every offset // at a time. Sequences are placed so that they start at every offset
// around the block boundaries of keys (16, 32) and values (8, 24), with // around the block boundaries of keys (16, 32) and values (8, 24), with
// text of several lengths after them. // text of several lengths after them.
const std::size_t prefixes[] = {0, 6, 7, 8, 13, 14, 15, 16, 21, 22, 23, 24, 29, 30, 31, 32}; const std::array<std::size_t, 16> prefixes = {{0, 6, 7, 8, 13, 14, 15, 16, 21, 22, 23, 24, 29, 30, 31, 32}};
const std::size_t suffixes[] = {0, 3, 17}; const std::array<std::size_t, 3> suffixes = {{0, 3, 17}};
const auto around = [&](const std::string & seq, std::size_t prefix, std::size_t suffix) const auto around = [&](const std::string & seq, std::size_t prefix, std::size_t suffix)
{ {
return std::string(prefix, 'a') + seq + std::string(suffix, 'b'); return std::string(prefix, 'a') + seq + std::string(suffix, 'b');
@@ -460,7 +461,7 @@ TEST_CASE("json_view builder: strings across vector blocks")
SECTION("three- and four-byte sequences") SECTION("three- and four-byte sequences")
{ {
const unsigned conts[] = {0x7F, 0x80, 0x8F, 0x90, 0x9F, 0xA0, 0xBF, 0xC0}; const std::array<unsigned, 8> conts = {{0x7F, 0x80, 0x8F, 0x90, 0x9F, 0xA0, 0xBF, 0xC0}};
for (unsigned lead = 0xE0; lead <= 0xF7; ++lead) for (unsigned lead = 0xE0; lead <= 0xF7; ++lead)
{ {
for (const unsigned b2 : conts) for (const unsigned b2 : conts)
@@ -485,21 +486,21 @@ TEST_CASE("json_view builder: strings across vector blocks")
SECTION("long runs of text with one damaged byte") SECTION("long runs of text with one damaged byte")
{ {
const char* const chars[] = {"a", "\xc3\xa9", "\xe3\x81\x82", "\xf0\x9f\x98\x80", "\xed\x9f\xbf", "\xef\xbf\xbf", "\xf4\x8f\xbf\xbf"}; const std::array<const char*, 7> chars = {{"a", "\xc3\xa9", "\xe3\x81\x82", "\xf0\x9f\x98\x80", "\xed\x9f\xbf", "\xef\xbf\xbf", "\xf4\x8f\xbf\xbf"}};
const char damage[] = {'\x80', '\xbf', '\xc0', '\xc1', '\xe0', '\xed', '\xf5', '\xff', '\x1f', '"', '\\'}; const std::array<char, 11> damage = {{'\x80', '\xbf', '\xc0', '\xc1', '\xe0', '\xed', '\xf5', '\xff', '\x1f', '"', '\\'}};
std::mt19937 rng(5295); std::mt19937 rng(5295); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible
for (int i = 0; i < 4000; ++i) for (int i = 0; i < 4000; ++i)
{ {
std::string text; std::string text;
const auto n = rng() % 60; const auto n = rng() % 60;
for (unsigned k = 0; k < n; ++k) for (unsigned k = 0; k < n; ++k)
{ {
text += chars[rng() % 7]; text += chars[rng() % chars.size()];
} }
check_string(text); check_string(text);
if (!text.empty()) if (!text.empty())
{ {
text[rng() % text.size()] = damage[rng() % 11]; text[rng() % text.size()] = damage[rng() % damage.size()];
check_string(text); check_string(text);
} }
} }