From da4ce8b8a3ee5d48583774df07bc5951a7a89804 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 13 Oct 2026 09:09:43 +0200 Subject: [PATCH] Skip the indentation after a newline 8 bytes at a time Pretty-printed JSON indents every line with a run of spaces, which skip_whitespace() read one character at a time. For contiguous input (bulk_scan) the lexer now skips the spaces after a newline 8 bytes at a time directly in the input buffer, including the leading spaces of the last, partial word. This bypasses get(), which is safe because bulk_scan implies the lazy position from the previous commit: nothing is counted per character. Compared to lazy positions alone, pretty-printed input (twitter, citm_catalog with indent 2 and 4) parses up to 13% faster and accept() gets up to 30% faster (GCC 13 and clang 18 on x86-64, Apple clang 21 on arm64). Minified input is unchanged within noise. Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/home/architecture.md | 2 +- include/nlohmann/detail/bit_ops.hpp | 19 ++++ include/nlohmann/detail/input/lexer.hpp | 43 +++++++++ single_include/nlohmann/json.hpp | 111 +++++++++++++++++++----- tests/src/unit-class_lexer.cpp | 4 + tests/src/unit-deserialization.cpp | 73 ++++++++++++++++ 6 files changed, 227 insertions(+), 25 deletions(-) diff --git a/docs/mkdocs/docs/home/architecture.md b/docs/mkdocs/docs/home/architecture.md index 41466f695..4975c1d1b 100644 --- a/docs/mkdocs/docs/home/architecture.md +++ b/docs/mkdocs/docs/home/architecture.md @@ -189,7 +189,7 @@ random-access input of single-byte characters: `count_consumed_lines()`, the lexer also derives the read position from the adapter and counts lines only when an error needs them, instead of counting characters and lines on every read. - `supports_bulk_scan`, `bulk_data()`, `bulk_remaining()`, and `bulk_skip()` let the lexer scan strings directly in - contiguous memory, several bytes at a time. + contiguous memory, several bytes at a time, and skip the spaces that indent a line in 8-byte steps. The function `input_adapter` picks the right adapter for the argument passed to `parse`, `accept`, `sax_parse`, or the `from_*` functions: diff --git a/include/nlohmann/detail/bit_ops.hpp b/include/nlohmann/detail/bit_ops.hpp index 9ccb82c66..8752dd74a 100644 --- a/include/nlohmann/detail/bit_ops.hpp +++ b/include/nlohmann/detail/bit_ops.hpp @@ -42,6 +42,25 @@ inline int count_leading_zeros(std::uint64_t x) noexcept #endif } +/// number of trailing zero bits of x (x != 0) +inline int count_trailing_zeros(std::uint64_t x) noexcept +{ +#if defined(__GNUC__) || defined(__clang__) + return __builtin_ctzll(x); +#else + int n = 0; + for (int shift = 32; shift != 0; shift >>= 1) + { + if ((x << (64 - shift)) == 0) + { + n += shift; + x >>= shift; + } + } + return n; +#endif +} + /// the 128-bit product of two 64-bit numbers struct uint128_parts { diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index af5907d78..55f09c3cc 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -17,6 +17,7 @@ #include // move #include // vector +#include #include #include #include @@ -2079,11 +2080,53 @@ scan_number_done: // into one that reloads the pointer from memory every character do { + if (current == '\n') + { + skip_indentation(std::integral_constant {}); + } get_ignoring_pending_unget(); } while (current_is_whitespace()); } + /*! + @brief contiguous input: skip the space indentation that follows a newline + 8 bytes at a time + + Pretty-printed JSON indents every line with a run of spaces, which the loop + in skip_whitespace() would otherwise read one character at a time. The + spaces are skipped 8 bytes at a time; the last 7 bytes of the input and any + other whitespace are left to that loop. Skipping bypasses get(), which is + fine because bulk_scan implies lazy_position: the position is derived from + the adapter, not counted per character. + */ + void skip_indentation(std::true_type /*bulk*/) + { + JSON_ASSERT(!next_unget); + const std::size_t remaining = ia.bulk_remaining(); + if (remaining < 8) + { + return; + } + const char* const data = reinterpret_cast(ia.bulk_data()); + std::size_t pos = 0; + for (; pos + 8 <= remaining; pos += 8) + { + // little-endian, so the first byte is the lowest + const std::uint64_t non_space = read_eight_bytes(data + pos) ^ 0x2020202020202020ull; + if (non_space != 0) + { + // skip the spaces before the first other byte of this word + pos += static_cast(count_trailing_zeros(non_space)) / 8; + break; + } + } + ia.bulk_skip(pos); + } + + /// streaming input: no bulk fast path + void skip_indentation(std::false_type /*bulk*/) const noexcept {} + token_type scan() { // initially, skip the BOM diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index b62b5569b..c87dfb578 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -8925,30 +8925,6 @@ NLOHMANN_JSON_NAMESPACE_END #include // move #include // vector -// #include - -// #include -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2021 The fast_float authors -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - - - -#include // array -#include // FLT_EVAL_METHOD -#include // localeconv -#include // size_t -#include // int64_t, uint64_t -#include // strtof, strtod, strtold -#include // memcpy -#include // numeric_limits -#include // string - // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -8995,6 +8971,25 @@ inline int count_leading_zeros(std::uint64_t x) noexcept #endif } +/// number of trailing zero bits of x (x != 0) +inline int count_trailing_zeros(std::uint64_t x) noexcept +{ +#if defined(__GNUC__) || defined(__clang__) + return __builtin_ctzll(x); +#else + int n = 0; + for (int shift = 32; shift != 0; shift >>= 1) + { + if ((x << (64 - shift)) == 0) + { + n += shift; + x >>= shift; + } + } + return n; +#endif +} + /// the 128-bit product of two 64-bit numbers struct uint128_parts { @@ -9042,6 +9037,32 @@ inline std::uint64_t read_eight_bytes(const char* p) noexcept } // namespace detail NLOHMANN_JSON_NAMESPACE_END +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2021 The fast_float authors +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // array +#include // FLT_EVAL_METHOD +#include // localeconv +#include // size_t +#include // int64_t, uint64_t +#include // strtof, strtod, strtold +#include // memcpy +#include // numeric_limits +#include // string + +// #include + // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -12500,11 +12521,53 @@ scan_number_done: // into one that reloads the pointer from memory every character do { + if (current == '\n') + { + skip_indentation(std::integral_constant {}); + } get_ignoring_pending_unget(); } while (current_is_whitespace()); } + /*! + @brief contiguous input: skip the space indentation that follows a newline + 8 bytes at a time + + Pretty-printed JSON indents every line with a run of spaces, which the loop + in skip_whitespace() would otherwise read one character at a time. The + spaces are skipped 8 bytes at a time; the last 7 bytes of the input and any + other whitespace are left to that loop. Skipping bypasses get(), which is + fine because bulk_scan implies lazy_position: the position is derived from + the adapter, not counted per character. + */ + void skip_indentation(std::true_type /*bulk*/) + { + JSON_ASSERT(!next_unget); + const std::size_t remaining = ia.bulk_remaining(); + if (remaining < 8) + { + return; + } + const char* const data = reinterpret_cast(ia.bulk_data()); + std::size_t pos = 0; + for (; pos + 8 <= remaining; pos += 8) + { + // little-endian, so the first byte is the lowest + const std::uint64_t non_space = read_eight_bytes(data + pos) ^ 0x2020202020202020ull; + if (non_space != 0) + { + // skip the spaces before the first other byte of this word + pos += static_cast(count_trailing_zeros(non_space)) / 8; + break; + } + } + ia.bulk_skip(pos); + } + + /// streaming input: no bulk fast path + void skip_indentation(std::false_type /*bulk*/) const noexcept {} + token_type scan() { // initially, skip the BOM diff --git a/tests/src/unit-class_lexer.cpp b/tests/src/unit-class_lexer.cpp index 860af81d6..34f3a04f0 100644 --- a/tests/src/unit-class_lexer.cpp +++ b/tests/src/unit-class_lexer.cpp @@ -938,6 +938,10 @@ TEST_CASE("Eisel-Lemire float conversion") const int k = i % 64; const std::uint64_t x = (std::uint64_t{1} << k) | (a & ((std::uint64_t{1} << k) - 1)); CHECK(nlohmann::detail::count_leading_zeros(x) == 63 - k); + + // the mirror image: bit k set, arbitrary bits above it + const std::uint64_t y = (std::uint64_t{1} << k) | (k == 63 ? 0 : (a << (k + 1))); + CHECK(nlohmann::detail::count_trailing_zeros(y) == k); } } diff --git a/tests/src/unit-deserialization.cpp b/tests/src/unit-deserialization.cpp index 4aa3a1b1e..e82685311 100644 --- a/tests/src/unit-deserialization.cpp +++ b/tests/src/unit-deserialization.cpp @@ -1314,3 +1314,76 @@ TEST_CASE("parse error positions are the same for contiguous and stream input") CHECK(stream_sax.error == expected); } } + +#if !defined(JSON_NOEXCEPTION) +TEST_CASE("indentation runs of every length around the 8-byte skip") +{ + // contiguous input skips the spaces after a newline 8 bytes at a time (see + // lexer::skip_indentation), stream input reads them one by one; results, + // byte offsets, and messages must be the same for runs that end anywhere + // relative to a word boundary, including at the end of the input + const auto parse_result = [](const std::string & input, bool contiguous) + { + try + { + if (contiguous) + { + return json::parse(input.data(), input.data() + input.size(), nullptr, true, true).dump(); + } + std::istringstream is(input); + return json::parse(is, nullptr, true, true).dump(); + } + catch (const json::parse_error& e) + { + return std::to_string(e.byte).append(" ").append(e.what()); + } + }; + + // replace every '~' in a pattern by an indentation run + const auto indented = [](const std::string & pattern, const std::string & indent) + { + std::string result; + for (const char c : pattern) + { + if (c == '~') + { + result += indent; + } + else + { + result += c; + } + } + return result; + }; + + const std::vector patterns = + { + "[\n~1,\n~2\n]", + "[\n~1,\n~x]", + "[\n~1,\n~", + "1\n~", + "[\n~\t1,\n~\r\n~2]", + "[\n~/* c */ 1]", + "[\n~1\n~2]", + }; + + for (std::size_t n = 0; n <= 33; ++n) + { + const std::string indent(n, ' '); + for (const auto& pattern : patterns) + { + const std::string input = indented(pattern, indent); + CAPTURE(n) + CAPTURE(input) + const std::string expected = parse_result(input, false); + CHECK(parse_result(input, true) == expected); + CHECK(json::accept(input.data(), input.data() + input.size(), true) == json::accept(input, true)); + } + } + + // and against fixed expectations, in case both paths broke the same way + CHECK(parse_result(indented("[\n~x]", std::string(16, ' ')), true) == "19 [json.exception.parse_error.101] parse error at line 2, column 17: syntax error while parsing value - invalid literal; last read: '[ x'"); + CHECK(parse_result(indented("[\n~1]", std::string(17, ' ')), true) == "[1]"); +} +#endif