wt/json
1
0
Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-10-13 10:14:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Skip the indentation after a newline 8 bytes at a time

Browse Source 
Pretty-printed JSON indents every line with a run of spaces, which
skip_whitespace() read one character at a time. For contiguous input
(bulk_scan) the lexer now skips the spaces after a newline 8 bytes at a
time directly in the input buffer, including the leading spaces of the
last, partial word. This bypasses get(), which is safe because
bulk_scan implies the lazy position from the previous commit: nothing
is counted per character.

Compared to lazy positions alone, pretty-printed input (twitter,
citm_catalog with indent 2 and 4) parses up to 13% faster and accept()
gets up to 30% faster (GCC 13 and clang 18 on x86-64, Apple clang 21 on
arm64). Minified input is unchanged within noise.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann
2026-10-13 11:24:24 +02:00
parent 1d4dfc3af0
commit da4ce8b8a3
6 changed files with 227 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/mkdocs/docs/home/architecture.md
+1 -1
View File
@@ -189,7 +189,7 @@ random-access input of single-byte characters:
`count_consumed_lines()`, the lexer also derives the read position from the adapter and counts lines only when an
error needs them, instead of counting characters and lines on every read.
- `supports_bulk_scan`, `bulk_data()`, `bulk_remaining()`, and `bulk_skip()` let the lexer scan strings directly in
contiguous memory, several bytes at a time.
contiguous memory, several bytes at a time, and skip the spaces that indent a line in 8-byte steps.
The function `input_adapter` picks the right adapter for the argument passed to `parse`, `accept`, `sax_parse`, or the
`from_*` functions:
include/nlohmann/detail/bit_ops.hpp
+19
View File
@@ -42,6 +42,25 @@ inline int count_leading_zeros(std::uint64_t x) noexcept
#endif
}
/// number of trailing zero bits of x (x != 0)
inline int count_trailing_zeros(std::uint64_t x) noexcept
{
#if defined(__GNUC__) || defined(__clang__)
return __builtin_ctzll(x);
#else
int n = 0;
for (int shift = 32; shift != 0; shift >>= 1)
{
if ((x << (64 - shift)) == 0)
{
n += shift;
x >>= shift;
}
}
return n;
#endif
}
/// the 128-bit product of two 64-bit numbers
struct uint128_parts
{
include/nlohmann/detail/input/lexer.hpp
+43
View File
@@ -17,6 +17,7 @@
#include <utility> // move
#include <vector> // vector
#include <nlohmann/detail/bit_ops.hpp>
#include <nlohmann/detail/input/input_adapters.hpp>
#include <nlohmann/detail/input/number_parse.hpp>
#include <nlohmann/detail/input/position_t.hpp>
@@ -2079,11 +2080,53 @@ scan_number_done:
// into one that reloads the pointer from memory every character
do
{
if (current == '\n')
{
skip_indentation(std::integral_constant<bool, bulk_scan> {});
}
get_ignoring_pending_unget();
}
while (current_is_whitespace());
}
/*!
@brief contiguous input: skip the space indentation that follows a newline
8 bytes at a time
Pretty-printed JSON indents every line with a run of spaces, which the loop
in skip_whitespace() would otherwise read one character at a time. The
spaces are skipped 8 bytes at a time; the last 7 bytes of the input and any
other whitespace are left to that loop. Skipping bypasses get(), which is
fine because bulk_scan implies lazy_position: the position is derived from
the adapter, not counted per character.
*/
void skip_indentation(std::true_type /*bulk*/)
{
JSON_ASSERT(!next_unget);
const std::size_t remaining = ia.bulk_remaining();
if (remaining < 8)
{
return;
}
const char* const data = reinterpret_cast<const char*>(ia.bulk_data());
std::size_t pos = 0;
for (; pos + 8 <= remaining; pos += 8)
{
// little-endian, so the first byte is the lowest
const std::uint64_t non_space = read_eight_bytes(data + pos) ^ 0x2020202020202020ull;
if (non_space != 0)
{
// skip the spaces before the first other byte of this word
pos += static_cast<std::size_t>(count_trailing_zeros(non_space)) / 8;
break;
}
}
ia.bulk_skip(pos);
}
/// streaming input: no bulk fast path
void skip_indentation(std::false_type /*bulk*/) const noexcept {}
token_type scan()
{
// initially, skip the BOM
single_include/nlohmann/json.hpp
Generated
+87 -24
View File
@@ -8925,30 +8925,6 @@ NLOHMANN_JSON_NAMESPACE_END
#include <utility> // move
#include <vector> // vector
// #include <nlohmann/detail/input/input_adapters.hpp>
// #include <nlohmann/detail/input/number_parse.hpp>
// __ _____ _____ _____
// __| | __| | | | JSON for Modern C++
// | | |__ | | | | | | version 3.12.0
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
//
// SPDX-FileCopyrightText: 2021 The fast_float authors <https://github.com/fastfloat/fast_float>
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
// SPDX-License-Identifier: MIT
#include <array> // array
#include <cfloat> // FLT_EVAL_METHOD
#include <clocale> // localeconv
#include <cstddef> // size_t
#include <cstdint> // int64_t, uint64_t
#include <cstdlib> // strtof, strtod, strtold
#include <cstring> // memcpy
#include <limits> // numeric_limits
#include <string> // string
// #include <nlohmann/detail/bit_ops.hpp>
// __ _____ _____ _____
// __| | __| | | | JSON for Modern C++
@@ -8995,6 +8971,25 @@ inline int count_leading_zeros(std::uint64_t x) noexcept
#endif
}
/// number of trailing zero bits of x (x != 0)
inline int count_trailing_zeros(std::uint64_t x) noexcept
{
#if defined(__GNUC__) || defined(__clang__)
return __builtin_ctzll(x);
#else
int n = 0;
for (int shift = 32; shift != 0; shift >>= 1)
{
if ((x << (64 - shift)) == 0)
{
n += shift;
x >>= shift;
}
}
return n;
#endif
}
/// the 128-bit product of two 64-bit numbers
struct uint128_parts
{
@@ -9042,6 +9037,32 @@ inline std::uint64_t read_eight_bytes(const char* p) noexcept
} // namespace detail
NLOHMANN_JSON_NAMESPACE_END
// #include <nlohmann/detail/input/input_adapters.hpp>
// #include <nlohmann/detail/input/number_parse.hpp>
// __ _____ _____ _____
// __| | __| | | | JSON for Modern C++
// | | |__ | | | | | | version 3.12.0
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
//
// SPDX-FileCopyrightText: 2021 The fast_float authors <https://github.com/fastfloat/fast_float>
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
// SPDX-License-Identifier: MIT
#include <array> // array
#include <cfloat> // FLT_EVAL_METHOD
#include <clocale> // localeconv
#include <cstddef> // size_t
#include <cstdint> // int64_t, uint64_t
#include <cstdlib> // strtof, strtod, strtold
#include <cstring> // memcpy
#include <limits> // numeric_limits
#include <string> // string
// #include <nlohmann/detail/bit_ops.hpp>
// #include <nlohmann/detail/input/pow5_table.hpp>
// __ _____ _____ _____
// __| | __| | | | JSON for Modern C++
@@ -12500,11 +12521,53 @@ scan_number_done:
// into one that reloads the pointer from memory every character
do
{
if (current == '\n')
{
skip_indentation(std::integral_constant<bool, bulk_scan> {});
}
get_ignoring_pending_unget();
}
while (current_is_whitespace());
}
/*!
@brief contiguous input: skip the space indentation that follows a newline
8 bytes at a time
Pretty-printed JSON indents every line with a run of spaces, which the loop
in skip_whitespace() would otherwise read one character at a time. The
spaces are skipped 8 bytes at a time; the last 7 bytes of the input and any
other whitespace are left to that loop. Skipping bypasses get(), which is
fine because bulk_scan implies lazy_position: the position is derived from
the adapter, not counted per character.
*/
void skip_indentation(std::true_type /*bulk*/)
{
JSON_ASSERT(!next_unget);
const std::size_t remaining = ia.bulk_remaining();
if (remaining < 8)
{
return;
}
const char* const data = reinterpret_cast<const char*>(ia.bulk_data());
std::size_t pos = 0;
for (; pos + 8 <= remaining; pos += 8)
{
// little-endian, so the first byte is the lowest
const std::uint64_t non_space = read_eight_bytes(data + pos) ^ 0x2020202020202020ull;
if (non_space != 0)
{
// skip the spaces before the first other byte of this word
pos += static_cast<std::size_t>(count_trailing_zeros(non_space)) / 8;
break;
}
}
ia.bulk_skip(pos);
}
/// streaming input: no bulk fast path
void skip_indentation(std::false_type /*bulk*/) const noexcept {}
token_type scan()
{
// initially, skip the BOM
tests/src/unit-class_lexer.cpp
+4
View File
@@ -938,6 +938,10 @@ TEST_CASE("Eisel-Lemire float conversion")
const int k = i % 64;
const std::uint64_t x = (std::uint64_t{1} << k) | (a & ((std::uint64_t{1} << k) - 1));
CHECK(nlohmann::detail::count_leading_zeros(x) == 63 - k);
// the mirror image: bit k set, arbitrary bits above it
const std::uint64_t y = (std::uint64_t{1} << k) | (k == 63 ? 0 : (a << (k + 1)));
CHECK(nlohmann::detail::count_trailing_zeros(y) == k);
}
}
tests/src/unit-deserialization.cpp
+73
View File
@@ -1314,3 +1314,76 @@ TEST_CASE("parse error positions are the same for contiguous and stream input")
CHECK(stream_sax.error == expected);
}
}
#if !defined(JSON_NOEXCEPTION)
TEST_CASE("indentation runs of every length around the 8-byte skip")
{
// contiguous input skips the spaces after a newline 8 bytes at a time (see
// lexer::skip_indentation), stream input reads them one by one; results,
// byte offsets, and messages must be the same for runs that end anywhere
// relative to a word boundary, including at the end of the input
const auto parse_result = [](const std::string & input, bool contiguous)
{
try
{
if (contiguous)
{
return json::parse(input.data(), input.data() + input.size(), nullptr, true, true).dump();
}
std::istringstream is(input);
return json::parse(is, nullptr, true, true).dump();
}
catch (const json::parse_error& e)
{
return std::to_string(e.byte).append(" ").append(e.what());
}
};
// replace every '~' in a pattern by an indentation run
const auto indented = [](const std::string & pattern, const std::string & indent)
{
std::string result;
for (const char c : pattern)
{
if (c == '~')
{
result += indent;
}
else
{
result += c;
}
}
return result;
};
const std::vector<std::string> patterns =
{
"[\n~1,\n~2\n]",
"[\n~1,\n~x]",
"[\n~1,\n~",
"1\n~",
"[\n~\t1,\n~\r\n~2]",
"[\n~/* c */ 1]",
"[\n~1\n~2]",
};
for (std::size_t n = 0; n <= 33; ++n)
{
const std::string indent(n, ' ');
for (const auto& pattern : patterns)
{
const std::string input = indented(pattern, indent);
CAPTURE(n)
CAPTURE(input)
const std::string expected = parse_result(input, false);
CHECK(parse_result(input, true) == expected);
CHECK(json::accept(input.data(), input.data() + input.size(), true) == json::accept(input, true));
}
}
// and against fixed expectations, in case both paths broke the same way
CHECK(parse_result(indented("[\n~x]", std::string(16, ' ')), true) == "19 [json.exception.parse_error.101] parse error at line 2, column 17: syntax error while parsing value - invalid literal; last read: '[<U+000A> x'");
CHECK(parse_result(indented("[\n~1]", std::string(17, ' ')), true) == "[1]");
}
#endif