From 24de5b2f5cfa0f0bf832ec07fc8c89632aa654bd Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 17 Oct 2026 20:10:10 +0200 Subject: [PATCH] Count lines incrementally when a recovering parse reports many errors With seekable input, the lexer derives line and column on demand (#5809) by counting the newlines in all input consumed so far. A SAX parser that recovers from errors asks for the position once per error, so input with an error at every character took quadratic time: "{" followed by 100000 commas took 85 s at -O0, and the parser tests timed out under Valgrind (1500 s, was 69 s on develop). get_position() now remembers what it counted and only scans the input consumed since the previous call; count_consumed_lines() takes the start of the range. The 100000 commas now take 0.17 s, unit-class_parser 0.9 s instead of 86 s. Over 69284 error messages from random multi-line inputs, the messages are identical to those before the change. Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/input_adapters.hpp | 21 +++++--- include/nlohmann/detail/input/lexer.hpp | 31 +++++++++-- single_include/nlohmann/json.hpp | 52 ++++++++++++++----- 3 files changed, 78 insertions(+), 26 deletions(-) diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index 2df78dabb..d36ff963a 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -314,17 +314,22 @@ class iterator_input_adapter return get_consumed_count() + eof_reads; } - // number of newlines in the consumed range [0, last_index); @a line_start - // is set to the index after the last of them (0 if there is none); only - // valid when supports_seek - std::size_t count_consumed_lines(std::size_t last_index, std::size_t& line_start) const + // number of newlines in the consumed range [first_index, last_index); if + // there is one, @a line_start is set to the index after the last of them, + // otherwise it is left unchanged; only valid when supports_seek + std::size_t count_consumed_lines(std::size_t first_index, std::size_t last_index, std::size_t& line_start) const { - const auto to = std::next(begin, static_cast::difference_type>(last_index)); + using difference_type = typename std::iterator_traits::difference_type; + const auto from = std::next(begin, static_cast(first_index)); + const auto to = std::next(from, static_cast(last_index - first_index)); const auto newline = char_traits::to_char_type('\n'); using reverse_it = std::reverse_iterator; - const auto last_newline = std::find(reverse_it(to), reverse_it(begin), newline); - line_start = static_cast(std::distance(begin, last_newline.base())); - return static_cast(std::count(begin, last_newline.base(), newline)); + const auto last_newline = std::find(reverse_it(to), reverse_it(from), newline); + if (last_newline.base() != from) + { + line_start = static_cast(std::distance(begin, last_newline.base())); + } + return static_cast(std::count(from, last_newline.base(), newline)); } // append the already-consumed characters in the half-open range diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index dfceb0862..530e65bfc 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -2012,17 +2012,25 @@ scan_number_done: return position.chars_read_total; } - /// seekable adapter: count the lines in the consumed input; O(input read - /// so far), but only called on errors + /// seekable adapter: count the lines in the consumed input. Only the + /// input consumed since the previous call is scanned, so a parse that + /// reports many errors (a SAX parser recovering from them) stays linear + /// in the input instead of rescanning it for every error. position_t get_position_impl(std::true_type /*lazy*/) const { position_t result; result.chars_read_total = get_chars_read_total(); // reads at EOF count as characters but are not part of the input const std::size_t consumed = (std::min)(result.chars_read_total, ia.get_consumed_count()); - std::size_t line_start = 0; - result.lines_read = ia.count_consumed_lines(consumed, line_start); - result.chars_read_current_line = result.chars_read_total - line_start; + if (JSON_HEDLEY_UNLIKELY(consumed < position_cache.consumed)) + { + // the consumed input never shrinks; recount from the start if it did + position_cache = {}; + } + position_cache.lines += ia.count_consumed_lines(position_cache.consumed, consumed, position_cache.line_start); + position_cache.consumed = consumed; + result.lines_read = position_cache.lines; + result.chars_read_current_line = result.chars_read_total - position_cache.line_start; return result; } @@ -2898,6 +2906,19 @@ scan_number_done: /// that unget() can restore the column instead of leaving it at 0 std::size_t chars_read_before_newline = 0; + /// seekable adapter: what get_position() counted last, so that the next + /// call only scans the input consumed since + struct position_cache_t + { + /// consumed characters scanned so far + std::size_t consumed = 0; + /// newlines among them + std::size_t lines = 0; + /// index after the last of these newlines + std::size_t line_start = 0; + }; + mutable position_cache_t position_cache {}; + /// raw input token string for error messages; only populated for streaming /// adapters (seekable adapters reconstruct it lazily via token_string_start) std::vector token_string {}; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 12daf54bd..8525f94fd 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -8624,17 +8624,22 @@ class iterator_input_adapter return get_consumed_count() + eof_reads; } - // number of newlines in the consumed range [0, last_index); @a line_start - // is set to the index after the last of them (0 if there is none); only - // valid when supports_seek - std::size_t count_consumed_lines(std::size_t last_index, std::size_t& line_start) const + // number of newlines in the consumed range [first_index, last_index); if + // there is one, @a line_start is set to the index after the last of them, + // otherwise it is left unchanged; only valid when supports_seek + std::size_t count_consumed_lines(std::size_t first_index, std::size_t last_index, std::size_t& line_start) const { - const auto to = std::next(begin, static_cast::difference_type>(last_index)); + using difference_type = typename std::iterator_traits::difference_type; + const auto from = std::next(begin, static_cast(first_index)); + const auto to = std::next(from, static_cast(last_index - first_index)); const auto newline = char_traits::to_char_type('\n'); using reverse_it = std::reverse_iterator; - const auto last_newline = std::find(reverse_it(to), reverse_it(begin), newline); - line_start = static_cast(std::distance(begin, last_newline.base())); - return static_cast(std::count(begin, last_newline.base(), newline)); + const auto last_newline = std::find(reverse_it(to), reverse_it(from), newline); + if (last_newline.base() != from) + { + line_start = static_cast(std::distance(begin, last_newline.base())); + } + return static_cast(std::count(from, last_newline.base(), newline)); } // append the already-consumed characters in the half-open range @@ -13209,17 +13214,25 @@ scan_number_done: return position.chars_read_total; } - /// seekable adapter: count the lines in the consumed input; O(input read - /// so far), but only called on errors + /// seekable adapter: count the lines in the consumed input. Only the + /// input consumed since the previous call is scanned, so a parse that + /// reports many errors (a SAX parser recovering from them) stays linear + /// in the input instead of rescanning it for every error. position_t get_position_impl(std::true_type /*lazy*/) const { position_t result; result.chars_read_total = get_chars_read_total(); // reads at EOF count as characters but are not part of the input const std::size_t consumed = (std::min)(result.chars_read_total, ia.get_consumed_count()); - std::size_t line_start = 0; - result.lines_read = ia.count_consumed_lines(consumed, line_start); - result.chars_read_current_line = result.chars_read_total - line_start; + if (JSON_HEDLEY_UNLIKELY(consumed < position_cache.consumed)) + { + // the consumed input never shrinks; recount from the start if it did + position_cache = {}; + } + position_cache.lines += ia.count_consumed_lines(position_cache.consumed, consumed, position_cache.line_start); + position_cache.consumed = consumed; + result.lines_read = position_cache.lines; + result.chars_read_current_line = result.chars_read_total - position_cache.line_start; return result; } @@ -14095,6 +14108,19 @@ scan_number_done: /// that unget() can restore the column instead of leaving it at 0 std::size_t chars_read_before_newline = 0; + /// seekable adapter: what get_position() counted last, so that the next + /// call only scans the input consumed since + struct position_cache_t + { + /// consumed characters scanned so far + std::size_t consumed = 0; + /// newlines among them + std::size_t lines = 0; + /// index after the last of these newlines + std::size_t line_start = 0; + }; + mutable position_cache_t position_cache {}; + /// raw input token string for error messages; only populated for streaming /// adapters (seekable adapters reconstruct it lazily via token_string_start) std::vector token_string {};