From 3dbed2a0cbf0b15751a8e9cb2d3796323e07bc67 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 13 Oct 2026 08:52:49 +0200 Subject: [PATCH] Derive the lexer position lazily for seekable input The lexer counted the read position on every character: two increments and a newline check per get(), all on members kept in memory. For seekable input (pointers, strings, other random-access byte ranges) the position can be derived instead: chars_read_total is the adapter's read count (including reads at EOF) minus a pending unget, and line and column are recomputed from the consumed input only when get_position() is called, i.e. on parse errors. JSON_DIAGNOSTIC_POSITIONS uses the new O(1) get_chars_read_total(). Stream, file, wide-string and user-defined adapters keep the eager counters. Parsing pretty-printed input gets 8-12% faster and accept() 12-27% (canada, citm_catalog, twitter; clang 21 arm64 and GCC 13 x86-64). Error messages, byte offsets, SAX positions and diagnostic positions are unchanged (differential comparison against develop on 548k inputs). Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/home/architecture.md | 4 +- .../nlohmann/detail/input/input_adapters.hpp | 27 ++- include/nlohmann/detail/input/json_sax.hpp | 6 +- include/nlohmann/detail/input/lexer.hpp | 115 ++++++++++--- include/nlohmann/detail/input/parser.hpp | 8 +- single_include/nlohmann/json.hpp | 156 ++++++++++++++---- tests/src/unit-deserialization.cpp | 69 ++++++++ 7 files changed, 328 insertions(+), 57 deletions(-) diff --git a/docs/mkdocs/docs/home/architecture.md b/docs/mkdocs/docs/home/architecture.md index ea47bab6e..41466f695 100644 --- a/docs/mkdocs/docs/home/architecture.md +++ b/docs/mkdocs/docs/home/architecture.md @@ -185,7 +185,9 @@ The lexer detects two optional extensions at compile time. Only `iterator_input_ random-access input of single-byte characters: - `supports_seek`, `get_consumed_count()`, and `copy_consumed_range()` let the lexer reconstruct already consumed input - for error messages instead of copying every character it reads. + for error messages instead of copying every character it reads. With `get_read_count()` and + `count_consumed_lines()`, the lexer also derives the read position from the adapter and counts lines only when an + error needs them, instead of counting characters and lines on every read. - `supports_bulk_scan`, `bulk_data()`, `bulk_remaining()`, and `bulk_skip()` let the lexer scan strings directly in contiguous memory, several bytes at a time. diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index 847890f7a..2df78dabb 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -8,12 +8,12 @@ #pragma once -#include // min +#include // count, find, min #include // array #include // size_t #include // uint32_t #include // strlen -#include // begin, end, iterator_traits, random_access_iterator_tag, distance, next +#include // begin, end, iterator_traits, random_access_iterator_tag, distance, next, reverse_iterator #include // streambuf #include // string, char_traits #include // enable_if, is_base_of, is_pointer, is_integral, remove_pointer @@ -297,6 +297,7 @@ class iterator_input_adapter return result; } + ++eof_reads; return char_traits::eof(); } @@ -306,6 +307,26 @@ class iterator_input_adapter return static_cast(std::distance(begin, current)); } + // number of get_character() calls so far, including those that returned + // EOF; the lexer derives its position from it (see lexer::lazy_position) + std::size_t get_read_count() const + { + return get_consumed_count() + eof_reads; + } + + // number of newlines in the consumed range [0, last_index); @a line_start + // is set to the index after the last of them (0 if there is none); only + // valid when supports_seek + std::size_t count_consumed_lines(std::size_t last_index, std::size_t& line_start) const + { + const auto to = std::next(begin, static_cast::difference_type>(last_index)); + const auto newline = char_traits::to_char_type('\n'); + using reverse_it = std::reverse_iterator; + const auto last_newline = std::find(reverse_it(to), reverse_it(begin), newline); + line_start = static_cast(std::distance(begin, last_newline.base())); + return static_cast(std::count(begin, last_newline.base(), newline)); + } + // append the already-consumed characters in the half-open range // [first_index, last_index) to @a out; only valid when supports_seek template @@ -422,6 +443,8 @@ class iterator_input_adapter IteratorType begin; IteratorType current; SentinelType end; + /// number of get_character() calls that returned EOF + std::size_t eof_reads = 0; template friend struct wide_string_input_helper; diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 816359a60..161568ded 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -217,7 +217,7 @@ struct diagnostic_positions { // Lexer has read the first character of the container, so // subtract 1 from the position to get the correct start position. - v.start_position = lexer->get_position() - 1; + v.start_position = lexer->get_chars_read_total() - 1; } } @@ -231,7 +231,7 @@ struct diagnostic_positions if (lexer) { // Lexer's position is past the closing brace or bracket, so set that as the end position. - v.end_position = lexer->get_position(); + v.end_position = lexer->get_chars_read_total(); } } @@ -247,7 +247,7 @@ struct diagnostic_positions // Lexer has read past the current field value, so set the end position to the current position. // The start position will be set below based on the length of the string representation // of the value. - v.end_position = lexer->get_position(); + v.end_position = lexer->get_chars_read_total(); switch (v.type()) { diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 361baa642..af5907d78 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -186,6 +186,15 @@ class lexer : public lexer_base static constexpr bool lazy_token_string = input_adapter_supports_seek(is_detected {}); + /// whether the read position is derived from the input adapter on demand + /// instead of being counted on every scanned character. Seekable adapters + /// know how much input they consumed and can revisit it, so + /// chars_read_total is the adapter's read count (minus a pending unget), + /// and line and column are recomputed from the consumed input only when + /// get_position() is called (errors). Counting per character kept two + /// increments and a newline check in the hot path of every get(). + static constexpr bool lazy_position = lazy_token_string; + /// whether a simulated unget can be passed on to the input adapter, which /// then leaves the character in the input; see /// input_adapter_supports_lookahead @@ -195,7 +204,9 @@ class lexer : public lexer_base /// whether string scanning may bulk-consume runs of ordinary characters /// directly from a contiguous input buffer (SWAR fast path). This requires /// the token to be reconstructible lazily (lazy_token_string), so bypassing - /// the per-character capture in get() cannot lose error diagnostics. + /// the per-character capture in get() cannot lose error diagnostics. It + /// also implies lazy_position, so the bulk paths need not advance the + /// position counters. static constexpr bool bulk_scan = lazy_token_string && input_adapter_supports_bulk_scan(is_detected {}); @@ -331,10 +342,6 @@ class lexer : public lexer_base } token_buffer.append(reinterpret_cast(data), pos); ia.bulk_skip(pos); - // the run contains no newline (all bytes < 0x20 are treated as special), - // so only the flat character counters advance - position.chars_read_total += pos; - position.chars_read_current_line += pos; } /// streaming input: no bulk fast path @@ -1567,8 +1574,6 @@ scan_number_done: if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized)) { ia.bulk_skip(len - 1); - position.chars_read_total += (len - 1); - position.chars_read_current_line += (len - 1); return integer_result; } // The value does not fit an integer, so this token converts as a @@ -1585,8 +1590,6 @@ scan_number_done: decimal_point_position = dot_index; ia.bulk_skip(len - 1); - position.chars_read_total += (len - 1); - position.chars_read_current_line += (len - 1); return convert_number(number_type, mantissa_end); } @@ -1637,7 +1640,7 @@ scan_number_done: #if JSON_DIAGNOSTIC_POSITIONS // the first character of the token has already been read, hence the -1 - token_start_position = position.chars_read_total - 1; + token_start_position = get_chars_read_total() - 1; #endif note_token_start(std::integral_constant {}); @@ -1690,6 +1693,15 @@ scan_number_done: /// per-character position counters (line-count-on-'\n' bookkeeping is /// handled afterwards, in track_after_read(), once `current` is known) void advance_position() noexcept + { + advance_position_impl(std::integral_constant {}); + } + + /// seekable adapter: the position is derived from the adapter on demand + void advance_position_impl(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: count the character + void advance_position_impl(std::false_type /*lazy*/) noexcept { ++position.chars_read_total; ++position.chars_read_current_line; @@ -1703,7 +1715,16 @@ scan_number_done: // seekable adapters reconstruct the token lazily on error (see // get_token_string), so the eager per-character copy is skipped capture_char(std::integral_constant {}); + track_newline(std::integral_constant {}); + return current; + } + /// seekable adapter: lines are counted on demand in get_position() + void track_newline(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: start a new line after reading a newline + void track_newline(std::false_type /*lazy*/) noexcept + { if (current == '\n') { ++position.lines_read; @@ -1712,8 +1733,6 @@ scan_number_done: chars_read_before_newline = position.chars_read_current_line; position.chars_read_current_line = 0; } - - return current; } /*! @@ -1757,12 +1776,22 @@ scan_number_done: We implement unget by setting variable next_unget to true. The input is not changed - we just simulate ungetting by modifying chars_read_total, chars_read_current_line, and token_string. The next call to get() will - behave as if the unget character is read again. + behave as if the unget character is read again. With lazy_position, the + pending next_unget is all get_chars_read_total() needs. */ void unget() { next_unget = true; + unget_position(std::integral_constant {}); + uncapture_char(std::integral_constant {}); + } + /// seekable adapter: get_chars_read_total() accounts for next_unget + void unget_position(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: step the counters back over the ungotten character + void unget_position(std::false_type /*lazy*/) noexcept + { --position.chars_read_total; // in case we "unget" a newline, we have to also decrement the lines_read @@ -1785,8 +1814,6 @@ scan_number_done: { --position.chars_read_current_line; } - - uncapture_char(std::integral_constant {}); } /// adapter without lookahead: nothing to do (see release_lookahead) @@ -1857,10 +1884,21 @@ scan_number_done: // diagnostics ///////////////////// - /// return position of last read token - constexpr position_t get_position() const noexcept + /*! + @brief return the number of characters read so far + + Unlike get_position(), this is O(1) for all adapters, so it may be called + for every value (JSON_DIAGNOSTIC_POSITIONS). + */ + std::size_t get_chars_read_total() const noexcept { - return position; + return get_chars_read_total_impl(std::integral_constant {}); + } + + /// return position of last read token + position_t get_position() const + { + return get_position_impl(std::integral_constant {}); } /*! @@ -1897,6 +1935,43 @@ scan_number_done: } #endif + private: + /// seekable adapter: every get() either read from the adapter (including + /// reads at EOF, which get() counts as characters as well) or replayed an + /// ungotten character, so the count is the adapter's read count minus a + /// still pending unget + std::size_t get_chars_read_total_impl(std::true_type /*lazy*/) const noexcept + { + return ia.get_read_count() - (next_unget ? 1u : 0u); + } + + /// streaming adapter: the counter is maintained by get() and unget() + std::size_t get_chars_read_total_impl(std::false_type /*lazy*/) const noexcept + { + return position.chars_read_total; + } + + /// seekable adapter: count the lines in the consumed input; O(input read + /// so far), but only called on errors + position_t get_position_impl(std::true_type /*lazy*/) const + { + position_t result; + result.chars_read_total = get_chars_read_total(); + // reads at EOF count as characters but are not part of the input + const std::size_t consumed = (std::min)(result.chars_read_total, ia.get_consumed_count()); + std::size_t line_start = 0; + result.lines_read = ia.count_consumed_lines(consumed, line_start); + result.chars_read_current_line = result.chars_read_total - line_start; + return result; + } + + /// streaming adapter: the position is maintained by get() and unget() + position_t get_position_impl(std::false_type /*lazy*/) const noexcept + { + return position; + } + + public: /// seekable adapter: rebuild the last read token from the input on demand const std::vector& collect_token_chars(std::vector& out, std::true_type /*lazy*/) const { @@ -2012,7 +2087,7 @@ scan_number_done: token_type scan() { // initially, skip the BOM - if (position.chars_read_total == 0 && !skip_bom()) + if (get_chars_read_total() == 0 && !skip_bom()) { error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given"; return token_type::parse_error; @@ -2039,7 +2114,7 @@ scan_number_done: token_type scan_expecting(token_type expected_type) { JSON_ASSERT(expected_type == token_type::name_separator || expected_type == token_type::value_separator); - JSON_ASSERT(position.chars_read_total > 0); + JSON_ASSERT(get_chars_read_total() > 0); const char_int_type expected_char = static_cast((expected_type == token_type::name_separator) ? ':' : ','); skip_whitespace(); if (JSON_HEDLEY_LIKELY(current == expected_char)) diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index 3a53ff6b3..9f1618c84 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -370,7 +370,7 @@ class parser } case token_type::end_of_input: { - if (JSON_HEDLEY_UNLIKELY(m_lexer.get_position().chars_read_total == 1)) + if (JSON_HEDLEY_UNLIKELY(m_lexer.get_chars_read_total() == 1)) { return syntax_error(*sax, "attempting to parse an empty input; check that your input string or stream contains the expected JSON"); } @@ -520,8 +520,10 @@ class parser { // MSVC 2015 reports C4100 (unreferenced parameter) when SAX::parse_error is static static_cast(sax); - return sax.parse_error(m_lexer.get_position(), m_lexer.get_token_string(), - parse_error::create(exception_id::syntax_error, m_lexer.get_position(), message, nullptr)); + // computing the position is O(input read so far) for seekable input + const position_t position = m_lexer.get_position(); + return sax.parse_error(position, m_lexer.get_token_string(), + parse_error::create(exception_id::syntax_error, position, message, nullptr)); } std::string exception_message(const token_type expected, const std::string& context) diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index b6b927835..b62b5569b 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -7963,12 +7963,12 @@ NLOHMANN_JSON_NAMESPACE_END -#include // min +#include // count, find, min #include // array #include // size_t #include // uint32_t #include // strlen -#include // begin, end, iterator_traits, random_access_iterator_tag, distance, next +#include // begin, end, iterator_traits, random_access_iterator_tag, distance, next, reverse_iterator #include // streambuf #include // string, char_traits #include // enable_if, is_base_of, is_pointer, is_integral, remove_pointer @@ -8257,6 +8257,7 @@ class iterator_input_adapter return result; } + ++eof_reads; return char_traits::eof(); } @@ -8266,6 +8267,26 @@ class iterator_input_adapter return static_cast(std::distance(begin, current)); } + // number of get_character() calls so far, including those that returned + // EOF; the lexer derives its position from it (see lexer::lazy_position) + std::size_t get_read_count() const + { + return get_consumed_count() + eof_reads; + } + + // number of newlines in the consumed range [0, last_index); @a line_start + // is set to the index after the last of them (0 if there is none); only + // valid when supports_seek + std::size_t count_consumed_lines(std::size_t last_index, std::size_t& line_start) const + { + const auto to = std::next(begin, static_cast::difference_type>(last_index)); + const auto newline = char_traits::to_char_type('\n'); + using reverse_it = std::reverse_iterator; + const auto last_newline = std::find(reverse_it(to), reverse_it(begin), newline); + line_start = static_cast(std::distance(begin, last_newline.base())); + return static_cast(std::count(begin, last_newline.base(), newline)); + } + // append the already-consumed characters in the half-open range // [first_index, last_index) to @a out; only valid when supports_seek template @@ -8382,6 +8403,8 @@ class iterator_input_adapter IteratorType begin; IteratorType current; SentinelType end; + /// number of get_character() calls that returned EOF + std::size_t eof_reads = 0; template friend struct wide_string_input_helper; @@ -10584,6 +10607,15 @@ class lexer : public lexer_base static constexpr bool lazy_token_string = input_adapter_supports_seek(is_detected {}); + /// whether the read position is derived from the input adapter on demand + /// instead of being counted on every scanned character. Seekable adapters + /// know how much input they consumed and can revisit it, so + /// chars_read_total is the adapter's read count (minus a pending unget), + /// and line and column are recomputed from the consumed input only when + /// get_position() is called (errors). Counting per character kept two + /// increments and a newline check in the hot path of every get(). + static constexpr bool lazy_position = lazy_token_string; + /// whether a simulated unget can be passed on to the input adapter, which /// then leaves the character in the input; see /// input_adapter_supports_lookahead @@ -10593,7 +10625,9 @@ class lexer : public lexer_base /// whether string scanning may bulk-consume runs of ordinary characters /// directly from a contiguous input buffer (SWAR fast path). This requires /// the token to be reconstructible lazily (lazy_token_string), so bypassing - /// the per-character capture in get() cannot lose error diagnostics. + /// the per-character capture in get() cannot lose error diagnostics. It + /// also implies lazy_position, so the bulk paths need not advance the + /// position counters. static constexpr bool bulk_scan = lazy_token_string && input_adapter_supports_bulk_scan(is_detected {}); @@ -10729,10 +10763,6 @@ class lexer : public lexer_base } token_buffer.append(reinterpret_cast(data), pos); ia.bulk_skip(pos); - // the run contains no newline (all bytes < 0x20 are treated as special), - // so only the flat character counters advance - position.chars_read_total += pos; - position.chars_read_current_line += pos; } /// streaming input: no bulk fast path @@ -11965,8 +11995,6 @@ scan_number_done: if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized)) { ia.bulk_skip(len - 1); - position.chars_read_total += (len - 1); - position.chars_read_current_line += (len - 1); return integer_result; } // The value does not fit an integer, so this token converts as a @@ -11983,8 +12011,6 @@ scan_number_done: decimal_point_position = dot_index; ia.bulk_skip(len - 1); - position.chars_read_total += (len - 1); - position.chars_read_current_line += (len - 1); return convert_number(number_type, mantissa_end); } @@ -12035,7 +12061,7 @@ scan_number_done: #if JSON_DIAGNOSTIC_POSITIONS // the first character of the token has already been read, hence the -1 - token_start_position = position.chars_read_total - 1; + token_start_position = get_chars_read_total() - 1; #endif note_token_start(std::integral_constant {}); @@ -12088,6 +12114,15 @@ scan_number_done: /// per-character position counters (line-count-on-'\n' bookkeeping is /// handled afterwards, in track_after_read(), once `current` is known) void advance_position() noexcept + { + advance_position_impl(std::integral_constant {}); + } + + /// seekable adapter: the position is derived from the adapter on demand + void advance_position_impl(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: count the character + void advance_position_impl(std::false_type /*lazy*/) noexcept { ++position.chars_read_total; ++position.chars_read_current_line; @@ -12101,7 +12136,16 @@ scan_number_done: // seekable adapters reconstruct the token lazily on error (see // get_token_string), so the eager per-character copy is skipped capture_char(std::integral_constant {}); + track_newline(std::integral_constant {}); + return current; + } + /// seekable adapter: lines are counted on demand in get_position() + void track_newline(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: start a new line after reading a newline + void track_newline(std::false_type /*lazy*/) noexcept + { if (current == '\n') { ++position.lines_read; @@ -12110,8 +12154,6 @@ scan_number_done: chars_read_before_newline = position.chars_read_current_line; position.chars_read_current_line = 0; } - - return current; } /*! @@ -12155,12 +12197,22 @@ scan_number_done: We implement unget by setting variable next_unget to true. The input is not changed - we just simulate ungetting by modifying chars_read_total, chars_read_current_line, and token_string. The next call to get() will - behave as if the unget character is read again. + behave as if the unget character is read again. With lazy_position, the + pending next_unget is all get_chars_read_total() needs. */ void unget() { next_unget = true; + unget_position(std::integral_constant {}); + uncapture_char(std::integral_constant {}); + } + /// seekable adapter: get_chars_read_total() accounts for next_unget + void unget_position(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: step the counters back over the ungotten character + void unget_position(std::false_type /*lazy*/) noexcept + { --position.chars_read_total; // in case we "unget" a newline, we have to also decrement the lines_read @@ -12183,8 +12235,6 @@ scan_number_done: { --position.chars_read_current_line; } - - uncapture_char(std::integral_constant {}); } /// adapter without lookahead: nothing to do (see release_lookahead) @@ -12255,10 +12305,21 @@ scan_number_done: // diagnostics ///////////////////// - /// return position of last read token - constexpr position_t get_position() const noexcept + /*! + @brief return the number of characters read so far + + Unlike get_position(), this is O(1) for all adapters, so it may be called + for every value (JSON_DIAGNOSTIC_POSITIONS). + */ + std::size_t get_chars_read_total() const noexcept { - return position; + return get_chars_read_total_impl(std::integral_constant {}); + } + + /// return position of last read token + position_t get_position() const + { + return get_position_impl(std::integral_constant {}); } /*! @@ -12295,6 +12356,43 @@ scan_number_done: } #endif + private: + /// seekable adapter: every get() either read from the adapter (including + /// reads at EOF, which get() counts as characters as well) or replayed an + /// ungotten character, so the count is the adapter's read count minus a + /// still pending unget + std::size_t get_chars_read_total_impl(std::true_type /*lazy*/) const noexcept + { + return ia.get_read_count() - (next_unget ? 1u : 0u); + } + + /// streaming adapter: the counter is maintained by get() and unget() + std::size_t get_chars_read_total_impl(std::false_type /*lazy*/) const noexcept + { + return position.chars_read_total; + } + + /// seekable adapter: count the lines in the consumed input; O(input read + /// so far), but only called on errors + position_t get_position_impl(std::true_type /*lazy*/) const + { + position_t result; + result.chars_read_total = get_chars_read_total(); + // reads at EOF count as characters but are not part of the input + const std::size_t consumed = (std::min)(result.chars_read_total, ia.get_consumed_count()); + std::size_t line_start = 0; + result.lines_read = ia.count_consumed_lines(consumed, line_start); + result.chars_read_current_line = result.chars_read_total - line_start; + return result; + } + + /// streaming adapter: the position is maintained by get() and unget() + position_t get_position_impl(std::false_type /*lazy*/) const noexcept + { + return position; + } + + public: /// seekable adapter: rebuild the last read token from the input on demand const std::vector& collect_token_chars(std::vector& out, std::true_type /*lazy*/) const { @@ -12410,7 +12508,7 @@ scan_number_done: token_type scan() { // initially, skip the BOM - if (position.chars_read_total == 0 && !skip_bom()) + if (get_chars_read_total() == 0 && !skip_bom()) { error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given"; return token_type::parse_error; @@ -12437,7 +12535,7 @@ scan_number_done: token_type scan_expecting(token_type expected_type) { JSON_ASSERT(expected_type == token_type::name_separator || expected_type == token_type::value_separator); - JSON_ASSERT(position.chars_read_total > 0); + JSON_ASSERT(get_chars_read_total() > 0); const char_int_type expected_char = static_cast((expected_type == token_type::name_separator) ? ':' : ','); skip_whitespace(); if (JSON_HEDLEY_LIKELY(current == expected_char)) @@ -12791,7 +12889,7 @@ struct diagnostic_positions { // Lexer has read the first character of the container, so // subtract 1 from the position to get the correct start position. - v.start_position = lexer->get_position() - 1; + v.start_position = lexer->get_chars_read_total() - 1; } } @@ -12805,7 +12903,7 @@ struct diagnostic_positions if (lexer) { // Lexer's position is past the closing brace or bracket, so set that as the end position. - v.end_position = lexer->get_position(); + v.end_position = lexer->get_chars_read_total(); } } @@ -12821,7 +12919,7 @@ struct diagnostic_positions // Lexer has read past the current field value, so set the end position to the current position. // The start position will be set below based on the length of the string representation // of the value. - v.end_position = lexer->get_position(); + v.end_position = lexer->get_chars_read_total(); switch (v.type()) { @@ -18664,7 +18762,7 @@ class parser } case token_type::end_of_input: { - if (JSON_HEDLEY_UNLIKELY(m_lexer.get_position().chars_read_total == 1)) + if (JSON_HEDLEY_UNLIKELY(m_lexer.get_chars_read_total() == 1)) { return syntax_error(*sax, "attempting to parse an empty input; check that your input string or stream contains the expected JSON"); } @@ -18814,8 +18912,10 @@ class parser { // MSVC 2015 reports C4100 (unreferenced parameter) when SAX::parse_error is static static_cast(sax); - return sax.parse_error(m_lexer.get_position(), m_lexer.get_token_string(), - parse_error::create(exception_id::syntax_error, m_lexer.get_position(), message, nullptr)); + // computing the position is O(input read so far) for seekable input + const position_t position = m_lexer.get_position(); + return sax.parse_error(position, m_lexer.get_token_string(), + parse_error::create(exception_id::syntax_error, position, message, nullptr)); } std::string exception_message(const token_type expected, const std::string& context) diff --git a/tests/src/unit-deserialization.cpp b/tests/src/unit-deserialization.cpp index 6222def34..fe8d5cf0e 100644 --- a/tests/src/unit-deserialization.cpp +++ b/tests/src/unit-deserialization.cpp @@ -35,6 +35,7 @@ using nlohmann::json; #include #include #include +#include #include "sax_event_loggers.hpp" @@ -1250,3 +1251,71 @@ TEST_CASE_TEMPLATE("deserialization of different character types (UTF-32)", T, c CHECK(json::sax_parse(v, &l)); CHECK(l.events.size() == 1); } + +TEST_CASE("parse error positions are the same for contiguous and stream input") +{ + // contiguous input derives the position from the input on error, stream + // input counts it while reading (see lexer::lazy_position); the cases + // cover newlines before the error, ungetting a newline after a number, + // newlines inside a string, a comment, and reads at the end of input + struct test_case + { + std::string input; + std::size_t byte; + std::string message; + }; + const std::vector cases = + { + {"[\n 1,\n 2\n", 12, "[json.exception.parse_error.101] parse error at line 4, column 1: syntax error while parsing array - unexpected end of input; expected ']'"}, + {"[1\n,2,\n]", 8, "[json.exception.parse_error.101] parse error at line 3, column 1: syntax error while parsing value - unexpected ']'; expected '[', '{', or a literal"}, + {"[1\n x]", 5, "[json.exception.parse_error.101] parse error at line 2, column 2: syntax error while parsing array - invalid literal; last read: '1 x'; expected ']'"}, + {"1\n2", 3, "[json.exception.parse_error.101] parse error at line 2, column 1: syntax error while parsing value - unexpected number literal; expected end of input"}, + { + "{\"a\":\n\"b\nc\"}", 9, R"([json.exception.parse_error.101] parse error at line 3, column 0: syntax error while parsing value - invalid string: control character U+000A (LF) must be escaped to \u000A or \n; last read: '"b')" + }, + {"/* a\n b */\n x", 13, "[json.exception.parse_error.101] parse error at line 3, column 2: syntax error while parsing value - invalid literal; last read: '/* a b */ x'"}, + {"\n\n", 3, "[json.exception.parse_error.101] parse error at line 3, column 1: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal"}, + {"[\"a\",\n 1e\n]", 10, "[json.exception.parse_error.101] parse error at line 3, column 0: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '1e'"}, + {"[tru\n]", 5, "[json.exception.parse_error.101] parse error at line 2, column 0: syntax error while parsing value - invalid literal; last read: '[tru'"}, + }; + + for (const auto& c : cases) + { + CAPTURE(c.input) + const std::string expected = std::to_string(c.byte) + " " + c.message; + std::string string_result; + std::string pointer_result; + std::string stream_result; + try + { + const json j = json::parse(c.input, nullptr, true, true); + static_cast(j); + } + catch (const json::parse_error& e) + { + string_result = std::to_string(e.byte) + " " + e.what(); + } + try + { + const json j = json::parse(c.input.data(), c.input.data() + c.input.size(), nullptr, true, true); + static_cast(j); + } + catch (const json::parse_error& e) + { + pointer_result = std::to_string(e.byte) + " " + e.what(); + } + try + { + std::istringstream is(c.input); + const json j = json::parse(is, nullptr, true, true); + static_cast(j); + } + catch (const json::parse_error& e) + { + stream_result = std::to_string(e.byte) + " " + e.what(); + } + CHECK(string_result == expected); + CHECK(pointer_result == expected); + CHECK(stream_result == expected); + } +}