Lexer, parser, stream adapter, SAX: remove checks that invariants make dead

Branch coverage of the test suite and of 24k mutated inputs showed
these conditions as never taken; each is implied by the code around it:

- lexer: `next_unget` in get_codepoint_bulk() and scan_literal_bulk().
  unget() is only called at the end of scan_number(), in scan_comment()
  and in skip_bom(); every token scan starts with a plain get() in
  skip_whitespace() that consumes a pending unget, and nothing between
  that get() and the bulk helpers ungets (now asserts, as in
  get_after_string_run())
- lexer: `number_type == value_integer` in convert_integer(): both
  callers exclude value_float and the type has three values
- parser: `!context.empty()` in exception_message(): all callers pass a
  literal
- input_stream_adapter: `refill` in get_character_slow(): the function
  is only called after get_character() found the get area exhausted,
  and the variable recomputed that comparison
- json_sax_dom_callback_parser: `ref_stack.back()` in key() (the null
  case returned a few lines earlier, and the callback in between cannot
  reach ref_stack), `is_structured()` in end_object() (non-null entries
  are only pushed for objects and arrays), `!key_stack.empty()` in
  current_key() (a kept object pushed its key before any value)

No change in behaviour: a build that aborts on each removed side passes
the test suite, the fuzz corpora and 40k random documents through every
input adapter in four macro configurations.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/input_adapters.hpp

@@ -545,12 +545,9 @@ class input_stream_adapter
     std::char_traits<char>::int_type get_character_slow()
     {
 #if !JSON_DISABLE_STREAM_BULK_SCAN
-        // sbumpc() refills the get area if it is exhausted
-        const bool refill = streambuf_access::get_ptr(sb) == streambuf_access::get_end(sb);
-        if (JSON_HEDLEY_UNLIKELY(refill))
-        {
-            save_token_segment();
-        }
+        // the get area is exhausted (get_character() handles the other case),
+        // so sbumpc() refills it
+        save_token_segment();
 #endif
 
         auto res = sb->sbumpc();
@@ -563,17 +560,13 @@ class input_stream_adapter
             is->clear(is->rdstate() | std::ios::eofbit);
         }
 #if !JSON_DISABLE_STREAM_BULK_SCAN
-        else if (JSON_HEDLEY_UNLIKELY(refill))
+        else
         {
             // the consumed character is at gptr() - 1, unless the streambuf
             // has no get area and returned it from uflow()
             const char* const next = streambuf_access::get_ptr(sb);
             after_refill(res, next != nullptr && next != streambuf_access::get_begin(sb));
         }
-        else
-        {
-            current_in_get_area = true; // LCOV_EXCL_LINE (called only when the get area is exhausted)
-        }
 #endif
         return res;
     }

include/nlohmann/detail/input/json_sax.hpp

@@ -1737,7 +1737,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state
         key_stack.push_back(val);
 
         // add discarded value at the given key and store the reference for later
-        if (keep && ref_stack.back())
+        if (keep)
         {
             auto& obj = *ref_stack.back()->m_data.m_value.object;
             // a single lookup: insert the placeholder, or find the existing member
@@ -1794,7 +1794,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state
 
         const string_t object_key = pop_container();
 
-        if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured())
+        if (!ref_stack.empty() && ref_stack.back())
         {
             // remove discarded value
             remove_discarded_value(*ref_stack.back(), object_key);
@@ -1960,8 +1960,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state
     */
     string_t current_key() const
     {
-        if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object()
-                && !key_stack.empty())
+        if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object())
         {
             return key_stack.back();
         }

include/nlohmann/detail/input/lexer.hpp

@@ -300,14 +300,16 @@ class lexer : public lexer_base<BasicJsonType>
     /// calls would (including the position and token bookkeeping of a
     /// streaming adapter, see bulk_consume) and leaves @a current holding the last of
     /// the 4 digits, just as the last such get() would; the codepoint is
-    /// written to @a out. Makes no state change and returns false for a
-    /// pending unget, fewer than 4 remaining bytes, or any of the 4 bytes not
-    /// being a hex digit, so the caller falls back unchanged to the
-    /// per-character loop, which then reports the same diagnostic (stopping
-    /// at the first invalid digit) as before this optimization.
+    /// written to @a out. Makes no state change and returns false for fewer
+    /// than 4 remaining bytes or any of the 4 bytes not being a hex digit, so
+    /// the caller falls back unchanged to the per-character loop, which then
+    /// reports the same diagnostic (stopping at the first invalid digit) as
+    /// before this optimization.
     bool get_codepoint_bulk(std::true_type /*bulk*/, int& out)
     {
-        if (next_unget || ia.bulk_remaining() < 4)
+        // no unget is pending inside a string (see get_after_string_run())
+        JSON_ASSERT(!next_unget);
+        if (ia.bulk_remaining() < 4)
         {
             return false;
         }
@@ -1409,7 +1411,7 @@ scan_number_done:
                 return token_type::value_unsigned;
             }
         }
-        else if (number_type == token_type::value_integer)
+        else // value_integer (the callers handle value_float themselves)
         {
             if (parse_integer_signed(first, last, value_integer))
             {
@@ -1780,7 +1782,10 @@ scan_number_done:
     /// the error at the right character
     JSON_HEDLEY_ALWAYS_INLINE bool scan_literal_bulk(const char* rest, const std::size_t length, std::true_type /*bulk*/)
     {
-        if (next_unget || ia.bulk_remaining() < length
+        // called right after scan_after_whitespace() read current (see
+        // get_after_string_run()), so no unget is pending
+        JSON_ASSERT(!next_unget);
+        if (ia.bulk_remaining() < length
                 || std::memcmp(ia.bulk_data(), rest, length) != 0)
         {
             return false;

include/nlohmann/detail/input/parser.hpp

@@ -575,10 +575,7 @@ class parser
     {
         std::string error_msg = "syntax error ";
 
-        if (!context.empty())
-        {
-            error_msg += concat("while parsing ", context, ' ');
-        }
+        error_msg += concat("while parsing ", context, ' ');
 
         error_msg += "- ";
 

single_include/nlohmann/json.hpp

@@ -9652,12 +9652,9 @@ class input_stream_adapter
     std::char_traits<char>::int_type get_character_slow()
     {
 #if !JSON_DISABLE_STREAM_BULK_SCAN
-        // sbumpc() refills the get area if it is exhausted
-        const bool refill = streambuf_access::get_ptr(sb) == streambuf_access::get_end(sb);
-        if (JSON_HEDLEY_UNLIKELY(refill))
-        {
-            save_token_segment();
-        }
+        // the get area is exhausted (get_character() handles the other case),
+        // so sbumpc() refills it
+        save_token_segment();
 #endif
 
         auto res = sb->sbumpc();
@@ -9670,17 +9667,13 @@ class input_stream_adapter
             is->clear(is->rdstate() | std::ios::eofbit);
         }
 #if !JSON_DISABLE_STREAM_BULK_SCAN
-        else if (JSON_HEDLEY_UNLIKELY(refill))
+        else
         {
             // the consumed character is at gptr() - 1, unless the streambuf
             // has no get area and returned it from uflow()
             const char* const next = streambuf_access::get_ptr(sb);
             after_refill(res, next != nullptr && next != streambuf_access::get_begin(sb));
         }
-        else
-        {
-            current_in_get_area = true; // LCOV_EXCL_LINE (called only when the get area is exhausted)
-        }
 #endif
         return res;
     }
@@ -13334,14 +13327,16 @@ class lexer : public lexer_base<BasicJsonType>
     /// calls would (including the position and token bookkeeping of a
     /// streaming adapter, see bulk_consume) and leaves @a current holding the last of
     /// the 4 digits, just as the last such get() would; the codepoint is
-    /// written to @a out. Makes no state change and returns false for a
-    /// pending unget, fewer than 4 remaining bytes, or any of the 4 bytes not
-    /// being a hex digit, so the caller falls back unchanged to the
-    /// per-character loop, which then reports the same diagnostic (stopping
-    /// at the first invalid digit) as before this optimization.
+    /// written to @a out. Makes no state change and returns false for fewer
+    /// than 4 remaining bytes or any of the 4 bytes not being a hex digit, so
+    /// the caller falls back unchanged to the per-character loop, which then
+    /// reports the same diagnostic (stopping at the first invalid digit) as
+    /// before this optimization.
     bool get_codepoint_bulk(std::true_type /*bulk*/, int& out)
     {
-        if (next_unget || ia.bulk_remaining() < 4)
+        // no unget is pending inside a string (see get_after_string_run())
+        JSON_ASSERT(!next_unget);
+        if (ia.bulk_remaining() < 4)
         {
             return false;
         }
@@ -14443,7 +14438,7 @@ scan_number_done:
                 return token_type::value_unsigned;
             }
         }
-        else if (number_type == token_type::value_integer)
+        else // value_integer (the callers handle value_float themselves)
         {
             if (parse_integer_signed(first, last, value_integer))
             {
@@ -14814,7 +14809,10 @@ scan_number_done:
     /// the error at the right character
     JSON_HEDLEY_ALWAYS_INLINE bool scan_literal_bulk(const char* rest, const std::size_t length, std::true_type /*bulk*/)
     {
-        if (next_unget || ia.bulk_remaining() < length
+        // called right after scan_after_whitespace() read current (see
+        // get_after_string_run()), so no unget is pending
+        JSON_ASSERT(!next_unget);
+        if (ia.bulk_remaining() < length
                 || std::memcmp(ia.bulk_data(), rest, length) != 0)
         {
             return false;
@@ -17371,7 +17369,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state
         key_stack.push_back(val);
 
         // add discarded value at the given key and store the reference for later
-        if (keep && ref_stack.back())
+        if (keep)
         {
             auto& obj = *ref_stack.back()->m_data.m_value.object;
             // a single lookup: insert the placeholder, or find the existing member
@@ -17428,7 +17426,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state
 
         const string_t object_key = pop_container();
 
-        if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured())
+        if (!ref_stack.empty() && ref_stack.back())
         {
             // remove discarded value
             remove_discarded_value(*ref_stack.back(), object_key);
@@ -17594,8 +17592,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state
     */
     string_t current_key() const
     {
-        if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object()
-                && !key_stack.empty())
+        if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object())
         {
             return key_stack.back();
         }
@@ -23739,10 +23736,7 @@ class parser
     {
         std::string error_msg = "syntax error ";
 
-        if (!context.empty())
-        {
-            error_msg += concat("while parsing ", context, ' ');
-        }
+        error_msg += concat("while parsing ", context, ' ');
 
         error_msg += "- ";
 

tests/src/unit-ordered_json.cpp

@@ -534,10 +534,9 @@ TEST_CASE("object_key_index")
 
         // the hashes are not distinct on purpose: elements with the same
         // hash are told apart by the caller
-        const auto hash_of = [](std::size_t position) -> std::size_t
+        const auto hash_of = [](std::size_t position)
         {
-            // truncated to 32 bits where std::size_t is that small
-            return static_cast<std::size_t>((position % 50) * 0x9E3779B97F4A7C15ull);
+            return (position % 50) * 0x9E3779B97F4A7C15ull;
         };
 
         std::vector<std::size_t> hashes;

tests/src/unit-regression1.cpp

@@ -1555,7 +1555,6 @@ TEST_CASE("regression tests 1")
         {
             my_json j = make();
 #ifdef JSON_NO_THREAD_LOCAL
-            // (set by the user, or by the library for clang with MinGW)
             // the objects below the outermost one are not copied by their
             // object_t here, but member by member, which keeps their
             // timestamps correct: "c" is not mistaken for "b" and is inserted