fix: leave the character that terminates a number in the input

Read the character following a number without consuming it, instead of
consuming it and putting it back. input_stream_adapter now peeks with
sgetc() and only steps over the character when the next one is requested
or when the adapter is destroyed, so releasing it cannot fail - no
putback position is required from the streambuf.

Suggested by gregmarr in #5344.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/lexer.hpp

@@ -125,20 +125,20 @@ constexpr bool input_adapter_supports_seek(std::false_type /*detected*/)
     return false;
 }
 
-// Detect whether an input adapter can return the character last read to the
-// input (see input_stream_adapter::supports_unget), detected like
-// supports_seek above.
+// Detect whether an input adapter reads with one character of lookahead that
+// can be left in the input (see input_stream_adapter::supports_lookahead),
+// detected like supports_seek above.
 template<typename InputAdapterType>
-using detect_supports_unget = decltype(InputAdapterType::supports_unget);
+using detect_supports_lookahead = decltype(InputAdapterType::supports_lookahead);
 
 template<typename InputAdapterType>
-constexpr bool input_adapter_supports_unget(std::true_type /*detected*/)
+constexpr bool input_adapter_supports_lookahead(std::true_type /*detected*/)
 {
-    return InputAdapterType::supports_unget;
+    return InputAdapterType::supports_lookahead;
 }
 
 template<typename InputAdapterType>
-constexpr bool input_adapter_supports_unget(std::false_type /*detected*/)
+constexpr bool input_adapter_supports_lookahead(std::false_type /*detected*/)
 {
     return false;
 }
@@ -164,10 +164,11 @@ class lexer : public lexer_base<BasicJsonType>
     static constexpr bool lazy_token_string =
         input_adapter_supports_seek<InputAdapterType>(is_detected<detect_supports_seek, InputAdapterType> {});
 
-    /// whether a pending simulated unget can be turned into a real unget on
-    /// the input adapter; see input_adapter_supports_unget
-    static constexpr bool can_unget_input =
-        input_adapter_supports_unget<InputAdapterType>(is_detected<detect_supports_unget, InputAdapterType> {});
+    /// whether a simulated unget can be passed on to the input adapter, which
+    /// then leaves the character in the input; see
+    /// input_adapter_supports_lookahead
+    static constexpr bool can_release_lookahead =
+        input_adapter_supports_lookahead<InputAdapterType>(is_detected<detect_supports_lookahead, InputAdapterType> {});
 
   public:
     using token_type = typename lexer_base<BasicJsonType>::token_type;
@@ -1479,23 +1480,19 @@ scan_number_done:
         uncapture_char(std::integral_constant<bool, lazy_token_string> {});
     }
 
-    /// adapter without unget support: nothing to do (see restore_pending_unget)
-    bool restore_pending_unget_impl(std::false_type /*can_unget*/) const noexcept
-    {
-        return false;
-    }
+    /// adapter without lookahead: nothing to do (see release_lookahead)
+    void release_lookahead_impl(std::false_type /*can_release*/) const noexcept {}
 
-    /// adapter with unget support: give back the character consumed but unread
-    bool restore_pending_unget_impl(std::true_type /*can_unget*/)
+    /// adapter with lookahead: leave the character in the input instead
+    void release_lookahead_impl(std::true_type /*can_release*/)
     {
-        if (!next_unget || current == char_traits<char_type>::eof())
+        if (next_unget)
         {
-            // nothing was consumed beyond the last token
-            return true;
+            // the character is read from the input again rather than replayed
+            // from current, so the adapter must not step over it
+            next_unget = false;
+            ia.release_lookahead();
         }
-
-        next_unget = false;
-        return ia.unget_character();
     }
 
     /// seekable adapter: nothing was captured, so nothing to undo
@@ -1562,25 +1559,26 @@ scan_number_done:
     }
 
     /*!
-    @brief turn a pending simulated unget into a real one on the input
+    @brief pass a pending simulated unget on to the input
 
     unget() only rewinds the lexer's own bookkeeping, so the character that
-    terminated the last token (e.g. the character after a number) stays
-    consumed from the input. Callers that hand the input back to the user
-    afterwards - operator>> and non-strict sax_parse - call this once when
-    scanning is done, so that the input is positioned right after the value.
+    terminated the last token (e.g. the character after a number) would still
+    be stepped over when the input adapter is done. Callers that hand the
+    input back to the user afterwards - operator>> and non-strict sax_parse -
+    call this once when scanning is done, so that the input is positioned
+    right after the value.
 
-    A pending unget of EOF must not be restored: EOF was never consumed. The
-    lexer must not read again after this call; next_unget is cleared so that
-    the restored character is not also replayed from @a current.
+    Adapters without lookahead (see input_adapter_supports_lookahead) are not
+    handed back to the user, so this is a no-op for them.
 
-    @return whether the input is positioned right after the last token; false
-            if the adapter cannot unget or the unget failed, in which case the
-            input is left as is (the pre-existing behaviour)
+    Scanning may continue after this call: @a next_unget is cleared, and the
+    character is read from the input again instead of being replayed from
+    @a current. A pending unget of EOF needs no special case, because reaching
+    EOF leaves no lookahead to release.
     */
-    bool restore_pending_unget()
+    void release_lookahead()
     {
-        return restore_pending_unget_impl(std::integral_constant<bool, can_unget_input> {});
+        release_lookahead_impl(std::integral_constant<bool, can_release_lookahead> {});
     }
 
     /// seekable adapter: rebuild the last read token from the input on demand