Scan the buffer of a FILE in bulk with glibc and the Apple C library

Parsing from a FILE* still read one character at a time through
getc_unlocked(), while a std::istream has its get area scanned in bulk.
file_input_adapter now offers the lexer the same kind of window into the
buffer of the FILE: it reads the fields that the C library's own
getc_unlocked() macro reads (glibc: _IO_read_ptr and _IO_read_end; Apple:
_p and _r) and consumes exactly the characters getc_unlocked() would
return, so the file is left where it was left before. The part of the
current token in a buffer that is about to be refilled is kept for error
messages, as for streams. The window needs the file locked for the whole
parse, so JSON_DISABLE_UNLOCKED_FILE_IO and MemorySanitizer builds turn it
off; uClibc, which defines __GLIBC__ but has other fields, does not get it.

parse() from a FILE* gets 2.0-2.6x as fast on an Apple M1 Max and 1.7-2.2x
on x86-64 (glibc, GCC 15 and Clang 21), accept() 2.9-4.4x; that is about as
fast as parsing from a std::ifstream. The new tests read with FILE buffers
of 1 to 70 bytes and unbuffered files, after ungetc(), with SAX positions
and with binary strings across the buffer's edges.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

tests/src/unit-file_input.cpp

@@ -225,6 +225,125 @@ TEST_CASE("parsing from a FILE*")
         check_unlocked(f.get());
     }
 
+#if !defined(JSON_NOEXCEPTION)
+    SECTION("tokens across the edges of the file's buffer")
+    {
+        // the adapter reads the buffer of the FILE directly where it can (see
+        // JSON_FILE_WINDOW): with buffers of 1 to 70 bytes and none, every
+        // token crosses an edge somewhere; the results, the error messages
+        // and their positions must be those of the string
+        const std::vector<std::string> documents =
+        {
+            input,
+            R"({"long": "a string that is longer than the buffer of the file, with \"escapes\" and \u00e4", "n": [123456789012345678901, -0.000123e-45, 1.5E+300, 0, -0]})",
+            "[\n    \"indented\",\n    \"caf\xc3\xa9 \xe2\x82\xac \xf0\x9f\x98\x80\",\n    {\n        \"key\": 1\n    }\n]",
+            R"({"a": [1, 2, 3], "b": "unterminated)",
+            R"([1, 2, 3, 1e999])",
+            R"({"a": "invalid escape \x"})",
+            "[\"invalid UTF-8 \xc3\x28\"]",
+            "[1, 2, 3] trailing",
+            R"([truex])",
+        };
+        for (const auto& document : documents)
+        {
+            std::string expected;
+            try
+            {
+                expected = json::parse(document).dump();
+            }
+            catch (const json::exception& e)
+            {
+                expected = e.what();
+            }
+            const bool expected_accept = json::accept(document);
+            for (int size = 0; size <= 70; ++size)
+            {
+                CAPTURE(document)
+                CAPTURE(size)
+                std::vector<char> buffer(static_cast<std::size_t>(size) + 1);
+                for (int pass = 0; pass < 2; ++pass)
+                {
+                    const file_ptr f = open_with(name, document);
+                    REQUIRE(f != nullptr);
+                    // size 0: unbuffered
+                    REQUIRE(std::setvbuf(f.get(), size == 0 ? nullptr : buffer.data(), size == 0 ? _IONBF : _IOFBF, static_cast<std::size_t>(size)) == 0);
+                    if (pass == 0)
+                    {
+                        std::string actual;
+                        try
+                        {
+                            actual = json::parse(f.get()).dump();
+                        }
+                        catch (const json::exception& e)
+                        {
+                            actual = e.what();
+                        }
+                        CHECK(actual == expected);
+                    }
+                    else
+                    {
+                        CHECK(json::accept(f.get()) == expected_accept);
+                    }
+                    check_unlocked(f.get());
+                }
+            }
+        }
+    }
+#endif
+
+#if !defined(JSON_NOEXCEPTION)
+    SECTION("characters pushed back with ungetc() before parsing")
+    {
+        // (glibc and Apple keep them in a separate buffer)
+        const file_ptr f = open_with(name, "2, 3]");
+        REQUIRE(f != nullptr);
+        REQUIRE(std::ungetc(',', f.get()) == ',');
+        REQUIRE(std::ungetc('1', f.get()) == '1');
+        REQUIRE(std::ungetc('[', f.get()) == '[');
+        CHECK(json::parse(f.get()) == json({1, 2, 3}));
+    }
+#endif
+
+    SECTION("SAX events across the edges of the file's buffer")
+    {
+        const std::string document = R"({"key": "a value that is longer than the buffer", "n": [1, -2.5e3, true, null]})";
+        std::vector<long> expected;
+        {
+            const file_ptr f = open_with(name, document);
+            REQUIRE(f != nullptr);
+            position_reading_sax sax(f.get());
+            CHECK(json::sax_parse(f.get(), &sax));
+            expected = sax.positions();
+        }
+        for (int size = 1; size <= 20; ++size)
+        {
+            CAPTURE(size)
+            std::vector<char> buffer(static_cast<std::size_t>(size));
+            const file_ptr f = open_with(name, document);
+            REQUIRE(f != nullptr);
+            REQUIRE(std::setvbuf(f.get(), buffer.data(), _IOFBF, buffer.size()) == 0);
+            position_reading_sax sax(f.get());
+            CHECK(json::sax_parse(f.get(), &sax));
+            // the events happen at the same positions of the file
+            CHECK(sax.positions() == expected);
+        }
+    }
+
+    SECTION("binary strings across the edges of the file's buffer")
+    {
+        const json value = {{"a string that is longer than the buffer of the file", "and a value that is longer, too"}, {"n", {1, 2}}};
+        const std::vector<std::uint8_t> cbor = json::to_cbor(value);
+        for (int size = 1; size <= 20; ++size)
+        {
+            CAPTURE(size)
+            std::vector<char> buffer(static_cast<std::size_t>(size));
+            const file_ptr f = open_with(name, std::string(cbor.begin(), cbor.end()));
+            REQUIRE(f != nullptr);
+            REQUIRE(std::setvbuf(f.get(), buffer.data(), _IOFBF, buffer.size()) == 0);
+            CHECK(json::from_cbor(f.get()) == value);
+        }
+    }
+
     SECTION("binary formats read from a FILE*")
     {
         const json j = {{"string", std::string(300, 'x')}, {"array", {1, 2, 3}}, {"number", 1.5}};