Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Niels Lohmann
|
9ed8f85b21 | ||
|
Niels Lohmann
|
88ab2d56f5 | ||
|
Niels Lohmann
|
5901d861ff |
No files matched your search
-3272
File diff suppressed because it is too large.
Load diff
@@ -1,4 +1,4 @@
|
||||
.PHONY: all amalgamate check-amalgamation check_build_options pretty install_astyle clean ChangeLog.md release update_hedley update_hedley_undef BUILD.bazel natvis macro_builder_check run_benchmarks pvs_studio serve_header reuse spdx
|
||||
.PHONY: all amalgamate check-amalgamation check_build_options pretty install_astyle clean release update_hedley update_hedley_undef BUILD.bazel natvis macro_builder_check run_benchmarks pvs_studio serve_header reuse spdx
|
||||
|
||||
##########################################################################
|
||||
# configuration
|
||||
@@ -40,7 +40,6 @@ MACRO_SCOPE_HPP=include/nlohmann/detail/macro_scope.hpp
|
||||
all:
|
||||
@echo "amalgamate - amalgamate files single_include/nlohmann/json{,_fwd,_literals,_view}.hpp from the include/nlohmann sources"
|
||||
@echo "BUILD.bazel - regenerate the Bazel BUILD file from the include/nlohmann sources"
|
||||
@echo "ChangeLog.md - generate ChangeLog file"
|
||||
@echo "check-amalgamation - check whether sources have been amalgamated and BUILD.bazel is up to date"
|
||||
@echo "check_build_options - check that the Meson build and the pkg-config files offer the options of the CMake target"
|
||||
@echo "clean - remove built files"
|
||||
@@ -175,22 +174,6 @@ check-amalgamation:
|
||||
BUILD.bazel:
|
||||
cmake -P cmake/scripts/gen_bazel_build_file.cmake
|
||||
|
||||
##########################################################################
|
||||
# ChangeLog
|
||||
##########################################################################
|
||||
|
||||
# Create a ChangeLog based on the git log using the GitHub Changelog Generator
|
||||
# (<https://github.com/github-changelog-generator/github-changelog-generator>).
|
||||
|
||||
# variable to control the diffs between the last released version and the current repository state
|
||||
NEXT_VERSION ?= "unreleased"
|
||||
|
||||
ChangeLog.md:
|
||||
github_changelog_generator -o ChangeLog.md --user nlohmann --project json --simple-list --release-url https://github.com/nlohmann/json/releases/tag/%s --future-release $(NEXT_VERSION)
|
||||
$(SED) -i 's|https://github.com/nlohmann/json/releases/tag/HEAD|https://github.com/nlohmann/json/tree/HEAD|' ChangeLog.md
|
||||
$(SED) -i '2i All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/).' ChangeLog.md
|
||||
|
||||
|
||||
##########################################################################
|
||||
# Release files
|
||||
##########################################################################
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
[](https://github.com/nlohmann/json/releases)
|
||||
<a href="https://github.com/nlohmann/json/releases">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="docs/mkdocs/docs/images/json-dark.svg">
|
||||
<img src="docs/mkdocs/docs/images/json-light.svg" alt="JSON for Modern C++: animated feature slideshow">
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
[](https://ci.appveyor.com/project/nlohmann/json)
|
||||
[](https://github.com/nlohmann/json/actions?query=workflow%3AUbuntu)
|
||||
@@ -39,6 +44,7 @@
|
||||
- [Conversions to/from arbitrary types](#arbitrary-types-conversions)
|
||||
- [Specializing enum conversion](#specializing-enum-conversion)
|
||||
- [Binary formats (BSON, CBOR, MessagePack, UBJSON, BJData, and BON8)](#binary-formats-bson-cbor-messagepack-ubjson-bjdata-and-bon8)
|
||||
- [Zero-copy views](#zero-copy-views)
|
||||
- [Customers](#customers)
|
||||
- [Ecosystem](#ecosystem)
|
||||
- [Supported compilers](#supported-compilers)
|
||||
@@ -68,7 +74,7 @@ Other aspects were not so important to us:
|
||||
|
||||
- **Memory efficiency**. Each JSON object has an overhead of one pointer (the maximal size of a union) and one enumeration element (1 byte). The default generalization uses the following C++ data types: `std::string` for strings, `int64_t`, `uint64_t` or `double` for numbers, `std::map` for objects, `std::vector` for arrays, and `bool` for Booleans. However, you can template the generalized class `basic_json` to your needs.
|
||||
|
||||
- **Speed**. There are certainly [faster JSON libraries](https://github.com/miloyip/nativejson-benchmark#parsing-time) out there. However, if your goal is to speed up your development by adding JSON support with a single header, then this library is the way to go. If you know how to use a `std::vector` or `std::map`, you are already set.
|
||||
- **Speed**. There are certainly [faster JSON libraries](https://github.com/miloyip/nativejson-benchmark#parsing-time) out there. However, if your goal is to speed up your development by adding JSON support with a single header, then this library is the way to go. If you know how to use a `std::vector` or `std::map`, you are already set. The [performance](https://json.nlohmann.me/features/performance/) page shows how to get the most out of the library.
|
||||
|
||||
See the [contribution guidelines](https://github.com/nlohmann/json/blob/develop/.github/CONTRIBUTING.md#please-dont) for more information.
|
||||
|
||||
@@ -1052,7 +1058,7 @@ struct less_than_32_serializer {
|
||||
Be **very** careful when reimplementing your serializer, you can stack overflow if you don't pay attention:
|
||||
|
||||
```cpp
|
||||
template <typename T, void>
|
||||
template <typename T, typename = void>
|
||||
struct bad_serializer
|
||||
{
|
||||
template <typename BasicJsonType>
|
||||
@@ -1063,10 +1069,10 @@ struct bad_serializer
|
||||
}
|
||||
|
||||
template <typename BasicJsonType>
|
||||
static void to_json(const BasicJsonType& j, T& value) {
|
||||
static void from_json(const BasicJsonType& j, T& value) {
|
||||
// this calls BasicJsonType::json_serializer<T>::from_json(j, value)
|
||||
// if BasicJsonType::json_serializer == bad_serializer ... oops!
|
||||
value = j.get<T>(); // oops!
|
||||
value = j.template get<T>(); // oops!
|
||||
}
|
||||
};
|
||||
```
|
||||
@@ -1769,7 +1775,7 @@ I deeply appreciate the help of the following people.
|
||||
330. [AniketDhemare](https://github.com/AniketDhemare) fixed a version in the documentation.
|
||||
331. [Philip Müller](https://github.com/philip-paul-mueller) fixed an example.
|
||||
332. [Leila Shcheglova](https://github.com/LeilaShcheglova) fixed a warning in a test.
|
||||
333. [Alex Prabhat Bara](https://github.com/alexprabhat99) fixed a function name in the documentation.
|
||||
333. [Alex Prabhat Bara](https://github.com/alexprabhat99) fixed a function name in the documentation and avoided temporary `basic_json` values for CBOR and MessagePack object keys.
|
||||
334. [laterlaugh](https://github.com/laterlaugh) fixed some typos.
|
||||
335. [Yuanhao Jia](https://github.com/MrJia1997) fixed the GDB pretty printer.
|
||||
336. [Fallen_Breath](https://github.com/Fallen-Breath) fixed an example for JSON Pointer.
|
||||
@@ -1847,7 +1853,7 @@ I deeply appreciate the help of the following people.
|
||||
408. [Luke Banicevic](https://github.com/banaboi) fixed corrupt BSON output for lengths exceeding `INT32_MAX`, cleaned up the BSON writer, and improved the documentation.
|
||||
409. [Patrick Armstrong](https://github.com/Patrick10199) updated the CBOR references and the half-precision float assertions.
|
||||
410. [Yash Bavadiya](https://github.com/xevrion) added checks to all BSON reads.
|
||||
411. [hum4nBeing](https://github.com/hum4nBeing) fixed the overflow handling of high-precision numbers in UBJSON.
|
||||
411. [hum4nBeing](https://github.com/hum4nBeing) and [AJ369ninja](https://github.com/AJ369ninja) fixed the overflow handling of high-precision numbers in UBJSON.
|
||||
412. [tomatotomata](https://github.com/tomatotomata) added checks for reading CBOR tagged subtypes.
|
||||
413. [YingqiDuan](https://github.com/YingqiDuan) documented the BSON interoperability.
|
||||
414. [KBS](https://github.com/youdie006) documented the standards compliance and the strictness of `parse()` and `operator>>`.
|
||||
@@ -1861,6 +1867,26 @@ I deeply appreciate the help of the following people.
|
||||
422. [Avionic Harshit](https://github.com/avionicharshit-byte) made `diff()` linear when an array shrinks.
|
||||
423. [Qatadaha Bin Matloob](https://github.com/qatcod) fixed comparisons between integers and floats and fixed unparsable BJData output.
|
||||
424. [Wu Shuwen](https://github.com/dajiaohuang) removed an unused include.
|
||||
425. [Alexander Lanin](https://github.com/AlexanderLanin) improved the error message of `get_to()` for `const` arguments.
|
||||
426. [satelliteprogrammer](https://github.com/satelliteprogrammer) added `merge_diff()` to create JSON Merge Patches (RFC 7396).
|
||||
427. [Jeremy Nimmer](https://github.com/jwnimmer-tri) removed `alwayslink` from the Bazel build.
|
||||
428. [Dadi Reddy Sai Praneeth Reddy](https://github.com/dsp0redy) made `to_msgpack()` throw when a size exceeds the MessagePack limit.
|
||||
429. [ayush-singh-0601](https://github.com/ayush-singh-0601) reduced the runtime of the test suite.
|
||||
430. [Kartikey Negi](https://github.com/ReturnKartikey) fixed the CBOR tag handling of tagged values that are not byte strings.
|
||||
431. [Michiel van Slobbe](https://github.com/mvanslobbe) sped up the parser by checking for the expected separator first.
|
||||
432. [Mohd Quamar Tyagi](https://github.com/Tyagiquamar) allowed SAX parsing of tagged CBOR.
|
||||
433. [fhgffy](https://github.com/fhgffy) rejected NUL bytes in UBJSON and BJData high-precision numbers and fixed a `-Wswitch-enum` warning.
|
||||
434. [Suyog Verma](https://github.com/commit-msuyog) used MSVC intrinsics for the 128-bit multiplication in the floating-point serialization.
|
||||
435. [Bas van Dijk](https://github.com/basdedas) fixed a TU-local exposure of `UTF8_ACCEPT` and `UTF8_REJECT` in C++20 modules.
|
||||
436. [Christoph Grüninger](https://github.com/gruenich) fixed a leak and a crash when an allocation throws during parsing.
|
||||
437. [Amelia Barnaby](https://github.com/ameliabarnabyhub) made serializing discarded values to binary formats throw instead of writing corrupt output.
|
||||
438. [Vesko Karaganev](https://github.com/veselink1) made the destructor use the provided allocator.
|
||||
439. [Michael Sam](https://github.com/michaelsam94) kept allocation failures from escaping the `noexcept` destructor.
|
||||
440. [Mihnea Magheru](https://github.com/sakuntalle) added a natvis visualizer for `detail::json_default_base`.
|
||||
441. [Joseph Demarest](https://github.com/JosephDemarest) rejected nested indefinite-length CBOR strings.
|
||||
442. [Afonso Januário](https://github.com/afonsojanu) made `std::hash<basic_json>` consistent with `operator==` for numbers.
|
||||
443. [Kartik Gupta](https://github.com/KartikeyG-world) limited the natvis namespace variants to two ABI tags.
|
||||
444. [manop55555](https://github.com/manop55555) reported a buffer over-read in `dump()` with large indentation values.
|
||||
|
||||
Thanks a lot for helping out! Please [let me know](mailto:mail@nlohmann.me) if I forgot someone.
|
||||
|
||||
@@ -1883,7 +1909,6 @@ The library itself consists of a single header file licensed under the MIT licen
|
||||
- [**cpplint**](https://github.com/cpplint/cpplint) for style checks
|
||||
- [**doctest**](https://github.com/doctest/doctest) for the unit tests
|
||||
- [**Flawfinder**](https://dwheeler.com/flawfinder/) for static analysis
|
||||
- [**GitHub Changelog Generator**](https://github.com/github-changelog-generator/github-changelog-generator) to generate the [ChangeLog](https://github.com/nlohmann/json/blob/develop/ChangeLog.md)
|
||||
- [**Google Benchmark**](https://github.com/google/benchmark) to implement the benchmarks
|
||||
- [**Hedley**](https://nemequ.github.io/hedley/) to avoid re-inventing several compiler-agnostic feature macros
|
||||
- [**include-what-you-use**](https://include-what-you-use.org) to check the included headers
|
||||
@@ -1950,7 +1975,7 @@ For more information, see [JSON With Commas and Comments (JWCC)](https://nigelta
|
||||
|
||||
By default, the library does not preserve the **insertion order of object elements**. This is standards-compliant, as the [JSON standard](https://tools.ietf.org/html/rfc8259.html) defines objects as "an unordered collection of zero or more name/value pairs".
|
||||
|
||||
If you do want to preserve the insertion order, you can try the type [`nlohmann::ordered_json`](https://github.com/nlohmann/json/issues/2179). Alternatively, you can use a more sophisticated ordered map like [`tsl::ordered_map`](https://github.com/Tessil/ordered-map) ([integration](https://github.com/nlohmann/json/issues/546#issuecomment-304447518)) or [`nlohmann::fifo_map`](https://github.com/nlohmann/fifo_map) ([integration](https://github.com/nlohmann/json/issues/485#issuecomment-333652309)).
|
||||
If you do want to preserve the insertion order, you can try the type [`nlohmann::ordered_json`](https://json.nlohmann.me/api/ordered_json/). Alternatively, you can use a more sophisticated ordered map like [`tsl::ordered_map`](https://github.com/Tessil/ordered-map) ([integration](https://github.com/nlohmann/json/issues/546#issuecomment-304447518)) or [`nlohmann::fifo_map`](https://github.com/nlohmann/fifo_map) ([integration](https://github.com/nlohmann/json/issues/485#issuecomment-333652309)).
|
||||
|
||||
See the [**documentation on object order**](https://json.nlohmann.me/features/object_order/) for more information.
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# hashes must list every dependency, so this file was generated with
|
||||
# uv pip compile requirements.in --generate-hashes --python-version 3.14 \
|
||||
# --python-platform x86_64-manylinux_2_39 --no-header --no-annotate
|
||||
# from a requirements.in containing only "semgrep==1.168.0". Dependabot keeps it current.
|
||||
# from a requirements.in containing only "semgrep==1.184.0". Dependabot keeps it current.
|
||||
annotated-types==0.8.0 \
|
||||
--hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \
|
||||
--hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0
|
||||
@@ -295,9 +295,9 @@ charset-normalizer==3.5.2 \
|
||||
--hash=sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424 \
|
||||
--hash=sha256:fd1fbe0f116b6e55da77aca2c6ddcddcfac2186cbf78bdebf40fc156efca389d \
|
||||
--hash=sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f
|
||||
click==8.1.8 \
|
||||
--hash=sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2 \
|
||||
--hash=sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a
|
||||
click==8.4.2 \
|
||||
--hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \
|
||||
--hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76
|
||||
click-option-group==0.5.9 \
|
||||
--hash=sha256:ad2599248bd373e2e19bec5407967c3eec1d0d4fc4a5e77b08a0481e75991080 \
|
||||
--hash=sha256:f94ed2bc4cf69052e0f29592bd1e771a1789bd7bfc482dd0bc482134aff95823
|
||||
@@ -403,9 +403,9 @@ jsonschema-specifications==2025.9.1 \
|
||||
markdown-it-py==4.2.0 \
|
||||
--hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \
|
||||
--hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a
|
||||
mcp==1.23.3 \
|
||||
--hash=sha256:32768af4b46a1b4f7df34e2bfdf5c6011e7b63d7f1b0e321d0fdef4cd6082031 \
|
||||
--hash=sha256:b3b0da2cc949950ce1259c7bfc1b081905a51916fcd7c8182125b85e70825201
|
||||
mcp==1.29.0 \
|
||||
--hash=sha256:52d01f334de1868cc3bb2d6604931126a67631f99a6c5d3b82ba47290315ec36 \
|
||||
--hash=sha256:f5a075bb611f23d6f4d080c6a1699fa62772eebc562ba9e66b306ddde1c755f7
|
||||
mdurl==0.1.2 \
|
||||
--hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \
|
||||
--hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba
|
||||
@@ -609,9 +609,9 @@ pydantic-settings==2.16.2 \
|
||||
pygments==2.21.0 \
|
||||
--hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \
|
||||
--hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c
|
||||
pyjwt==2.13.0 \
|
||||
--hash=sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423 \
|
||||
--hash=sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728
|
||||
pyjwt==2.15.1 \
|
||||
--hash=sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193 \
|
||||
--hash=sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8
|
||||
python-dotenv==1.2.4 \
|
||||
--hash=sha256:42269a8a5b3fd54ffa6f3d84b18abed50064717576b4ecf03dc4a55d8aa04fdc \
|
||||
--hash=sha256:f0d53e69935a851c0dcc78f3ab7aaccd8cabef0b92382b576b824212902873c0
|
||||
@@ -824,21 +824,23 @@ ruamel-yaml-clib==0.2.15 \
|
||||
semantic-version==2.10.0 \
|
||||
--hash=sha256:bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c \
|
||||
--hash=sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177
|
||||
semgrep==1.168.0 \
|
||||
--hash=sha256:09dfacb0530ed4a17bd2deb7914e9a25fc3581d5d84d5365cdac77bbebed8081 \
|
||||
--hash=sha256:1a5d6b5b47347b22bcb1d6f15a5e51a21e734244884624c3494e09bc1de955cd \
|
||||
--hash=sha256:86e99b095b80492b6cfe5087bf4a3b0175a1132c9c9dcd25a96c3745cc58d85a \
|
||||
--hash=sha256:a072b1734b5c54e39cbbe957b10cf0b83a114b9b9fc762b4ea51afe2f114cd11 \
|
||||
--hash=sha256:c4a664f4dda097fbdf657d64f79217e94c4d87502c16c991fac569f07aaecb8e \
|
||||
--hash=sha256:d288699c4c056cc3d7cf82830e00f9c07fac33227c3a5290a2113bd1be63da46 \
|
||||
--hash=sha256:e6b85d84e815ff86f56dcb6b4a6370b98f1bc0e402144daafa9b41aa00ea9a90 \
|
||||
--hash=sha256:fa22f0a41ee3857ecec1b3883b2920b1d41d07ef1bdd7e216da3c519210ae5fb
|
||||
semgrep==1.184.0 \
|
||||
--hash=sha256:0ce3064175a44236dd9fd35603627e3f56139ac61e13ca967bae2b76b2cec9bf \
|
||||
--hash=sha256:0fc8a58e9a62de59ce644dde792dfc36764245d0139ab85632f66a8e0846a8c5 \
|
||||
--hash=sha256:3c86b2961b4663d1338fb965de038c7869571661971b78e18e9346ab57e1e20a \
|
||||
--hash=sha256:42a59e95433fd05a0908f8083e25656c90994c65549e483e0de8a1a0004a7adf \
|
||||
--hash=sha256:cf82d77eb911f921902f3e9d098fb97604c5419c0a87cf7de7ab2944b05e2ceb \
|
||||
--hash=sha256:d847abb8fb6fdde77d9e30a6bc869dbfcb68eff4142dcd77897c52635a7600d9 \
|
||||
--hash=sha256:f794d7c6d33126b6881f80d069583ef35b52bd796d882a5fa249ce1124aba3f3
|
||||
sse-starlette==3.7.0 \
|
||||
--hash=sha256:326416da2ab52e9a41b3f409d3ff897eda45aa68c9f3b7b17d0913aab8d219c3 \
|
||||
--hash=sha256:f1d3136caa12d129284fca3d2ded79bc9af943320b6472712955fea52a6d3eea
|
||||
starlette==1.10.0 \
|
||||
--hash=sha256:478c0ab3b4ced6e82719849c118fa406f0278a3499c1a898cd15cd31bdc38ce4 \
|
||||
--hash=sha256:b8399098d4c1e3522734f7235dc79b1fc967f96cf139c81e0fa30479e0685f9b
|
||||
structlog==26.1.0 \
|
||||
--hash=sha256:e081a26d6c373e6d201eca24eede26d8ffab07f88f477822e679183428d3d91e \
|
||||
--hash=sha256:f63a716cbd1b1291cf7661de7794b455acfa4c43c5bcf1630e6ad5ddc1adb3b7
|
||||
tomli==2.4.1 \
|
||||
--hash=sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853 \
|
||||
--hash=sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe \
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.6 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.6 MiB |
@@ -324,23 +324,15 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief vector_utf8_valid_blocks() from the first block that is not ASCII
|
||||
@brief skip 16-byte blocks of well-formed UTF-8 at p (a character boundary)
|
||||
|
||||
@a block is that block, and @a first is where the check began (only ASCII up
|
||||
to @a block). Returns where fewer than 16 bytes are left, or @a first if the
|
||||
check found an error in any block.
|
||||
|
||||
From a block that is not ASCII on, eight blocks are checked without testing
|
||||
for ASCII again: in text with a few non-ASCII characters (accented letters),
|
||||
every other block has one, and the test would be a mispredicted branch for
|
||||
most blocks. After them, ASCII blocks are skipped with one test again.
|
||||
|
||||
The errors of all blocks are collected and tested once at the end. A test
|
||||
after each block lets the compiler advance the position with a conditional
|
||||
select on its result (Apple Clang 17 does), and each block then waits for the
|
||||
check of the block before: 28 cycles for 16 bytes instead of 4 on Apple M1.
|
||||
Like vector_utf8_run, but quotes, backslashes, and control characters are
|
||||
ordinary characters. Returns the start of the first block in which the check
|
||||
found an error (which may belong to a sequence begun in the block before) or
|
||||
where fewer than 16 bytes are left; a sequence may be cut at that point, so the
|
||||
caller resumes at its start, one sequence at a time.
|
||||
*/
|
||||
JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vector_utf8_check_blocks(const unsigned char* first, const unsigned char* block, const unsigned char* e) noexcept
|
||||
JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vector_utf8_valid_blocks(const unsigned char* block, const unsigned char* e) noexcept
|
||||
{
|
||||
using lookup = utf8_lookup4<>;
|
||||
#if JSON_SCAN_NEON
|
||||
@@ -348,29 +340,27 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vect
|
||||
const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low.data());
|
||||
const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high.data());
|
||||
uint8x16_t prev = vdupq_n_u8(0);
|
||||
uint8x16_t error = vdupq_n_u8(0);
|
||||
while (e - block >= 16)
|
||||
{
|
||||
const uint8x16_t ahead = vld1q_u8(block);
|
||||
if (vmaxvq_u8(vorrq_u8(ahead, prev)) < 0x80)
|
||||
const uint8x16_t in = vld1q_u8(block);
|
||||
if (vmaxvq_u8(vorrq_u8(in, prev)) < 0x80)
|
||||
{
|
||||
// ASCII here, and no sequence from the block before to finish
|
||||
prev = ahead;
|
||||
prev = in;
|
||||
block += 16;
|
||||
continue;
|
||||
}
|
||||
for (int blocks = 8; blocks != 0 && e - block >= 16; --blocks)
|
||||
const uint8x16_t prev1 = vextq_u8(prev, in, 15);
|
||||
const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4)));
|
||||
const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80)));
|
||||
const uint8x16_t err = veorq_u8(vandq_u8(must23, vdupq_n_u8(0x80)), sc);
|
||||
if (vmaxvq_u8(err) != 0)
|
||||
{
|
||||
const uint8x16_t in = vld1q_u8(block);
|
||||
const uint8x16_t prev1 = vextq_u8(prev, in, 15);
|
||||
const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4)));
|
||||
const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80)));
|
||||
error = vorrq_u8(error, veorq_u8(vandq_u8(must23, vdupq_n_u8(0x80)), sc));
|
||||
prev = in;
|
||||
block += 16;
|
||||
break;
|
||||
}
|
||||
prev = in;
|
||||
block += 16;
|
||||
}
|
||||
return vmaxvq_u8(error) != 0 ? first : block;
|
||||
#else
|
||||
const __m128i t1h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_high.data()))); // NOLINT(bugprone-casting-through-void)
|
||||
const __m128i t1l = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_low.data()))); // NOLINT(bugprone-casting-through-void)
|
||||
@@ -378,62 +368,30 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vect
|
||||
const __m128i nibble = _mm_set1_epi8(0x0F);
|
||||
const __m128i zero = _mm_setzero_si128();
|
||||
__m128i prev = zero;
|
||||
__m128i error = zero;
|
||||
while (e - block >= 16)
|
||||
{
|
||||
const __m128i ahead = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(block))); // NOLINT(bugprone-casting-through-void)
|
||||
if (_mm_movemask_epi8(_mm_or_si128(ahead, prev)) == 0)
|
||||
const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(block))); // NOLINT(bugprone-casting-through-void)
|
||||
if (_mm_movemask_epi8(_mm_or_si128(in, prev)) == 0)
|
||||
{
|
||||
prev = ahead;
|
||||
prev = in;
|
||||
block += 16;
|
||||
continue;
|
||||
}
|
||||
for (int blocks = 8; blocks != 0 && e - block >= 16; --blocks)
|
||||
const __m128i prev1 = _mm_alignr_epi8(in, prev, 15);
|
||||
const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)),
|
||||
_mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))),
|
||||
_mm_shuffle_epi8(t2h, _mm_and_si128(_mm_srli_epi16(in, 4), nibble)));
|
||||
const __m128i must23 = _mm_or_si128(_mm_subs_epu8(_mm_alignr_epi8(in, prev, 14), _mm_set1_epi8(0xE0 - 0x80)),
|
||||
_mm_subs_epu8(_mm_alignr_epi8(in, prev, 13), _mm_set1_epi8(0xF0 - 0x80)));
|
||||
const __m128i err = _mm_xor_si128(_mm_and_si128(must23, _mm_set1_epi8(static_cast<char>(-128))), sc);
|
||||
if (_mm_movemask_epi8(_mm_cmpeq_epi8(err, zero)) != 0xFFFF)
|
||||
{
|
||||
const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(block))); // NOLINT(bugprone-casting-through-void)
|
||||
const __m128i prev1 = _mm_alignr_epi8(in, prev, 15);
|
||||
const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)),
|
||||
_mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))),
|
||||
_mm_shuffle_epi8(t2h, _mm_and_si128(_mm_srli_epi16(in, 4), nibble)));
|
||||
const __m128i must23 = _mm_or_si128(_mm_subs_epu8(_mm_alignr_epi8(in, prev, 14), _mm_set1_epi8(0xE0 - 0x80)),
|
||||
_mm_subs_epu8(_mm_alignr_epi8(in, prev, 13), _mm_set1_epi8(0xF0 - 0x80)));
|
||||
error = _mm_or_si128(error, _mm_xor_si128(_mm_and_si128(must23, _mm_set1_epi8(static_cast<char>(-128))), sc));
|
||||
prev = in;
|
||||
block += 16;
|
||||
}
|
||||
}
|
||||
return _mm_movemask_epi8(_mm_cmpeq_epi8(error, zero)) != 0xFFFF ? first : block;
|
||||
#endif
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief skip 16-byte blocks of well-formed UTF-8 at p (a character boundary)
|
||||
|
||||
Like vector_utf8_run, but quotes, backslashes, and control characters are
|
||||
ordinary characters. Returns where fewer than 16 bytes are left; a sequence may
|
||||
be cut at that point, so the caller resumes at its start, one sequence at a
|
||||
time. If the check found an error in any block, @a block itself is returned,
|
||||
for the caller to find the error one sequence at a time.
|
||||
|
||||
ASCII blocks need one test each and none of the tables of the UTF-8 check,
|
||||
which starts at the first block that is not ASCII (see
|
||||
vector_utf8_check_blocks()). Out of line, like the check.
|
||||
*/
|
||||
JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vector_utf8_valid_blocks(const unsigned char* block, const unsigned char* e) noexcept
|
||||
{
|
||||
const unsigned char* const first = block;
|
||||
while (e - block >= 16)
|
||||
{
|
||||
#if JSON_SCAN_NEON
|
||||
if (JSON_HEDLEY_UNLIKELY(vmaxvq_u8(vld1q_u8(block)) >= 0x80))
|
||||
#else
|
||||
if (JSON_HEDLEY_UNLIKELY(_mm_movemask_epi8(_mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(block)))) != 0)) // NOLINT(bugprone-casting-through-void)
|
||||
#endif
|
||||
{
|
||||
return vector_utf8_check_blocks(first, block, e);
|
||||
break;
|
||||
}
|
||||
prev = in;
|
||||
block += 16;
|
||||
}
|
||||
#endif
|
||||
return block;
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -393,7 +393,7 @@ struct depth_counter<false>
|
||||
|
||||
/*!
|
||||
@brief where the double x nearest to a token w * 10^q lies, for 10^15 <= w <
|
||||
10^17 and -17 <= q <= -1 (Eisel-Lemire, then a fixed-point position)
|
||||
10^17 and -16 <= q <= -1 (Eisel-Lemire, then a fixed-point position)
|
||||
|
||||
dump() writes the shortest decimal in the rounding interval of x (the closest
|
||||
one if there are several, see zmij::to_shortest()). Its digits are those of a
|
||||
@@ -1258,13 +1258,15 @@ class view_serializer
|
||||
}
|
||||
for (; p != stop && batch.count != float_batch::capacity(); ++p)
|
||||
{
|
||||
// a token of the source: 16 or 17 digits, at least 8 after the
|
||||
// point, no exponent, 32 bytes to read
|
||||
// a token of the source: 16 or 17 digits, 8 to 16 after the
|
||||
// point, no exponent, 32 bytes to read (17 after the point, with
|
||||
// no digit before it, only in a damaged image loaded with
|
||||
// image_check::bounds: beyond the range of locate_token())
|
||||
const unsigned frac_digits = p->extra >> 8u;
|
||||
const unsigned int_digits = p->extra & 0xFFu;
|
||||
const unsigned count = int_digits + frac_digits;
|
||||
if (p->kind == node_kind::number_float && (p->flags & node_flags::storage) == 0
|
||||
&& count - 16u <= 1u && frac_digits >= 8 && src_end - (src + p->off) >= 33)
|
||||
&& count - 16u <= 1u && frac_digits - 8u <= 8u && src_end - (src + p->off) >= 33)
|
||||
{
|
||||
const auto* const token = reinterpret_cast<const unsigned char*>(src + p->off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
|
||||
const unsigned sign = token[0] == '-' ? 1 : 0;
|
||||
|
||||
Generated
+32
-74
@@ -12593,23 +12593,15 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief vector_utf8_valid_blocks() from the first block that is not ASCII
|
||||
@brief skip 16-byte blocks of well-formed UTF-8 at p (a character boundary)
|
||||
|
||||
@a block is that block, and @a first is where the check began (only ASCII up
|
||||
to @a block). Returns where fewer than 16 bytes are left, or @a first if the
|
||||
check found an error in any block.
|
||||
|
||||
From a block that is not ASCII on, eight blocks are checked without testing
|
||||
for ASCII again: in text with a few non-ASCII characters (accented letters),
|
||||
every other block has one, and the test would be a mispredicted branch for
|
||||
most blocks. After them, ASCII blocks are skipped with one test again.
|
||||
|
||||
The errors of all blocks are collected and tested once at the end. A test
|
||||
after each block lets the compiler advance the position with a conditional
|
||||
select on its result (Apple Clang 17 does), and each block then waits for the
|
||||
check of the block before: 28 cycles for 16 bytes instead of 4 on Apple M1.
|
||||
Like vector_utf8_run, but quotes, backslashes, and control characters are
|
||||
ordinary characters. Returns the start of the first block in which the check
|
||||
found an error (which may belong to a sequence begun in the block before) or
|
||||
where fewer than 16 bytes are left; a sequence may be cut at that point, so the
|
||||
caller resumes at its start, one sequence at a time.
|
||||
*/
|
||||
JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vector_utf8_check_blocks(const unsigned char* first, const unsigned char* block, const unsigned char* e) noexcept
|
||||
JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vector_utf8_valid_blocks(const unsigned char* block, const unsigned char* e) noexcept
|
||||
{
|
||||
using lookup = utf8_lookup4<>;
|
||||
#if JSON_SCAN_NEON
|
||||
@@ -12617,29 +12609,27 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vect
|
||||
const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low.data());
|
||||
const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high.data());
|
||||
uint8x16_t prev = vdupq_n_u8(0);
|
||||
uint8x16_t error = vdupq_n_u8(0);
|
||||
while (e - block >= 16)
|
||||
{
|
||||
const uint8x16_t ahead = vld1q_u8(block);
|
||||
if (vmaxvq_u8(vorrq_u8(ahead, prev)) < 0x80)
|
||||
const uint8x16_t in = vld1q_u8(block);
|
||||
if (vmaxvq_u8(vorrq_u8(in, prev)) < 0x80)
|
||||
{
|
||||
// ASCII here, and no sequence from the block before to finish
|
||||
prev = ahead;
|
||||
prev = in;
|
||||
block += 16;
|
||||
continue;
|
||||
}
|
||||
for (int blocks = 8; blocks != 0 && e - block >= 16; --blocks)
|
||||
const uint8x16_t prev1 = vextq_u8(prev, in, 15);
|
||||
const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4)));
|
||||
const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80)));
|
||||
const uint8x16_t err = veorq_u8(vandq_u8(must23, vdupq_n_u8(0x80)), sc);
|
||||
if (vmaxvq_u8(err) != 0)
|
||||
{
|
||||
const uint8x16_t in = vld1q_u8(block);
|
||||
const uint8x16_t prev1 = vextq_u8(prev, in, 15);
|
||||
const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4)));
|
||||
const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80)));
|
||||
error = vorrq_u8(error, veorq_u8(vandq_u8(must23, vdupq_n_u8(0x80)), sc));
|
||||
prev = in;
|
||||
block += 16;
|
||||
break;
|
||||
}
|
||||
prev = in;
|
||||
block += 16;
|
||||
}
|
||||
return vmaxvq_u8(error) != 0 ? first : block;
|
||||
#else
|
||||
const __m128i t1h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_high.data()))); // NOLINT(bugprone-casting-through-void)
|
||||
const __m128i t1l = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_low.data()))); // NOLINT(bugprone-casting-through-void)
|
||||
@@ -12647,62 +12637,30 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vect
|
||||
const __m128i nibble = _mm_set1_epi8(0x0F);
|
||||
const __m128i zero = _mm_setzero_si128();
|
||||
__m128i prev = zero;
|
||||
__m128i error = zero;
|
||||
while (e - block >= 16)
|
||||
{
|
||||
const __m128i ahead = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(block))); // NOLINT(bugprone-casting-through-void)
|
||||
if (_mm_movemask_epi8(_mm_or_si128(ahead, prev)) == 0)
|
||||
const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(block))); // NOLINT(bugprone-casting-through-void)
|
||||
if (_mm_movemask_epi8(_mm_or_si128(in, prev)) == 0)
|
||||
{
|
||||
prev = ahead;
|
||||
prev = in;
|
||||
block += 16;
|
||||
continue;
|
||||
}
|
||||
for (int blocks = 8; blocks != 0 && e - block >= 16; --blocks)
|
||||
const __m128i prev1 = _mm_alignr_epi8(in, prev, 15);
|
||||
const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)),
|
||||
_mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))),
|
||||
_mm_shuffle_epi8(t2h, _mm_and_si128(_mm_srli_epi16(in, 4), nibble)));
|
||||
const __m128i must23 = _mm_or_si128(_mm_subs_epu8(_mm_alignr_epi8(in, prev, 14), _mm_set1_epi8(0xE0 - 0x80)),
|
||||
_mm_subs_epu8(_mm_alignr_epi8(in, prev, 13), _mm_set1_epi8(0xF0 - 0x80)));
|
||||
const __m128i err = _mm_xor_si128(_mm_and_si128(must23, _mm_set1_epi8(static_cast<char>(-128))), sc);
|
||||
if (_mm_movemask_epi8(_mm_cmpeq_epi8(err, zero)) != 0xFFFF)
|
||||
{
|
||||
const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(block))); // NOLINT(bugprone-casting-through-void)
|
||||
const __m128i prev1 = _mm_alignr_epi8(in, prev, 15);
|
||||
const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)),
|
||||
_mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))),
|
||||
_mm_shuffle_epi8(t2h, _mm_and_si128(_mm_srli_epi16(in, 4), nibble)));
|
||||
const __m128i must23 = _mm_or_si128(_mm_subs_epu8(_mm_alignr_epi8(in, prev, 14), _mm_set1_epi8(0xE0 - 0x80)),
|
||||
_mm_subs_epu8(_mm_alignr_epi8(in, prev, 13), _mm_set1_epi8(0xF0 - 0x80)));
|
||||
error = _mm_or_si128(error, _mm_xor_si128(_mm_and_si128(must23, _mm_set1_epi8(static_cast<char>(-128))), sc));
|
||||
prev = in;
|
||||
block += 16;
|
||||
}
|
||||
}
|
||||
return _mm_movemask_epi8(_mm_cmpeq_epi8(error, zero)) != 0xFFFF ? first : block;
|
||||
#endif
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief skip 16-byte blocks of well-formed UTF-8 at p (a character boundary)
|
||||
|
||||
Like vector_utf8_run, but quotes, backslashes, and control characters are
|
||||
ordinary characters. Returns where fewer than 16 bytes are left; a sequence may
|
||||
be cut at that point, so the caller resumes at its start, one sequence at a
|
||||
time. If the check found an error in any block, @a block itself is returned,
|
||||
for the caller to find the error one sequence at a time.
|
||||
|
||||
ASCII blocks need one test each and none of the tables of the UTF-8 check,
|
||||
which starts at the first block that is not ASCII (see
|
||||
vector_utf8_check_blocks()). Out of line, like the check.
|
||||
*/
|
||||
JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vector_utf8_valid_blocks(const unsigned char* block, const unsigned char* e) noexcept
|
||||
{
|
||||
const unsigned char* const first = block;
|
||||
while (e - block >= 16)
|
||||
{
|
||||
#if JSON_SCAN_NEON
|
||||
if (JSON_HEDLEY_UNLIKELY(vmaxvq_u8(vld1q_u8(block)) >= 0x80))
|
||||
#else
|
||||
if (JSON_HEDLEY_UNLIKELY(_mm_movemask_epi8(_mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(block)))) != 0)) // NOLINT(bugprone-casting-through-void)
|
||||
#endif
|
||||
{
|
||||
return vector_utf8_check_blocks(first, block, e);
|
||||
break;
|
||||
}
|
||||
prev = in;
|
||||
block += 16;
|
||||
}
|
||||
#endif
|
||||
return block;
|
||||
}
|
||||
#endif
|
||||
|
||||
Generated
+6
-4
@@ -8841,7 +8841,7 @@ struct depth_counter<false>
|
||||
|
||||
/*!
|
||||
@brief where the double x nearest to a token w * 10^q lies, for 10^15 <= w <
|
||||
10^17 and -17 <= q <= -1 (Eisel-Lemire, then a fixed-point position)
|
||||
10^17 and -16 <= q <= -1 (Eisel-Lemire, then a fixed-point position)
|
||||
|
||||
dump() writes the shortest decimal in the rounding interval of x (the closest
|
||||
one if there are several, see zmij::to_shortest()). Its digits are those of a
|
||||
@@ -9706,13 +9706,15 @@ class view_serializer
|
||||
}
|
||||
for (; p != stop && batch.count != float_batch::capacity(); ++p)
|
||||
{
|
||||
// a token of the source: 16 or 17 digits, at least 8 after the
|
||||
// point, no exponent, 32 bytes to read
|
||||
// a token of the source: 16 or 17 digits, 8 to 16 after the
|
||||
// point, no exponent, 32 bytes to read (17 after the point, with
|
||||
// no digit before it, only in a damaged image loaded with
|
||||
// image_check::bounds: beyond the range of locate_token())
|
||||
const unsigned frac_digits = p->extra >> 8u;
|
||||
const unsigned int_digits = p->extra & 0xFFu;
|
||||
const unsigned count = int_digits + frac_digits;
|
||||
if (p->kind == node_kind::number_float && (p->flags & node_flags::storage) == 0
|
||||
&& count - 16u <= 1u && frac_digits >= 8 && src_end - (src + p->off) >= 33)
|
||||
&& count - 16u <= 1u && frac_digits - 8u <= 8u && src_end - (src + p->off) >= 33)
|
||||
{
|
||||
const auto* const token = reinterpret_cast<const unsigned char*>(src + p->off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
|
||||
const unsigned sign = token[0] == '-' ? 1 : 0;
|
||||
|
||||
@@ -1165,6 +1165,44 @@ TEST_CASE("json_view images: check")
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("float tokens with 17 digits after the point and none before it")
|
||||
{
|
||||
// No JSON number has this layout. dump() batches tokens of 16 or 17
|
||||
// digits, but only those with 8 to 16 fraction digits are in its
|
||||
// range: 17 read past the table of powers of five (caught by a
|
||||
// bounds-checked std::array), and on x86-64 wrote 18 digits
|
||||
// (the assertion of write_decimal()).
|
||||
const std::vector<std::uint8_t> img = json_document::parse("[1.2345678901234567]" + std::string(40, ' ')).save();
|
||||
node n = node_at(img, 1);
|
||||
REQUIRE(n.len == 18);
|
||||
n.extra = 17u << 8u;
|
||||
std::vector<std::uint8_t> b = img;
|
||||
set_node(b, 1, n);
|
||||
std::memcpy(b.data() + text_at(img) + n.off, ".99999999999999999", 18);
|
||||
const json_document d = json_document::load(b, image_check::bounds);
|
||||
const std::string dumped = d.root().dump();
|
||||
CAPTURE(dumped)
|
||||
CHECK(json::parse(dumped)[0].is_number());
|
||||
CHECK(dumped == d.root().materialize().dump());
|
||||
|
||||
// (the input of OSS-Fuzz issue 583898810: the token is " " and
|
||||
// 15 nines, then "1{")
|
||||
const std::vector<std::uint8_t> fuzzed =
|
||||
{
|
||||
0x4E, 0x4A, 0x56, 0x49, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x22, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x07, 0x10, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x20, 0x20, 0x20, 0x20,
|
||||
0x20, 0x39, 0x39, 0x39, 0x39, 0x39, 0x39, 0x39, 0x39, 0x39, 0x39, 0x39, 0x39, 0x39, 0x39, 0x39,
|
||||
0x31, 0x7B, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
|
||||
0x20, 0x20, 0x00, 0x00
|
||||
};
|
||||
CHECK(load_result(fuzzed, image_check::full) == check_failed);
|
||||
const json_document f = json_document::load(fuzzed, image_check::bounds);
|
||||
CHECK_NOTHROW(f.root().dump());
|
||||
}
|
||||
|
||||
SECTION("nodes that share a range")
|
||||
{
|
||||
// [big string, then n strings made to point to the big string]: every
|
||||
|
||||
@@ -1669,60 +1669,3 @@ TEST_CASE("UTF-8 check of whole strings")
|
||||
CHECK(valid > 20000);
|
||||
CHECK(valid < 80000);
|
||||
}
|
||||
|
||||
TEST_CASE("UTF-8 check of long strings")
|
||||
{
|
||||
// Where a vector check is available, valid_utf8_string() skips ASCII 16
|
||||
// bytes at a time, checks eight blocks from the first one that is not
|
||||
// ASCII without testing for ASCII in between, and collects the errors of
|
||||
// all blocks until the end. A sequence of each length and each kind of
|
||||
// ill-formed sequence is put at every offset of strings that are longer
|
||||
// than eight blocks: before, inside, at the end of, and after these
|
||||
// blocks, and across every block boundary.
|
||||
const char* const pieces[] = // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
|
||||
{
|
||||
"\xC3\xA9", "\xE2\x82\xAC", "\xE0\xA0\x80", "\xEF\xBF\xBF", "\xF0\x9F\x98\x80", "\xF4\x8F\xBF\xBF",
|
||||
"\x80", "\xC0\xAF", "\xC3", "\xE2\x82", "\xED\xA0\x80", "\xF0\x9F", "\xF0\x9F\x98", "\xF4\x90\x80\x80", "\xFF"
|
||||
};
|
||||
// the text around the piece: ASCII only, a sequence at the start (so that
|
||||
// the check of eight blocks starts there), and a sequence in every block
|
||||
const char* const fillers[] = {"", "\xC3\xA9", "\xE2\x82\xAC"}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
|
||||
std::size_t valid = 0;
|
||||
std::size_t invalid = 0;
|
||||
for (const char* const piece : pieces)
|
||||
{
|
||||
for (std::size_t filler = 0; filler < 3; ++filler)
|
||||
{
|
||||
std::string base;
|
||||
if (filler == 1)
|
||||
{
|
||||
base += fillers[1];
|
||||
}
|
||||
while (base.size() < 300)
|
||||
{
|
||||
base += (filler == 2 && base.size() % 16 == 5) ? std::string(fillers[2]) : std::string("x");
|
||||
}
|
||||
for (std::size_t offset = 0; offset <= base.size(); ++offset)
|
||||
{
|
||||
// (not inside a sequence of the text around the piece)
|
||||
if (offset < base.size() && (static_cast<unsigned char>(base[offset]) & 0xC0u) == 0x80u)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
std::string s = base;
|
||||
s.insert(offset, piece);
|
||||
const auto* data = reinterpret_cast<const unsigned char*>(s.data());
|
||||
const bool expected = nlohmann::detail::is_valid_utf8(s);
|
||||
(expected ? valid : invalid) += 1;
|
||||
CAPTURE(offset)
|
||||
CAPTURE(filler)
|
||||
CAPTURE(piece)
|
||||
CHECK(nlohmann::detail::valid_utf8_string(data, s.size()) == expected);
|
||||
CHECK((nlohmann::detail::valid_utf8_prefix(data, s.size()) == s.size()) == expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
// the six well-formed pieces are valid at every offset, the others at none
|
||||
CHECK(valid * 9 == invalid * 6);
|
||||
CHECK(valid > 5000);
|
||||
}
|
||||
Reference in new issue
Block a user