diff --git a/.github/workflows/cancel_closed_pr_runs.yml b/.github/workflows/cancel_closed_pr_runs.yml index 91005de52..fd30c5047 100644 --- a/.github/workflows/cancel_closed_pr_runs.yml +++ b/.github/workflows/cancel_closed_pr_runs.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/check_amalgamation.yml b/.github/workflows/check_amalgamation.yml index 253531e40..4fbec781d 100644 --- a/.github/workflows/check_amalgamation.yml +++ b/.github/workflows/check_amalgamation.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit @@ -46,7 +46,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/check_docs_links.yml b/.github/workflows/check_docs_links.yml index f7e28ab69..0ec9778e6 100644 --- a/.github/workflows/check_docs_links.yml +++ b/.github/workflows/check_docs_links.yml @@ -20,7 +20,7 @@ jobs: timeout-minutes: 30 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index f5feb9a8f..02a5a8173 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 61036d79d..99541a96f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -27,7 +27,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/comment_check_amalgamation.yml b/.github/workflows/comment_check_amalgamation.yml index ba2fcad7b..558d5fba4 100644 --- a/.github/workflows/comment_check_amalgamation.yml +++ b/.github/workflows/comment_check_amalgamation.yml @@ -20,7 +20,7 @@ jobs: pull-requests: write steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 27c229572..8034aa1c8 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/flawfinder.yml b/.github/workflows/flawfinder.yml index ffc8c2a43..2fae15234 100644 --- a/.github/workflows/flawfinder.yml +++ b/.github/workflows/flawfinder.yml @@ -31,7 +31,7 @@ jobs: security-events: write steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/json_view_benchmarks.yml b/.github/workflows/json_view_benchmarks.yml index d87357c42..49876a07b 100644 --- a/.github/workflows/json_view_benchmarks.yml +++ b/.github/workflows/json_view_benchmarks.yml @@ -54,7 +54,7 @@ jobs: timeout-minutes: 60 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 85c7a513d..004039586 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/publish_documentation.yml b/.github/workflows/publish_documentation.yml index ad53308d7..ba8f435bf 100644 --- a/.github/workflows/publish_documentation.yml +++ b/.github/workflows/publish_documentation.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 62e1d504a..47cde1be7 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -40,7 +40,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 8b1e6ed5e..ce5c115b4 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -36,7 +36,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index da7d91fce..b624f9f40 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index c93c0a706..581fb6081 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -25,7 +25,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -38,7 +38,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Check that Meson and pkg-config offer the CMake options run: make check_build_options - name: Install Meson @@ -80,7 +80,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit @@ -92,7 +92,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -112,7 +112,7 @@ jobs: target: [ci_test_single_header, ci_cppcheck, ci_cpplint, ci_reproducible_tests, ci_non_git_tests, ci_offline_testdata, ci_reuse_compliance, ci_test_valgrind] steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit @@ -123,7 +123,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -142,7 +142,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -168,7 +168,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -178,7 +178,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit @@ -244,7 +244,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: CXX=g++-${{ matrix.compiler }} cmake -S . -B build -DJSON_CI=On - name: Build @@ -262,7 +262,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -279,7 +279,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Disable filesystem support (std::filesystem is unusable) for clang 7 / 8 / 9 / 10 run: echo "JSON_FORCED_GLOBAL_COMPILE_OPTIONS=-DJSON_HAS_FILESYSTEM=0;-DJSON_HAS_EXPERIMENTAL_FILESYSTEM=0" >> "$GITHUB_ENV" if: ${{ matrix.compiler == '7' || matrix.compiler == '8' || matrix.compiler == '9' || matrix.compiler == '10' }} @@ -299,7 +299,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -319,7 +319,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build with libc++ @@ -346,7 +346,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -366,7 +366,7 @@ jobs: # import-std support. Its opt-in token is CMake-version-specific, so pin # CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt. - name: Get pinned CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 with: cmakeVersion: 4.3.4 # Clang: the std library module is provided by libc++ (the image's libstdc++ @@ -392,7 +392,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -409,7 +409,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -423,7 +423,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -433,7 +433,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit @@ -443,7 +443,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Run CMake run: cmake -S . -B build -DCMAKE_TOOLCHAIN_FILE=$EMSDK/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -GNinja - name: Build @@ -456,7 +456,7 @@ jobs: target: [ci_test_examples, ci_test_build_documentation, ci_test_documentation_mermaid] steps: - name: Harden Runner - uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 with: egress-policy: audit @@ -484,7 +484,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Check that the job runs natively on arm64 run: uname -m && test "$(uname -m)" = aarch64 - name: Run CMake diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 59647ed94..b9f9d0251 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -90,7 +90,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 - name: Set extra CXX_FLAGS for latest std_version # /wd5285 silences C5285 emitted by the bundled third-party doctest.h, which # specializes std::tuple (newly diagnosed by the VS2026 v145 toolset) @@ -216,7 +216,7 @@ jobs: # import-std support. Its opt-in token is CMake-version-specific, so pin # CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt. - name: Get pinned CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + uses: lukka/get-cmake@f86dd60acae0d132056913e248bdabf0e0580b7a # v4.4.5 with: cmakeVersion: 4.3.4 # CMAKE_CXX_FLAGS replaces CMake's MSVC defaults (which include /EHsc), and diff --git a/cmake/ci.cmake b/cmake/ci.cmake index 0d4b3db92..b41ab8b62 100644 --- a/cmake/ci.cmake +++ b/cmake/ci.cmake @@ -348,20 +348,20 @@ add_custom_target(ci_test_no_thread_local add_custom_target(ci_test_coverage COMMAND CXX=g++ ${CMAKE_COMMAND} - -DCMAKE_BUILD_TYPE=Debug -GNinja -DCMAKE_CXX_FLAGS="--coverage;-fprofile-arcs;-ftest-coverage" + -DCMAKE_BUILD_TYPE=Debug -GNinja -DCMAKE_CXX_FLAGS="--coverage;-fprofile-arcs;-ftest-coverage;-fvar-tracking-assignments" -DJSON_BuildTests=ON -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_coverage COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_coverage COMMAND cd ${PROJECT_BINARY_DIR}/build_coverage && ${CMAKE_CTEST_COMMAND} --parallel ${N} --output-on-failure COMMAND CXX=g++ ${CMAKE_COMMAND} - -DCMAKE_BUILD_TYPE=Debug -GNinja -DCMAKE_CXX_FLAGS="-m32;--coverage;-fprofile-arcs;-ftest-coverage" + -DCMAKE_BUILD_TYPE=Debug -GNinja -DCMAKE_CXX_FLAGS="-m32;--coverage;-fprofile-arcs;-ftest-coverage;-fvar-tracking-assignments" -DJSON_BuildTests=ON -DJSON_32bitTest=ONLY -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_coverage32 COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_coverage32 COMMAND cd ${PROJECT_BINARY_DIR}/build_coverage32 && ${CMAKE_CTEST_COMMAND} --parallel ${N} --output-on-failure - COMMAND ${LCOV_TOOL} --directory . --capture --output-file json.info --rc branch_coverage=1 --rc geninfo_unexecuted_blocks=1 --ignore-errors mismatch --ignore-errors unused --ignore-errors inconsistent + COMMAND ${LCOV_TOOL} --directory . --capture --output-file json.info --rc branch_coverage=1 --ignore-errors mismatch --ignore-errors unused --ignore-errors inconsistent COMMAND ${LCOV_TOOL} -e json.info ${SRC_FILES} --output-file json.info.filtered --rc branch_coverage=1 --ignore-errors unused --ignore-errors inconsistent COMMAND ${CMAKE_SOURCE_DIR}/tests/thirdparty/imapdl/filterbr.py json.info.filtered > json.info.filtered.noexcept COMMAND ${GENHTML_TOOL} --title "JSON for Modern C++" --legend --demangle-cpp --output-directory html --show-details --branch-coverage --ignore-errors inconsistent json.info.filtered.noexcept diff --git a/cmake/test.cmake b/cmake/test.cmake index 5922c5106..9a42ac263 100644 --- a/cmake/test.cmake +++ b/cmake/test.cmake @@ -178,8 +178,10 @@ function(_json_test_add_test test_name file main cxx_standard) COMMAND ${memcheck_command} $ ${DOCTEST_TEST_FILTER} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} ) + # under Valgrind the slowest tests take close to 25 minutes, so ctest's + # default timeout of 1500 seconds leaves no headroom set_tests_properties(${test_target}_valgrind PROPERTIES - LABELS "valgrind" FIXTURES_REQUIRED TEST_DATA + LABELS "valgrind" FIXTURES_REQUIRED TEST_DATA TIMEOUT 3600 ) endif() endfunction() diff --git a/docs/mkdocs/docs/api/basic_json/get.md b/docs/mkdocs/docs/api/basic_json/get.md index 016595d5f..06e34218a 100644 --- a/docs/mkdocs/docs/api/basic_json/get.md +++ b/docs/mkdocs/docs/api/basic_json/get.md @@ -120,6 +120,16 @@ overload (3). See [Number conversion](../../features/types/number_handling.md#number-conversion) for more information. +!!! note "Booleans as numbers" + + A JSON boolean converts to `#!cpp 0` or `#!cpp 1` for every arithmetic type **except** [`number_integer_t`](number_integer_t.md), + [`number_unsigned_t`](number_unsigned_t.md) and [`number_float_t`](number_float_t.md) (by default `#!cpp std::int64_t`, + `#!cpp std::uint64_t`, and `#!cpp double`), for which `#!cpp get()` throws + [`type_error.302`](../../home/exceptions.md#jsonexceptiontype_error302) ("type must be number, but is boolean"). So + `#!cpp json(true).get()` is `#!cpp 1`, while `#!cpp json(true).get()` and + `#!cpp json(true).get()` throw. This is the behavior of all 3.x versions; + [`basic_json_view::get()`](../basic_json_view/get.md) behaves the same. + !!! note "`std::optional` conversions" Prior to version 3.13.0, `#!cpp get>()` (and other conversions to `std::optional`) failed to diff --git a/docs/mkdocs/docs/api/basic_json/insert.md b/docs/mkdocs/docs/api/basic_json/insert.md index 12619a2d8..721cb8ae7 100644 --- a/docs/mkdocs/docs/api/basic_json/insert.md +++ b/docs/mkdocs/docs/api/basic_json/insert.md @@ -65,7 +65,12 @@ the same index, which is now a different value. ## Exception safety -Strong exception safety: if an exception occurs, the original value stays intact. +Overload (1) gives the strong guarantee: if an exception is thrown, there are no changes in the JSON value. + +Overloads (2) to (5) give the basic guarantee: if an exception is thrown while the elements or members are copied +(for example, `std::bad_alloc`), the JSON value may be partially modified, but it stays a valid array or object. + +The exceptions listed below are thrown before the JSON value is changed. ## Exceptions diff --git a/docs/mkdocs/docs/api/basic_json/patch_inplace.md b/docs/mkdocs/docs/api/basic_json/patch_inplace.md index ff65e8dbd..c3cd4f530 100644 --- a/docs/mkdocs/docs/api/basic_json/patch_inplace.md +++ b/docs/mkdocs/docs/api/basic_json/patch_inplace.md @@ -15,7 +15,9 @@ the patch. This function applies a JSON patch in place and returns void. ## Exception safety -No guarantees, value may be corrupted by an unsuccessful patch operation. +No guarantees: if an operation fails, the operations before it stay applied, and a failing `move` operation may already +have removed the value at its "from" location. Use [`patch`](patch.md) if the value must be unchanged after a failed +patch. ## Exceptions @@ -62,8 +64,19 @@ the JSON value, the complexity can usually be neglected. ## Notes Unlike [`patch`](patch.md), `patch_inplace` applies the operation "in place" and no copy of the JSON value is created. -That makes it faster for large documents by avoiding the copy. However, the JSON value might be corrupted if the -function throws an exception. +That makes it faster for large documents by avoiding the copy. + +!!! warning "Not atomic" + + `patch_inplace` applies the operations one after the other and does not undo them when one fails. After an + exception, the value holds the result of the operations before the failing one, and a failing `move` operation + (a "remove" followed by an "add") may already have removed the value at its "from" location; see the second example + below. [RFC 6902](https://tools.ietf.org/html/rfc6902#section-5) requires that a patch with a failing operation is + not deemed successful, but leaves open what happens to the document. + + If the value must stay unchanged when the patch fails, use [`patch`](patch.md), which applies the patch to a copy: + `#!cpp j = j.patch(json_patch);`. Validating the patch beforehand is not enough, as "test" operations and missing + paths are only detected while the patch is applied. Duplicate member names inside an operation object are resolved by the parser before the patch is applied (the last one is kept, see [`object_t`](object_t.md#behavior)), so `#!json {"op": "add", "op": "remove", "path": "/a"}` is a diff --git a/docs/mkdocs/docs/api/basic_json_view/get.md b/docs/mkdocs/docs/api/basic_json_view/get.md index 7ee7952bf..1739d372b 100644 --- a/docs/mkdocs/docs/api/basic_json_view/get.md +++ b/docs/mkdocs/docs/api/basic_json_view/get.md @@ -11,7 +11,8 @@ For the types below, the conversion works directly on the flat index — no `Bas - `#!cpp bool` - arithmetic types other than `#!cpp bool` (for a boolean, `#!cpp 0`/`#!cpp 1`, exactly as for - [`BasicJsonType::get()`](../basic_json/get.md)) + [`BasicJsonType::get()`](../basic_json/get.md#notes), except for `number_integer_t`, `number_unsigned_t` and + `number_float_t`, which throw `type_error.302` for a boolean, as `BasicJsonType::get()` does) - `#!cpp std::nullptr_t` - `#!cpp std::basic_string` (including `string_t`) — a copy of the string - [`string_view_t`](index.md#member-types) — **no copy**: the returned view points into the document's diff --git a/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md b/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md index 8a772791a..4dac9f9ff 100644 --- a/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md +++ b/docs/mkdocs/docs/api/macros/json_diagnostic_positions.md @@ -35,6 +35,9 @@ Note that enabling this macro increases the size of every JSON value by two `std overhead to parsing, copying JSON value objects, and the generation of error messages for exceptions. It also causes these values to be reported in those error messages. +If [`JSON_DIAGNOSTICS`](json_diagnostics.md) is also set, the message contains the path and the byte range, such as +`(/1) (bytes 4-8)`. An exception about the root value has no path and only the range, such as `(bytes 0-4)`. + ## Default definition The default value is `0` (position diagnostics are switched off). diff --git a/docs/mkdocs/docs/api/macros/json_disable_unlocked_file_io.md b/docs/mkdocs/docs/api/macros/json_disable_unlocked_file_io.md index 2c266ba2d..031566cfe 100644 --- a/docs/mkdocs/docs/api/macros/json_disable_unlocked_file_io.md +++ b/docs/mkdocs/docs/api/macros/json_disable_unlocked_file_io.md @@ -5,7 +5,9 @@ ``` When defined to `1`, parsing from a `#!cpp FILE*` reads every character with `std::fgetc`, as in version 3.12.0 and -earlier, instead of locking the file once and reading it with the unlocked stdio functions. +earlier, instead of locking the file once and reading it with the unlocked stdio functions. This also turns off the +scan of the `FILE`'s buffer in bulk (see below), which [`JSON_DISABLE_STREAM_BULK_SCAN`](json_disable_stream_bulk_scan.md) +does not affect. It does not change any result: values, exceptions, and the position of the file after parsing are the same either way. Only the speed and the locking of the file differ (see the notes on locking below). @@ -37,9 +39,17 @@ The default value is `0`: the file is locked once and read with the unlocked std | Microsoft (MSVC, clang-cl) | `_lock_file`, `_fgetc_nolock`, `_unlock_file` | | others (e.g., musl, MinGW, newlib) | `std::fgetc`, as before | - This makes parsing from a `FILE*` 3 to 4 times as fast with the Apple C library and 1.1 to 1.3 times as fast with - glibc, whose locking is cheaper. The binary formats ([`from_cbor`](../basic_json/from_cbor.md), - [`from_msgpack`](../basic_json/from_msgpack.md), ...) read from a `FILE*` the same way. + With glibc and the Apple C library, the parser also scans the characters that the `FILE` already holds in its + buffer in bulk, as it scans the get area of a stream (see + [`JSON_DISABLE_STREAM_BULK_SCAN`](json_disable_stream_bulk_scan.md)): it reads the same fields of the `FILE` that + the C library's own `getc_unlocked` reads, and it consumes exactly the characters that `getc_unlocked` would have + returned, so the file is left where it was left before. + + Locking the file once makes parsing from a `FILE*` 3 to 4 times as fast with the Apple C library and 1.1 to 1.3 + times as fast with glibc, whose locking is cheaper. Scanning the buffer in bulk makes `parse()` another 1.7 to 2.6 + times as fast (and `accept()` 2.9 to 4.4 times), about as fast as parsing from a `std::ifstream`. The binary + formats ([`from_cbor`](../basic_json/from_cbor.md), [`from_msgpack`](../basic_json/from_msgpack.md), ...) read + from a `FILE*` the same way. !!! note "Locking" diff --git a/docs/mkdocs/docs/features/files.md b/docs/mkdocs/docs/features/files.md new file mode 100644 index 000000000..3833f13db --- /dev/null +++ b/docs/mkdocs/docs/features/files.md @@ -0,0 +1,130 @@ +# Reading and writing files + +The library has no functions that take a file name. It reads from and writes to streams, so files are handled with +`#!cpp std::ifstream` and `#!cpp std::ofstream` (or a `#!cpp FILE*` when reading). This page collects the recipes for +the common cases and the mistakes that show up again and again. + +## Reading a JSON file + +Pass the stream to [`parse`](../api/basic_json/parse.md) directly, and check that the file was opened: + +```cpp +#include +#include +#include +#include + +using json = nlohmann::json; + +json read_json_file(const std::string& path) +{ + std::ifstream f(path); + if (!f.is_open()) + { + throw std::runtime_error("cannot open " + path); + } + return json::parse(f); +} +``` + +- **Check that the file is open.** A stream that failed to open looks like an empty input, so `parse` reports + [`parse_error.101`](../home/exceptions.md#jsonexceptionparse_error101) ("unexpected end of input") at line 1, + column 1 instead of saying that the file is missing. A relative path is resolved against the working directory of the + running program, which is often not the directory of the source file (see the + [FAQ](../home/faq.md#parsing-a-file-fails-with-unexpected-end-of-input)). +- **Don't read the file into a string first.** Parsing from a `#!cpp std::ifstream` is only about 10 to 20 % slower + than reading the whole file into a string with one `read()` and parsing that, and it needs no memory for the copy (see + [input sources and performance](parsing/index.md#input-sources-and-performance)). The common way to read a file with + `#!cpp std::istreambuf_iterator` is so slow that it takes about twice as long as parsing from the stream. Read + the file into a string only if you need the text itself, for example for an error message. This advice is for + `#!cpp json::parse`. [`json_document`](json_view.md) reads a stream into a buffer that it owns anyway, so there is no + copy to save: reading the file into a `#!cpp std::string` yourself and passing that as an lvalue, which the document + borrows, was about 6 to 12 % faster than parsing from the stream, with the same single buffer. +- `parse` requires the file to hold exactly one JSON value. Files with one value per line are read with + [JSON Lines](parsing/json_lines.md); files too large for memory are covered in + [Large, streamed, and partial input](parsing/large_and_partial_input.md). + +## Writing a JSON file + +[`dump`](../api/basic_json/dump.md) the value into a string first, then write the string: + +```cpp +void write_json_file(const std::string& path, const json& j) +{ + const std::string text = j.dump(4) + '\n'; // may throw: nothing is written yet + + std::ofstream f(path, std::ios::binary); + if (!f.is_open()) + { + throw std::runtime_error("cannot open " + path); + } + f << text; + f.close(); + if (f.fail()) + { + throw std::runtime_error("cannot write " + path); + } +} +``` + +- **Serialize before opening the file.** Opening a `#!cpp std::ofstream` truncates the file. `dump` and + [`operator<<`](../api/operator_ltlt.md) throw [`type_error.316`](../home/exceptions.md#jsonexceptiontype_error316) + when a string is not valid UTF-8 (see [handling invalid UTF-8](serialization.md#handling-invalid-utf-8)), and + `#!cpp f << std::setw(4) << j` may already have written part of the value when it throws. Serializing into a string + first leaves the old file untouched in that case. +- **Check the stream after closing it.** A full disk or a missing permission sets the stream's error state; it does not + throw unless you enabled exceptions on the stream with `exceptions()`. +- `std::ios::binary` writes the text as it is. Without it, Windows writes `\r\n` for every newline of a pretty-printed + value. + +## Replacing a file without corrupting it + +The library never writes a file itself, so this section is not about the library. It is general C++ advice for a +problem users of the library keep running into. + +A program that crashes or is killed while it writes leaves a truncated file behind. To make sure readers see either the +old or the new contents, write to a temporary file in the same directory and rename it over the original +(C++17, `#!cpp `): + +```cpp +#include + +void replace_json_file(const std::filesystem::path& path, const json& j) +{ + std::filesystem::path tmp = path; + tmp += ".tmp"; // use a unique name if several processes may write the file + + write_json_file(tmp.string(), j); + std::filesystem::rename(tmp, path); // replaces path +} +``` + +The temporary file must be in the same directory (more precisely, on the same file system) as the target: a rename +across file systems does not fall back to a copy, it fails and throws `#!cpp std::filesystem::filesystem_error`. On +POSIX systems, the rename atomically replaces the file. Standard C++ has no way to flush a file to the disk (`fsync`), +so this protects against a crashed program, but not necessarily against a power loss. + +## Binary formats + +The [binary formats](binary_formats/index.md) work the same way, with the streams opened in binary mode: + +```cpp +// read +std::ifstream in("data.cbor", std::ios::binary); +json j = json::from_cbor(in); + +// write +const std::vector bytes = json::to_cbor(j); +std::ofstream out("data.cbor", std::ios::binary); +out.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); +``` + +Without `std::ios::binary`, Windows turns the bytes `0x0D 0x0A` into `0x0A` and may stop at a byte `0x1A` while reading, +and turns `0x0A` into `0x0D 0x0A` while writing, which corrupts the data. + +## See also + +- [Parsing](parsing/index.md) - the inputs `parse` accepts +- [Serialization](serialization.md) - the options of `dump` +- [`parse`](../api/basic_json/parse.md), [`dump`](../api/basic_json/dump.md) +- [`operator>>`](../api/operator_gtgt.md), [`operator<<`](../api/operator_ltlt.md) diff --git a/docs/mkdocs/docs/features/index.md b/docs/mkdocs/docs/features/index.md index f9b0f20cc..d8c6c9367 100644 --- a/docs/mkdocs/docs/features/index.md +++ b/docs/mkdocs/docs/features/index.md @@ -46,6 +46,8 @@ C++ types, and finally serialize it again. [BJData](binary_formats/bjdata.md), [BON8](binary_formats/bon8.md), [BSON](binary_formats/bson.md), [CBOR](binary_formats/cbor.md), [MessagePack](binary_formats/messagepack.md), or [UBJSON](binary_formats/ubjson.md). - [Binary values](binary_values.md) — store and exchange raw byte sequences. +- [Reading and writing files](files.md) — read and write files safely, including replacing a file without + corrupting it. ## How values are stored and configured diff --git a/docs/mkdocs/docs/features/json_patch.md b/docs/mkdocs/docs/features/json_patch.md index ea22b4e15..d9e96c831 100644 --- a/docs/mkdocs/docs/features/json_patch.md +++ b/docs/mkdocs/docs/features/json_patch.md @@ -29,7 +29,8 @@ other three continue on its result. [`patch`](../api/basic_json/patch.md) leaves the original value unchanged and returns the patched result as a copy. If the document is large and the original value is no longer needed, - [`patch_inplace`](../api/basic_json/patch_inplace.md) applies the same operations in place instead. + [`patch_inplace`](../api/basic_json/patch_inplace.md) applies the same operations in place instead. Unlike + `patch`, it is not atomic: if an operation fails, the operations before it stay applied. !!! note "Duplicate member names" diff --git a/docs/mkdocs/docs/features/json_view.md b/docs/mkdocs/docs/features/json_view.md index 44bf581a5..5ab2281f0 100644 --- a/docs/mkdocs/docs/features/json_view.md +++ b/docs/mkdocs/docs/features/json_view.md @@ -160,7 +160,9 @@ document: `#!cpp auto v = json_document::parse(text).root();` does not compile. wide. The limit is 16 bytes below 2^32, because the parser reads in 16-byte steps, and a position one step past the end must still fit. - **A stream is always read to its end.** There is no partial/streaming read of an `#!cpp std::istream`: the stream - is read to its end into a buffer the document owns, and parsing starts after that. + is read to its end into a buffer the document owns, and parsing starts after that. For a file, reading it into a + `#!cpp std::string` yourself and parsing that (an lvalue is borrowed) is therefore not more expensive in memory, and + was about 6 to 12 % faster than parsing from an `#!cpp std::ifstream`. - **Fewer source positions after edits.** With [`JSON_DIAGNOSTIC_POSITIONS`](../api/macros/json_diagnostic_positions.md) enabled, [`materialize()`](../api/basic_json_view/materialize.md) sets the same positions as `#!cpp basic_json::parse()`. diff --git a/docs/mkdocs/docs/features/parsing/index.md b/docs/mkdocs/docs/features/parsing/index.md index 0a5b7e4a8..1bc654e01 100644 --- a/docs/mkdocs/docs/features/parsing/index.md +++ b/docs/mkdocs/docs/features/parsing/index.md @@ -35,6 +35,8 @@ std::ifstream f("example.json"); json data = json::parse(f); ``` +See [Reading and writing files](../files.md) for checking that the file was opened and for writing files safely. + The input must be encoded in UTF-8; other encodings are not supported. The exception is wide-character input, which is read as UTF-16 or UTF-32 — depending on the character type — and converted to UTF-8; see the [`parse`](../../api/basic_json/parse.md) page for an example. A single input may contain only one JSON value. @@ -75,7 +77,7 @@ them faster than others: |--------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------| | strings, string views, string literals, contiguous containers, pointer or iterator ranges over contiguous memory | the whole input is scanned in bulk: runs of string characters, numbers, and indentation are recognized several bytes at a time | | `#!cpp std::istream` (`#!cpp std::ifstream`, `#!cpp std::istringstream`, ...) | the characters the stream buffer holds are scanned in bulk like a string; the stream buffer is asked for more when they are used up | -| `#!cpp FILE*` | one character at a time, with the file locked once for the whole parse | +| `#!cpp FILE*` | with glibc and the Apple C library, the characters the `FILE` holds in its buffer are scanned in bulk like a stream's; elsewhere one character at a time; the file is locked once for the whole parse | | other iterator ranges (e.g., `#!cpp std::list`, `#!cpp std::istreambuf_iterator`), wide strings | one character at a time | Parsing a file from a `#!cpp std::ifstream` is therefore only about 10 to 20 % slower than reading the whole file into a @@ -84,6 +86,9 @@ ahead: after parsing, it is positioned exactly as if every character had been re [`operator>>`](../../api/operator_gtgt.md) can still read several values from one stream (see also [JSON Lines](json_lines.md)). +This does not hold for [`json_document`](../json_view.md): it reads a stream into a buffer it owns, so the stream saves +no memory, and reading the file into a `#!cpp std::string` yourself and borrowing it is about 6 to 12 % faster. + Two macros switch these optimizations off without changing any result: [`JSON_DISABLE_STREAM_BULK_SCAN`](../../api/macros/json_disable_stream_bulk_scan.md) for stream buffers that do not follow the usual `std::streambuf` contract, and diff --git a/docs/mkdocs/docs/features/performance.md b/docs/mkdocs/docs/features/performance.md index ffea9c215..3d0714ca9 100644 --- a/docs/mkdocs/docs/features/performance.md +++ b/docs/mkdocs/docs/features/performance.md @@ -27,8 +27,8 @@ A `#!cpp std::istream` (including `#!cpp std::ifstream`) gets the same bulk scan characters its stream buffer already holds (the *get area*) and asks the stream buffer for more only when they are used up. Nothing is read ahead, so the stream is left exactly where it would be after reading one character at a time, and [`operator>>`](../api/operator_gtgt.md) can still read several values from one stream. A stream buffer without a get -area is read one character at a time. A `#!cpp FILE*` is still read one character at a time, but `file_input_adapter` -locks the file once per parse instead of once per character. See +area is read one character at a time. A `#!cpp FILE*` is locked once per parse instead of once per character, and with +glibc and the Apple C library, the characters in its buffer are scanned in bulk as a stream's get area is. See [input sources and performance](parsing/index.md#input-sources-and-performance) for the full table. According to the description of [#5828](https://github.com/nlohmann/json/pull/5828), parsing from a stream is within about 10 to 20 % of parsing an equivalent `#!cpp std::string`. @@ -39,7 +39,9 @@ iterator pair over non-contiguous but random-access storage (e.g. `#!cpp std::de optimization but not the second, since the byte-scanning fast path requires contiguous storage. Practically: a file can be passed directly to `parse`, which also avoids the memory for a copy of the file. Reading it -into a string first is no longer necessary for speed: +into a string first is no longer necessary for speed (this is for `#!cpp json::parse`; a +[`json_document`](json_view.md) reads a stream into an owned buffer anyway, so for it reading the file into a string +and borrowing that is the faster way): ```cpp // fast: the stream buffer's get area is scanned in bulk diff --git a/docs/mkdocs/docs/features/serialization.md b/docs/mkdocs/docs/features/serialization.md index c1b23442c..fd482cb73 100644 --- a/docs/mkdocs/docs/features/serialization.md +++ b/docs/mkdocs/docs/features/serialization.md @@ -17,6 +17,9 @@ To write a value directly to a stream (for example, a file or `#!cpp std::cout`) std::cout << j << std::endl; ``` +To write a file without leaving a truncated file behind when serialization fails, see +[Reading and writing files](files.md#writing-a-json-file). + !!! note "String, not raw value" `dump` always returns a **JSON text**. Serializing a JSON string therefore includes the surrounding quotes and diff --git a/docs/mkdocs/docs/features/types/template_parameters.md b/docs/mkdocs/docs/features/types/template_parameters.md index f3868f594..6ae6820f5 100644 --- a/docs/mkdocs/docs/features/types/template_parameters.md +++ b/docs/mkdocs/docs/features/types/template_parameters.md @@ -406,6 +406,7 @@ using array_t = ArrayType>; | `#!cpp operator<<(std::ostream&, const json_pointer&)` | streamability to `#!cpp std::ostream` | | [`to_string`](../../api/basic_json/to_string.md) | conversion of `StringType` to `#!cpp std::string` (the function returns a `#!cpp std::string`) | | exception messages | `data()` and `size()`, or `begin()` and `end()` | +| [`basic_json_view::dump`](../../api/basic_json_view/dump.md) | `reserve(n)` and `resize(n)` (one argument); `capacity()` is **not** required | ### Compatible types diff --git a/docs/mkdocs/docs/home/faq.md b/docs/mkdocs/docs/home/faq.md index c43acef37..d06819569 100644 --- a/docs/mkdocs/docs/home/faq.md +++ b/docs/mkdocs/docs/home/faq.md @@ -110,7 +110,7 @@ in the source files that need it, instantiate `basic_json` once, and consider pr The parser did not see any character: the file was most likely not opened. A relative path is resolved against the working directory of the running program, which is often the build directory of your IDE, not the source directory. Check `is_open()` and try an absolute path. Other causes are an empty file and a stream that was already read. See -[`parse_error.101`](exceptions.md#jsonexceptionparse_error101). +[`parse_error.101`](exceptions.md#jsonexceptionparse_error101) and [Reading and writing files](../features/files.md). ### Input that arrives in chunks diff --git a/docs/mkdocs/docs/images/compile_times.svg b/docs/mkdocs/docs/images/compile_times.svg index e8e904a60..302817e1c 100644 --- a/docs/mkdocs/docs/images/compile_times.svg +++ b/docs/mkdocs/docs/images/compile_times.svg @@ -1,21 +1,21 @@ - + Change in compile time against the default configuration at -O0, Apple clang and G​CC 16 -Grouped horizontal bars, negative is faster. json_fwd.hpp instead of json.hpp on a header-only translation unit: Apple clang -62 %, G​CC 16 -82 %; extern template on a parsing translation unit: Apple clang -7 %, G​CC 16 -21 %; JSON_NO_AUTOMATIC_UDLS on a model translation unit: Apple clang -11 %, G​CC 16 -11 %; JSON_NO_AUTOMATIC_UDLS on a parsing translation unit: Apple clang -1 %, G​CC 16 -2 %; JSON_HAS_FILESYSTEM=0 + JSON_NO_IO on a model translation unit: Apple clang -8 %, G​CC 16 -11 %; JSON_HAS_FILESYSTEM=0 on a model translation unit: Apple clang -5 %, G​CC 16 -9 %; JSON_NO_IO on a model translation unit: Apple clang -2 %, G​CC 16 -2 %; JSON_USE_GLOBAL_UDLS=0 on a model translation unit: Apple clang 0 %, G​CC 16 -1 %. +Grouped horizontal bars, negative is faster. json_fwd.hpp instead of json.hpp on a header-only translation unit: Apple clang -62 %, G​CC 16 -82 %; extern template on a parsing translation unit: Apple clang -7 %, G​CC 16 -21 %; JSON_NO_AUTOMATIC_UDLS on a model translation unit: Apple clang -11 %, G​CC 16 -11 %; JSON_NO_AUTOMATIC_UDLS on a parsing translation unit: Apple clang -1 %, G​CC 16 -2 %; JSON_HAS_FILESYSTEM=0 + JSON_NO_IO on a model translation unit: Apple clang -8 %, G​CC 16 -11 %; JSON_HAS_FILESYSTEM=0 on a model translation unit: Apple clang -5 %, G​CC 16 -9 %; JSON_HAS_STD_FORMAT=0 on a model translation unit: Apple clang -1 %, G​CC 16 -4 %; JSON_HAS_RANGES=0 on a model translation unit: Apple clang -4 %, G​CC 16 -1 %; JSON_NO_IO on a model translation unit: Apple clang -2 %, G​CC 16 -2 %; JSON_USE_GLOBAL_UDLS=0 on a model translation unit: Apple clang 0 %, G​CC 16 -1 %. Apple clang G​CC 16 - -−80 % - -−60 % - -−40 % - -−20 % -0 % -← faster -slower → + +−80 % + +−60 % + +−40 % + +−20 % +0 % +← faster +slower → json_fwd.hpp instead of json.hpp · header-only TU json_fwd.hpp instead of json.hpp, header-only TU (Apple clang): −62 % −62 % @@ -46,15 +46,25 @@ −5 % JSON_HAS_FILESYSTEM=0, model TU (G​CC 16): −9 % −9 % -JSON_NO_IO · model TU* -JSON_NO_IO, model TU (Apple clang): −2 % -−2 % -JSON_NO_IO, model TU (G​CC 16): −2 % -−2 % -JSON_USE_GLOBAL_UDLS=0 · model TU* -JSON_USE_GLOBAL_UDLS=0, model TU (Apple clang): 0 % -0 % -JSON_USE_GLOBAL_UDLS=0, model TU (G​CC 16): −1 % +JSON_HAS_STD_FORMAT=0 · model TU +JSON_HAS_STD_FORMAT=0, model TU (Apple clang): −1 % +−1 % +JSON_HAS_STD_FORMAT=0, model TU (G​CC 16): −4 % +−4 % +JSON_HAS_RANGES=0 · model TU +JSON_HAS_RANGES=0, model TU (Apple clang): −4 % +−4 % +JSON_HAS_RANGES=0, model TU (G​CC 16): −1 % −1 % - +JSON_NO_IO · model TU* +JSON_NO_IO, model TU (Apple clang): −2 % +−2 % +JSON_NO_IO, model TU (G​CC 16): −2 % +−2 % +JSON_USE_GLOBAL_UDLS=0 · model TU* +JSON_USE_GLOBAL_UDLS=0, model TU (Apple clang): 0 % +0 % +JSON_USE_GLOBAL_UDLS=0, model TU (G​CC 16): −1 % +−1 % + diff --git a/docs/mkdocs/docs/integration/compile_times.md b/docs/mkdocs/docs/integration/compile_times.md index 8798931cb..699b21f05 100644 --- a/docs/mkdocs/docs/integration/compile_times.md +++ b/docs/mkdocs/docs/integration/compile_times.md @@ -11,6 +11,11 @@ that cost, ordered by how much they typically save. 16.2.0 on macOS (Apple M1 Max). All configurations were run interleaved, in alternating order. They show the order of magnitude to expect; measure your own code before and after a change. + The two sections on `std::format` and ranges apply to C++20 only and were measured with `-std=c++20`, so their + baselines are slower than the others. Their differences are mostly within the run-to-run noise of a few percent: + two complete runs of the same configuration differed by up to 10% in the median. The tables show the later run (one + outlier cell was repeated). + The translation units are listed below. They replace the ones used for earlier versions of this page, whose sources were not published, so the numbers differ from earlier versions. @@ -66,7 +71,8 @@ that cost, ordered by how much they typically save. --8<-- "images/compile_times.svg"

Change in compile time at `-O0` against the default configuration of the same -translation unit, taken from the tables below (medians of nine runs, see above). * computed from the milliseconds in +translation unit, taken from the tables below (medians of nine runs, see above; the `std::format` and ranges rows are +measured with `-std=c++20`). * computed from the milliseconds in [Options without effect on compile times](#options-without-effect-on-compile-times).

## Include `json_fwd.hpp` in headers @@ -214,6 +220,88 @@ from and to `std::string`; whether that fallback exists depends on the platform' (`std::wstring` on Windows), so check code that converts paths. `JSON_NO_IO` additionally removes all stream functions (see [Options without effect](#options-without-effect-on-compile-times)). +## Switch off `std::format` support + +With C++20 and a standard library that provides ``, the library includes it to provide a +[`std::formatter`](../api/basic_json/std_formatter.md) specialization (see +[`JSON_HAS_STD_FORMAT`](../api/macros/json_has_std_format.md)). If you do not format JSON values with `std::format`, +switch the detection off for the whole project: + +```cmake +target_compile_definitions(my_target PRIVATE JSON_HAS_STD_FORMAT=0) +``` + +| Compiler | Translation unit | Default (`-O0` / `-O2`) | `std::format` off (`-O0` / `-O2`) | Change | +|-------------|------------------|------------------------:|----------------------------------:|-----------:| +| Apple clang | model | 1283 ms / 1544 ms | 1269 ms / 1539 ms | −1% / 0% | +| GCC 16 | model | 2026 ms / 2482 ms | 1944 ms / 2416 ms | −4% / −3% | +| Apple clang | parsing | 1350 ms / 2278 ms | 1347 ms / 2300 ms | 0% / +1% | +| GCC 16 | parsing | 2715 ms / 4351 ms | 2644 ms / 4287 ms | −3% / −1% | + +

Compile time with `JSON_HAS_STD_FORMAT=0` against the default configuration, both with `-std=c++20`, at `-O0` and `-O2`.

+ +The saving is small, and only GCC shows it: libstdc++'s `` is an additional header that nothing else in the +translation unit includes, so the macro removes it (and the headers it pulls in). With libc++ (Apple clang), other +standard headers include `` anyway, so the macro only removes the library's own formatter, and the difference is +within the noise. + +The trade-off: the `std::formatter` specialization is removed, so `#!cpp std::format("{}", j)`, +and `#!cpp std::format_to` with a JSON value no longer compile (in a test with both compilers on +macOS, the error was that `std::formatter` must be specialized for the type). Use [`dump()`](../api/basic_json/dump.md), +or the stream operators, instead. + +## Switch off ranges support + +With C++20 and a standard library that provides ``, the library includes it to declare the +[`items()`](../api/basic_json/items.md) proxy a *borrowed range*, and uses the `std::ranges` concepts and algorithms for +construction from range views and for iterator input (see [`JSON_HAS_RANGES`](../api/macros/json_has_ranges.md)). If you +do not need these, switch the detection off for the whole project: + +```cmake +target_compile_definitions(my_target PRIVATE JSON_HAS_RANGES=0) +``` + +| Compiler | Translation unit | Default (`-O0` / `-O2`) | Ranges off (`-O0` / `-O2`) | Change | +|-------------|------------------|------------------------:|---------------------------:|-----------:| +| Apple clang | model | 1283 ms / 1544 ms | 1237 ms / 1495 ms | −4% / −3% | +| GCC 16 | model | 2026 ms / 2482 ms | 2004 ms / 2464 ms | −1% / −1% | +| Apple clang | parsing | 1350 ms / 2278 ms | 1311 ms / 2284 ms | −3% / 0% | +| GCC 16 | parsing | 2715 ms / 4351 ms | 2705 ms / 4427 ms | 0% / +2% | + +

Compile time with `JSON_HAS_RANGES=0` against the default configuration, both with `-std=c++20`, at `-O0` and `-O2`.

+ +The saving is small as well, and with GCC it is not distinguishable from the noise: its `` includes ``, +so with `std::format` support left on, part of the ranges machinery is still parsed. +Both macros together save more than either alone: + +```cmake +target_compile_definitions(my_target PRIVATE JSON_HAS_STD_FORMAT=0 JSON_HAS_RANGES=0) +``` + +| Compiler | Translation unit | Default (`-O0` / `-O2`) | Both off (`-O0` / `-O2`) | Change | +|-------------|------------------|------------------------:|-------------------------:|-----------:| +| Apple clang | model | 1283 ms / 1544 ms | 1223 ms / 1504 ms | −5% / −3% | +| GCC 16 | model | 2026 ms / 2482 ms | 1914 ms / 2378 ms | −6% / −4% | +| Apple clang | parsing | 1350 ms / 2278 ms | 1311 ms / 2271 ms | −3% / 0% | +| GCC 16 | parsing | 2715 ms / 4351 ms | 2635 ms / 4260 ms | −3% / −2% | + +

Compile time with `JSON_HAS_STD_FORMAT=0` and `JSON_HAS_RANGES=0` against the default configuration, both with `-std=c++20`, at `-O0` and `-O2`.

+ +The trade-offs of `JSON_HAS_RANGES=0` are larger than those of the other options on this page: + +- `items()` is no longer a borrowed range. On a temporary, `#!cpp std::ranges::begin(make_json().items())` no longer + compiles, and `std::ranges` algorithms such as `#!cpp std::ranges::find_if(make_json().items(), pred)` return + `std::ranges::dangling` instead of an iterator, so code that uses the result no longer compiles. Calling them on + `#!cpp j.items()` for a named `j` is not affected. +- [`JSON_HAS_RANGE_VIEW_CONVERSION`](../api/macros/json_has_range_view_conversion.md) becomes `0`, so a JSON array can no + longer be constructed from a `std::ranges::view` such as the result of `std::views::filter`. The assignment + `#!cpp json j = v | std::views::filter(pred);` no longer compiles. +- Parsing from an iterator pair loses the C++20 fast path for contiguous iterators that are not pointers (for + example those of `std::vector`) and falls back to byte-at-a-time scanning. The result is the same; parsing may be + slower (not measured here). + +The first two were checked with both compilers on macOS; the third is read from the library's source. + ## Use C++20 modules With a toolchain that supports named modules, `import nlohmann.json;` compiles the library once into a module and @@ -343,6 +431,8 @@ and −2% / 0% (GCC). To get a saving, combine it with - [`JSON_HAS_FILESYSTEM`](../api/macros/json_has_filesystem.md) - control the conversions from and to `std::filesystem::path` +- [`JSON_HAS_STD_FORMAT`](../api/macros/json_has_std_format.md) - control the `std::formatter` specialization +- [`JSON_HAS_RANGES`](../api/macros/json_has_ranges.md) - control the use of `std::ranges` - [`JSON_NO_AUTOMATIC_UDLS`](../api/macros/json_no_automatic_udls.md) - do not include the user-defined string literals automatically - [Modules](../features/modules.md) - C++20 module support diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index b0c3d1a1f..c8ef68238 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -130,6 +130,7 @@ nav: - features/element_access/unchecked_access.md - features/element_access/checked_access.md - features/element_access/default_value.md + - features/files.md - features/iterators.md - features/merge_patch.md - features/json_patch.md diff --git a/docs/mkdocs/requirements.txt b/docs/mkdocs/requirements.txt index 245ebc387..783f9d88c 100644 --- a/docs/mkdocs/requirements.txt +++ b/docs/mkdocs/requirements.txt @@ -1,8 +1,8 @@ wheel==0.48.0 mkdocs==1.6.1 # documentation framework -mkdocs-git-revision-date-localized-plugin==1.6.0 # plugin "git-revision-date-localized" -mkdocs-material==9.7.7 # theme for mkdocs +mkdocs-git-revision-date-localized-plugin==1.6.2 # plugin "git-revision-date-localized" +mkdocs-material==9.8.0 # theme for mkdocs mkdocs-material-extensions==1.3.1 # extensions mkdocs-minify-plugin==0.8.0 # plugin "minify" mkdocs-redirects==1.2.3 # plugin "redirects" diff --git a/docs/mkdocs/scripts/figures/charts.py b/docs/mkdocs/scripts/figures/charts.py index 6f1c4dc3e..d0440977c 100644 --- a/docs/mkdocs/scripts/figures/charts.py +++ b/docs/mkdocs/scripts/figures/charts.py @@ -73,6 +73,8 @@ COMPILE = [ ("JSON_NO_AUTOMATIC_UDLS", "parsing TU", -1, -2, False), ("JSON_HAS_FILESYSTEM=0 + JSON_NO_IO", "model TU", -8, -11, False), ("JSON_HAS_FILESYSTEM=0", "model TU", -5, -9, False), + ("JSON_HAS_STD_FORMAT=0", "model TU", -1, -4, False), + ("JSON_HAS_RANGES=0", "model TU", -4, -1, False), # computed from the ms in "Options without effect on compile times": 1034/1051, 1756/1801, 1055/1051, 1783/1801 ("JSON_NO_IO", "model TU", round((1034 / 1051 - 1) * 100), round((1756 / 1801 - 1) * 100), True), ("JSON_USE_GLOBAL_UDLS=0", "model TU", round((1055 / 1051 - 1) * 100), round((1783 / 1801 - 1) * 100), True), diff --git a/include/nlohmann/detail/conversions/from_json.hpp b/include/nlohmann/detail/conversions/from_json.hpp index 8fb9d149b..09f0713ea 100644 --- a/include/nlohmann/detail/conversions/from_json.hpp +++ b/include/nlohmann/detail/conversions/from_json.hpp @@ -91,7 +91,7 @@ const typename BasicJsonType::binary_t& binary_value(const BasicJsonType& j) } template::value, int> = 0> -typename View::basic_json_type::binary_t binary_value(const View& /*unused*/) +typename View::basic_json_type::binary_t binary_value(const View& /*unused*/) // LCOV_EXCL_LINE { return {}; // LCOV_EXCL_LINE } diff --git a/include/nlohmann/detail/exceptions.hpp b/include/nlohmann/detail/exceptions.hpp index 144bb770c..fc9359882 100644 --- a/include/nlohmann/detail/exceptions.hpp +++ b/include/nlohmann/detail/exceptions.hpp @@ -198,18 +198,19 @@ class exception : public std::exception // all other parent types (primitives) contribute no token } - if (tokens.empty()) + // the root value has no path, but still has a byte range + std::string prefix; + if (!tokens.empty()) { - return ""; + const auto str = std::accumulate(tokens.rbegin(), tokens.rend(), std::string{}, + [](const std::string & a, const std::string & b) + { + return concat(a, '/', detail::escape(b)); + }); + prefix = concat('(', str, ") "); } - auto str = std::accumulate(tokens.rbegin(), tokens.rend(), std::string{}, - [](const std::string & a, const std::string & b) - { - return concat(a, '/', detail::escape(b)); - }); - - return concat('(', str, ") ", get_byte_positions(leaf_element)); + return concat(prefix, get_byte_positions(leaf_element)); #else return get_byte_positions(leaf_element); #endif @@ -222,7 +223,8 @@ class exception : public std::exception template static std::string get_byte_positions(const BasicJsonType* leaf_element) { - if ((leaf_element->start_pos() != std::string::npos) && (leaf_element->end_pos() != std::string::npos)) + // a value-initialized iterator has no value (invalid_iterator.212) + if (leaf_element != nullptr && leaf_element->start_pos() != std::string::npos && leaf_element->end_pos() != std::string::npos) { return concat("(bytes ", std::to_string(leaf_element->start_pos()), "-", std::to_string(leaf_element->end_pos()), ") "); } diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 0ec6fedc6..0f7f6ba05 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -371,8 +371,8 @@ class binary_reader nested document could declare a length that disagrees with where its terminator actually falls and quietly hand the bytes in between to the enclosing document. A well-formed document is at least 5 bytes (the prefix - plus the terminator); the equality also rejects those impossible sizes, - since at least 5 bytes are always consumed. + plus the terminator); open_bson_document() rejects a smaller size before + it is stored, so @a document_size is never negative here. When recovering from errors, a document whose size does not match is accepted: its terminator was found, so everything in it has been read. A @@ -386,7 +386,7 @@ class binary_reader */ bool check_bson_document_size(const std::size_t document_start, const std::int32_t document_size, const bool size_repaired) { - if (JSON_HEDLEY_UNLIKELY(!size_repaired && (document_size < 0 || static_cast(document_size) != chars_read - document_start))) + if (JSON_HEDLEY_UNLIKELY(!size_repaired && static_cast(document_size) != chars_read - document_start)) { static_cast(bson_document_size_error(document_size, chars_read - document_start)); return repair_requested(); @@ -1236,7 +1236,7 @@ class binary_reader { std::uint64_t len{}; std::size_t size{}; - return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_cbor_array(size); + return get_cbor_argument(len) && get_container_size(len, size, "array") && enter_cbor_array(size); } case 0x9F: // array (indefinite length) @@ -1276,7 +1276,7 @@ class binary_reader { std::uint64_t len{}; std::size_t size{}; - return get_cbor_argument(len) && get_cbor_container_size(len, size, "map") && enter_object(size); + return get_cbor_argument(len) && get_container_size(len, size, "map") && enter_object(size); } case 0xBF: // map (indefinite length) @@ -1940,7 +1940,7 @@ class binary_reader } /*! - @brief narrow a definite CBOR array/map length to std::size_t + @brief narrow a definite CBOR or MessagePack array/map length to std::size_t A definite length is rejected if it does not fit in std::size_t or if it equals detail::unknown_size(), which is reserved to mark an indefinite- @@ -1953,7 +1953,7 @@ class binary_reader @param[in] context "array" or "map", for the error message @return whether the length is usable */ - bool get_cbor_container_size(const std::uint64_t len, std::size_t& result, const char* context) + bool get_container_size(const std::uint64_t len, std::size_t& result, const char* context) { if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(len) || len == detail::unknown_size())) { @@ -2096,14 +2096,18 @@ class binary_reader Only the start of an array is tried, which covers arrays of numbers (e.g. coordinates) but costs arrays of other values only one call, not one - per element. + per element. An array of one element is not tried at all: that element is + mostly an object or an array (`[[...]]`, the single ring of a GeoJSON + polygon), for which the call would find no scalar, while a single scalar, + which the run would read faster, is rare. The benchmarks + `FromBinaryShape/nested` and `FromBinaryShape/singles` show both sides. @param[in] len the number of elements @return whether the SAX parser accepted the array and the values read */ bool enter_cbor_array(const std::size_t len) { - return enter_array(len) && (len == 0 || parse_cbor_scalar_run(std::integral_constant {})); + return enter_array(len) && (len <= 1 || parse_cbor_scalar_run(std::integral_constant {})); } /// input that is not contiguous: every value is read by parse_cbor_value @@ -2666,7 +2670,8 @@ class binary_reader case 0xDD: // array 32 { std::uint32_t len{}; - return get_number(len) && enter_msgpack_array(conditional_static_cast(len)); + std::size_t size{}; + return get_number(len) && get_container_size(len, size, "array") && enter_msgpack_array(size); } case 0xDE: // map 16 @@ -2678,7 +2683,8 @@ class binary_reader case 0xDF: // map 32 { std::uint32_t len{}; - return get_number(len) && enter_object(conditional_static_cast(len)); + std::size_t size{}; + return get_number(len) && get_container_size(len, size, "map") && enter_object(size); } // negative fixint @@ -3128,12 +3134,16 @@ class binary_reader /*! @brief open a MessagePack array and read the scalars it begins with (see @ref parse_msgpack_scalar_run) + + An array of one element is not tried, for the reason given at + @ref enter_cbor_array. + @param[in] len the number of elements @return whether the SAX parser accepted the array and the values read */ bool enter_msgpack_array(const std::size_t len) { - return enter_array(len) && (len == 0 || parse_msgpack_scalar_run(std::integral_constant {})); + return enter_array(len) && (len <= 1 || parse_msgpack_scalar_run(std::integral_constant {})); } /// input that is not contiguous: every value is read by parse_msgpack_value @@ -3596,16 +3606,13 @@ class binary_reader { if (size_and_type.second != 0) { - if (size_and_type.second != 'N') + for (std::size_t i = 0; i < size_and_type.first; ++i) { - for (std::size_t i = 0; i < size_and_type.first; ++i) + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, size_and_type.second))) { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, size_and_type.second))) - { - return false; - } - dim.push_back(dimlen); + return false; } + dim.push_back(dimlen); } } else @@ -3635,6 +3642,27 @@ class binary_reader return true; } + /*! + @brief store the count of an optimized container (or the length of a string) + + A count equal to npos is rejected: npos marks a container whose count is + not known, so a count of that value would be read as "no count" (compare + get_container_size()). No container can hold that many elements. + + @param[in] count the count read from the input + @param[out] result set to @a count unless it is rejected + @return whether @a count is usable + */ + bool store_ubjson_count(const std::size_t count, std::size_t& result) + { + if (JSON_HEDLEY_UNLIKELY(count == npos)) + { + return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); + } + result = count; + return true; + } + /*! @brief read a UBJSON/BJData optimized-container count of a signed marker type ('i', 'I', 'l', 'L') and narrow it to std::size_t @@ -3665,8 +3693,7 @@ class binary_reader { return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } - result = static_cast(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char - return true; + return store_ubjson_count(static_cast(number), result); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char } /*! @@ -3743,8 +3770,7 @@ class binary_reader { return false; } - result = conditional_static_cast(number); - return true; + return store_ubjson_count(conditional_static_cast(number), result); } case 'M': @@ -3762,8 +3788,7 @@ class binary_reader { return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } - result = detail::conditional_static_cast(number); - return true; + return store_ubjson_count(detail::conditional_static_cast(number), result); } case '[': @@ -3809,11 +3834,9 @@ class binary_reader // follow the documented _ArrayType_, _ArraySize_, _ArrayData_ order if (ndarray_dtype != 0) { + // every optimized type marker is_valid_optimized_type() accepts for BJData has a name const char* type_name = bjd_type_name(ndarray_dtype); - if (JSON_HEDLEY_UNLIKELY(type_name == nullptr)) - { - return unexpected_byte("invalid byte", "type"); - } + JSON_ASSERT(type_name != nullptr); string_t type_key = "_ArrayType_"; string_t type = type_name; // sax->string() takes a reference @@ -3898,12 +3921,6 @@ class binary_reader if (current == '$') { result.second = get(); // must not ignore 'N', because 'N' maybe the type - if (input_format == input_format_t::bjdata - && JSON_HEDLEY_UNLIKELY(is_bjd_excluded_optimized_type(result.second))) - { - return optimized_type_error(); - } - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof("type"))) { return false; @@ -3931,7 +3948,7 @@ class binary_reader // an ndarray was read here only if the flag flipped; when it was // seeded true, get_ubjson_size_value() already rejected the nested // dimension vector - if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) + if (is_ndarray && !inside_ndarray) { result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters } @@ -3941,7 +3958,7 @@ class binary_reader if (current == '#') { const bool is_error = get_ubjson_size_value(result.first, is_ndarray); - if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) + if (is_ndarray && !inside_ndarray) { return last_byte_error(exception_id::unexpected_byte, "ndarray requires both type and size", "size"); } @@ -4123,7 +4140,6 @@ class binary_reader JSON_HEDLEY_NEVER_INLINE bool parse_ubjson_typed_run(std::size_t& remaining, const char_int_type type, std::true_type /*bulk*/) { - const bool bjdata = input_format == input_format_t::bjdata; switch (type) { case 'U': @@ -4137,11 +4153,11 @@ class binary_reader case 'L': return parse_ubjson_typed_run(remaining); case 'u': - return !bjdata || parse_ubjson_typed_run(remaining); + return parse_ubjson_typed_run(remaining); case 'm': - return !bjdata || parse_ubjson_typed_run(remaining); + return parse_ubjson_typed_run(remaining); case 'M': - return !bjdata || parse_ubjson_typed_run(remaining); + return parse_ubjson_typed_run(remaining); case 'd': return parse_ubjson_typed_run(remaining); case 'D': @@ -4222,7 +4238,7 @@ class binary_reader // if bit-8 of size_and_type.second is set to 1, encode bjdata ndarray as an object in JData annotated array format (https://github.com/NeuroJSON/jdata): // {"_ArrayType_" : "typeid", "_ArraySize_" : [n1, n2, ...], "_ArrayData_" : [v1, v2, ...]} - if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) + if ((size_and_type.second & (1 << 8)) != 0) { size_and_type.second &= ~(static_cast(1) << 8); // use bit 8 to indicate ndarray, here we remove the bit to restore the type marker @@ -4283,7 +4299,7 @@ class binary_reader } // If BJData type marker is 'B' decode as binary - if (input_format == input_format_t::bjdata && size_and_type.first != npos && size_and_type.second == 'B') + if (size_and_type.first != npos && size_and_type.second == 'B') { binary_t result; return get_binary(size_and_type.first, result) && sax->binary(result); @@ -4332,7 +4348,7 @@ class binary_reader } // do not accept ND-array size in objects in BJData - if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) + if ((size_and_type.second & (1 << 8)) != 0) { return last_byte_error(exception_id::unexpected_byte, "BJData object does not support ND-array size in optimized format", "object"); } @@ -5610,6 +5626,14 @@ class binary_reader const char* context, ContainerType& result) { + // a string that is in the block of input as a whole is appended from + // it, without the zero-filling resize() below + constexpr bool from_block = bulk_scan && std::is_same::value; + if (append_from_block(len, result, std::integral_constant {})) + { + return true; + } + // upper bound on the number of bytes read (and allocated) per chunk constexpr std::size_t chunk_size = 4096; @@ -5646,6 +5670,36 @@ class binary_reader return true; } + /// append the @a len bytes at the current position to @a result, if the + /// input holds them in one block; false (with nothing read) otherwise + template + bool append_from_block(const NumberType len, string_t& result, std::true_type /*block*/) + { + if (len <= 0) + { + return false; + } + // (len > 0, so the unsigned type holds it; casting a signed char + // straight to std::size_t would trip bugprone-signed-char-misuse) + const auto ulen = static_cast::type>(len); + const std::size_t remaining = ia.bulk_remaining(); + if (static_cast(ulen) > static_cast(remaining)) + { + return false; // (the loop reports the premature end) + } + const auto n = static_cast(ulen); + result.append(reinterpret_cast(ia.bulk_data()), n); + ia.bulk_skip(n); + chars_read += n; + return true; + } + + template + static bool append_from_block(const NumberType /*len*/, ContainerType& /*result*/, std::false_type /*block*/) noexcept + { + return false; + } + /*! @brief report an error to the SAX parser and remember its answer @@ -6132,8 +6186,9 @@ class binary_reader JSON_PRIVATE_UNLESS_TESTED: /*! @brief check whether @a marker is a type marker that may follow '$' - UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C - (BJData's excluded markers are rejected separately). + UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C. + The markers BJData excludes (F H N S T Z [ {) are the ones + binary_writer::is_bjdata_excluded_type_marker() lists; keep the two in sync. */ bool is_valid_optimized_type(const char_int_type marker) const noexcept { @@ -6168,20 +6223,6 @@ class binary_reader } } - /*! - @brief whether @a marker is excluded from BJData's optimized ND-array types - @return whether @a marker is one of 'F', 'H', 'N', 'S', 'T', 'Z', '[', '{' - - Mirrors binary_writer's @ref binary_writer::is_bjdata_excluded_type_marker - "is_bjdata_excluded_type_marker()`, which encodes the same list the other - way; keep the two in sync. - */ - static constexpr bool is_bjd_excluded_optimized_type(const char_int_type marker) noexcept - { - return marker == '[' || marker == '{' || marker == 'S' || marker == 'H' - || marker == 'T' || marker == 'F' || marker == 'N' || marker == 'Z'; - } - /*! @brief look up the ND-array element type name for a BJData dtype marker @return the type name ("uint8", "int8", ...), or nullptr if @a marker does diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index b1b92256c..4545c3e5c 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -58,6 +58,19 @@ #define JSON_UNLOCKED_FILE_IO 0 #endif #undef JSON_UNLOCKED_FILE_IO_MSAN + + // Whether file_input_adapter scans the buffer of the FILE directly, as + // getc_unlocked() reads it: the fields it uses are those that the C + // library's own getc_unlocked() macro uses (glibc: _IO_read_ptr and + // _IO_read_end; Apple: _p and _r). It needs the file locked for the whole + // parse, as JSON_UNLOCKED_FILE_IO == 1 has it (so JSON_DISABLE_UNLOCKED_FILE_IO + // and MemorySanitizer builds turn it off). + // (uClibc defines __GLIBC__ as well, but its FILE has other fields) + #if JSON_UNLOCKED_FILE_IO == 1 && ((defined(__GLIBC__) && !defined(__UCLIBC__)) || defined(__APPLE__)) + #define JSON_FILE_WINDOW 1 + #else + #define JSON_FILE_WINDOW 0 + #endif #endif // JSON_NO_IO NLOHMANN_JSON_NAMESPACE_BEGIN @@ -179,8 +192,12 @@ class token_carry_buffer char* m_capacity_end = nullptr; }; /*! -Input adapter for stdio file access. This adapter read only 1 byte and do not use any - buffer. This adapter is a very low level adapter. +Input adapter for stdio file access. The file is locked for the lifetime of the +adapter and read with the unlocked stdio functions (see +JSON_DISABLE_UNLOCKED_FILE_IO). With glibc and the Apple C library, the +characters in the buffer of the FILE are also scanned in bulk, as the get area +of a stream is (see input_stream_adapter): exactly the characters getc_unlocked() +would return are consumed, so the file is left where it was left before. */ class file_input_adapter { @@ -199,6 +216,12 @@ class file_input_adapter file_input_adapter(const file_input_adapter&) = delete; file_input_adapter(file_input_adapter&& rhs) noexcept : m_file(rhs.m_file) +#if JSON_FILE_WINDOW + , token_carry(std::move(rhs.token_carry)) + , token_segment(rhs.token_segment) + , current_in_buffer(rhs.current_in_buffer) + , tracking_token(rhs.tracking_token) +#endif { // the lock is owned by the thread, not the adapter: hand it over rhs.m_file = nullptr; @@ -210,6 +233,20 @@ class file_input_adapter unlock(); } +#if JSON_FILE_WINDOW + std::char_traits::int_type get_character() + { + // what getc_unlocked() does, while the buffer has characters + const char* const next = buffer_ptr(); + if (JSON_HEDLEY_LIKELY(next != buffer_end())) + { + current_in_buffer = true; + buffer_bump(1); + return std::char_traits::to_int_type(*next); + } + return get_character_slow(); + } +#else std::char_traits::int_type get_character() noexcept { // the file is locked for the lifetime of the adapter, so the @@ -222,6 +259,7 @@ class file_input_adapter return std::fgetc(m_file); #endif } +#endif // returns the number of characters successfully read template @@ -230,7 +268,134 @@ class file_input_adapter return fread(dest, 1, sizeof(T) * count, m_file); } +#if JSON_FILE_WINDOW + // The lexer may scan the buffer of the FILE directly, as the window of + // input_stream_adapter: the buffer ends wherever the C library's buffer + // ends, and is refilled by the next get_character(). + static constexpr bool supports_bulk_scan = true; + static constexpr bool bulk_window = true; + + // Pointer to the next unread character; only valid when bulk_remaining() > 0. + const char* bulk_data() const + { + return buffer_ptr(); + } + + // Number of unread characters in the buffer. + std::size_t bulk_remaining() const + { + if (!current_in_buffer) + { + return 0; + } + return static_cast(buffer_end() - buffer_ptr()); + } + + // Consume @a n characters previously inspected via bulk_data(). + void bulk_skip(std::size_t n) + { + JSON_ASSERT(n <= bulk_remaining()); + buffer_bump(n); + } + + // The adapter keeps the current token for the lexer's error messages, as + // input_stream_adapter does. + static constexpr bool supports_token_tracking = true; + + void begin_token() noexcept + { + tracking_token = true; + token_carry.clear(); + token_segment = buffer_ptr(); + } + + void mark_token_start(std::char_traits::int_type c) + { + tracking_token = true; + token_carry.clear(); + if (current_in_buffer) + { + token_segment = buffer_ptr() - 1; + } + else + { + token_segment = nullptr; + token_carry.push_back(std::char_traits::to_char_type(c)); + } + } + + void copy_token(std::vector& out, bool drop_last) const + { + token_carry.copy_to(out); + if (token_segment != nullptr) + { + out.insert(out.end(), token_segment, buffer_ptr()); + } + if (drop_last && !out.empty()) + { + out.pop_back(); + } + } +#endif + private: +#if JSON_FILE_WINDOW + // the buffer of the FILE: [buffer_ptr(), buffer_end()) are the characters + // getc_unlocked() would return next +#if defined(__GLIBC__) + const char* buffer_ptr() const noexcept + { + return m_file->_IO_read_ptr; + } + const char* buffer_end() const noexcept + { + return m_file->_IO_read_end; + } + void buffer_bump(std::size_t n) noexcept + { + m_file->_IO_read_ptr += n; + } +#else + const char* buffer_ptr() const noexcept + { + return reinterpret_cast(m_file->_p); + } + const char* buffer_end() const noexcept + { + return m_file->_r > 0 ? reinterpret_cast(m_file->_p) + m_file->_r : reinterpret_cast(m_file->_p); + } + void buffer_bump(std::size_t n) noexcept + { + m_file->_p += n; + m_file->_r -= static_cast(n); + } +#endif + + // the buffer is exhausted: getc_unlocked() refills it + JSON_HEDLEY_NEVER_INLINE std::char_traits::int_type get_character_slow() + { + if (token_segment != nullptr) + { + token_carry.append(token_segment, buffer_end()); + token_segment = nullptr; + } + const auto res = getc_unlocked(m_file); + if (res == EOF) + { + current_in_buffer = false; + return std::char_traits::eof(); + } + // the character was taken from the refilled buffer, at buffer_ptr() - 1 + current_in_buffer = true; + if (tracking_token) + { + token_segment = buffer_ptr() - 1; + } + return res; + } + +#endif + // Lock the file once for the whole parse instead of once per character. // The lock is recursive, so fread() and the user's own stdio calls on the // same thread keep working. @@ -259,6 +424,16 @@ class file_input_adapter /// the file pointer to read from std::FILE* m_file; +#if JSON_FILE_WINDOW + /// the part of the current token that was in earlier buffers + token_carry_buffer token_carry{}; // NOLINT(readability-redundant-member-init) + /// start of the current token's part in the buffer, or nullptr + const char* token_segment = nullptr; + /// whether the character last returned by get_character() is in the buffer + bool current_in_buffer = false; + /// whether a token is tracked at all; the binary readers never start one + bool tracking_token = false; +#endif }; /*! @@ -545,12 +720,9 @@ class input_stream_adapter std::char_traits::int_type get_character_slow() { #if !JSON_DISABLE_STREAM_BULK_SCAN - // sbumpc() refills the get area if it is exhausted - const bool refill = streambuf_access::get_ptr(sb) == streambuf_access::get_end(sb); - if (JSON_HEDLEY_UNLIKELY(refill)) - { - save_token_segment(); - } + // the get area is exhausted (get_character() handles the other case), + // so sbumpc() refills it + save_token_segment(); #endif auto res = sb->sbumpc(); @@ -563,17 +735,13 @@ class input_stream_adapter is->clear(is->rdstate() | std::ios::eofbit); } #if !JSON_DISABLE_STREAM_BULK_SCAN - else if (JSON_HEDLEY_UNLIKELY(refill)) + else { // the consumed character is at gptr() - 1, unless the streambuf // has no get area and returned it from uflow() const char* const next = streambuf_access::get_ptr(sb); after_refill(res, next != nullptr && next != streambuf_access::get_begin(sb)); } - else - { - current_in_get_area = true; - } #endif return res; } diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 7fd5f279f..bad8c62ca 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -911,7 +911,8 @@ class json_sax_dom_parser : public json_sax_dom_error_state { hashes.push_back(hasher(element.first)); } - ordered_data.key_indexes.emplace_back(new object_key_index(std::move(hashes))); + std::unique_ptr index(new object_key_index(std::move(hashes))); // NOLINT(cppcoreguidelines-owning-memory,modernize-make-unique): std::make_unique is C++14 + ordered_data.key_indexes.push_back(std::move(index)); f.ordered.index = ordered_data.key_indexes.back().get(); } @@ -1006,7 +1007,7 @@ class json_sax_dom_parser : public json_sax_dom_error_state ordered_data.members[f.ordered.dest_member].second = std::move(value); } - static void place_member(const frame& /*f*/, BasicJsonType&& /*value*/, std::false_type /*indexed*/) noexcept {} + static void place_member(const frame& /*f*/, BasicJsonType&& /*value*/, std::false_type /*indexed*/) noexcept {} // LCOV_EXCL_LINE /// store the members of a new object directly in it bool start_stored_object(std::size_t len) @@ -1310,7 +1311,7 @@ class json_sax_dom_parser : public json_sax_dom_error_state { // a new member is null and holds nothing: construct the value // over it instead of destroying and swapping - ::new (static_cast(&member)) BasicJsonType(std::forward(v)); + construct_member(member, std::forward(v), std::is_nothrow_constructible {}); } else { @@ -1319,6 +1320,26 @@ class json_sax_dom_parser : public json_sax_dom_error_state } } + /// constructing the value cannot throw (for example a number or a boolean): construct it in place + template + JSON_HEDLEY_ALWAYS_INLINE + static void construct_member(BasicJsonType& member, Value&& v, std::true_type /*nothrow*/) + { + ::new (static_cast(&member)) BasicJsonType(std::forward(v)); + } + + /// constructing the value can throw (for example a container or a string), and a + /// constructor that was interrupted must not have touched the member (#5985): + /// create the value first, then move it over the member (the move constructor is + /// noexcept) + template + JSON_HEDLEY_ALWAYS_INLINE + static void construct_member(BasicJsonType& member, Value&& v, std::false_type /*may throw*/) + { + BasicJsonType value(std::forward(v)); + ::new (static_cast(&member)) BasicJsonType(std::move(value)); + } + /// a custom base class may care about the lifetime of the value template static void set_member(BasicJsonType& member, Value&& v, std::false_type /*custom base*/) @@ -1744,7 +1765,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state key_stack.push_back(val); // add discarded value at the given key and store the reference for later - if (keep && ref_stack.back()) + if (keep) { auto& obj = *ref_stack.back()->m_data.m_value.object; // a single lookup: insert the placeholder, or find the existing member @@ -1801,7 +1822,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state const string_t object_key = pop_container(); - if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured()) + if (!ref_stack.empty() && ref_stack.back()) { // remove discarded value remove_discarded_value(*ref_stack.back(), object_key); @@ -1967,8 +1988,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state */ string_t current_key() const { - if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object() - && !key_stack.empty()) + if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object()) { return key_stack.back(); } diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index cd102b4db..f86987a9a 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -300,14 +300,16 @@ class lexer : public lexer_base /// calls would (including the position and token bookkeeping of a /// streaming adapter, see bulk_consume) and leaves @a current holding the last of /// the 4 digits, just as the last such get() would; the codepoint is - /// written to @a out. Makes no state change and returns false for a - /// pending unget, fewer than 4 remaining bytes, or any of the 4 bytes not - /// being a hex digit, so the caller falls back unchanged to the - /// per-character loop, which then reports the same diagnostic (stopping - /// at the first invalid digit) as before this optimization. + /// written to @a out. Makes no state change and returns false for fewer + /// than 4 remaining bytes or any of the 4 bytes not being a hex digit, so + /// the caller falls back unchanged to the per-character loop, which then + /// reports the same diagnostic (stopping at the first invalid digit) as + /// before this optimization. bool get_codepoint_bulk(std::true_type /*bulk*/, int& out) { - if (next_unget || ia.bulk_remaining() < 4) + // no unget is pending inside a string (see get_after_string_run()) + JSON_ASSERT(!next_unget); + if (ia.bulk_remaining() < 4) { return false; } @@ -452,11 +454,10 @@ class lexer : public lexer_base template char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant keep) { - // a pending unget must be consumed through the normal path first - if (next_unget) - { - return get(); - } + // scan_string() is entered from scan_after_whitespace(), whose current + // was read by skip_whitespace() (its first get() clears a pending + // unget), and nothing in scan_string() calls unget() + JSON_ASSERT(!next_unget); const std::size_t remaining = ia.bulk_remaining(); if (remaining == 0) { @@ -1447,7 +1448,7 @@ scan_number_done: return token_type::value_unsigned; } } - else if (number_type == token_type::value_integer) + else // value_integer (the callers handle value_float themselves) { if (parse_integer_signed(first, last, value_integer)) { @@ -1553,11 +1554,9 @@ scan_number_done: */ token_type scan_number_bulk_contiguous() { - // a pending unget offsets the buffer position from current; fall back - if (next_unget) - { - return token_type::uninitialized; - } + // called right after scan_after_whitespace() read current (see + // get_after_string_run()), so no unget is pending + JSON_ASSERT(!next_unget); const std::size_t rem = ia.bulk_remaining(); if (rem == 0) { @@ -1578,11 +1577,7 @@ scan_number_done: if (data[0] == '-') { number_type = token_type::value_integer; - i = 1; - if (i >= avail) - { - return token_type::uninitialized; - } + i = 1; // avail >= 2: rem == 0 returned above } const std::size_t integer_begin = i; if (data[i] == '0') @@ -1825,7 +1820,10 @@ scan_number_done: /// the error at the right character JSON_HEDLEY_ALWAYS_INLINE bool scan_literal_bulk(const char* rest, const std::size_t length, std::true_type /*bulk*/) { - if (next_unget || ia.bulk_remaining() < length + // called right after scan_after_whitespace() read current (see + // get_after_string_run()), so no unget is pending + JSON_ASSERT(!next_unget); + if (ia.bulk_remaining() < length || std::memcmp(ia.bulk_data(), rest, length) != 0) { return false; diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index 6382449c8..2831019d8 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -1281,14 +1281,7 @@ class parser std::string exception_message(const token_type expected, const std::string& context) { - std::string error_msg = "syntax error "; - - if (!context.empty()) - { - error_msg += concat("while parsing ", context, ' '); - } - - error_msg += "- "; + std::string error_msg = concat("syntax error while parsing ", context, " - "); if (last_token == token_type::parse_error) { @@ -1306,7 +1299,7 @@ class parser } return error_msg; - } + } // LCOV_EXCL_LINE (exception cleanup only) private: /// callback function diff --git a/include/nlohmann/detail/iterators/iter_impl.hpp b/include/nlohmann/detail/iterators/iter_impl.hpp index 84a850356..6e6cc0ef6 100644 --- a/include/nlohmann/detail/iterators/iter_impl.hpp +++ b/include/nlohmann/detail/iterators/iter_impl.hpp @@ -108,6 +108,9 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); + // m_it is already value-initialized, so this is not needed for + // correctness; it is kept because GCC generates faster loops with it + // (iterating a parsed document is 0.5-2.7 % slower without it, #6005) if (m_object->is_object()) { m_it.object_iterator = typename object_t::iterator(); diff --git a/include/nlohmann/detail/json_pointer.hpp b/include/nlohmann/detail/json_pointer.hpp index 6df49fbc0..ec30e17c5 100644 --- a/include/nlohmann/detail/json_pointer.hpp +++ b/include/nlohmann/detail/json_pointer.hpp @@ -304,7 +304,7 @@ class json_pointer result.reference_tokens.insert(result.reference_tokens.end(), lhs.reference_tokens.begin(), lhs.reference_tokens.end()); result.reference_tokens.insert(result.reference_tokens.end(), rhs.reference_tokens.begin(), rhs.reference_tokens.end()); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief create a new JSON pointer by appending the unescaped token at the end of the JSON pointer /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ @@ -315,7 +315,7 @@ class json_pointer result.reference_tokens.insert(result.reference_tokens.end(), lhs.reference_tokens.begin(), lhs.reference_tokens.end()); result.reference_tokens.push_back(std::move(token)); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief create a new JSON pointer by appending the array-index-token at the end of the JSON pointer /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ @@ -2180,7 +2180,7 @@ class json_pointer { // create an entry in the array current = ¤t->operator[](array_index(token)); - break; + break; // LCOV_EXCL_LINE (needs an array index with 10 or more digits: more than 1e9 elements) } /* @@ -2217,7 +2217,7 @@ class json_pointer json_pointer result; result.reference_tokens = reference_tokens; return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) json_pointer convert()&& { diff --git a/include/nlohmann/detail/macro_unscope.hpp b/include/nlohmann/detail/macro_unscope.hpp index 68a668915..f27de01d2 100644 --- a/include/nlohmann/detail/macro_unscope.hpp +++ b/include/nlohmann/detail/macro_unscope.hpp @@ -23,6 +23,7 @@ #undef JSON_DISABLE_ENUM_SERIALIZATION #undef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION #undef JSON_UNLOCKED_FILE_IO +#undef JSON_FILE_WINDOW #undef JSON_DTOA_SSE2 #undef JSON_DTOA_NEON #undef JSON_LITTLE_ENDIAN_TARGET diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 0c3c823ac..dda9c3758 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -833,7 +833,7 @@ class binary_writer while (n != 0); std::reverse(result.begin(), result.end()); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /*! @brief the decimal digits of the integer @a n, with a leading '-' if it @@ -860,7 +860,7 @@ class binary_writer } std::reverse(result.begin(), result.end()); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /*! @brief throws because the integer @a j is too large for @a format @@ -1792,7 +1792,7 @@ class binary_writer std::memcpy(out, &n, sizeof(NumberType)); if (!is_little_endian) { - std::reverse(out, out + sizeof(NumberType)); + std::reverse(out, out + sizeof(NumberType)); // LCOV_EXCL_LINE (big-endian hosts only) } } @@ -1921,7 +1921,7 @@ class binary_writer const string_t& value = *j.m_data.m_value.string; if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(value.size() + 1ul))) { - return false; + return false; // LCOV_EXCL_LINE (a string of 2 GiB or more) } out[0] = to_char_type(0x02); store_little_endian(p, static_cast(value.size() + 1ul)); @@ -2378,7 +2378,7 @@ class binary_writer const std::size_t size = oa.written() - pos; if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(size))) { - return false; + return false; // LCOV_EXCL_LINE (a document of 2 GiB or more) } const auto le_size = static_cast(size); oa.overwrite(pos, is_little_endian ? le_size : byteswap(le_size)); @@ -2422,7 +2422,6 @@ class binary_writer nothing large has been written then, and the caller writes the document in two passes, which reports that error */ - template::value, int>::type = 0> bool write_bson_document_single_pass(const BasicJsonType& document) { const std::size_t start = oa.written(); @@ -2507,7 +2506,7 @@ class binary_writer { if (JSON_HEDLEY_UNLIKELY(!close_bson_container(nested_size_pos))) { - return false; + return false; // LCOV_EXCL_LINE (a document of 2 GiB or more) } continue; } @@ -2522,7 +2521,7 @@ class binary_writer if (JSON_HEDLEY_UNLIKELY(!close_bson_container(current.size_slot))) { - return false; + return false; // LCOV_EXCL_LINE (a document of 2 GiB or more) } if (parents.empty()) { @@ -2779,10 +2778,10 @@ class binary_writer Selected through masks rather than branches, as the type and the sign of the values in an array of mixed signs alternate unpredictably (non-negative - integers are number_unsigned after parsing). Up to 64 bits, the sign is bit - 63 of the result (a narrower integer is sign-extended); computing it with - masks keeps GCC from branching on the type. A wider integer has to be - compared. + integers are number_unsigned after parsing). The sign is bit 63 of the + result (a narrower integer is sign-extended); computing it with masks keeps + GCC from branching on the type. Only called if @ref integers_fit_64_bits(); + a wider number type is written with range checks on the whole value. */ static std::uint64_t integer_bits(const BasicJsonType& j, std::uint64_t& sign_mask) noexcept { @@ -2790,9 +2789,7 @@ class binary_writer const std::uint64_t raw = is_signed ? static_cast(j.m_data.m_value.number_integer) : static_cast(j.m_data.m_value.number_unsigned); - sign_mask = sizeof(typename BasicJsonType::number_integer_t) <= sizeof(std::uint64_t) - ? (std::uint64_t{0} - static_cast(is_signed)) & (std::uint64_t{0} - (raw >> 63u)) - : std::uint64_t{0} - static_cast(is_signed && j.m_data.m_value.number_integer < 0); + sign_mask = (std::uint64_t{0} - static_cast(is_signed)) & (std::uint64_t{0} - (raw >> 63u)); return raw; } @@ -3129,22 +3126,19 @@ class binary_writer // negative fixnum write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)()) { // int 8 oa.write_character(to_char_type(0xD0)); write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)()) { // int 16 oa.write_character(to_char_type(0xD1)); write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)()) { // int 32 oa.write_character(to_char_type(0xD2)); @@ -4060,7 +4054,7 @@ class binary_writer Containers, strings, high-precision numbers, booleans and null cannot be declared as the single type of an optimized container in BJData; such a container is written unoptimized. The reader rejects them with the same - list (binary_reader::is_bjd_excluded_optimized_type()). + list (binary_reader::is_valid_optimized_type()). */ static constexpr bool is_bjdata_excluded_type_marker(const CharType marker) noexcept { @@ -4159,57 +4153,34 @@ class binary_writer } /*! - @brief validate (dry_run) or write one BJData ND-array element of integer dtype @a T - @return whether @a el's value is in range of @a T; always true when @a dry_run is false - */ - template - bool write_bjdata_ndarray_element(const BasicJsonType& el, const bool dry_run) - { - if (dry_run) - { - return bjdata_ndarray_value_in_range(el); - } - using storage_type = typename std::conditional::value, std::uint64_t, std::int64_t>::type; - write_number(static_cast(el.template get()), true); - return true; - } - - /*! - @brief validate (dry_run) or write one BJData ND-array element of dtype 'd' (single precision) + @brief whether the BJData ND-array element @a el can be written with dtype 'd' (single precision) @return whether @a el's value survives narrowing to float and back without any change - (so the ND-array round-trips exactly), or is infinite or NaN; always true when - @a dry_run is false + (so the ND-array round-trips exactly), or is infinite or NaN */ - bool write_bjdata_ndarray_float_element(const BasicJsonType& el, const bool dry_run) + static bool bjdata_ndarray_float_in_range(const BasicJsonType& el) noexcept { const auto dval = static_cast(el.m_data.m_value.number_float); - if (dry_run) - { #ifdef __GNUC__ - JSON_HEDLEY_DIAGNOSTIC_PUSH - JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif - // a value that would be rounded (rather than exactly represented) by the - // narrowing to float is treated like an out-of-range integer element; this - // is the same criterion write_compact_float() uses for CBOR/MessagePack - const bool in_range = std::isnan(dval) || - (dval >= static_cast(std::numeric_limits::lowest()) && - dval <= static_cast((std::numeric_limits::max)()) && - static_cast(static_cast(dval)) == dval) || - std::isinf(dval); + // a value that would be rounded (rather than exactly represented) by the + // narrowing to float is treated like an out-of-range integer element; this + // is the same criterion write_compact_float() uses for CBOR/MessagePack + return std::isnan(dval) || + (dval >= static_cast(std::numeric_limits::lowest()) && + dval <= static_cast((std::numeric_limits::max)()) && + static_cast(static_cast(dval)) == dval) || + std::isinf(dval); #ifdef __GNUC__ - JSON_HEDLEY_DIAGNOSTIC_POP + JSON_HEDLEY_DIAGNOSTIC_POP #endif - return in_range; - } - write_number(static_cast(dval), true); - return true; } /*! @brief write the elements of an ND-array as the number kind @a dtype names (see @ref write_bjdata_ndarray), after they have been found to be - in range by a dry run of @ref write_bjdata_ndarray_elements + in range by @ref bjdata_ndarray_elements_in_range The elements are encoded in blocks into space reserved from the sink. */ @@ -4291,20 +4262,13 @@ class binary_writer } /*! - @brief validate or write every element of a BJData ND-array's `_ArrayData_` + @brief whether every element of a BJData ND-array's `_ArrayData_` is in range for its dtype @param[in] array_data the `_ArrayData_` array @param[in] dtype the ND-array dtype marker, as returned by bjdata_ndarray_type_marker() - @param[in] dry_run true to only range-check each element, false to write it - @return whether every element is in range for @a dtype (always true when @a dry_run is false) + @return whether every element is in range for @a dtype */ - bool write_bjdata_ndarray_elements(const BasicJsonType& array_data, const CharType dtype, const bool dry_run) + static bool bjdata_ndarray_elements_in_range(const BasicJsonType& array_data, const CharType dtype) { - if (!dry_run) - { - write_bjdata_ndarray_values(array_data, dtype); - return true; - } - for (const auto& el : array_data) { bool ok = true; @@ -4312,43 +4276,39 @@ class binary_writer { case 'U': case 'B': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'C': // BJData limits a char to 0..127 (the intersection of the int8 and uint8 ranges) ok = bjdata_ndarray_value_in_range(el) && bjdata_ndarray_value_in_range(el); break; case 'i': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'u': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'I': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'm': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'l': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'M': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'L': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'd': - ok = write_bjdata_ndarray_float_element(el, dry_run); + ok = bjdata_ndarray_float_in_range(el); break; case 'D': default: // 'D' (double) already spans the full range of number_float_t - if (!dry_run) - { - write_number(el.template get(), true); - } break; } if (!ok) @@ -4435,7 +4395,7 @@ class binary_writer const auto dim = el.template get(); if (!value_in_range_of(dim)) { - return true; + return true; // LCOV_EXCL_LINE (size_t is narrower than 64 bits) } const auto dim_size = static_cast(dim); @@ -4485,7 +4445,7 @@ class binary_writer // wrap (integers) or overflow to infinity (the "single" precision // float) instead of being reported, so such an object falls back to // a plain object encoding as well - if (!write_bjdata_ndarray_elements(array_data, dtype, true)) + if (!bjdata_ndarray_elements_in_range(array_data, dtype)) { return true; } @@ -4497,7 +4457,7 @@ class binary_writer write_ubjson(array_size, use_count, use_type, true, true, bjdata_version); - write_bjdata_ndarray_elements(array_data, dtype, false); + write_bjdata_ndarray_values(array_data, dtype); return false; } @@ -4823,8 +4783,8 @@ class binary_writer run are encoded into space reserved from the sink, as in @ref write_array_elements; an integer that BON8 cannot write (an unsigned one above the int64 range, or one of a number type wider than 64 bits) throws - from write_bon8_value, like any other value that is not a number. Kept out - of line, like @ref write_cbor_array. + out_of_range.407, as write_bon8_value does. Kept out of line, like + @ref write_cbor_array. */ template JSON_HEDLEY_NEVER_INLINE @@ -4874,10 +4834,10 @@ class binary_writer string_open = false; } - if (unencodable) + if (JSON_HEDLEY_UNLIKELY(unencodable)) { - write_bon8_value(*it, string_open, depth + 1); - ++it; + // an integer beyond the int64 range, as write_bon8_value would report it + throw_integer_out_of_range(*it, input_format_t::bon8, "int64"); } } } @@ -4912,14 +4872,13 @@ class binary_writer break; case value_t::number_integer: - if (JSON_HEDLEY_LIKELY(integer_fits(value.m_data.m_value.number_integer))) + if (JSON_HEDLEY_UNLIKELY(!integer_fits(value.m_data.m_value.number_integer))) { - write_bon8_integer(static_cast(value.m_data.m_value.number_integer)); - string_open = false; - break; + // beyond 64 bits (number types wider than that) + throw_integer_out_of_range(value, input_format_t::bon8, "int64"); } - // beyond 64 bits (number types wider than that): throws - write_bon8_value(value, string_open, depth + 1); + write_bon8_integer(static_cast(value.m_data.m_value.number_integer)); + string_open = false; break; case value_t::number_float: @@ -4928,14 +4887,13 @@ class binary_writer break; case value_t::number_unsigned: - if (JSON_HEDLEY_LIKELY(integer_fits_max(value.m_data.m_value.number_unsigned))) + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(value.m_data.m_value.number_unsigned))) { - write_bon8_integer(static_cast(value.m_data.m_value.number_unsigned)); - string_open = false; - break; + // too large for BON8 + throw_integer_out_of_range(value, input_format_t::bon8, "int64"); } - // too large for BON8: throws - write_bon8_value(value, string_open, depth + 1); + write_bon8_integer(static_cast(value.m_data.m_value.number_unsigned)); + string_open = false; break; case value_t::object: diff --git a/include/nlohmann/detail/output/output_adapters.hpp b/include/nlohmann/detail/output/output_adapters.hpp index e9e57ab9a..b1ea305b9 100644 --- a/include/nlohmann/detail/output/output_adapters.hpp +++ b/include/nlohmann/detail/output/output_adapters.hpp @@ -116,6 +116,10 @@ inline void copy_short(CharType* dst, const CharType* src, const std::size_t len /// the number of characters an empty vector is grown to by output_vector_sink constexpr std::size_t min_vector_sink_capacity = 64; +/// the number of characters output_vector_sink makes writable ahead of its +/// position at a time: 64 KiB of the vector's capacity, not all of it +constexpr std::size_t vector_sink_window = 65536; + /// @brief non-virtual output sink writing into a std::vector /// /// This sink is not part of the virtual output_adapter_protocol hierarchy: it is @@ -129,7 +133,8 @@ constexpr std::size_t min_vector_sink_capacity = 64; /// encoding them: insert() is an out-of-line call into a routine that can insert /// anywhere, and resize() is one as well with libstdc++. The sink therefore keeps /// its own write position: it grows the vector geometrically ahead of the -/// position and copies each write to it, so a write is a capacity check and a +/// position (making @ref vector_sink_window characters of it writable at a +/// time) and copies each write to it, so a write is a capacity check and a /// short copy. The destructor shrinks the vector to the bytes actually written, /// also when a write throws; shrinking a vector never allocates. The sink owns /// that trim, so it can be moved but not copied, and the vector must not be used @@ -277,9 +282,17 @@ class output_vector_sink // reallocate for every few bytes const std::size_t written = position(); const std::size_t needed = written + length; - v->resize((std::max)(min_vector_sink_capacity, needed <= v->capacity() - ? v->capacity() - : (std::max)(needed, v->capacity() * 2))); + const std::size_t capacity = (std::max)(min_vector_sink_capacity, needed <= v->capacity() + ? v->capacity() + : (std::max)(needed, v->capacity() * 2)); + v->reserve(capacity); + // resize() zeroes the characters it adds. Zeroing all of a doubled + // capacity, megabytes of a fresh allocation, at once faults in its + // pages and evicts them from the cache before the writer gets there, + // so its stores miss again; a window ahead of the position is still + // cached when they come (the binary writers on a long array of + // numbers: 4-26 % faster on x86, 1-2.5 % on Apple M1) + v->resize((std::min)(capacity, (std::max)(needed, written + vector_sink_window))); cursor = v->data() + written; limit = v->data() + v->size(); } diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 4656dbdf1..235e7bafd 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -8,7 +8,7 @@ #pragma once -#include // remove, fill, find, none_of, min +#include // copy, none_of, min, search #include // array #include // localeconv, lconv #include // isfinite @@ -1113,36 +1113,21 @@ class serializer { // the code point is at least 0x60 (smaller ones are handled // above) and is not printable ASCII (which the scanner - // copies): it is DEL (0x7F) under EnsureAscii or a non-ASCII - // character + // copies). Under EnsureAscii it is DEL (0x7F): every + // well-formed multi-byte character is escaped by + // write_u_escapes() above, so decode() only ever accepts a + // single byte here, and nothing of it was buffered if (EnsureAscii) { - JSON_ASSERT(codepoint >= 0x7F); - if (error_handler == error_handler_t::keep) - { - // this character was buffered as raw bytes below - // in case it turned out to be part of an - // ill-formed sequence (which is kept as is); now - // that it decoded to a well-formed code point, - // undo that and \u-escape it like any other - // character instead - bytes = bytes_after_last_accept; - } - if (codepoint <= 0xFFFF) - { - write_u_escape(bytes, static_cast(codepoint)); - } - else - { - write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); - write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); - } + JSON_ASSERT(codepoint == 0x7F); + JSON_ASSERT(bytes == bytes_after_last_accept); + write_u_escape(bytes, static_cast(codepoint)); } else { // copy byte to buffer (all previous bytes of the // sequence have been copied in the default case below) - string_buffer[bytes++] = s[i]; + string_buffer[bytes++] = s[i]; // LCOV_EXCL_LINE (well-formed sequences are copied in runs) } reserve_string_buffer(bytes); @@ -2095,31 +2080,35 @@ class serializer // check if the buffer was large enough JSON_ASSERT(static_cast(len) < number_buffer.size()); - // look up the locale's thousands separator and decimal point now, - // matching what snprintf_float() just used (see lexer::get_decimal_point()) + // look up the locale's decimal point now, matching what + // snprintf_float() just used (see get_decimal_point() in + // number_parse.hpp); it is a string, which can be longer than one + // byte (U+066B in ps_AF, #5993). + // A thousands separator needs no handling: %g never groups digits. const auto* loc = std::localeconv(); JSON_ASSERT(loc != nullptr); - const char thousands_sep = (loc->thousands_sep == nullptr) ? '\0' : *loc->thousands_sep; - const char decimal_point = (loc->decimal_point == nullptr) ? '\0' : *loc->decimal_point; - - // erase thousands separators - if (thousands_sep != '\0') - { - // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep); - std::fill(end, number_buffer.end(), '\0'); - JSON_ASSERT((end - number_buffer.begin()) <= len); - len = (end - number_buffer.begin()); - } + const char* const decimal_point = (loc->decimal_point == nullptr) ? "" : loc->decimal_point; + const std::size_t decimal_point_len = std::char_traits::length(decimal_point); // convert decimal point to '.' - if (decimal_point != '\0' && decimal_point != '.') + if ((decimal_point_len > 1) || (decimal_point_len == 1 && decimal_point[0] != '.')) { - // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point); - if (dec_pos != number_buffer.end()) + const auto first = number_buffer.begin(); + const auto last = first + len; + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::search returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto dec_pos = std::search(first, last, decimal_point, decimal_point + decimal_point_len); + if (dec_pos != last) { *dec_pos = '.'; + if (decimal_point_len > 1) + { + // move the fraction and exponent behind the '.' (std::copy + // allows the overlap, as the destination is before the source) + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::copy returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto end = std::copy(dec_pos + static_cast(decimal_point_len), last, dec_pos + 1); + *end = '\0'; + len = end - first; + } } } diff --git a/include/nlohmann/detail/string_concat.hpp b/include/nlohmann/detail/string_concat.hpp index 74002c878..d7ad45af3 100644 --- a/include/nlohmann/detail/string_concat.hpp +++ b/include/nlohmann/detail/string_concat.hpp @@ -139,7 +139,7 @@ inline OutStringType concat(Args && ... args) str.reserve(concat_length(args...)); concat_into(str, std::forward(args)...); return str; -} +} // LCOV_EXCL_LINE (exception cleanup only) } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/string_escape.hpp b/include/nlohmann/detail/string_escape.hpp index 1a4b6c501..3e6dc13ae 100644 --- a/include/nlohmann/detail/string_escape.hpp +++ b/include/nlohmann/detail/string_escape.hpp @@ -83,10 +83,6 @@ template inline void unescape(StringType& s) { auto pos = s.find_first_of('~', 0); - if (pos == StringType::npos) - { - return; - } StringType result; result.reserve(s.size()); diff --git a/include/nlohmann/detail/string_utils.hpp b/include/nlohmann/detail/string_utils.hpp index 4b1213c80..353fe5aa9 100644 --- a/include/nlohmann/detail/string_utils.hpp +++ b/include/nlohmann/detail/string_utils.hpp @@ -37,7 +37,7 @@ StringType to_string(std::size_t value) StringType result; int_to_string(result, value); return result; -} +} // LCOV_EXCL_LINE (exception cleanup only) /// @return a byte as two uppercase hexadecimal digits inline std::string hex_byte(const std::uint8_t byte) @@ -47,7 +47,7 @@ inline std::string hex_byte(const std::uint8_t byte) result[0] = nibble_to_hex[byte / 16]; result[1] = nibble_to_hex[byte % 16]; return result; -} +} // LCOV_EXCL_LINE (exception cleanup only) /*! @brief write a UTF-16 code unit as four lowercase hexadecimal digits @@ -391,7 +391,7 @@ inline StringType sanitize_utf8(const StringType& s, const error_handler_t error } return result; -} +} // LCOV_EXCL_LINE (exception cleanup only) } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/view/builder.hpp b/include/nlohmann/detail/view/builder.hpp index 938d6d73b..14be4ea7c 100644 --- a/include/nlohmann/detail/view/builder.hpp +++ b/include/nlohmann/detail/view/builder.hpp @@ -1168,19 +1168,20 @@ indent_done: if (negative) { ++p; + if (NLOHMANN_VIEW_UNLIKELY(cur() < '0' || cur() > '9')) + { + return fail(error_code::number_after_minus); + } } const unsigned char* const int_start = p; if (cur() == '0') { ++p; } - else if (NLOHMANN_VIEW_LIKELY(cur() >= '1' && cur() <= '9')) - { - p = skip_digits(p + 1, e); - } else { - return fail(error_code::number_after_minus); + // (a digit: the dispatch or the check after the sign saw it) + p = skip_digits(p + 1, e); } const auto int_digits = static_cast(p - int_start); std::size_t frac_digits = 0; diff --git a/include/nlohmann/detail/view/compare.hpp b/include/nlohmann/detail/view/compare.hpp index 96a14efbb..2d2c238b9 100644 --- a/include/nlohmann/detail/view/compare.hpp +++ b/include/nlohmann/detail/view/compare.hpp @@ -397,9 +397,9 @@ class view_side return m_view.size(); } - string_view_t string() const + string_view_t string() const noexcept { - return m_view.get_string(); + return {m_view.m_doc->str(*m_view.m_node), m_view.m_node->len}; } bool boolean() const noexcept @@ -428,28 +428,19 @@ class view_side return string_view_t(m_view.m_doc->str(*m_view.m_node), m_view.m_node->len); } - /// a number, boolean, or null as a basic_json value (no allocation) + /// a number as a basic_json value (no allocation); only called for numbers BasicJsonType scalar() const { - switch (m_view.type()) + const value_t t = m_view.type(); + if (t == value_t::number_integer) { - case value_t::number_integer: - return BasicJsonType(m_view.template get()); - case value_t::number_unsigned: - return BasicJsonType(m_view.template get()); - case value_t::number_float: - return BasicJsonType(m_view.template get()); - case value_t::boolean: - return BasicJsonType(m_view.template get()); - case value_t::null: - case value_t::object: - case value_t::array: - case value_t::string: - case value_t::binary: - case value_t::discarded: - default: - return BasicJsonType(nullptr); + return BasicJsonType(m_view.template get()); } + if (t == value_t::number_unsigned) + { + return BasicJsonType(m_view.template get()); + } + return BasicJsonType(m_view.template get()); } /// start at the first element of this array @@ -531,9 +522,9 @@ class json_side return m_json->size(); } - string_view_t string() const + string_view_t string() const noexcept { - const auto& s = m_json->template get_ref(); + const auto& s = *m_json->template get_ptr(); return string_view_t(s.data(), s.size()); } diff --git a/include/nlohmann/detail/view/edit.hpp b/include/nlohmann/detail/view/edit.hpp index b572b4848..8ddf3f849 100644 --- a/include/nlohmann/detail/view/edit.hpp +++ b/include/nlohmann/detail/view/edit.hpp @@ -96,11 +96,11 @@ NLOHMANN_VIEW_NOINLINE inline void check_utf8_scan(const unsigned char* p, std:: } if (*q >= 0x80) { - check_utf8_bytewise(p, n, static_cast(q - p)); - return; // not reached: the scan stopped at ill-formed UTF-8 + break; // ill-formed UTF-8: reported below } ++q; // a quote, backslash, or control character } + check_utf8_bytewise(p, n, static_cast(q - p)); } inline void check_utf8(const char* s, std::size_t n) diff --git a/include/nlohmann/detail/view/edit_storage.hpp b/include/nlohmann/detail/view/edit_storage.hpp index e5b7ddb90..34b804c46 100644 --- a/include/nlohmann/detail/view/edit_storage.hpp +++ b/include/nlohmann/detail/view/edit_storage.hpp @@ -158,7 +158,7 @@ inline void set_moved(document_data& d, node* n, node* seq, std::size_t cap) e.moved_cap[n->off] = cap; return; } - reserve_moved(d); + // every caller has called reserve_moved() (see above), so neither push_back() can throw e.moved.push_back(seq); e.moved_cap.push_back(cap); n->off = static_cast(e.moved.size() - 1); @@ -243,7 +243,7 @@ inline node* find_parent(const document_data& d, const node* target) } if (target == lo) { - return nullptr; + return nullptr; // LCOV_EXCL_LINE (the root, or a new value's root, which is linked) } for (;;) { diff --git a/include/nlohmann/detail/view/errors.hpp b/include/nlohmann/detail/view/errors.hpp index f4ab8db01..295eea4cb 100644 --- a/include/nlohmann/detail/view/errors.hpp +++ b/include/nlohmann/detail/view/errors.hpp @@ -185,10 +185,6 @@ void value_range(const document_data& d, const node* n, std::size_t& start, std: end = p.end; return; } - if ((n->flags & node_flags::is_new) != 0 || n->off >= d.size) - { - return; - } // the open brackets: n, then the last element as long as it is a non-empty container std::size_t closing = 0; std::size_t pos = std::string::npos; @@ -234,8 +230,7 @@ void value_range(const document_data& d, const node* n, std::size_t& start, std: /// the context basic_json puts in front of the message of an exception about /// a value: "(/path) " with JSON_DIAGNOSTICS (none for the root), then -/// "(bytes a-b) " with JSON_DIAGNOSTIC_POSITIONS (with JSON_DIAGNOSTICS, only -/// if there is a path) +/// "(bytes a-b) " with JSON_DIAGNOSTIC_POSITIONS (also for the root) /// /// This and the throw helpers that take an error_site are templates, so that /// the path and range code is instantiated only where an exception can be @@ -257,18 +252,22 @@ std::string diagnostics(const error_site site) #if JSON_DIAGNOSTICS std::vector tokens; const bool found = edited ? value_path(d, site.value(), tokens) : value_path(d, site.value(), tokens); - if (!found || tokens.empty()) + if (!found) { - // context is still empty; returning it keeps the copy elided (-Wnrvo) + // a value that is not found has no path, and with it no bytes return context; } - std::string path; - for (const std::string& token : tokens) + if (!tokens.empty()) { - path.push_back('/'); - path += detail::escape(token); + // the root value has no path, but still has a byte range + std::string path; + for (const std::string& token : tokens) + { + path.push_back('/'); + path += detail::escape(token); + } + context = concat('(', path, ") "); } - context = concat('(', path, ") "); #endif #if JSON_DIAGNOSTIC_POSITIONS std::size_t start = 0; @@ -420,10 +419,12 @@ struct throwing_sax return true; } + // LCOV_EXCL_START (throw_parse_failure parses JSON only) bool binary(binary_t& /*unused*/) { return true; } + // LCOV_EXCL_STOP bool start_object(std::size_t /*unused*/) { diff --git a/include/nlohmann/detail/view/image.hpp b/include/nlohmann/detail/view/image.hpp index 9e57082de..e6680730f 100644 --- a/include/nlohmann/detail/view/image.hpp +++ b/include/nlohmann/detail/view/image.hpp @@ -787,10 +787,6 @@ inline void load_image(document_data& d, const std::uint8_t* image, std::size_t d.edits.reset(); d.base[2] = nullptr; d.owned.clear(); - if (d.owned_image.empty() || image != d.owned_image.data()) - { - d.owned_image.clear(); - } d.arena.clear(); d.indexes.clear(); d.index_slots.clear(); diff --git a/include/nlohmann/detail/view/materialize.hpp b/include/nlohmann/detail/view/materialize.hpp index 1dcd1940c..71b29a13c 100644 --- a/include/nlohmann/detail/view/materialize.hpp +++ b/include/nlohmann/detail/view/materialize.hpp @@ -65,14 +65,9 @@ struct position_source return token{end - start}; } - /// the range [first, last), or none if either is npos + /// the range [first, last) void set(std::size_t first, std::size_t last) noexcept { - if (first == std::string::npos || last == std::string::npos) - { - clear(); - return; - } start = first; end = last; } @@ -191,11 +186,11 @@ struct position_source first = n.off; last = n.off + 4u; break; - case node_kind::object: + case node_kind::object: // LCOV_EXCL_LINE case node_kind::array: case node_kind::link: default: - break; + break; // LCOV_EXCL_LINE } // (a crafted image can point past the text) if (last == std::string::npos || last > d.size) diff --git a/include/nlohmann/detail/view/serializer.hpp b/include/nlohmann/detail/view/serializer.hpp index 2bfa2d2c0..deccec8e4 100644 --- a/include/nlohmann/detail/view/serializer.hpp +++ b/include/nlohmann/detail/view/serializer.hpp @@ -47,6 +47,7 @@ class output_buffer : m_out(sized(out, estimate)) , m_pos(&m_out[0]) , m_end(m_pos + m_out.size()) + , m_reserved((std::max)(estimate, m_out.size())) {} void finish() @@ -116,11 +117,12 @@ class output_buffer const auto used = static_cast(m_pos - m_out.data()); // (a step does not go beyond the reserved estimate, so that a good // estimate is never copied to a larger allocation; not a clamp: the - // lower bound used + n + 256 can exceed capacity(), and then it must win) - const std::size_t size = (std::max)((std::min)(m_out.size() + step, m_out.capacity()), used + n + 256); - if (size > m_out.capacity()) + // lower bound used + n + 256 can exceed m_reserved, and then it must win) + const std::size_t size = (std::max)((std::min)(m_out.size() + step, m_reserved), used + n + 256); + if (size > m_reserved) { - m_out.reserve((std::max)(m_out.capacity() * 2, size)); + m_reserved = (std::max)(m_reserved * 2, size); + m_out.reserve(m_reserved); } m_out.resize(size); m_pos = &m_out[0] + used; @@ -130,6 +132,9 @@ class output_buffer StringType& m_out; char* m_pos; char* m_end; + /// what was last passed to reserve(), a lower bound of the capacity + /// (StringType need not have a capacity() member) + std::size_t m_reserved; }; /// The length of the run at s that dump() writes unchanged without @@ -641,7 +646,7 @@ class view_serializer { invalid_utf8(s, n.len); // (the scan stopped at ill-formed UTF-8) } - write_codepoint(s[i], s + i, 1); // a quote, a backslash, or a control character + write_escaped(s[i]); // a quote, a backslash, or a control character ++i; } m_out.put('"'); @@ -1389,20 +1394,10 @@ class view_serializer /// a float node as dump() writes it void write_float_node(const node& n) { - write_float_node(n, fast_double {}); - } - - void write_float_node(const node& n, std::false_type /*other*/) - { + // (only for types other than double: doubles go through write_double_ahead()) write_float(float_value(m_doc, n)); } - void write_float_node(const node& n, std::true_type /*double*/) - { - m_out.reserve(64); - m_out.set_cursor(write_double_at(m_out.cursor(), n)); - } - /*! @brief (doubles) the float at n as dump() writes it, at w (64 bytes of room) @@ -1497,10 +1492,11 @@ class view_serializer { const string_t dumped = BasicJsonType(string_t(reinterpret_cast(s), n)).dump(); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) static_cast(dumped); - } + } // LCOV_EXCL_LINE (dump() throws: the callers pass only ill-formed UTF-8) - template - void write_codepoint(std::uint32_t codepoint, const unsigned char* bytes, std::size_t len) + /// write the escape sequence of a quote, a backslash or a control + /// character (the only bytes a run of plain output stops at) + void write_escaped(std::uint32_t codepoint) { switch (codepoint) { @@ -1528,20 +1524,7 @@ class view_serializer default: break; } - if (codepoint <= 0x1F || (EnsureAscii && codepoint >= 0x7F)) - { - if (codepoint <= 0xFFFF) - { - write_u_escape(codepoint); - } - else - { - write_u_escape(0xD7C0u + (codepoint >> 10u)); - write_u_escape(0xDC00u + (codepoint & 0x3FFu)); - } - return; - } - m_out.put(reinterpret_cast(bytes), len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) LCOV_EXCL_LINE (printable characters are copied in runs) + write_u_escape(codepoint); // (a control character) } void write_u_escape(std::uint32_t u) diff --git a/include/nlohmann/detail/view/value.hpp b/include/nlohmann/detail/view/value.hpp index ffdcc3605..81b175e2b 100644 --- a/include/nlohmann/detail/view/value.hpp +++ b/include/nlohmann/detail/view/value.hpp @@ -33,12 +33,37 @@ namespace view template struct value_tag {}; +/// whether basic_json reads T with a dedicated from_json() overload that does +/// not accept a boolean: exactly number_integer_t, number_unsigned_t and +/// number_float_t (the other arithmetic types convert a boolean to 0 or 1) +template +struct rejects_boolean : std::integral_constant < bool, + std::is_same::value + || std::is_same::value + || std::is_same::value > +{}; + +/// a boolean as T: 0 or 1 +template +NLOHMANN_VIEW_ALWAYS_INLINE T boolean_as(const document_data& /*d*/, const node& n, std::false_type /*rejects*/) +{ + return static_cast((n.flags & node_flags::is_true) != 0); +} + +/// a boolean as number_integer_t, number_unsigned_t or number_float_t: type_error.302, as in basic_json +template +[[noreturn]] NLOHMANN_VIEW_ALWAYS_INLINE T boolean_as(const document_data& d, const node& n, std::true_type /*rejects*/) +{ + throw_type_error(error_site(&d, &n), exception_id::type_mismatch, "type must be number, but is ", value_type_name(static_cast(n.kind))); +} + /*! @brief the number or boolean of a node converted to an arithmetic type As basic_json's get() for arithmetic types: integers and floats are -converted with static_cast, booleans give 0 or 1, and other types throw -type_error.302. +converted with static_cast, booleans give 0 or 1 (but throw type_error.302 for +number_integer_t, number_unsigned_t and number_float_t, as basic_json does), +and other types throw type_error.302. */ template NLOHMANN_VIEW_ALWAYS_INLINE T arithmetic_value(const document_data& d, const node& n) @@ -52,7 +77,7 @@ NLOHMANN_VIEW_ALWAYS_INLINE T arithmetic_value(const document_data& d, const nod case value_t::number_float: return static_cast(float_value(d, n)); case value_t::boolean: - return static_cast((n.flags & node_flags::is_true) != 0); + return boolean_as(d, n, rejects_boolean {}); case value_t::null: case value_t::object: case value_t::array: diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 3eee1ec92..040743923 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -400,7 +400,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}}; #elif defined(__GNUC__) || defined(__GNUG__) result["compiler"] = {{"family", "gcc"}, {"version", detail::concat( - detail::to_string(static_cast(__GNUC__)), '.', + detail::to_string(static_cast(__GNUC__)), '.', // LCOV_EXCL_LINE detail::to_string(static_cast(__GNUC_MINOR__)), '.', detail::to_string(static_cast(__GNUC_PATCHLEVEL__))) } @@ -543,6 +543,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec //////////////////////// JSON_PRIVATE_UNLESS_TESTED: + // the type and value of a basic_json without its base class (defined + // below); the destruction helpers of json_value use it as a stand-in + struct data; + /*! @brief a JSON value @@ -672,14 +676,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec case value_t::binary: binary = create(); break; - case value_t::null: + case value_t::null: // LCOV_EXCL_LINE case value_t::boolean: case value_t::number_integer: case value_t::number_unsigned: case value_t::number_float: case value_t::discarded: default: - break; + break; // LCOV_EXCL_LINE } } @@ -719,27 +723,28 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec explicit json_value(array_t* value) noexcept : array(value) {} private: - // raw, allocation-free transfer of m_data from src to dst: no - // set_parents()/assert_invariant() (the former is O(#children) per - // call under JSON_DIAGNOSTICS, which would make the walk below - // quadratic); dst takes ownership, src is left as value_t::null. - static void take(basic_json& dst, basic_json& src) noexcept + // raw, allocation-free transfer of the type and value from src to + // dst: no set_parents()/assert_invariant() (the former is + // O(#children) per call under JSON_DIAGNOSTICS, which would make the + // walk below quadratic); dst takes ownership, src is left as + // value_t::null. + static void take(data& dst, data& src) noexcept { - dst.m_data.m_type = src.m_data.m_type; - dst.m_data.m_value = src.m_data.m_value; - src.m_data.m_type = value_t::null; + dst.m_type = src.m_type; + dst.m_value = src.m_value; + src.m_type = value_t::null; } // true if v is not an array/object, or is an already-empty one - static bool has_no_children(const basic_json& v) noexcept + static bool has_no_children(const data& v) noexcept { - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - return v.m_data.m_value.array->empty(); + return v.m_value.array->empty(); } - if (v.m_data.m_type == value_t::object) + if (v.m_type == value_t::object) { - return v.m_data.m_value.object->empty(); + return v.m_value.object->empty(); } return true; } @@ -748,21 +753,21 @@ private: // its own, i.e. v can be freed in one go (its destructor then // destroys scalars and empty containers only, so it recurses at most // one level deep); stops at the first structured child it meets - static bool all_children_leaves(const basic_json& v) noexcept + static bool all_children_leaves(const data& v) noexcept { // array_t only needs forward iterators (see custom_array_type) - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - const auto& a = *v.m_data.m_value.array; + const auto& a = *v.m_value.array; return std::all_of(a.begin(), a.end(), [](const basic_json & el) { - return has_no_children(el); + return has_no_children(el.m_data); }); } - const auto& o = *v.m_data.m_value.object; + const auto& o = *v.m_value.object; return std::all_of(o.begin(), o.end(), [](const typename object_t::value_type & el) { - return has_no_children(el.second); + return has_no_children(el.second.m_data); }); } @@ -794,30 +799,30 @@ private: // the child of a non-empty array/object v that the walk in // destroy_container() continues with (see walk_child_it() above) - static basic_json& walk_child(basic_json& v) + static data& walk_child(data& v) { - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - return v.m_data.m_value.array->back(); + return v.m_value.array->back().m_data; } - JSON_ASSERT(v.m_data.m_type == value_t::object); - return walk_child_it(*v.m_data.m_value.object)->second; + JSON_ASSERT(v.m_type == value_t::object); + return walk_child_it(*v.m_value.object)->second.m_data; } // removes walk_child(v) from a non-empty array/object v; this never // allocates, and since it is only ever called when that child is a // scalar or an already-empty array/object, destroying it never // recurses more than one level deep (see destroy() below) - static void pop_walk_child(basic_json& v) + static void pop_walk_child(data& v) { - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - v.m_data.m_value.array->pop_back(); + v.m_value.array->pop_back(); } else { - JSON_ASSERT(v.m_data.m_type == value_t::object); - v.m_data.m_value.object->erase(walk_child_it(*v.m_data.m_value.object)); + JSON_ASSERT(v.m_type == value_t::object); + v.m_value.object->erase(walk_child_it(*v.m_value.object)); } } @@ -826,23 +831,23 @@ private: // recurses at most one level deep); this is the same allocator-based // free the old recursive implementation used, just factored out so // every level of the walk in destroy() can share it - static void free_container(basic_json& v) noexcept + static void free_container(data& v) noexcept { JSON_ASSERT(has_no_children(v) || all_children_leaves(v)); - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { AllocatorType alloc; - std::allocator_traits::destroy(alloc, v.m_data.m_value.array); - std::allocator_traits::deallocate(alloc, v.m_data.m_value.array, 1); + std::allocator_traits::destroy(alloc, v.m_value.array); + std::allocator_traits::deallocate(alloc, v.m_value.array, 1); } else { - JSON_ASSERT(v.m_data.m_type == value_t::object); + JSON_ASSERT(v.m_type == value_t::object); AllocatorType alloc; - std::allocator_traits::destroy(alloc, v.m_data.m_value.object); - std::allocator_traits::deallocate(alloc, v.m_data.m_value.object, 1); + std::allocator_traits::destroy(alloc, v.m_value.object); + std::allocator_traits::deallocate(alloc, v.m_value.object, 1); } - v.m_data.m_type = value_t::null; // avoid a double free if v is later destructed + v.m_type = value_t::null; // avoid a double free if v is later destructed } public: @@ -873,14 +878,15 @@ public: // This json_value is not itself a basic_json, so the top-level // container is first moved into a local stand-in ("cur"); a - // default-constructed basic_json has a null pointer in its - // m_value (see data::m_value's initializer), so swapping it - // with *this leaves this union's own pointer null, and it is - // never looked at or freed a second time. - basic_json cur; - cur.m_data.m_type = t; - using std::swap; - swap(cur.m_data.m_value, *this); + // default-constructed data has a null pointer in its m_value + // (see data::m_value's initializer), so swapping it with *this + // leaves this union's own pointer null, and it is never looked + // at or freed a second time. The stand-ins are data, not + // basic_json, so that destroying a value never constructs a + // CustomBaseClass, whose constructor may allocate (#5995). + data cur; + cur.m_type = t; + std::swap(cur.m_value, *this); destroy_bounded(cur, 0); } @@ -889,7 +895,7 @@ public: // The first recursion_depth_limit() levels are destroyed by plain // recursion, which is the fastest way and cannot throw; anything // nested deeper is handed to destroy_walk(), which uses no call stack. - static void destroy_bounded(basic_json& v, std::size_t depth) noexcept + static void destroy_bounded(data& v, std::size_t depth) noexcept { if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) { @@ -897,23 +903,23 @@ public: return; } - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - for (auto& child : *v.m_data.m_value.array) + for (auto& child : *v.m_value.array) { if (child.m_data.m_type == value_t::array || child.m_data.m_type == value_t::object) { - destroy_bounded(child, depth + 1); + destroy_bounded(child.m_data, depth + 1); } } } else { - for (auto& child : *v.m_data.m_value.object) + for (auto& child : *v.m_value.object) { if (child.second.m_data.m_type == value_t::array || child.second.m_data.m_type == value_t::object) { - destroy_bounded(child.second, depth + 1); + destroy_bounded(child.second.m_data, depth + 1); } } } @@ -940,18 +946,15 @@ public: // array/object, which neither allocates nor recurses more // than one level deep. // - // This json_value is not itself a basic_json, so the - // top-level container is first moved into a local stand-in - // ("cur"); a default-constructed basic_json has a null - // pointer in its m_value (see data::m_value's initializer), - // so swapping it with *this leaves this union's own pointer - // null, and it is never looked at or freed a second time. - static void destroy_walk(basic_json& top) noexcept + // Like destroy_container(), the walk keeps the containers it + // holds in data stand-ins (cur, prev, gp, tmp), which never + // construct a CustomBaseClass. + static void destroy_walk(data& top) noexcept { - basic_json cur; + data cur; take(cur, top); - basic_json prev; // value_t::null: no parent + data prev; // value_t::null: no parent // true right after cur was entered: its children have not been // looked at yet. If none of them has children of its own, cur is @@ -965,7 +968,7 @@ public: fresh = false; if (has_no_children(cur) || (fresh_now && all_children_leaves(cur))) { - if (prev.m_data.m_type == value_t::null) + if (prev.m_type == value_t::null) { free_container(cur); return; // back at the top with nothing left to do @@ -974,7 +977,7 @@ public: // ascend: detach the grandparent link from prev's // walk_child() slot, drop that (now null) slot, free cur // (it is empty), then move up one level - basic_json gp; + data gp; take(gp, walk_child(prev)); pop_walk_child(prev); @@ -985,7 +988,7 @@ public: continue; } - basic_json& cur_child_ref = walk_child(cur); + data& cur_child_ref = walk_child(cur); if (has_no_children(cur_child_ref)) { @@ -997,7 +1000,7 @@ public: // descend into the non-empty child, reversing the // link: its slot takes over prev, and the child becomes // the new cur - basic_json tmp; + data tmp; take(tmp, cur_child_ref); take(cur_child_ref, prev); take(prev, cur); @@ -1034,14 +1037,14 @@ public: destroy_container(t); break; - case value_t::null: + case value_t::null: // LCOV_EXCL_LINE case value_t::boolean: case value_t::number_integer: case value_t::number_unsigned: case value_t::number_float: case value_t::discarded: default: - break; + break; // LCOV_EXCL_LINE } } }; @@ -1214,9 +1217,9 @@ public: return is_array() ? array_capacity() : object_capacity(); } #else - static constexpr std::size_t container_capacity() noexcept + static constexpr std::size_t container_capacity() noexcept // LCOV_EXCL_LINE { - return detail::unknown_size(); + return detail::unknown_size(); // LCOV_EXCL_LINE } #endif @@ -1402,7 +1405,7 @@ public: /// into this switch is worth a few percent when copying a value made /// mostly of numbers JSON_HEDLEY_ALWAYS_INLINE - static void copy_leaf_value(const basic_json& src, basic_json& dst) + static void copy_leaf_value(const basic_json& src, basic_json& dst) // LCOV_EXCL_LINE { switch (src.m_data.m_type) { @@ -1576,7 +1579,7 @@ public: auto src_it = src_object.cbegin(); for (auto& element : *dst.m_data.m_value.object) { - if (JSON_HEDLEY_LIKELY(src_it != src_object.cend() && copy_keys_equal(src_it->first, element.first))) + if (JSON_HEDLEY_LIKELY(copy_keys_equal(src_it->first, element.first))) { copy_shallow(src_it->second, element.second, worklist); ++src_it; @@ -4915,6 +4918,30 @@ public: return {it, res.second}; } + private: + /// Sets the parents of a value when it goes out of scope, also when an + /// exception is thrown: a container insertion that throws may already + /// have added elements, whose parents must be set for the value to stay + /// valid (basic exception guarantee). Does nothing without JSON_DIAGNOSTICS. + class set_parents_on_exit + { + public: + explicit set_parents_on_exit(basic_json* v) noexcept : m_value(v) {} + set_parents_on_exit(const set_parents_on_exit&) = delete; + set_parents_on_exit& operator=(const set_parents_on_exit&) = delete; + set_parents_on_exit(set_parents_on_exit&&) = delete; + set_parents_on_exit& operator=(set_parents_on_exit&&) = delete; + + ~set_parents_on_exit() + { + m_value->set_parents(); + } + + private: + basic_json* m_value; + }; + + public: /// Helper for insertion of an iterator /// @note: This uses std::distance to support GCC 4.8, /// see https://github.com/nlohmann/json/pull/1257 @@ -4924,6 +4951,7 @@ public: iterator result(this); JSON_ASSERT(m_data.m_value.array != nullptr); + const set_parents_on_exit parents(this); auto insert_pos = std::distance(m_data.m_value.array->begin(), pos.m_it.array_iterator); m_data.m_value.array->insert(pos.m_it.array_iterator, std::forward(args)...); result.m_it.array_iterator = m_data.m_value.array->begin() + insert_pos; @@ -4932,7 +4960,6 @@ public: // result.m_it.array_iterator = m_data.m_value.array->insert(pos.m_it.array_iterator, cnt, val); // but the return value of insert is missing in GCC 4.8, so it is written this way instead. - set_parents(); return result; } @@ -4952,8 +4979,16 @@ public: throw_iterator_from_other_value(this); } - // insert to array and return iterator - return insert_iterator(pos, val); + // vector::insert has no effects if copying a single element to the + // end throws; elsewhere, it may already have shifted the elements, so + // copy first and insert by move, which cannot throw but by allocating + if (pos.m_it.array_iterator == m_data.m_value.array->end()) + { + return insert_iterator(pos, val); + } + + basic_json tmp(val); + return insert_iterator(pos, std::move(tmp)); } /// @brief inserts element into array @@ -5085,8 +5120,8 @@ public: detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterators first and last must point to objects", this); } + const set_parents_on_exit parents(this); m_data.m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator); - set_parents(); } /// @brief updates a JSON object from another object, overwriting existing keys @@ -5354,8 +5389,7 @@ public: /// @name lexicographical comparison operators /// @{ - // the same-type cases for non-owning scalars, shared by JSON_IMPLEMENT_OPERATOR - // and JSON_IMPLEMENT_SCALAR_FAST + // the same-type cases for non-owning scalars, used by JSON_IMPLEMENT_SCALAR_FAST #define JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ case value_t::null: \ return (null_result); \ @@ -5377,7 +5411,7 @@ public: // because any negative signed value is smaller than any unsigned value. // Otherwise, the non-negative signed value is cast to unsigned before the // comparison to avoid wraparound. -#define JSON_IMPLEMENT_OPERATOR(op, null_result, unordered_result, default_result, deep_result, may_descend) \ +#define JSON_IMPLEMENT_OPERATOR(op, unordered_result, default_result, deep_result, may_descend) \ const auto lhs_type = lhs.type(); \ const auto rhs_type = rhs.type(); \ \ @@ -5405,8 +5439,15 @@ public: return (*lhs.m_data.m_value.object) op (*rhs.m_data.m_value.object); \ } \ \ - JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ - \ + /* same-type scalars never get here: the inline fast path of the operator handles them */ \ + case value_t::null: \ + case value_t::boolean: \ + case value_t::number_integer: \ + case value_t::number_unsigned: \ + case value_t::number_float: \ + JSON_ASSERT(false); /* NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) */ \ + return (unordered_result); \ + \ case value_t::string: \ return (*lhs.m_data.m_value.string) op (*rhs.m_data.m_value.string); \ \ @@ -5446,7 +5487,7 @@ public: ? (number_integer_t(-1) op number_integer_t(1)) \ : (static_cast(lhs.m_data.m_value.number_integer) op rhs.m_data.m_value.number_unsigned); \ } \ - else if(compares_unordered(lhs, rhs))\ + else if(lhs_type == value_t::discarded || rhs_type == value_t::discarded)\ {\ return (unordered_result);\ }\ @@ -5514,7 +5555,7 @@ public: JSON_HEDLEY_DIAGNOSTIC_PUSH JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif - JSON_IMPLEMENT_OPERATOR( ==, true, false, false, + JSON_IMPLEMENT_OPERATOR( ==, false, false, compare_iteratively(lhs, rhs, false) == compare_result::equal, true) #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_POP @@ -5528,7 +5569,6 @@ public: // default_result is used if we cannot compare values. In that case, // we compare types. JSON_IMPLEMENT_OPERATOR(<=>, // *NOPAD* - std::partial_ordering::equivalent, std::partial_ordering::unordered, lhs_type <=> rhs_type, // *NOPAD* to_partial_ordering(compare_iteratively(lhs, rhs, false)), true) @@ -5540,7 +5580,7 @@ public: // default_result is used if we cannot compare values. In that case, // we compare types. Note we have to call the operator explicitly, // because MSVC has problems otherwise. - JSON_IMPLEMENT_OPERATOR( <, false, false, operator<(lhs_type, rhs_type), + JSON_IMPLEMENT_OPERATOR( <, false, operator<(lhs_type, rhs_type), compare_iteratively(lhs, rhs, true) == compare_result::less, false) } #endif @@ -5555,7 +5595,7 @@ public: /// their inlined code at each call site small. Binary values are /// rare and left to equal_general(). JSON_HEDLEY_ALWAYS_INLINE - static int equal_leaf(const_reference lhs, const_reference rhs) noexcept + static int equal_leaf(const_reference lhs, const_reference rhs) noexcept // LCOV_EXCL_LINE { #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_PUSH @@ -6864,7 +6904,7 @@ public: basic_json result(value_t::object); json_pointer::flatten("", *this, result); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief unflatten a previously flattened JSON value /// @sa https://json.nlohmann.me/api/basic_json/unflatten/ @@ -7577,7 +7617,7 @@ public: result.append(path.data(), path.size()); diff_append_key(result, key); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief the path of the element @a index of the value at @a path static string_t diff_index_path(const string_t& path, const std::size_t index) @@ -7587,7 +7627,7 @@ public: result.append(path.data(), path.size()); diff_append_index(result, index); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief append the "remove" operations for the elements of array /// @a source from @a index on, and the "add" operations for the @@ -7747,7 +7787,7 @@ public: // operations if (source == target) { - return false; + return false; // LCOV_EXCL_LINE (equal objects with the same members in the same order return early) } // slow path: the common keys are in a different relative diff --git a/include/nlohmann/json_view.hpp b/include/nlohmann/json_view.hpp index 043d09036..64aa8f723 100644 --- a/include/nlohmann/json_view.hpp +++ b/include/nlohmann/json_view.hpp @@ -1705,9 +1705,11 @@ class basic_json_document { // (as before: the library's adapter decides what a null stream yields) detail::view::append_adapter(buf, detail::input_adapter(file)); - return; } - detail::view::append_file(buf, file); + else + { + detail::view::append_file(buf, file); + } } #endif diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 37c19cc79..8e740c4aa 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -3648,10 +3648,6 @@ template inline void unescape(StringType& s) { auto pos = s.find_first_of('~', 0); - if (pos == StringType::npos) - { - return; - } StringType result; result.reserve(s.size()); @@ -5527,7 +5523,7 @@ inline OutStringType concat(Args && ... args) str.reserve(concat_length(args...)); concat_into(str, std::forward(args)...); return str; -} +} // LCOV_EXCL_LINE (exception cleanup only) } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -5706,18 +5702,19 @@ class exception : public std::exception // all other parent types (primitives) contribute no token } - if (tokens.empty()) + // the root value has no path, but still has a byte range + std::string prefix; + if (!tokens.empty()) { - return ""; + const auto str = std::accumulate(tokens.rbegin(), tokens.rend(), std::string{}, + [](const std::string & a, const std::string & b) + { + return concat(a, '/', detail::escape(b)); + }); + prefix = concat('(', str, ") "); } - auto str = std::accumulate(tokens.rbegin(), tokens.rend(), std::string{}, - [](const std::string & a, const std::string & b) - { - return concat(a, '/', detail::escape(b)); - }); - - return concat('(', str, ") ", get_byte_positions(leaf_element)); + return concat(prefix, get_byte_positions(leaf_element)); #else return get_byte_positions(leaf_element); #endif @@ -5730,7 +5727,8 @@ class exception : public std::exception template static std::string get_byte_positions(const BasicJsonType* leaf_element) { - if ((leaf_element->start_pos() != std::string::npos) && (leaf_element->end_pos() != std::string::npos)) + // a value-initialized iterator has no value (invalid_iterator.212) + if (leaf_element != nullptr && leaf_element->start_pos() != std::string::npos && leaf_element->end_pos() != std::string::npos) { return concat("(bytes ", std::to_string(leaf_element->start_pos()), "-", std::to_string(leaf_element->end_pos()), ") "); } @@ -6247,7 +6245,7 @@ const typename BasicJsonType::binary_t& binary_value(const BasicJsonType& j) } template::value, int> = 0> -typename View::basic_json_type::binary_t binary_value(const View& /*unused*/) +typename View::basic_json_type::binary_t binary_value(const View& /*unused*/) // LCOV_EXCL_LINE { return {}; // LCOV_EXCL_LINE } @@ -7174,7 +7172,7 @@ StringType to_string(std::size_t value) StringType result; int_to_string(result, value); return result; -} +} // LCOV_EXCL_LINE (exception cleanup only) /// @return a byte as two uppercase hexadecimal digits inline std::string hex_byte(const std::uint8_t byte) @@ -7184,7 +7182,7 @@ inline std::string hex_byte(const std::uint8_t byte) result[0] = nibble_to_hex[byte / 16]; result[1] = nibble_to_hex[byte % 16]; return result; -} +} // LCOV_EXCL_LINE (exception cleanup only) /*! @brief write a UTF-16 code unit as four lowercase hexadecimal digits @@ -7528,7 +7526,7 @@ inline StringType sanitize_utf8(const StringType& s, const error_handler_t error } return result; -} +} // LCOV_EXCL_LINE (exception cleanup only) } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -9181,6 +9179,19 @@ NLOHMANN_JSON_NAMESPACE_END #define JSON_UNLOCKED_FILE_IO 0 #endif #undef JSON_UNLOCKED_FILE_IO_MSAN + + // Whether file_input_adapter scans the buffer of the FILE directly, as + // getc_unlocked() reads it: the fields it uses are those that the C + // library's own getc_unlocked() macro uses (glibc: _IO_read_ptr and + // _IO_read_end; Apple: _p and _r). It needs the file locked for the whole + // parse, as JSON_UNLOCKED_FILE_IO == 1 has it (so JSON_DISABLE_UNLOCKED_FILE_IO + // and MemorySanitizer builds turn it off). + // (uClibc defines __GLIBC__ as well, but its FILE has other fields) + #if JSON_UNLOCKED_FILE_IO == 1 && ((defined(__GLIBC__) && !defined(__UCLIBC__)) || defined(__APPLE__)) + #define JSON_FILE_WINDOW 1 + #else + #define JSON_FILE_WINDOW 0 + #endif #endif // JSON_NO_IO NLOHMANN_JSON_NAMESPACE_BEGIN @@ -9302,8 +9313,12 @@ class token_carry_buffer char* m_capacity_end = nullptr; }; /*! -Input adapter for stdio file access. This adapter read only 1 byte and do not use any - buffer. This adapter is a very low level adapter. +Input adapter for stdio file access. The file is locked for the lifetime of the +adapter and read with the unlocked stdio functions (see +JSON_DISABLE_UNLOCKED_FILE_IO). With glibc and the Apple C library, the +characters in the buffer of the FILE are also scanned in bulk, as the get area +of a stream is (see input_stream_adapter): exactly the characters getc_unlocked() +would return are consumed, so the file is left where it was left before. */ class file_input_adapter { @@ -9322,6 +9337,12 @@ class file_input_adapter file_input_adapter(const file_input_adapter&) = delete; file_input_adapter(file_input_adapter&& rhs) noexcept : m_file(rhs.m_file) +#if JSON_FILE_WINDOW + , token_carry(std::move(rhs.token_carry)) + , token_segment(rhs.token_segment) + , current_in_buffer(rhs.current_in_buffer) + , tracking_token(rhs.tracking_token) +#endif { // the lock is owned by the thread, not the adapter: hand it over rhs.m_file = nullptr; @@ -9333,6 +9354,20 @@ class file_input_adapter unlock(); } +#if JSON_FILE_WINDOW + std::char_traits::int_type get_character() + { + // what getc_unlocked() does, while the buffer has characters + const char* const next = buffer_ptr(); + if (JSON_HEDLEY_LIKELY(next != buffer_end())) + { + current_in_buffer = true; + buffer_bump(1); + return std::char_traits::to_int_type(*next); + } + return get_character_slow(); + } +#else std::char_traits::int_type get_character() noexcept { // the file is locked for the lifetime of the adapter, so the @@ -9345,6 +9380,7 @@ class file_input_adapter return std::fgetc(m_file); #endif } +#endif // returns the number of characters successfully read template @@ -9353,7 +9389,134 @@ class file_input_adapter return fread(dest, 1, sizeof(T) * count, m_file); } +#if JSON_FILE_WINDOW + // The lexer may scan the buffer of the FILE directly, as the window of + // input_stream_adapter: the buffer ends wherever the C library's buffer + // ends, and is refilled by the next get_character(). + static constexpr bool supports_bulk_scan = true; + static constexpr bool bulk_window = true; + + // Pointer to the next unread character; only valid when bulk_remaining() > 0. + const char* bulk_data() const + { + return buffer_ptr(); + } + + // Number of unread characters in the buffer. + std::size_t bulk_remaining() const + { + if (!current_in_buffer) + { + return 0; + } + return static_cast(buffer_end() - buffer_ptr()); + } + + // Consume @a n characters previously inspected via bulk_data(). + void bulk_skip(std::size_t n) + { + JSON_ASSERT(n <= bulk_remaining()); + buffer_bump(n); + } + + // The adapter keeps the current token for the lexer's error messages, as + // input_stream_adapter does. + static constexpr bool supports_token_tracking = true; + + void begin_token() noexcept + { + tracking_token = true; + token_carry.clear(); + token_segment = buffer_ptr(); + } + + void mark_token_start(std::char_traits::int_type c) + { + tracking_token = true; + token_carry.clear(); + if (current_in_buffer) + { + token_segment = buffer_ptr() - 1; + } + else + { + token_segment = nullptr; + token_carry.push_back(std::char_traits::to_char_type(c)); + } + } + + void copy_token(std::vector& out, bool drop_last) const + { + token_carry.copy_to(out); + if (token_segment != nullptr) + { + out.insert(out.end(), token_segment, buffer_ptr()); + } + if (drop_last && !out.empty()) + { + out.pop_back(); + } + } +#endif + private: +#if JSON_FILE_WINDOW + // the buffer of the FILE: [buffer_ptr(), buffer_end()) are the characters + // getc_unlocked() would return next +#if defined(__GLIBC__) + const char* buffer_ptr() const noexcept + { + return m_file->_IO_read_ptr; + } + const char* buffer_end() const noexcept + { + return m_file->_IO_read_end; + } + void buffer_bump(std::size_t n) noexcept + { + m_file->_IO_read_ptr += n; + } +#else + const char* buffer_ptr() const noexcept + { + return reinterpret_cast(m_file->_p); + } + const char* buffer_end() const noexcept + { + return m_file->_r > 0 ? reinterpret_cast(m_file->_p) + m_file->_r : reinterpret_cast(m_file->_p); + } + void buffer_bump(std::size_t n) noexcept + { + m_file->_p += n; + m_file->_r -= static_cast(n); + } +#endif + + // the buffer is exhausted: getc_unlocked() refills it + JSON_HEDLEY_NEVER_INLINE std::char_traits::int_type get_character_slow() + { + if (token_segment != nullptr) + { + token_carry.append(token_segment, buffer_end()); + token_segment = nullptr; + } + const auto res = getc_unlocked(m_file); + if (res == EOF) + { + current_in_buffer = false; + return std::char_traits::eof(); + } + // the character was taken from the refilled buffer, at buffer_ptr() - 1 + current_in_buffer = true; + if (tracking_token) + { + token_segment = buffer_ptr() - 1; + } + return res; + } + +#endif + // Lock the file once for the whole parse instead of once per character. // The lock is recursive, so fread() and the user's own stdio calls on the // same thread keep working. @@ -9382,6 +9545,16 @@ class file_input_adapter /// the file pointer to read from std::FILE* m_file; +#if JSON_FILE_WINDOW + /// the part of the current token that was in earlier buffers + token_carry_buffer token_carry{}; // NOLINT(readability-redundant-member-init) + /// start of the current token's part in the buffer, or nullptr + const char* token_segment = nullptr; + /// whether the character last returned by get_character() is in the buffer + bool current_in_buffer = false; + /// whether a token is tracked at all; the binary readers never start one + bool tracking_token = false; +#endif }; /*! @@ -9668,12 +9841,9 @@ class input_stream_adapter std::char_traits::int_type get_character_slow() { #if !JSON_DISABLE_STREAM_BULK_SCAN - // sbumpc() refills the get area if it is exhausted - const bool refill = streambuf_access::get_ptr(sb) == streambuf_access::get_end(sb); - if (JSON_HEDLEY_UNLIKELY(refill)) - { - save_token_segment(); - } + // the get area is exhausted (get_character() handles the other case), + // so sbumpc() refills it + save_token_segment(); #endif auto res = sb->sbumpc(); @@ -9686,17 +9856,13 @@ class input_stream_adapter is->clear(is->rdstate() | std::ios::eofbit); } #if !JSON_DISABLE_STREAM_BULK_SCAN - else if (JSON_HEDLEY_UNLIKELY(refill)) + else { // the consumed character is at gptr() - 1, unless the streambuf // has no get area and returned it from uflow() const char* const next = streambuf_access::get_ptr(sb); after_refill(res, next != nullptr && next != streambuf_access::get_begin(sb)); } - else - { - current_in_get_area = true; - } #endif return res; } @@ -13355,14 +13521,16 @@ class lexer : public lexer_base /// calls would (including the position and token bookkeeping of a /// streaming adapter, see bulk_consume) and leaves @a current holding the last of /// the 4 digits, just as the last such get() would; the codepoint is - /// written to @a out. Makes no state change and returns false for a - /// pending unget, fewer than 4 remaining bytes, or any of the 4 bytes not - /// being a hex digit, so the caller falls back unchanged to the - /// per-character loop, which then reports the same diagnostic (stopping - /// at the first invalid digit) as before this optimization. + /// written to @a out. Makes no state change and returns false for fewer + /// than 4 remaining bytes or any of the 4 bytes not being a hex digit, so + /// the caller falls back unchanged to the per-character loop, which then + /// reports the same diagnostic (stopping at the first invalid digit) as + /// before this optimization. bool get_codepoint_bulk(std::true_type /*bulk*/, int& out) { - if (next_unget || ia.bulk_remaining() < 4) + // no unget is pending inside a string (see get_after_string_run()) + JSON_ASSERT(!next_unget); + if (ia.bulk_remaining() < 4) { return false; } @@ -13507,11 +13675,10 @@ class lexer : public lexer_base template char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant keep) { - // a pending unget must be consumed through the normal path first - if (next_unget) - { - return get(); - } + // scan_string() is entered from scan_after_whitespace(), whose current + // was read by skip_whitespace() (its first get() clears a pending + // unget), and nothing in scan_string() calls unget() + JSON_ASSERT(!next_unget); const std::size_t remaining = ia.bulk_remaining(); if (remaining == 0) { @@ -14502,7 +14669,7 @@ scan_number_done: return token_type::value_unsigned; } } - else if (number_type == token_type::value_integer) + else // value_integer (the callers handle value_float themselves) { if (parse_integer_signed(first, last, value_integer)) { @@ -14608,11 +14775,9 @@ scan_number_done: */ token_type scan_number_bulk_contiguous() { - // a pending unget offsets the buffer position from current; fall back - if (next_unget) - { - return token_type::uninitialized; - } + // called right after scan_after_whitespace() read current (see + // get_after_string_run()), so no unget is pending + JSON_ASSERT(!next_unget); const std::size_t rem = ia.bulk_remaining(); if (rem == 0) { @@ -14633,11 +14798,7 @@ scan_number_done: if (data[0] == '-') { number_type = token_type::value_integer; - i = 1; - if (i >= avail) - { - return token_type::uninitialized; - } + i = 1; // avail >= 2: rem == 0 returned above } const std::size_t integer_begin = i; if (data[i] == '0') @@ -14880,7 +15041,10 @@ scan_number_done: /// the error at the right character JSON_HEDLEY_ALWAYS_INLINE bool scan_literal_bulk(const char* rest, const std::size_t length, std::true_type /*bulk*/) { - if (next_unget || ia.bulk_remaining() < length + // called right after scan_after_whitespace() read current (see + // get_after_string_run()), so no unget is pending + JSON_ASSERT(!next_unget); + if (ia.bulk_remaining() < length || std::memcmp(ia.bulk_data(), rest, length) != 0) { return false; @@ -17215,7 +17379,8 @@ class json_sax_dom_parser : public json_sax_dom_error_state { hashes.push_back(hasher(element.first)); } - ordered_data.key_indexes.emplace_back(new object_key_index(std::move(hashes))); + std::unique_ptr index(new object_key_index(std::move(hashes))); // NOLINT(cppcoreguidelines-owning-memory,modernize-make-unique): std::make_unique is C++14 + ordered_data.key_indexes.push_back(std::move(index)); f.ordered.index = ordered_data.key_indexes.back().get(); } @@ -17310,7 +17475,7 @@ class json_sax_dom_parser : public json_sax_dom_error_state ordered_data.members[f.ordered.dest_member].second = std::move(value); } - static void place_member(const frame& /*f*/, BasicJsonType&& /*value*/, std::false_type /*indexed*/) noexcept {} + static void place_member(const frame& /*f*/, BasicJsonType&& /*value*/, std::false_type /*indexed*/) noexcept {} // LCOV_EXCL_LINE /// store the members of a new object directly in it bool start_stored_object(std::size_t len) @@ -17614,7 +17779,7 @@ class json_sax_dom_parser : public json_sax_dom_error_state { // a new member is null and holds nothing: construct the value // over it instead of destroying and swapping - ::new (static_cast(&member)) BasicJsonType(std::forward(v)); + construct_member(member, std::forward(v), std::is_nothrow_constructible {}); } else { @@ -17623,6 +17788,26 @@ class json_sax_dom_parser : public json_sax_dom_error_state } } + /// constructing the value cannot throw (for example a number or a boolean): construct it in place + template + JSON_HEDLEY_ALWAYS_INLINE + static void construct_member(BasicJsonType& member, Value&& v, std::true_type /*nothrow*/) + { + ::new (static_cast(&member)) BasicJsonType(std::forward(v)); + } + + /// constructing the value can throw (for example a container or a string), and a + /// constructor that was interrupted must not have touched the member (#5985): + /// create the value first, then move it over the member (the move constructor is + /// noexcept) + template + JSON_HEDLEY_ALWAYS_INLINE + static void construct_member(BasicJsonType& member, Value&& v, std::false_type /*may throw*/) + { + BasicJsonType value(std::forward(v)); + ::new (static_cast(&member)) BasicJsonType(std::move(value)); + } + /// a custom base class may care about the lifetime of the value template static void set_member(BasicJsonType& member, Value&& v, std::false_type /*custom base*/) @@ -18048,7 +18233,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state key_stack.push_back(val); // add discarded value at the given key and store the reference for later - if (keep && ref_stack.back()) + if (keep) { auto& obj = *ref_stack.back()->m_data.m_value.object; // a single lookup: insert the placeholder, or find the existing member @@ -18105,7 +18290,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state const string_t object_key = pop_container(); - if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured()) + if (!ref_stack.empty() && ref_stack.back()) { // remove discarded value remove_discarded_value(*ref_stack.back(), object_key); @@ -18271,8 +18456,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state */ string_t current_key() const { - if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object() - && !key_stack.empty()) + if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object()) { return key_stack.back(); } @@ -19044,8 +19228,8 @@ class binary_reader nested document could declare a length that disagrees with where its terminator actually falls and quietly hand the bytes in between to the enclosing document. A well-formed document is at least 5 bytes (the prefix - plus the terminator); the equality also rejects those impossible sizes, - since at least 5 bytes are always consumed. + plus the terminator); open_bson_document() rejects a smaller size before + it is stored, so @a document_size is never negative here. When recovering from errors, a document whose size does not match is accepted: its terminator was found, so everything in it has been read. A @@ -19059,7 +19243,7 @@ class binary_reader */ bool check_bson_document_size(const std::size_t document_start, const std::int32_t document_size, const bool size_repaired) { - if (JSON_HEDLEY_UNLIKELY(!size_repaired && (document_size < 0 || static_cast(document_size) != chars_read - document_start))) + if (JSON_HEDLEY_UNLIKELY(!size_repaired && static_cast(document_size) != chars_read - document_start)) { static_cast(bson_document_size_error(document_size, chars_read - document_start)); return repair_requested(); @@ -19909,7 +20093,7 @@ class binary_reader { std::uint64_t len{}; std::size_t size{}; - return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_cbor_array(size); + return get_cbor_argument(len) && get_container_size(len, size, "array") && enter_cbor_array(size); } case 0x9F: // array (indefinite length) @@ -19949,7 +20133,7 @@ class binary_reader { std::uint64_t len{}; std::size_t size{}; - return get_cbor_argument(len) && get_cbor_container_size(len, size, "map") && enter_object(size); + return get_cbor_argument(len) && get_container_size(len, size, "map") && enter_object(size); } case 0xBF: // map (indefinite length) @@ -20613,7 +20797,7 @@ class binary_reader } /*! - @brief narrow a definite CBOR array/map length to std::size_t + @brief narrow a definite CBOR or MessagePack array/map length to std::size_t A definite length is rejected if it does not fit in std::size_t or if it equals detail::unknown_size(), which is reserved to mark an indefinite- @@ -20626,7 +20810,7 @@ class binary_reader @param[in] context "array" or "map", for the error message @return whether the length is usable */ - bool get_cbor_container_size(const std::uint64_t len, std::size_t& result, const char* context) + bool get_container_size(const std::uint64_t len, std::size_t& result, const char* context) { if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(len) || len == detail::unknown_size())) { @@ -20769,14 +20953,18 @@ class binary_reader Only the start of an array is tried, which covers arrays of numbers (e.g. coordinates) but costs arrays of other values only one call, not one - per element. + per element. An array of one element is not tried at all: that element is + mostly an object or an array (`[[...]]`, the single ring of a GeoJSON + polygon), for which the call would find no scalar, while a single scalar, + which the run would read faster, is rare. The benchmarks + `FromBinaryShape/nested` and `FromBinaryShape/singles` show both sides. @param[in] len the number of elements @return whether the SAX parser accepted the array and the values read */ bool enter_cbor_array(const std::size_t len) { - return enter_array(len) && (len == 0 || parse_cbor_scalar_run(std::integral_constant {})); + return enter_array(len) && (len <= 1 || parse_cbor_scalar_run(std::integral_constant {})); } /// input that is not contiguous: every value is read by parse_cbor_value @@ -21339,7 +21527,8 @@ class binary_reader case 0xDD: // array 32 { std::uint32_t len{}; - return get_number(len) && enter_msgpack_array(conditional_static_cast(len)); + std::size_t size{}; + return get_number(len) && get_container_size(len, size, "array") && enter_msgpack_array(size); } case 0xDE: // map 16 @@ -21351,7 +21540,8 @@ class binary_reader case 0xDF: // map 32 { std::uint32_t len{}; - return get_number(len) && enter_object(conditional_static_cast(len)); + std::size_t size{}; + return get_number(len) && get_container_size(len, size, "map") && enter_object(size); } // negative fixint @@ -21801,12 +21991,16 @@ class binary_reader /*! @brief open a MessagePack array and read the scalars it begins with (see @ref parse_msgpack_scalar_run) + + An array of one element is not tried, for the reason given at + @ref enter_cbor_array. + @param[in] len the number of elements @return whether the SAX parser accepted the array and the values read */ bool enter_msgpack_array(const std::size_t len) { - return enter_array(len) && (len == 0 || parse_msgpack_scalar_run(std::integral_constant {})); + return enter_array(len) && (len <= 1 || parse_msgpack_scalar_run(std::integral_constant {})); } /// input that is not contiguous: every value is read by parse_msgpack_value @@ -22269,16 +22463,13 @@ class binary_reader { if (size_and_type.second != 0) { - if (size_and_type.second != 'N') + for (std::size_t i = 0; i < size_and_type.first; ++i) { - for (std::size_t i = 0; i < size_and_type.first; ++i) + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, size_and_type.second))) { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, size_and_type.second))) - { - return false; - } - dim.push_back(dimlen); + return false; } + dim.push_back(dimlen); } } else @@ -22308,6 +22499,27 @@ class binary_reader return true; } + /*! + @brief store the count of an optimized container (or the length of a string) + + A count equal to npos is rejected: npos marks a container whose count is + not known, so a count of that value would be read as "no count" (compare + get_container_size()). No container can hold that many elements. + + @param[in] count the count read from the input + @param[out] result set to @a count unless it is rejected + @return whether @a count is usable + */ + bool store_ubjson_count(const std::size_t count, std::size_t& result) + { + if (JSON_HEDLEY_UNLIKELY(count == npos)) + { + return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); + } + result = count; + return true; + } + /*! @brief read a UBJSON/BJData optimized-container count of a signed marker type ('i', 'I', 'l', 'L') and narrow it to std::size_t @@ -22338,8 +22550,7 @@ class binary_reader { return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } - result = static_cast(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char - return true; + return store_ubjson_count(static_cast(number), result); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char } /*! @@ -22416,8 +22627,7 @@ class binary_reader { return false; } - result = conditional_static_cast(number); - return true; + return store_ubjson_count(conditional_static_cast(number), result); } case 'M': @@ -22435,8 +22645,7 @@ class binary_reader { return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } - result = detail::conditional_static_cast(number); - return true; + return store_ubjson_count(detail::conditional_static_cast(number), result); } case '[': @@ -22482,11 +22691,9 @@ class binary_reader // follow the documented _ArrayType_, _ArraySize_, _ArrayData_ order if (ndarray_dtype != 0) { + // every optimized type marker is_valid_optimized_type() accepts for BJData has a name const char* type_name = bjd_type_name(ndarray_dtype); - if (JSON_HEDLEY_UNLIKELY(type_name == nullptr)) - { - return unexpected_byte("invalid byte", "type"); - } + JSON_ASSERT(type_name != nullptr); string_t type_key = "_ArrayType_"; string_t type = type_name; // sax->string() takes a reference @@ -22571,12 +22778,6 @@ class binary_reader if (current == '$') { result.second = get(); // must not ignore 'N', because 'N' maybe the type - if (input_format == input_format_t::bjdata - && JSON_HEDLEY_UNLIKELY(is_bjd_excluded_optimized_type(result.second))) - { - return optimized_type_error(); - } - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof("type"))) { return false; @@ -22604,7 +22805,7 @@ class binary_reader // an ndarray was read here only if the flag flipped; when it was // seeded true, get_ubjson_size_value() already rejected the nested // dimension vector - if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) + if (is_ndarray && !inside_ndarray) { result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters } @@ -22614,7 +22815,7 @@ class binary_reader if (current == '#') { const bool is_error = get_ubjson_size_value(result.first, is_ndarray); - if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray) + if (is_ndarray && !inside_ndarray) { return last_byte_error(exception_id::unexpected_byte, "ndarray requires both type and size", "size"); } @@ -22796,7 +22997,6 @@ class binary_reader JSON_HEDLEY_NEVER_INLINE bool parse_ubjson_typed_run(std::size_t& remaining, const char_int_type type, std::true_type /*bulk*/) { - const bool bjdata = input_format == input_format_t::bjdata; switch (type) { case 'U': @@ -22810,11 +23010,11 @@ class binary_reader case 'L': return parse_ubjson_typed_run(remaining); case 'u': - return !bjdata || parse_ubjson_typed_run(remaining); + return parse_ubjson_typed_run(remaining); case 'm': - return !bjdata || parse_ubjson_typed_run(remaining); + return parse_ubjson_typed_run(remaining); case 'M': - return !bjdata || parse_ubjson_typed_run(remaining); + return parse_ubjson_typed_run(remaining); case 'd': return parse_ubjson_typed_run(remaining); case 'D': @@ -22895,7 +23095,7 @@ class binary_reader // if bit-8 of size_and_type.second is set to 1, encode bjdata ndarray as an object in JData annotated array format (https://github.com/NeuroJSON/jdata): // {"_ArrayType_" : "typeid", "_ArraySize_" : [n1, n2, ...], "_ArrayData_" : [v1, v2, ...]} - if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) + if ((size_and_type.second & (1 << 8)) != 0) { size_and_type.second &= ~(static_cast(1) << 8); // use bit 8 to indicate ndarray, here we remove the bit to restore the type marker @@ -22956,7 +23156,7 @@ class binary_reader } // If BJData type marker is 'B' decode as binary - if (input_format == input_format_t::bjdata && size_and_type.first != npos && size_and_type.second == 'B') + if (size_and_type.first != npos && size_and_type.second == 'B') { binary_t result; return get_binary(size_and_type.first, result) && sax->binary(result); @@ -23005,7 +23205,7 @@ class binary_reader } // do not accept ND-array size in objects in BJData - if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) + if ((size_and_type.second & (1 << 8)) != 0) { return last_byte_error(exception_id::unexpected_byte, "BJData object does not support ND-array size in optimized format", "object"); } @@ -24283,6 +24483,14 @@ class binary_reader const char* context, ContainerType& result) { + // a string that is in the block of input as a whole is appended from + // it, without the zero-filling resize() below + constexpr bool from_block = bulk_scan && std::is_same::value; + if (append_from_block(len, result, std::integral_constant {})) + { + return true; + } + // upper bound on the number of bytes read (and allocated) per chunk constexpr std::size_t chunk_size = 4096; @@ -24319,6 +24527,36 @@ class binary_reader return true; } + /// append the @a len bytes at the current position to @a result, if the + /// input holds them in one block; false (with nothing read) otherwise + template + bool append_from_block(const NumberType len, string_t& result, std::true_type /*block*/) + { + if (len <= 0) + { + return false; + } + // (len > 0, so the unsigned type holds it; casting a signed char + // straight to std::size_t would trip bugprone-signed-char-misuse) + const auto ulen = static_cast::type>(len); + const std::size_t remaining = ia.bulk_remaining(); + if (static_cast(ulen) > static_cast(remaining)) + { + return false; // (the loop reports the premature end) + } + const auto n = static_cast(ulen); + result.append(reinterpret_cast(ia.bulk_data()), n); + ia.bulk_skip(n); + chars_read += n; + return true; + } + + template + static bool append_from_block(const NumberType /*len*/, ContainerType& /*result*/, std::false_type /*block*/) noexcept + { + return false; + } + /*! @brief report an error to the SAX parser and remember its answer @@ -24805,8 +25043,9 @@ class binary_reader JSON_PRIVATE_UNLESS_TESTED: /*! @brief check whether @a marker is a type marker that may follow '$' - UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C - (BJData's excluded markers are rejected separately). + UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C. + The markers BJData excludes (F H N S T Z [ {) are the ones + binary_writer::is_bjdata_excluded_type_marker() lists; keep the two in sync. */ bool is_valid_optimized_type(const char_int_type marker) const noexcept { @@ -24841,20 +25080,6 @@ class binary_reader } } - /*! - @brief whether @a marker is excluded from BJData's optimized ND-array types - @return whether @a marker is one of 'F', 'H', 'N', 'S', 'T', 'Z', '[', '{' - - Mirrors binary_writer's @ref binary_writer::is_bjdata_excluded_type_marker - "is_bjdata_excluded_type_marker()`, which encodes the same list the other - way; keep the two in sync. - */ - static constexpr bool is_bjd_excluded_optimized_type(const char_int_type marker) noexcept - { - return marker == '[' || marker == '{' || marker == 'S' || marker == 'H' - || marker == 'T' || marker == 'F' || marker == 'N' || marker == 'Z'; - } - /*! @brief look up the ND-array element type name for a BJData dtype marker @return the type name ("uint8", "int8", ...), or nullptr if @a marker does @@ -26205,14 +26430,7 @@ class parser std::string exception_message(const token_type expected, const std::string& context) { - std::string error_msg = "syntax error "; - - if (!context.empty()) - { - error_msg += concat("while parsing ", context, ' '); - } - - error_msg += "- "; + std::string error_msg = concat("syntax error while parsing ", context, " - "); if (last_token == token_type::parse_error) { @@ -26230,7 +26448,7 @@ class parser } return error_msg; - } + } // LCOV_EXCL_LINE (exception cleanup only) private: /// callback function @@ -26544,6 +26762,9 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); + // m_it is already value-initialized, so this is not needed for + // correctness; it is kept because GCC generates faster loops with it + // (iterating a parsed document is 0.5-2.7 % slower without it, #6005) if (m_object->is_object()) { m_it.object_iterator = typename object_t::iterator(); @@ -27547,7 +27768,7 @@ class json_pointer result.reference_tokens.insert(result.reference_tokens.end(), lhs.reference_tokens.begin(), lhs.reference_tokens.end()); result.reference_tokens.insert(result.reference_tokens.end(), rhs.reference_tokens.begin(), rhs.reference_tokens.end()); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief create a new JSON pointer by appending the unescaped token at the end of the JSON pointer /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ @@ -27558,7 +27779,7 @@ class json_pointer result.reference_tokens.insert(result.reference_tokens.end(), lhs.reference_tokens.begin(), lhs.reference_tokens.end()); result.reference_tokens.push_back(std::move(token)); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief create a new JSON pointer by appending the array-index-token at the end of the JSON pointer /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ @@ -29423,7 +29644,7 @@ class json_pointer { // create an entry in the array current = ¤t->operator[](array_index(token)); - break; + break; // LCOV_EXCL_LINE (needs an array index with 10 or more digits: more than 1e9 elements) } /* @@ -29460,7 +29681,7 @@ class json_pointer json_pointer result; result.reference_tokens = reference_tokens; return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) json_pointer convert()&& { @@ -29904,6 +30125,10 @@ inline void copy_short(CharType* dst, const CharType* src, const std::size_t len /// the number of characters an empty vector is grown to by output_vector_sink constexpr std::size_t min_vector_sink_capacity = 64; +/// the number of characters output_vector_sink makes writable ahead of its +/// position at a time: 64 KiB of the vector's capacity, not all of it +constexpr std::size_t vector_sink_window = 65536; + /// @brief non-virtual output sink writing into a std::vector /// /// This sink is not part of the virtual output_adapter_protocol hierarchy: it is @@ -29917,7 +30142,8 @@ constexpr std::size_t min_vector_sink_capacity = 64; /// encoding them: insert() is an out-of-line call into a routine that can insert /// anywhere, and resize() is one as well with libstdc++. The sink therefore keeps /// its own write position: it grows the vector geometrically ahead of the -/// position and copies each write to it, so a write is a capacity check and a +/// position (making @ref vector_sink_window characters of it writable at a +/// time) and copies each write to it, so a write is a capacity check and a /// short copy. The destructor shrinks the vector to the bytes actually written, /// also when a write throws; shrinking a vector never allocates. The sink owns /// that trim, so it can be moved but not copied, and the vector must not be used @@ -30065,9 +30291,17 @@ class output_vector_sink // reallocate for every few bytes const std::size_t written = position(); const std::size_t needed = written + length; - v->resize((std::max)(min_vector_sink_capacity, needed <= v->capacity() - ? v->capacity() - : (std::max)(needed, v->capacity() * 2))); + const std::size_t capacity = (std::max)(min_vector_sink_capacity, needed <= v->capacity() + ? v->capacity() + : (std::max)(needed, v->capacity() * 2)); + v->reserve(capacity); + // resize() zeroes the characters it adds. Zeroing all of a doubled + // capacity, megabytes of a fresh allocation, at once faults in its + // pages and evicts them from the cache before the writer gets there, + // so its stores miss again; a window ahead of the position is still + // cached when they come (the binary writers on a long array of + // numbers: 4-26 % faster on x86, 1-2.5 % on Apple M1) + v->resize((std::min)(capacity, (std::max)(needed, written + vector_sink_window))); cursor = v->data() + written; limit = v->data() + v->size(); } @@ -31230,7 +31464,7 @@ class binary_writer while (n != 0); std::reverse(result.begin(), result.end()); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /*! @brief the decimal digits of the integer @a n, with a leading '-' if it @@ -31257,7 +31491,7 @@ class binary_writer } std::reverse(result.begin(), result.end()); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /*! @brief throws because the integer @a j is too large for @a format @@ -32189,7 +32423,7 @@ class binary_writer std::memcpy(out, &n, sizeof(NumberType)); if (!is_little_endian) { - std::reverse(out, out + sizeof(NumberType)); + std::reverse(out, out + sizeof(NumberType)); // LCOV_EXCL_LINE (big-endian hosts only) } } @@ -32318,7 +32552,7 @@ class binary_writer const string_t& value = *j.m_data.m_value.string; if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(value.size() + 1ul))) { - return false; + return false; // LCOV_EXCL_LINE (a string of 2 GiB or more) } out[0] = to_char_type(0x02); store_little_endian(p, static_cast(value.size() + 1ul)); @@ -32775,7 +33009,7 @@ class binary_writer const std::size_t size = oa.written() - pos; if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(size))) { - return false; + return false; // LCOV_EXCL_LINE (a document of 2 GiB or more) } const auto le_size = static_cast(size); oa.overwrite(pos, is_little_endian ? le_size : byteswap(le_size)); @@ -32819,7 +33053,6 @@ class binary_writer nothing large has been written then, and the caller writes the document in two passes, which reports that error */ - template::value, int>::type = 0> bool write_bson_document_single_pass(const BasicJsonType& document) { const std::size_t start = oa.written(); @@ -32904,7 +33137,7 @@ class binary_writer { if (JSON_HEDLEY_UNLIKELY(!close_bson_container(nested_size_pos))) { - return false; + return false; // LCOV_EXCL_LINE (a document of 2 GiB or more) } continue; } @@ -32919,7 +33152,7 @@ class binary_writer if (JSON_HEDLEY_UNLIKELY(!close_bson_container(current.size_slot))) { - return false; + return false; // LCOV_EXCL_LINE (a document of 2 GiB or more) } if (parents.empty()) { @@ -33176,10 +33409,10 @@ class binary_writer Selected through masks rather than branches, as the type and the sign of the values in an array of mixed signs alternate unpredictably (non-negative - integers are number_unsigned after parsing). Up to 64 bits, the sign is bit - 63 of the result (a narrower integer is sign-extended); computing it with - masks keeps GCC from branching on the type. A wider integer has to be - compared. + integers are number_unsigned after parsing). The sign is bit 63 of the + result (a narrower integer is sign-extended); computing it with masks keeps + GCC from branching on the type. Only called if @ref integers_fit_64_bits(); + a wider number type is written with range checks on the whole value. */ static std::uint64_t integer_bits(const BasicJsonType& j, std::uint64_t& sign_mask) noexcept { @@ -33187,9 +33420,7 @@ class binary_writer const std::uint64_t raw = is_signed ? static_cast(j.m_data.m_value.number_integer) : static_cast(j.m_data.m_value.number_unsigned); - sign_mask = sizeof(typename BasicJsonType::number_integer_t) <= sizeof(std::uint64_t) - ? (std::uint64_t{0} - static_cast(is_signed)) & (std::uint64_t{0} - (raw >> 63u)) - : std::uint64_t{0} - static_cast(is_signed && j.m_data.m_value.number_integer < 0); + sign_mask = (std::uint64_t{0} - static_cast(is_signed)) & (std::uint64_t{0} - (raw >> 63u)); return raw; } @@ -33526,22 +33757,19 @@ class binary_writer // negative fixnum write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)()) { // int 8 oa.write_character(to_char_type(0xD0)); write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)()) { // int 16 oa.write_character(to_char_type(0xD1)); write_number(static_cast(j.m_data.m_value.number_integer)); } - else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && - j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) + else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)()) { // int 32 oa.write_character(to_char_type(0xD2)); @@ -34457,7 +34685,7 @@ class binary_writer Containers, strings, high-precision numbers, booleans and null cannot be declared as the single type of an optimized container in BJData; such a container is written unoptimized. The reader rejects them with the same - list (binary_reader::is_bjd_excluded_optimized_type()). + list (binary_reader::is_valid_optimized_type()). */ static constexpr bool is_bjdata_excluded_type_marker(const CharType marker) noexcept { @@ -34556,57 +34784,34 @@ class binary_writer } /*! - @brief validate (dry_run) or write one BJData ND-array element of integer dtype @a T - @return whether @a el's value is in range of @a T; always true when @a dry_run is false - */ - template - bool write_bjdata_ndarray_element(const BasicJsonType& el, const bool dry_run) - { - if (dry_run) - { - return bjdata_ndarray_value_in_range(el); - } - using storage_type = typename std::conditional::value, std::uint64_t, std::int64_t>::type; - write_number(static_cast(el.template get()), true); - return true; - } - - /*! - @brief validate (dry_run) or write one BJData ND-array element of dtype 'd' (single precision) + @brief whether the BJData ND-array element @a el can be written with dtype 'd' (single precision) @return whether @a el's value survives narrowing to float and back without any change - (so the ND-array round-trips exactly), or is infinite or NaN; always true when - @a dry_run is false + (so the ND-array round-trips exactly), or is infinite or NaN */ - bool write_bjdata_ndarray_float_element(const BasicJsonType& el, const bool dry_run) + static bool bjdata_ndarray_float_in_range(const BasicJsonType& el) noexcept { const auto dval = static_cast(el.m_data.m_value.number_float); - if (dry_run) - { #ifdef __GNUC__ - JSON_HEDLEY_DIAGNOSTIC_PUSH - JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif - // a value that would be rounded (rather than exactly represented) by the - // narrowing to float is treated like an out-of-range integer element; this - // is the same criterion write_compact_float() uses for CBOR/MessagePack - const bool in_range = std::isnan(dval) || - (dval >= static_cast(std::numeric_limits::lowest()) && - dval <= static_cast((std::numeric_limits::max)()) && - static_cast(static_cast(dval)) == dval) || - std::isinf(dval); + // a value that would be rounded (rather than exactly represented) by the + // narrowing to float is treated like an out-of-range integer element; this + // is the same criterion write_compact_float() uses for CBOR/MessagePack + return std::isnan(dval) || + (dval >= static_cast(std::numeric_limits::lowest()) && + dval <= static_cast((std::numeric_limits::max)()) && + static_cast(static_cast(dval)) == dval) || + std::isinf(dval); #ifdef __GNUC__ - JSON_HEDLEY_DIAGNOSTIC_POP + JSON_HEDLEY_DIAGNOSTIC_POP #endif - return in_range; - } - write_number(static_cast(dval), true); - return true; } /*! @brief write the elements of an ND-array as the number kind @a dtype names (see @ref write_bjdata_ndarray), after they have been found to be - in range by a dry run of @ref write_bjdata_ndarray_elements + in range by @ref bjdata_ndarray_elements_in_range The elements are encoded in blocks into space reserved from the sink. */ @@ -34688,20 +34893,13 @@ class binary_writer } /*! - @brief validate or write every element of a BJData ND-array's `_ArrayData_` + @brief whether every element of a BJData ND-array's `_ArrayData_` is in range for its dtype @param[in] array_data the `_ArrayData_` array @param[in] dtype the ND-array dtype marker, as returned by bjdata_ndarray_type_marker() - @param[in] dry_run true to only range-check each element, false to write it - @return whether every element is in range for @a dtype (always true when @a dry_run is false) + @return whether every element is in range for @a dtype */ - bool write_bjdata_ndarray_elements(const BasicJsonType& array_data, const CharType dtype, const bool dry_run) + static bool bjdata_ndarray_elements_in_range(const BasicJsonType& array_data, const CharType dtype) { - if (!dry_run) - { - write_bjdata_ndarray_values(array_data, dtype); - return true; - } - for (const auto& el : array_data) { bool ok = true; @@ -34709,43 +34907,39 @@ class binary_writer { case 'U': case 'B': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'C': // BJData limits a char to 0..127 (the intersection of the int8 and uint8 ranges) ok = bjdata_ndarray_value_in_range(el) && bjdata_ndarray_value_in_range(el); break; case 'i': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'u': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'I': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'm': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'l': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'M': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'L': - ok = write_bjdata_ndarray_element(el, dry_run); + ok = bjdata_ndarray_value_in_range(el); break; case 'd': - ok = write_bjdata_ndarray_float_element(el, dry_run); + ok = bjdata_ndarray_float_in_range(el); break; case 'D': default: // 'D' (double) already spans the full range of number_float_t - if (!dry_run) - { - write_number(el.template get(), true); - } break; } if (!ok) @@ -34832,7 +35026,7 @@ class binary_writer const auto dim = el.template get(); if (!value_in_range_of(dim)) { - return true; + return true; // LCOV_EXCL_LINE (size_t is narrower than 64 bits) } const auto dim_size = static_cast(dim); @@ -34882,7 +35076,7 @@ class binary_writer // wrap (integers) or overflow to infinity (the "single" precision // float) instead of being reported, so such an object falls back to // a plain object encoding as well - if (!write_bjdata_ndarray_elements(array_data, dtype, true)) + if (!bjdata_ndarray_elements_in_range(array_data, dtype)) { return true; } @@ -34894,7 +35088,7 @@ class binary_writer write_ubjson(array_size, use_count, use_type, true, true, bjdata_version); - write_bjdata_ndarray_elements(array_data, dtype, false); + write_bjdata_ndarray_values(array_data, dtype); return false; } @@ -35220,8 +35414,8 @@ class binary_writer run are encoded into space reserved from the sink, as in @ref write_array_elements; an integer that BON8 cannot write (an unsigned one above the int64 range, or one of a number type wider than 64 bits) throws - from write_bon8_value, like any other value that is not a number. Kept out - of line, like @ref write_cbor_array. + out_of_range.407, as write_bon8_value does. Kept out of line, like + @ref write_cbor_array. */ template JSON_HEDLEY_NEVER_INLINE @@ -35271,10 +35465,10 @@ class binary_writer string_open = false; } - if (unencodable) + if (JSON_HEDLEY_UNLIKELY(unencodable)) { - write_bon8_value(*it, string_open, depth + 1); - ++it; + // an integer beyond the int64 range, as write_bon8_value would report it + throw_integer_out_of_range(*it, input_format_t::bon8, "int64"); } } } @@ -35309,14 +35503,13 @@ class binary_writer break; case value_t::number_integer: - if (JSON_HEDLEY_LIKELY(integer_fits(value.m_data.m_value.number_integer))) + if (JSON_HEDLEY_UNLIKELY(!integer_fits(value.m_data.m_value.number_integer))) { - write_bon8_integer(static_cast(value.m_data.m_value.number_integer)); - string_open = false; - break; + // beyond 64 bits (number types wider than that) + throw_integer_out_of_range(value, input_format_t::bon8, "int64"); } - // beyond 64 bits (number types wider than that): throws - write_bon8_value(value, string_open, depth + 1); + write_bon8_integer(static_cast(value.m_data.m_value.number_integer)); + string_open = false; break; case value_t::number_float: @@ -35325,14 +35518,13 @@ class binary_writer break; case value_t::number_unsigned: - if (JSON_HEDLEY_LIKELY(integer_fits_max(value.m_data.m_value.number_unsigned))) + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(value.m_data.m_value.number_unsigned))) { - write_bon8_integer(static_cast(value.m_data.m_value.number_unsigned)); - string_open = false; - break; + // too large for BON8 + throw_integer_out_of_range(value, input_format_t::bon8, "int64"); } - // too large for BON8: throws - write_bon8_value(value, string_open, depth + 1); + write_bon8_integer(static_cast(value.m_data.m_value.number_unsigned)); + string_open = false; break; case value_t::object: @@ -35898,7 +36090,7 @@ NLOHMANN_JSON_NAMESPACE_END -#include // remove, fill, find, none_of, min +#include // copy, none_of, min, search #include // array #include // localeconv, lconv #include // isfinite @@ -38785,36 +38977,21 @@ class serializer { // the code point is at least 0x60 (smaller ones are handled // above) and is not printable ASCII (which the scanner - // copies): it is DEL (0x7F) under EnsureAscii or a non-ASCII - // character + // copies). Under EnsureAscii it is DEL (0x7F): every + // well-formed multi-byte character is escaped by + // write_u_escapes() above, so decode() only ever accepts a + // single byte here, and nothing of it was buffered if (EnsureAscii) { - JSON_ASSERT(codepoint >= 0x7F); - if (error_handler == error_handler_t::keep) - { - // this character was buffered as raw bytes below - // in case it turned out to be part of an - // ill-formed sequence (which is kept as is); now - // that it decoded to a well-formed code point, - // undo that and \u-escape it like any other - // character instead - bytes = bytes_after_last_accept; - } - if (codepoint <= 0xFFFF) - { - write_u_escape(bytes, static_cast(codepoint)); - } - else - { - write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); - write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); - } + JSON_ASSERT(codepoint == 0x7F); + JSON_ASSERT(bytes == bytes_after_last_accept); + write_u_escape(bytes, static_cast(codepoint)); } else { // copy byte to buffer (all previous bytes of the // sequence have been copied in the default case below) - string_buffer[bytes++] = s[i]; + string_buffer[bytes++] = s[i]; // LCOV_EXCL_LINE (well-formed sequences are copied in runs) } reserve_string_buffer(bytes); @@ -39767,31 +39944,35 @@ class serializer // check if the buffer was large enough JSON_ASSERT(static_cast(len) < number_buffer.size()); - // look up the locale's thousands separator and decimal point now, - // matching what snprintf_float() just used (see lexer::get_decimal_point()) + // look up the locale's decimal point now, matching what + // snprintf_float() just used (see get_decimal_point() in + // number_parse.hpp); it is a string, which can be longer than one + // byte (U+066B in ps_AF, #5993). + // A thousands separator needs no handling: %g never groups digits. const auto* loc = std::localeconv(); JSON_ASSERT(loc != nullptr); - const char thousands_sep = (loc->thousands_sep == nullptr) ? '\0' : *loc->thousands_sep; - const char decimal_point = (loc->decimal_point == nullptr) ? '\0' : *loc->decimal_point; - - // erase thousands separators - if (thousands_sep != '\0') - { - // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep); - std::fill(end, number_buffer.end(), '\0'); - JSON_ASSERT((end - number_buffer.begin()) <= len); - len = (end - number_buffer.begin()); - } + const char* const decimal_point = (loc->decimal_point == nullptr) ? "" : loc->decimal_point; + const std::size_t decimal_point_len = std::char_traits::length(decimal_point); // convert decimal point to '.' - if (decimal_point != '\0' && decimal_point != '.') + if ((decimal_point_len > 1) || (decimal_point_len == 1 && decimal_point[0] != '.')) { - // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point); - if (dec_pos != number_buffer.end()) + const auto first = number_buffer.begin(); + const auto last = first + len; + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::search returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto dec_pos = std::search(first, last, decimal_point, decimal_point + decimal_point_len); + if (dec_pos != last) { *dec_pos = '.'; + if (decimal_point_len > 1) + { + // move the fraction and exponent behind the '.' (std::copy + // allows the overlap, as the destination is before the source) + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::copy returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto end = std::copy(dec_pos + static_cast(decimal_point_len), last, dec_pos + 1); + *end = '\0'; + len = end - first; + } } } @@ -40655,7 +40836,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}}; #elif defined(__GNUC__) || defined(__GNUG__) result["compiler"] = {{"family", "gcc"}, {"version", detail::concat( - detail::to_string(static_cast(__GNUC__)), '.', + detail::to_string(static_cast(__GNUC__)), '.', // LCOV_EXCL_LINE detail::to_string(static_cast(__GNUC_MINOR__)), '.', detail::to_string(static_cast(__GNUC_PATCHLEVEL__))) } @@ -40798,6 +40979,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec //////////////////////// JSON_PRIVATE_UNLESS_TESTED: + // the type and value of a basic_json without its base class (defined + // below); the destruction helpers of json_value use it as a stand-in + struct data; + /*! @brief a JSON value @@ -40927,14 +41112,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec case value_t::binary: binary = create(); break; - case value_t::null: + case value_t::null: // LCOV_EXCL_LINE case value_t::boolean: case value_t::number_integer: case value_t::number_unsigned: case value_t::number_float: case value_t::discarded: default: - break; + break; // LCOV_EXCL_LINE } } @@ -40974,27 +41159,28 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec explicit json_value(array_t* value) noexcept : array(value) {} private: - // raw, allocation-free transfer of m_data from src to dst: no - // set_parents()/assert_invariant() (the former is O(#children) per - // call under JSON_DIAGNOSTICS, which would make the walk below - // quadratic); dst takes ownership, src is left as value_t::null. - static void take(basic_json& dst, basic_json& src) noexcept + // raw, allocation-free transfer of the type and value from src to + // dst: no set_parents()/assert_invariant() (the former is + // O(#children) per call under JSON_DIAGNOSTICS, which would make the + // walk below quadratic); dst takes ownership, src is left as + // value_t::null. + static void take(data& dst, data& src) noexcept { - dst.m_data.m_type = src.m_data.m_type; - dst.m_data.m_value = src.m_data.m_value; - src.m_data.m_type = value_t::null; + dst.m_type = src.m_type; + dst.m_value = src.m_value; + src.m_type = value_t::null; } // true if v is not an array/object, or is an already-empty one - static bool has_no_children(const basic_json& v) noexcept + static bool has_no_children(const data& v) noexcept { - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - return v.m_data.m_value.array->empty(); + return v.m_value.array->empty(); } - if (v.m_data.m_type == value_t::object) + if (v.m_type == value_t::object) { - return v.m_data.m_value.object->empty(); + return v.m_value.object->empty(); } return true; } @@ -41003,21 +41189,21 @@ private: // its own, i.e. v can be freed in one go (its destructor then // destroys scalars and empty containers only, so it recurses at most // one level deep); stops at the first structured child it meets - static bool all_children_leaves(const basic_json& v) noexcept + static bool all_children_leaves(const data& v) noexcept { // array_t only needs forward iterators (see custom_array_type) - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - const auto& a = *v.m_data.m_value.array; + const auto& a = *v.m_value.array; return std::all_of(a.begin(), a.end(), [](const basic_json & el) { - return has_no_children(el); + return has_no_children(el.m_data); }); } - const auto& o = *v.m_data.m_value.object; + const auto& o = *v.m_value.object; return std::all_of(o.begin(), o.end(), [](const typename object_t::value_type & el) { - return has_no_children(el.second); + return has_no_children(el.second.m_data); }); } @@ -41049,30 +41235,30 @@ private: // the child of a non-empty array/object v that the walk in // destroy_container() continues with (see walk_child_it() above) - static basic_json& walk_child(basic_json& v) + static data& walk_child(data& v) { - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - return v.m_data.m_value.array->back(); + return v.m_value.array->back().m_data; } - JSON_ASSERT(v.m_data.m_type == value_t::object); - return walk_child_it(*v.m_data.m_value.object)->second; + JSON_ASSERT(v.m_type == value_t::object); + return walk_child_it(*v.m_value.object)->second.m_data; } // removes walk_child(v) from a non-empty array/object v; this never // allocates, and since it is only ever called when that child is a // scalar or an already-empty array/object, destroying it never // recurses more than one level deep (see destroy() below) - static void pop_walk_child(basic_json& v) + static void pop_walk_child(data& v) { - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - v.m_data.m_value.array->pop_back(); + v.m_value.array->pop_back(); } else { - JSON_ASSERT(v.m_data.m_type == value_t::object); - v.m_data.m_value.object->erase(walk_child_it(*v.m_data.m_value.object)); + JSON_ASSERT(v.m_type == value_t::object); + v.m_value.object->erase(walk_child_it(*v.m_value.object)); } } @@ -41081,23 +41267,23 @@ private: // recurses at most one level deep); this is the same allocator-based // free the old recursive implementation used, just factored out so // every level of the walk in destroy() can share it - static void free_container(basic_json& v) noexcept + static void free_container(data& v) noexcept { JSON_ASSERT(has_no_children(v) || all_children_leaves(v)); - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { AllocatorType alloc; - std::allocator_traits::destroy(alloc, v.m_data.m_value.array); - std::allocator_traits::deallocate(alloc, v.m_data.m_value.array, 1); + std::allocator_traits::destroy(alloc, v.m_value.array); + std::allocator_traits::deallocate(alloc, v.m_value.array, 1); } else { - JSON_ASSERT(v.m_data.m_type == value_t::object); + JSON_ASSERT(v.m_type == value_t::object); AllocatorType alloc; - std::allocator_traits::destroy(alloc, v.m_data.m_value.object); - std::allocator_traits::deallocate(alloc, v.m_data.m_value.object, 1); + std::allocator_traits::destroy(alloc, v.m_value.object); + std::allocator_traits::deallocate(alloc, v.m_value.object, 1); } - v.m_data.m_type = value_t::null; // avoid a double free if v is later destructed + v.m_type = value_t::null; // avoid a double free if v is later destructed } public: @@ -41128,14 +41314,15 @@ public: // This json_value is not itself a basic_json, so the top-level // container is first moved into a local stand-in ("cur"); a - // default-constructed basic_json has a null pointer in its - // m_value (see data::m_value's initializer), so swapping it - // with *this leaves this union's own pointer null, and it is - // never looked at or freed a second time. - basic_json cur; - cur.m_data.m_type = t; - using std::swap; - swap(cur.m_data.m_value, *this); + // default-constructed data has a null pointer in its m_value + // (see data::m_value's initializer), so swapping it with *this + // leaves this union's own pointer null, and it is never looked + // at or freed a second time. The stand-ins are data, not + // basic_json, so that destroying a value never constructs a + // CustomBaseClass, whose constructor may allocate (#5995). + data cur; + cur.m_type = t; + std::swap(cur.m_value, *this); destroy_bounded(cur, 0); } @@ -41144,7 +41331,7 @@ public: // The first recursion_depth_limit() levels are destroyed by plain // recursion, which is the fastest way and cannot throw; anything // nested deeper is handed to destroy_walk(), which uses no call stack. - static void destroy_bounded(basic_json& v, std::size_t depth) noexcept + static void destroy_bounded(data& v, std::size_t depth) noexcept { if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) { @@ -41152,23 +41339,23 @@ public: return; } - if (v.m_data.m_type == value_t::array) + if (v.m_type == value_t::array) { - for (auto& child : *v.m_data.m_value.array) + for (auto& child : *v.m_value.array) { if (child.m_data.m_type == value_t::array || child.m_data.m_type == value_t::object) { - destroy_bounded(child, depth + 1); + destroy_bounded(child.m_data, depth + 1); } } } else { - for (auto& child : *v.m_data.m_value.object) + for (auto& child : *v.m_value.object) { if (child.second.m_data.m_type == value_t::array || child.second.m_data.m_type == value_t::object) { - destroy_bounded(child.second, depth + 1); + destroy_bounded(child.second.m_data, depth + 1); } } } @@ -41195,18 +41382,15 @@ public: // array/object, which neither allocates nor recurses more // than one level deep. // - // This json_value is not itself a basic_json, so the - // top-level container is first moved into a local stand-in - // ("cur"); a default-constructed basic_json has a null - // pointer in its m_value (see data::m_value's initializer), - // so swapping it with *this leaves this union's own pointer - // null, and it is never looked at or freed a second time. - static void destroy_walk(basic_json& top) noexcept + // Like destroy_container(), the walk keeps the containers it + // holds in data stand-ins (cur, prev, gp, tmp), which never + // construct a CustomBaseClass. + static void destroy_walk(data& top) noexcept { - basic_json cur; + data cur; take(cur, top); - basic_json prev; // value_t::null: no parent + data prev; // value_t::null: no parent // true right after cur was entered: its children have not been // looked at yet. If none of them has children of its own, cur is @@ -41220,7 +41404,7 @@ public: fresh = false; if (has_no_children(cur) || (fresh_now && all_children_leaves(cur))) { - if (prev.m_data.m_type == value_t::null) + if (prev.m_type == value_t::null) { free_container(cur); return; // back at the top with nothing left to do @@ -41229,7 +41413,7 @@ public: // ascend: detach the grandparent link from prev's // walk_child() slot, drop that (now null) slot, free cur // (it is empty), then move up one level - basic_json gp; + data gp; take(gp, walk_child(prev)); pop_walk_child(prev); @@ -41240,7 +41424,7 @@ public: continue; } - basic_json& cur_child_ref = walk_child(cur); + data& cur_child_ref = walk_child(cur); if (has_no_children(cur_child_ref)) { @@ -41252,7 +41436,7 @@ public: // descend into the non-empty child, reversing the // link: its slot takes over prev, and the child becomes // the new cur - basic_json tmp; + data tmp; take(tmp, cur_child_ref); take(cur_child_ref, prev); take(prev, cur); @@ -41289,14 +41473,14 @@ public: destroy_container(t); break; - case value_t::null: + case value_t::null: // LCOV_EXCL_LINE case value_t::boolean: case value_t::number_integer: case value_t::number_unsigned: case value_t::number_float: case value_t::discarded: default: - break; + break; // LCOV_EXCL_LINE } } }; @@ -41469,9 +41653,9 @@ public: return is_array() ? array_capacity() : object_capacity(); } #else - static constexpr std::size_t container_capacity() noexcept + static constexpr std::size_t container_capacity() noexcept // LCOV_EXCL_LINE { - return detail::unknown_size(); + return detail::unknown_size(); // LCOV_EXCL_LINE } #endif @@ -41657,7 +41841,7 @@ public: /// into this switch is worth a few percent when copying a value made /// mostly of numbers JSON_HEDLEY_ALWAYS_INLINE - static void copy_leaf_value(const basic_json& src, basic_json& dst) + static void copy_leaf_value(const basic_json& src, basic_json& dst) // LCOV_EXCL_LINE { switch (src.m_data.m_type) { @@ -41831,7 +42015,7 @@ public: auto src_it = src_object.cbegin(); for (auto& element : *dst.m_data.m_value.object) { - if (JSON_HEDLEY_LIKELY(src_it != src_object.cend() && copy_keys_equal(src_it->first, element.first))) + if (JSON_HEDLEY_LIKELY(copy_keys_equal(src_it->first, element.first))) { copy_shallow(src_it->second, element.second, worklist); ++src_it; @@ -45170,6 +45354,30 @@ public: return {it, res.second}; } + private: + /// Sets the parents of a value when it goes out of scope, also when an + /// exception is thrown: a container insertion that throws may already + /// have added elements, whose parents must be set for the value to stay + /// valid (basic exception guarantee). Does nothing without JSON_DIAGNOSTICS. + class set_parents_on_exit + { + public: + explicit set_parents_on_exit(basic_json* v) noexcept : m_value(v) {} + set_parents_on_exit(const set_parents_on_exit&) = delete; + set_parents_on_exit& operator=(const set_parents_on_exit&) = delete; + set_parents_on_exit(set_parents_on_exit&&) = delete; + set_parents_on_exit& operator=(set_parents_on_exit&&) = delete; + + ~set_parents_on_exit() + { + m_value->set_parents(); + } + + private: + basic_json* m_value; + }; + + public: /// Helper for insertion of an iterator /// @note: This uses std::distance to support GCC 4.8, /// see https://github.com/nlohmann/json/pull/1257 @@ -45179,6 +45387,7 @@ public: iterator result(this); JSON_ASSERT(m_data.m_value.array != nullptr); + const set_parents_on_exit parents(this); auto insert_pos = std::distance(m_data.m_value.array->begin(), pos.m_it.array_iterator); m_data.m_value.array->insert(pos.m_it.array_iterator, std::forward(args)...); result.m_it.array_iterator = m_data.m_value.array->begin() + insert_pos; @@ -45187,7 +45396,6 @@ public: // result.m_it.array_iterator = m_data.m_value.array->insert(pos.m_it.array_iterator, cnt, val); // but the return value of insert is missing in GCC 4.8, so it is written this way instead. - set_parents(); return result; } @@ -45207,8 +45415,16 @@ public: throw_iterator_from_other_value(this); } - // insert to array and return iterator - return insert_iterator(pos, val); + // vector::insert has no effects if copying a single element to the + // end throws; elsewhere, it may already have shifted the elements, so + // copy first and insert by move, which cannot throw but by allocating + if (pos.m_it.array_iterator == m_data.m_value.array->end()) + { + return insert_iterator(pos, val); + } + + basic_json tmp(val); + return insert_iterator(pos, std::move(tmp)); } /// @brief inserts element into array @@ -45340,8 +45556,8 @@ public: detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterators first and last must point to objects", this); } + const set_parents_on_exit parents(this); m_data.m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator); - set_parents(); } /// @brief updates a JSON object from another object, overwriting existing keys @@ -45609,8 +45825,7 @@ public: /// @name lexicographical comparison operators /// @{ - // the same-type cases for non-owning scalars, shared by JSON_IMPLEMENT_OPERATOR - // and JSON_IMPLEMENT_SCALAR_FAST + // the same-type cases for non-owning scalars, used by JSON_IMPLEMENT_SCALAR_FAST #define JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ case value_t::null: \ return (null_result); \ @@ -45632,7 +45847,7 @@ public: // because any negative signed value is smaller than any unsigned value. // Otherwise, the non-negative signed value is cast to unsigned before the // comparison to avoid wraparound. -#define JSON_IMPLEMENT_OPERATOR(op, null_result, unordered_result, default_result, deep_result, may_descend) \ +#define JSON_IMPLEMENT_OPERATOR(op, unordered_result, default_result, deep_result, may_descend) \ const auto lhs_type = lhs.type(); \ const auto rhs_type = rhs.type(); \ \ @@ -45660,8 +45875,15 @@ public: return (*lhs.m_data.m_value.object) op (*rhs.m_data.m_value.object); \ } \ \ - JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ - \ + /* same-type scalars never get here: the inline fast path of the operator handles them */ \ + case value_t::null: \ + case value_t::boolean: \ + case value_t::number_integer: \ + case value_t::number_unsigned: \ + case value_t::number_float: \ + JSON_ASSERT(false); /* NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) */ \ + return (unordered_result); \ + \ case value_t::string: \ return (*lhs.m_data.m_value.string) op (*rhs.m_data.m_value.string); \ \ @@ -45701,7 +45923,7 @@ public: ? (number_integer_t(-1) op number_integer_t(1)) \ : (static_cast(lhs.m_data.m_value.number_integer) op rhs.m_data.m_value.number_unsigned); \ } \ - else if(compares_unordered(lhs, rhs))\ + else if(lhs_type == value_t::discarded || rhs_type == value_t::discarded)\ {\ return (unordered_result);\ }\ @@ -45769,7 +45991,7 @@ public: JSON_HEDLEY_DIAGNOSTIC_PUSH JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif - JSON_IMPLEMENT_OPERATOR( ==, true, false, false, + JSON_IMPLEMENT_OPERATOR( ==, false, false, compare_iteratively(lhs, rhs, false) == compare_result::equal, true) #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_POP @@ -45783,7 +46005,6 @@ public: // default_result is used if we cannot compare values. In that case, // we compare types. JSON_IMPLEMENT_OPERATOR(<=>, // *NOPAD* - std::partial_ordering::equivalent, std::partial_ordering::unordered, lhs_type <=> rhs_type, // *NOPAD* to_partial_ordering(compare_iteratively(lhs, rhs, false)), true) @@ -45795,7 +46016,7 @@ public: // default_result is used if we cannot compare values. In that case, // we compare types. Note we have to call the operator explicitly, // because MSVC has problems otherwise. - JSON_IMPLEMENT_OPERATOR( <, false, false, operator<(lhs_type, rhs_type), + JSON_IMPLEMENT_OPERATOR( <, false, operator<(lhs_type, rhs_type), compare_iteratively(lhs, rhs, true) == compare_result::less, false) } #endif @@ -45810,7 +46031,7 @@ public: /// their inlined code at each call site small. Binary values are /// rare and left to equal_general(). JSON_HEDLEY_ALWAYS_INLINE - static int equal_leaf(const_reference lhs, const_reference rhs) noexcept + static int equal_leaf(const_reference lhs, const_reference rhs) noexcept // LCOV_EXCL_LINE { #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_PUSH @@ -47119,7 +47340,7 @@ public: basic_json result(value_t::object); json_pointer::flatten("", *this, result); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief unflatten a previously flattened JSON value /// @sa https://json.nlohmann.me/api/basic_json/unflatten/ @@ -47832,7 +48053,7 @@ public: result.append(path.data(), path.size()); diff_append_key(result, key); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief the path of the element @a index of the value at @a path static string_t diff_index_path(const string_t& path, const std::size_t index) @@ -47842,7 +48063,7 @@ public: result.append(path.data(), path.size()); diff_append_index(result, index); return result; - } + } // LCOV_EXCL_LINE (exception cleanup only) /// @brief append the "remove" operations for the elements of array /// @a source from @a index on, and the "add" operations for the @@ -48002,7 +48223,7 @@ public: // operations if (source == target) { - return false; + return false; // LCOV_EXCL_LINE (equal objects with the same members in the same order return early) } // slow path: the common keys are in a different relative @@ -48767,6 +48988,7 @@ struct formatter // NOLINT(cert-dcl58-c #undef JSON_DISABLE_ENUM_SERIALIZATION #undef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION #undef JSON_UNLOCKED_FILE_IO +#undef JSON_FILE_WINDOW #undef JSON_DTOA_SSE2 #undef JSON_DTOA_NEON #undef JSON_LITTLE_ENDIAN_TARGET diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 0ee79fcc1..597822933 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -2473,19 +2473,20 @@ indent_done: if (negative) { ++p; + if (NLOHMANN_VIEW_UNLIKELY(cur() < '0' || cur() > '9')) + { + return fail(error_code::number_after_minus); + } } const unsigned char* const int_start = p; if (cur() == '0') { ++p; } - else if (NLOHMANN_VIEW_LIKELY(cur() >= '1' && cur() <= '9')) - { - p = skip_digits(p + 1, e); - } else { - return fail(error_code::number_after_minus); + // (a digit: the dispatch or the check after the sign saw it) + p = skip_digits(p + 1, e); } const auto int_digits = static_cast(p - int_start); std::size_t frac_digits = 0; @@ -3549,9 +3550,9 @@ class view_side return m_view.size(); } - string_view_t string() const + string_view_t string() const noexcept { - return m_view.get_string(); + return {m_view.m_doc->str(*m_view.m_node), m_view.m_node->len}; } bool boolean() const noexcept @@ -3580,28 +3581,19 @@ class view_side return string_view_t(m_view.m_doc->str(*m_view.m_node), m_view.m_node->len); } - /// a number, boolean, or null as a basic_json value (no allocation) + /// a number as a basic_json value (no allocation); only called for numbers BasicJsonType scalar() const { - switch (m_view.type()) + const value_t t = m_view.type(); + if (t == value_t::number_integer) { - case value_t::number_integer: - return BasicJsonType(m_view.template get()); - case value_t::number_unsigned: - return BasicJsonType(m_view.template get()); - case value_t::number_float: - return BasicJsonType(m_view.template get()); - case value_t::boolean: - return BasicJsonType(m_view.template get()); - case value_t::null: - case value_t::object: - case value_t::array: - case value_t::string: - case value_t::binary: - case value_t::discarded: - default: - return BasicJsonType(nullptr); + return BasicJsonType(m_view.template get()); } + if (t == value_t::number_unsigned) + { + return BasicJsonType(m_view.template get()); + } + return BasicJsonType(m_view.template get()); } /// start at the first element of this array @@ -3683,9 +3675,9 @@ class json_side return m_json->size(); } - string_view_t string() const + string_view_t string() const noexcept { - const auto& s = m_json->template get_ref(); + const auto& s = *m_json->template get_ptr(); return string_view_t(s.data(), s.size()); } @@ -4478,14 +4470,9 @@ struct position_source return token{end - start}; } - /// the range [first, last), or none if either is npos + /// the range [first, last) void set(std::size_t first, std::size_t last) noexcept { - if (first == std::string::npos || last == std::string::npos) - { - clear(); - return; - } start = first; end = last; } @@ -4604,11 +4591,11 @@ struct position_source first = n.off; last = n.off + 4u; break; - case node_kind::object: + case node_kind::object: // LCOV_EXCL_LINE case node_kind::array: case node_kind::link: default: - break; + break; // LCOV_EXCL_LINE } // (a crafted image can point past the text) if (last == std::string::npos || last > d.size) @@ -5027,10 +5014,6 @@ void value_range(const document_data& d, const node* n, std::size_t& start, std: end = p.end; return; } - if ((n->flags & node_flags::is_new) != 0 || n->off >= d.size) - { - return; - } // the open brackets: n, then the last element as long as it is a non-empty container std::size_t closing = 0; std::size_t pos = std::string::npos; @@ -5076,8 +5059,7 @@ void value_range(const document_data& d, const node* n, std::size_t& start, std: /// the context basic_json puts in front of the message of an exception about /// a value: "(/path) " with JSON_DIAGNOSTICS (none for the root), then -/// "(bytes a-b) " with JSON_DIAGNOSTIC_POSITIONS (with JSON_DIAGNOSTICS, only -/// if there is a path) +/// "(bytes a-b) " with JSON_DIAGNOSTIC_POSITIONS (also for the root) /// /// This and the throw helpers that take an error_site are templates, so that /// the path and range code is instantiated only where an exception can be @@ -5099,18 +5081,22 @@ std::string diagnostics(const error_site site) #if JSON_DIAGNOSTICS std::vector tokens; const bool found = edited ? value_path(d, site.value(), tokens) : value_path(d, site.value(), tokens); - if (!found || tokens.empty()) + if (!found) { - // context is still empty; returning it keeps the copy elided (-Wnrvo) + // a value that is not found has no path, and with it no bytes return context; } - std::string path; - for (const std::string& token : tokens) + if (!tokens.empty()) { - path.push_back('/'); - path += detail::escape(token); + // the root value has no path, but still has a byte range + std::string path; + for (const std::string& token : tokens) + { + path.push_back('/'); + path += detail::escape(token); + } + context = concat('(', path, ") "); } - context = concat('(', path, ") "); #endif #if JSON_DIAGNOSTIC_POSITIONS std::size_t start = 0; @@ -5262,10 +5248,12 @@ struct throwing_sax return true; } + // LCOV_EXCL_START (throw_parse_failure parses JSON only) bool binary(binary_t& /*unused*/) { return true; } + // LCOV_EXCL_STOP bool start_object(std::size_t /*unused*/) { @@ -5498,7 +5486,7 @@ inline void set_moved(document_data& d, node* n, node* seq, std::size_t cap) e.moved_cap[n->off] = cap; return; } - reserve_moved(d); + // every caller has called reserve_moved() (see above), so neither push_back() can throw e.moved.push_back(seq); e.moved_cap.push_back(cap); n->off = static_cast(e.moved.size() - 1); @@ -5583,7 +5571,7 @@ inline node* find_parent(const document_data& d, const node* target) } if (target == lo) { - return nullptr; + return nullptr; // LCOV_EXCL_LINE (the root, or a new value's root, which is linked) } for (;;) { @@ -6204,11 +6192,11 @@ NLOHMANN_VIEW_NOINLINE inline void check_utf8_scan(const unsigned char* p, std:: } if (*q >= 0x80) { - check_utf8_bytewise(p, n, static_cast(q - p)); - return; // not reached: the scan stopped at ill-formed UTF-8 + break; // ill-formed UTF-8: reported below } ++q; // a quote, backslash, or control character } + check_utf8_bytewise(p, n, static_cast(q - p)); } inline void check_utf8(const char* s, std::size_t n) @@ -7802,10 +7790,6 @@ inline void load_image(document_data& d, const std::uint8_t* image, std::size_t d.edits.reset(); d.base[2] = nullptr; d.owned.clear(); - if (d.owned_image.empty() || image != d.owned_image.data()) - { - d.owned_image.clear(); - } d.arena.clear(); d.indexes.clear(); d.index_slots.clear(); @@ -8521,6 +8505,7 @@ class output_buffer : m_out(sized(out, estimate)) , m_pos(&m_out[0]) , m_end(m_pos + m_out.size()) + , m_reserved((std::max)(estimate, m_out.size())) {} void finish() @@ -8590,11 +8575,12 @@ class output_buffer const auto used = static_cast(m_pos - m_out.data()); // (a step does not go beyond the reserved estimate, so that a good // estimate is never copied to a larger allocation; not a clamp: the - // lower bound used + n + 256 can exceed capacity(), and then it must win) - const std::size_t size = (std::max)((std::min)(m_out.size() + step, m_out.capacity()), used + n + 256); - if (size > m_out.capacity()) + // lower bound used + n + 256 can exceed m_reserved, and then it must win) + const std::size_t size = (std::max)((std::min)(m_out.size() + step, m_reserved), used + n + 256); + if (size > m_reserved) { - m_out.reserve((std::max)(m_out.capacity() * 2, size)); + m_reserved = (std::max)(m_reserved * 2, size); + m_out.reserve(m_reserved); } m_out.resize(size); m_pos = &m_out[0] + used; @@ -8604,6 +8590,9 @@ class output_buffer StringType& m_out; char* m_pos; char* m_end; + /// what was last passed to reserve(), a lower bound of the capacity + /// (StringType need not have a capacity() member) + std::size_t m_reserved; }; /// The length of the run at s that dump() writes unchanged without @@ -9115,7 +9104,7 @@ class view_serializer { invalid_utf8(s, n.len); // (the scan stopped at ill-formed UTF-8) } - write_codepoint(s[i], s + i, 1); // a quote, a backslash, or a control character + write_escaped(s[i]); // a quote, a backslash, or a control character ++i; } m_out.put('"'); @@ -9863,20 +9852,10 @@ class view_serializer /// a float node as dump() writes it void write_float_node(const node& n) { - write_float_node(n, fast_double {}); - } - - void write_float_node(const node& n, std::false_type /*other*/) - { + // (only for types other than double: doubles go through write_double_ahead()) write_float(float_value(m_doc, n)); } - void write_float_node(const node& n, std::true_type /*double*/) - { - m_out.reserve(64); - m_out.set_cursor(write_double_at(m_out.cursor(), n)); - } - /*! @brief (doubles) the float at n as dump() writes it, at w (64 bytes of room) @@ -9971,10 +9950,11 @@ class view_serializer { const string_t dumped = BasicJsonType(string_t(reinterpret_cast(s), n)).dump(); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) static_cast(dumped); - } + } // LCOV_EXCL_LINE (dump() throws: the callers pass only ill-formed UTF-8) - template - void write_codepoint(std::uint32_t codepoint, const unsigned char* bytes, std::size_t len) + /// write the escape sequence of a quote, a backslash or a control + /// character (the only bytes a run of plain output stops at) + void write_escaped(std::uint32_t codepoint) { switch (codepoint) { @@ -10002,20 +9982,7 @@ class view_serializer default: break; } - if (codepoint <= 0x1F || (EnsureAscii && codepoint >= 0x7F)) - { - if (codepoint <= 0xFFFF) - { - write_u_escape(codepoint); - } - else - { - write_u_escape(0xD7C0u + (codepoint >> 10u)); - write_u_escape(0xDC00u + (codepoint & 0x3FFu)); - } - return; - } - m_out.put(reinterpret_cast(bytes), len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) LCOV_EXCL_LINE (printable characters are copied in runs) + write_u_escape(codepoint); // (a control character) } void write_u_escape(std::uint32_t u) @@ -10077,12 +10044,37 @@ namespace view template struct value_tag {}; +/// whether basic_json reads T with a dedicated from_json() overload that does +/// not accept a boolean: exactly number_integer_t, number_unsigned_t and +/// number_float_t (the other arithmetic types convert a boolean to 0 or 1) +template +struct rejects_boolean : std::integral_constant < bool, + std::is_same::value + || std::is_same::value + || std::is_same::value > +{}; + +/// a boolean as T: 0 or 1 +template +NLOHMANN_VIEW_ALWAYS_INLINE T boolean_as(const document_data& /*d*/, const node& n, std::false_type /*rejects*/) +{ + return static_cast((n.flags & node_flags::is_true) != 0); +} + +/// a boolean as number_integer_t, number_unsigned_t or number_float_t: type_error.302, as in basic_json +template +[[noreturn]] NLOHMANN_VIEW_ALWAYS_INLINE T boolean_as(const document_data& d, const node& n, std::true_type /*rejects*/) +{ + throw_type_error(error_site(&d, &n), exception_id::type_mismatch, "type must be number, but is ", value_type_name(static_cast(n.kind))); +} + /*! @brief the number or boolean of a node converted to an arithmetic type As basic_json's get() for arithmetic types: integers and floats are -converted with static_cast, booleans give 0 or 1, and other types throw -type_error.302. +converted with static_cast, booleans give 0 or 1 (but throw type_error.302 for +number_integer_t, number_unsigned_t and number_float_t, as basic_json does), +and other types throw type_error.302. */ template NLOHMANN_VIEW_ALWAYS_INLINE T arithmetic_value(const document_data& d, const node& n) @@ -10096,7 +10088,7 @@ NLOHMANN_VIEW_ALWAYS_INLINE T arithmetic_value(const document_data& d, const nod case value_t::number_float: return static_cast(float_value(d, n)); case value_t::boolean: - return static_cast((n.flags & node_flags::is_true) != 0); + return boolean_as(d, n, rejects_boolean {}); case value_t::null: case value_t::object: case value_t::array: @@ -11809,9 +11801,11 @@ class basic_json_document { // (as before: the library's adapter decides what a null stream yields) detail::view::append_adapter(buf, detail::input_adapter(file)); - return; } - detail::view::append_file(buf, file); + else + { + detail::view::append_file(buf, file); + } } #endif diff --git a/tests/benchmarks/src/benchmarks.cpp b/tests/benchmarks/src/benchmarks.cpp index 0d9e06d21..9e38a420c 100644 --- a/tests/benchmarks/src/benchmarks.cpp +++ b/tests/benchmarks/src/benchmarks.cpp @@ -629,6 +629,22 @@ static json make_containers() return j; } +/// many arrays of one number each: one-element arrays are left to the general +/// path (CBOR and MessagePack), which reads a lone number slower than the loop +/// for arrays of scalars would +static json make_singles() +{ + json data = json::array(); + for (std::size_t i = 0; i < 100000; ++i) + { + data.push_back(json::array({i})); + } + + json j = json::object(); + j["data"] = std::move(data); + return j; +} + /// one flat array of numbers: the scalar decoding path, which must not move static json make_scalars() { @@ -676,6 +692,8 @@ BENCHMARK_CAPTURE(FromBinaryShape, containers / ubjson_optimized, make_container BENCHMARK_CAPTURE(FromBinaryShape, containers / bjdata, make_containers, binary_format::bjdata); BENCHMARK_CAPTURE(FromBinaryShape, containers / bson, make_containers, binary_format::bson); BENCHMARK_CAPTURE(FromBinaryShape, containers / bon8, make_containers, binary_format::bon8); +BENCHMARK_CAPTURE(FromBinaryShape, singles / cbor, make_singles, binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryShape, singles / msgpack, make_singles, binary_format::msgpack); // BSON names every array element, so a large array measures key generation // rather than scalar decoding and is left out here BENCHMARK_CAPTURE(FromBinaryShape, scalars / cbor, make_scalars, binary_format::cbor); diff --git a/tests/src/unit-32bit.cpp b/tests/src/unit-32bit.cpp index 97549c999..e4e2dea6d 100644 --- a/tests/src/unit-32bit.cpp +++ b/tests/src/unit-32bit.cpp @@ -97,3 +97,23 @@ TEST_CASE("CBOR") } } } + +TEST_CASE("MessagePack") +{ + SECTION("parse errors") + { + SECTION("array/map size equal to the unknown-size sentinel") + { + // on 32-bit platforms an array 32/map 32 length of 0xFFFFFFFF aliases unknown_size() + std::vector const varr = {0xDD, 0xFF, 0xFF, 0xFF, 0xFF}; + std::vector const vmap = {0xDF, 0xFF, 0xFF, 0xFF, 0xFF}; + + json _; + CHECK_THROWS_WITH_AS(_ = json::from_msgpack(varr), "[json.exception.out_of_range.408] syntax error while parsing MessagePack size: excessive array size", json::out_of_range&); + CHECK(json::from_msgpack(varr, true, false).is_discarded()); + + CHECK_THROWS_WITH_AS(_ = json::from_msgpack(vmap), "[json.exception.out_of_range.408] syntax error while parsing MessagePack size: excessive map size", json::out_of_range&); + CHECK(json::from_msgpack(vmap, true, false).is_discarded()); + } + } +} diff --git a/tests/src/unit-allocator.cpp b/tests/src/unit-allocator.cpp index 4062870d5..9862e8c5e 100644 --- a/tests/src/unit-allocator.cpp +++ b/tests/src/unit-allocator.cpp @@ -12,6 +12,8 @@ #include using nlohmann::json; +#include +#include #include #if JSON_HAS_RANGES #include @@ -790,7 +792,7 @@ TEST_CASE("a failed allocation leaves the value unchanged") #if JSON_HAS_RANGES && !defined(__MINGW32__) const std::vector numbers = {1, 2}; next_construct_fails = true; - CHECK_THROWS_AS(nlohmann::to_json(j, numbers | std::views::filter([](int /*unused*/) + CHECK_THROWS_AS(nlohmann::to_json(j, numbers | std::views::filter([](int /*unused*/) noexcept { return true; })), std::bad_alloc&); @@ -804,3 +806,410 @@ TEST_CASE("a failed allocation leaves the value unchanged") #endif } #endif + +// the no-exceptions CI job has no std::bad_alloc to catch; with iterator +// debugging (MSVC STL debug builds) containers allocate inside noexcept +// functions, so failing an allocation there terminates the program +#if !defined(JSON_NOEXCEPTION) && !(defined(_ITERATOR_DEBUG_LEVEL) && _ITERATOR_DEBUG_LEVEL > 0) +namespace +{ +// whether every child of j names j as its parent (always true without +// JSON_DIAGNOSTICS, where no parents are stored) +template +bool children_name_their_parent(const BasicJsonType& j) +{ +#if JSON_DIAGNOSTICS + if (j.is_array()) + { + for (const auto& element : *j.m_data.m_value.array) + { + if (element.m_parent != &j) + { + return false; + } + } + } + else if (j.is_object()) + { + for (const auto& element : *j.m_data.m_value.object) + { + if (element.second.m_parent != &j) + { + return false; + } + } + } +#else + static_cast(j); +#endif + return true; +} + +enum class guarantee +{ + basic, // the value stays valid and keeps its type + strong // the value is unchanged +}; + +// Runs op on a fresh value make() with the 0th, 1st, 2nd, ... allocation made +// by the library failing in turn, until op completes. Every failed attempt +// must have thrown std::bad_alloc and left the value valid, with all parent +// pointers intact (copying a value asserts the invariant), and for the strong +// guarantee exactly as it was. +template +void check_exception_guarantee(guarantee g, + const std::function& make, + const std::function& op) +{ + fail_at_alloc_call = -1; + + bool completed = false; + std::size_t failures = 0; + for (std::size_t n = 0; !completed; ++n) + { + CAPTURE(n) + REQUIRE(n < 1000); + + BasicJsonType j = make(); + const std::string before = j.dump(); + const auto type_before = j.type(); + + alloc_call_count = 0; + fail_at_alloc_call = static_cast(n); + try + { + op(j); + completed = true; + } + catch (const std::bad_alloc&) + { + ++failures; + } + fail_at_alloc_call = -1; + + if (!completed) + { + CHECK((j.type() == type_before)); + CHECK(children_name_their_parent(j)); + const BasicJsonType copy(j); // NOLINT(performance-unnecessary-copy-initialization): the copy is the point, it allocates + CHECK(copy.dump() == j.dump()); + if (g == guarantee::strong) + { + CHECK(j.dump() == before); + } + } + } + + // the operation allocates, so at least one attempt must have failed + CHECK(failures > 0); +} + +template +void check_insert_exception_guarantees() +{ + using json_t = BasicJsonType; + using make_t = std::function; + + // strings of this length do not fit into the small string buffer, so + // copying them allocates + const json_t val = std::string(64, 'x'); + const json_t source = json_t::array({std::string(64, 'p'), std::string(64, 'q'), std::string(64, 'r')}); + + for (const bool spare_capacity : + { + false, true + }) + { + CAPTURE(spare_capacity) + + // [ "aaa...", "bbb...", "ccc..." ], with a spare slot if requested + const make_t make = [spare_capacity]() + { + json_t j = json_t::array({std::string(64, 'a'), std::string(64, 'b'), std::string(64, 'c')}); + if (spare_capacity) + { + j.push_back(std::string(64, 'd')); + j.erase(3); + } + return json_t(std::move(j)); + }; + + for (const bool at_end : + { + false, true + }) + { + CAPTURE(at_end) + + SECTION("insert(pos, const basic_json&)") + { + check_exception_guarantee(guarantee::strong, make, [&](json_t& j) + { + j.insert(at_end ? j.end() : j.begin() + 1, val); + }); + } + + SECTION("insert(pos, basic_json&&)") + { + check_exception_guarantee(guarantee::strong, make, [&](json_t& j) + { + j.insert(at_end ? j.end() : j.begin() + 1, json_t(val)); + }); + } + + SECTION("insert(pos, size_type, const basic_json&)") + { + check_exception_guarantee(guarantee::basic, make, [&](json_t& j) + { + j.insert(at_end ? j.end() : j.begin() + 1, 3, val); + }); + } + + SECTION("insert(pos, const_iterator, const_iterator)") + { + check_exception_guarantee(guarantee::basic, make, [&](json_t& j) + { + j.insert(at_end ? j.end() : j.begin() + 1, source.begin(), source.end()); + }); + } + + SECTION("insert(pos, initializer_list_t)") + { + check_exception_guarantee(guarantee::basic, make, [&](json_t& j) + { + j.insert(at_end ? j.end() : j.begin() + 1, {val, val}); + }); + } + } + } + + SECTION("insert(const_iterator, const_iterator) for objects") + { + const json_t members = {{"m", std::string(64, 'm')}, {"n", std::string(64, 'n')}, {"o", std::string(64, 'o')}}; + + SECTION("no member exists yet") + { + check_exception_guarantee(guarantee::basic, []() + { + return json_t({{"a", 1}}); + }, [&](json_t& j) + { + j.insert(members.begin(), members.end()); + }); + } + + SECTION("some members exist already") + { + check_exception_guarantee(guarantee::basic, []() + { + return json_t({{"a", 1}, {"n", std::string(64, 'x')}}); + }, [&](json_t& j) + { + j.insert(members.begin(), members.end()); + }); + } + } +} +} // namespace + +TEST_CASE("insert() provides its documented exception guarantees (#5986)") +{ + SECTION("std::map-backed object_t") + { + check_insert_exception_guarantees>(); + } + + SECTION("ordered_map-backed object_t") + { + check_insert_exception_guarantees>(); + } +} +#endif + +#if !defined(JSON_NOEXCEPTION) +namespace +{ +// the allocation numbered sweep_fail_at (counted from 1) throws; 0 never fails +std::size_t sweep_allocations = 0; +std::size_t sweep_fail_at = 0; +std::ptrdiff_t sweep_live = 0; + +template +struct sweep_allocator +{ + using value_type = T; + // GCC 4.8's std::map takes these from the allocator instead of from + // std::allocator_traits + using pointer = T*; + using const_pointer = const T*; + using reference = T&; + using const_reference = const T&; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + + sweep_allocator() = default; + template sweep_allocator(const sweep_allocator& /*unused*/) noexcept {} + + T* allocate(std::size_t n) + { + if (sweep_fail_at != 0 && ++sweep_allocations == sweep_fail_at) + { + throw std::bad_alloc(); + } + ++sweep_live; + return std::allocator().allocate(n); + } + + void deallocate(T* p, std::size_t n) noexcept + { + --sweep_live; + std::allocator().deallocate(p, n); + } + + // GCC 4.8's std::map calls these on the allocator directly instead of + // going through std::allocator_traits + std::size_t max_size() const noexcept + { + return (std::numeric_limits::max)() / sizeof(T); + } + + template + void construct(U* p, Args&& ... args) noexcept(noexcept(::new (static_cast(p)) U(std::forward(args)...))) + { + ::new (static_cast(p)) U(std::forward(args)...); + } + + // libc++ probes whether construct() is usable for const objects (when + // checking for trivial copies); this must not be viable then + template + void construct(const U* /*unused*/, const Args& ... /*unused*/) = delete; + + template + void destroy(U* p) noexcept + { + static_cast(p); // unused for trivially destructible types + p->~U(); + } + + template + struct rebind + { + using other = sweep_allocator; + }; + + bool operator==(const sweep_allocator& /*unused*/) const noexcept + { + return true; + } + bool operator!=(const sweep_allocator& /*unused*/) const noexcept + { + return false; + } +}; + +// run make() with the allocation numbered k failing, for k = 1, 2, ...: every +// failure must be a std::bad_alloc and must not leave anything behind, and +// the first k without a failure must yield the value made without failures +template +void sweep_allocations_of(const Make& make) +{ + sweep_fail_at = 0; + const BasicJsonType expected = make(); + const std::ptrdiff_t live_before = sweep_live; + + std::size_t failures = 0; + for (std::size_t k = 1; k < 10000; ++k) + { + sweep_allocations = 0; + sweep_fail_at = k; + try + { + const BasicJsonType result = make(); + sweep_fail_at = 0; + CHECK(result == expected); + break; + } + catch (const std::bad_alloc&) + { + sweep_fail_at = 0; + ++failures; + } + CHECK(sweep_live == live_before); + } + sweep_fail_at = 0; + CHECK(failures > 10); + CHECK(sweep_live == live_before); +} +} // namespace + +TEST_CASE("parse() survives a failing allocation at every point (#5985)") +{ + // strings beyond the small string buffer, nested objects and arrays + const std::string text = R"({"a":"a long string that does not fit into the small string buffer1","b":[1,2,[3,{"k":"another string that is longer than any small string buffer"}]],"c":{"x":1,"y":{"z":[]},"w":"yet another string that does not fit into a small buffer"},"d":1,"b":[[],{}]})"; + + using sweep_json = nlohmann::json::with_allocator_t; + using sweep_ordered_json = nlohmann::ordered_json::with_allocator_t; + + SECTION("json") + { + sweep_allocations_of([&] + { + return sweep_json::parse(text); + }); + } + + SECTION("ordered_json") + { + sweep_allocations_of([&] + { + return sweep_ordered_json::parse(text); + }); + } + + // With iterator debugging (MSVC STL debug builds, also used by clang-cl), + // containers allocate a debug proxy through the allocator inside their + // noexcept default and move constructors, so failing that allocation + // terminates the program instead of throwing std::bad_alloc. Nothing to + // check there. +#if !(defined(_ITERATOR_DEBUG_LEVEL) && _ITERATOR_DEBUG_LEVEL > 0) + // Clang 3.4 and 3.5 leak memory when an element constructor throws + // inside libstdc++'s containers (a std::map copy, a std::vector + // reallocation), without any json value involved. +#if !(defined(__clang__) && !defined(__apple_build_version__) && __clang_major__ == 3 && __clang_minor__ < 6) + SECTION("parser callback") + { + const auto keep_all = [](int /*depth*/, sweep_json::parse_event_t /*event*/, sweep_json& /*parsed*/) noexcept + { + return true; + }; + sweep_allocations_of([&] + { + return sweep_json::parse(text, keep_all); + }); + + const auto drop_d = [](int /*depth*/, sweep_json::parse_event_t event, sweep_json & parsed) + { + return event != sweep_json::parse_event_t::key || parsed != "d"; + }; + sweep_allocations_of([&] + { + return sweep_json::parse(text, drop_d); + }); + } + + SECTION("sax_parse into an existing value") + { + // a failed parse leaves the target in a state that is safe to destroy + using sax_t = nlohmann::detail::json_sax_dom_parser>; + sweep_allocations_of([&]() -> sweep_json + { + sweep_json value; + sax_t sax(value, false); + static_cast(sweep_json::sax_parse(text, &sax)); + return value; + }); + } +#endif +#endif +} +#endif diff --git a/tests/src/unit-alt-string.cpp b/tests/src/unit-alt-string.cpp index b2832dcdf..e34afe4a1 100644 --- a/tests/src/unit-alt-string.cpp +++ b/tests/src/unit-alt-string.cpp @@ -10,6 +10,7 @@ #include "doctest_compatibility.h" #include +#include #include #include @@ -523,4 +524,56 @@ TEST_CASE("alternative string type") } } +TEST_CASE("alternative string type with json_view") +{ + // regression test for #5994: basic_json_view::dump() called + // StringType::capacity(), which alt_string (and the documented StringType + // requirements) do not provide + using alt_document = nlohmann::basic_json_document; + using alt_editable_document = nlohmann::basic_json_document; + + const char* text = R"({"a":[1,"x\u00e4"],"b":{"c":true,"d":null},"e":2.5})"; + const alt_json j = alt_json::parse(text); + + SECTION("read-only document") + { + const alt_document d = alt_document::parse(text); + CHECK(d.root().dump() == j.dump()); + CHECK(d.root().dump(2) == j.dump(2)); + CHECK(d.root().dump(-1, ' ', true) == j.dump(-1, ' ', true)); + CHECK(d.root().materialize() == j); + CHECK(d.root().contains(alt_string("b"))); + CHECK(d.root()[alt_string("b")][alt_string("c")].dump() == "true"); + } + + SECTION("long output") + { + // more than one growth step of the output buffer + alt_json big = alt_json::array(); + for (int i = 0; i < 20000; ++i) + { + big.push_back(alt_json::parse(text)); + } + const alt_string big_text = big.dump(); + const alt_document d = alt_document::parse(big_text); + CHECK(d.root().dump() == big_text); + CHECK(d.root().dump(4) == big.dump(4)); + } + + SECTION("editable document") + { + alt_editable_document d = alt_editable_document::parse(text); + d.set(d.root(), "f", "new"); + d.push_back(d.root()[alt_string("a")], 3); + alt_json expected = j; + expected["f"] = "new"; + expected["a"].push_back(3); + CHECK(d.root().materialize() == expected); + CHECK(alt_json::parse(d.root().dump()) == expected); + + const alt_editable_document loaded = alt_editable_document::load(d.save()); + CHECK(loaded.root().materialize() == expected); + } +} + DOCTEST_CLANG_SUPPRESS_WARNING_POP diff --git a/tests/src/unit-binary_writer_sinks.cpp b/tests/src/unit-binary_writer_sinks.cpp index 1e18c3f15..5638435ca 100644 --- a/tests/src/unit-binary_writer_sinks.cpp +++ b/tests/src/unit-binary_writer_sinks.cpp @@ -14,7 +14,9 @@ using nlohmann::json; #include #include #include +#include #include +#include #include #include @@ -48,6 +50,14 @@ std::vector test_values() large_payloads.push_back(json::binary(std::vector(static_cast(100 * i), static_cast(i)))); } + // several times vector_sink_window characters in every format: the vector + // sink makes the capacity it grows to writable a window at a time + json many_windows = json::array(); + for (std::uint64_t i = 0; i < 30000; ++i) + { + many_windows.push_back((i * 0x9E3779B97F4A7C15u) >> 1u); + } + return { json(nullptr), json(true), json(false), @@ -59,7 +69,7 @@ std::vector test_values() json::array(), json::object(), json::array({1, 2, 3}), json({{"a", 1}, {"b", nullptr}}), json({{"nested", {{"deep", json::array({1, "two", 3.0, nullptr})}}}}), - big_array, big_object, large_payloads, + big_array, big_object, large_payloads, many_windows, json::binary(std::vector(5000, 0x42)) }; } @@ -767,3 +777,69 @@ TEST_CASE("a write that throws leaves the same bytes in every kind of output") } } #endif + +#if !defined(JSON_NOEXCEPTION) +namespace +{ + +// an adapter whose every write fails +class failing_adapter : public nlohmann::detail::output_adapter_protocol +{ + public: + [[noreturn]] void write_character(char /*c*/) override + { + throw std::runtime_error("write failed"); + } + + [[noreturn]] void write_characters(const char* /*s*/, std::size_t /*length*/) override + { + throw std::runtime_error("write failed"); + } +}; + +} // namespace +#endif + +TEST_CASE("output adapters") +{ + SECTION("writing a single character") + { + // the writers pass whole buffers on, but an adapter takes single + // characters as well + std::vector as_vector; + nlohmann::detail::output_vector_adapter vector_adapter(as_vector); + vector_adapter.write_character('a'); + vector_adapter.write_character('b'); + CHECK(as_vector == std::vector({'a', 'b'})); + + std::ostringstream as_stream; + nlohmann::detail::output_stream_adapter stream_adapter(as_stream); + stream_adapter.write_character('c'); + stream_adapter.write_character('d'); + CHECK(as_stream.str() == "cd"); + + std::string as_string; + nlohmann::detail::output_string_adapter string_adapter(as_string); + string_adapter.write_character('e'); + string_adapter.write_character('f'); + CHECK(as_string == "ef"); + } + +#if !defined(JSON_NOEXCEPTION) + SECTION("a sink that is destroyed with characters left over") + { + // the destructor passes the pending characters on, as a write that + // threw leaves them there; the adapter failing then must not escape + { + nlohmann::detail::output_adapter_sink sink(std::make_shared()); + sink.write_character('x'); + } + + // while flush() reports the failure + nlohmann::detail::output_adapter_sink sink(std::make_shared()); + sink.write_character('x'); + CHECK_THROWS_AS(sink.flush(), std::runtime_error&); + // (the character counts as written, so there is nothing left for the destructor) + } +#endif +} diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index 04ee3acc4..dd5831dbd 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -78,23 +78,25 @@ TEST_CASE("BJData") { SECTION("binary_reader BJData lookup tables") { - // both lookups are static member functions + // bjd_type_name() is static; is_valid_optimized_type() needs a BJData reader std::vector const data; using reader_t = nlohmann::detail::binary_reader; - // the excluded optimized-type markers must match binary_writer's - // is_bjdata_excluded_type_marker(), which encodes the same 8 markers + // the markers BJData excludes as optimized types must match + // binary_writer's is_bjdata_excluded_type_marker(), which encodes the + // same 8 markers + const reader_t reader(nlohmann::detail::input_adapter(data), json::input_format_t::bjdata); for (const char marker : - {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z' + {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z', 'x' }) { - CHECK(reader_t::is_bjd_excluded_optimized_type(static_cast(marker))); + CHECK(!reader.is_valid_optimized_type(static_cast(marker))); } for (const char marker : - {'U', 'i', 'u', 'I', 'm', 'l', 'M', 'L', 'd', 'D', 'C', 'B', 'x' + {'U', 'i', 'u', 'I', 'm', 'l', 'M', 'L', 'd', 'D', 'C', 'B', 'h' }) { - CHECK(!reader_t::is_bjd_excluded_optimized_type(static_cast(marker))); + CHECK(reader.is_valid_optimized_type(static_cast(marker))); } // every dtype marker must round-trip to its ND-array type name @@ -2312,6 +2314,23 @@ TEST_CASE("BJData") CHECK_FALSE(json::sax_parse(v, &scp, json::input_format_t::bjdata)); } + SECTION("every event of an ndarray of chars") + { + // the elements of an ndarray of 'C' are read one by one + std::vector const v = {'[', '$', 'C', '#', '[', '$', 'U', '#', 'U', 2, 2, 2, 'a', 'b', 'c', 'd'}; + // start_object, key, string, key, start_array, 2 numbers, end_array, + // key, start_array, 4 chars, end_array, end_object + constexpr int events = 16; + for (int allowed = 0; allowed < events; ++allowed) + { + CAPTURE(allowed) + SaxCountdown scp(allowed); + CHECK_FALSE(json::sax_parse(v, &scp, json::input_format_t::bjdata)); + } + SaxCountdown scp(events); + CHECK(json::sax_parse(v, &scp, json::input_format_t::bjdata)); + } + SECTION("start_array() in ndarray _ArrayData_") { std::vector const v = {'[', '$', 'U', '#', '[', 'i', 2, 'i', 3, ']', 6, 5, 4, 3, 2, 1}; @@ -3095,6 +3114,15 @@ TEST_CASE("BJData") json _; CHECK_THROWS_WITH(_ = json::from_bjdata(v), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing BJData char: byte after 'C' must be in range 0x00..0x7F; last byte: 0x82"); } + + SECTION("ndarray of chars cut off by the end of the input") + { + // 2x2 chars announced, three present + std::vector const v = {'[', '$', 'C', '#', '[', '$', 'U', '#', 'U', 2, 2, 2, 'a', 'b', 'c'}; + json _; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(v), "[json.exception.parse_error.110] parse error at byte 16: syntax error while parsing BJData char: unexpected end of input", json::parse_error&); + CHECK(json::from_bjdata(v, true, false).is_discarded()); + } } SECTION("byte") @@ -3211,14 +3239,34 @@ TEST_CASE("BJData") #endif CHECK(json::from_bjdata(vM, true, false).is_discarded()); -#if SIZE_MAX != 0xffffffff - CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vMX), "[json.exception.out_of_range.408] syntax error while parsing BJData size: excessive ndarray size caused overflow", json::out_of_range&); -#else + // the first dimension is 2^64-1, which is rejected as a count (see below) CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vMX), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&); -#endif CHECK(json::from_bjdata(vMX, true, false).is_discarded()); } + SECTION("optimized array: count equal to SIZE_MAX") + { + // the reader marks a container without a count by SIZE_MAX, so + // a count of that value must not be read as "no count" + // (it used to parse as [1]) + json _; +#if SIZE_MAX != 0xffffffff + std::vector const vM = {'[', '#', 'M', 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'}; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vM), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&); + CHECK(json::from_bjdata(vM, true, false).is_discarded()); + // one less is an ordinary excessive count + std::vector const vM1 = {'[', '#', 'M', 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'}; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vM1), "[json.exception.out_of_range.408] excessive array size: 18446744073709551614", json::out_of_range&); +#else + std::vector const vm = {'[', '#', 'm', 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'}; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vm), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&); + CHECK(json::from_bjdata(vm, true, false).is_discarded()); + std::vector const vL = {'[', '#', 'L', 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 'i', 0x01, ']'}; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vL), "[json.exception.out_of_range.408] syntax error while parsing BJData size: integer value overflow", json::out_of_range&); + CHECK(json::from_bjdata(vL, true, false).is_discarded()); +#endif + } + SECTION("optimized array: integer value overflow") { #if SIZE_MAX == 0xffffffff diff --git a/tests/src/unit-bon8.cpp b/tests/src/unit-bon8.cpp index 1e24d32d4..85c9fa821 100644 --- a/tests/src/unit-bon8.cpp +++ b/tests/src/unit-bon8.cpp @@ -329,6 +329,9 @@ TEST_CASE("BON8") check_bon8({false, nullptr, true}, {0x83, 0xF8, 0xFA, 0xF9}); check_bon8({false, nullptr, true, 1.0}, {0x84, 0xF8, 0xFA, 0xF9, 0xFD}); check_bon8({false, nullptr, true, 1.0, json::array(), 0.0}, {0x85, 0xF8, 0xFA, 0xF9, 0xFD, 0x80, 0xFC, 0xFE}); + // an array that starts with a number: -1, 0, and 1 have bytes of their own + check_bon8({-1.0, 0.0, 1.0}, {0x83, 0xFB, 0xFC, 0xFD}); + check_bon8({1.0, -1.0, -0.0}, {0x83, 0xFD, 0xFB, 0x8E, 0x80, 0x00, 0x00, 0x00}); check_bon8({{{1}}}, {0x81, 0x81, 0x81, 0x91}); check_bon8({{{"foo"}}}, {0x81, 0x81, 0x81, 'f', 'o', 'o', 0xFF}); check_bon8({{{""}}}, {0x81, 0x81, 0x81, 0xFF}); diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index ac8e2b24f..f7957d566 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -82,8 +82,20 @@ class huge_string_t : public std::string return result; } + // returns a copy of @a s whose size() pretends to be @a n + static huge_string_t as_size(const std::string& s, const size_type n) + { + huge_string_t result(s); + result.fake_size = n; + return result; + } + size_type size() const noexcept { + if (fake_size != 0) + { + return fake_size; + } if (pretend_huge) { // one byte more than the BSON length field can represent @@ -94,6 +106,7 @@ class huge_string_t : public std::string private: bool pretend_huge = false; + size_type fake_size = 0; }; using huge_string_json = nlohmann::json::with_string_t; @@ -268,6 +281,17 @@ TEST_CASE("BSON") CHECK_THROWS_WITH_AS(huge_string_json::to_bson(j), "[json.exception.out_of_range.412] BSON length 2147483661 exceeds maximum of 2147483647", huge_string_json::out_of_range&); } + SECTION("an array element that no longer fits the document") + { + // the string fits the length field, but not the document that + // holds it; nothing of it is written + const auto limit = static_cast((std::numeric_limits::max)()); + huge_string_json j; + j["a"] = huge_string_json::array({huge_string_t::as_size("value", limit - 8)}); + + CHECK_THROWS_AS(huge_string_json::to_bson(j), huge_string_json::out_of_range&); + } + SECTION("document") { // an oversized string nested one level deep makes the @@ -899,6 +923,38 @@ TEST_CASE("BSON") CHECK_THROWS_AS(_ = json::from_bson(input), json::parse_error&); } + SECTION("binary member with subtype 0x02 (old binary) with a truncated inner length") + { + // the inner length field is cut off by the end of the input + std::vector const input = + { + 0x0F, 0x00, 0x00, 0x00, // size (little endian) + 0x05, 'a', 0x00, // entry: binary + 0x08, 0x00, 0x00, 0x00, // size of binary (little endian) + 0x02, // "old binary" subtype + 0x01 // first byte of the inner length + }; + + json _; + CHECK_THROWS_WITH_AS(_ = json::from_bson(input), "[json.exception.parse_error.110] parse error at byte 14: syntax error while parsing BSON number: unexpected end of input", json::parse_error&); + CHECK(json::from_bson(input, true, false).is_discarded()); + } + + SECTION("string member that is cut off by the end of the input") + { + std::vector const input = + { + 0x0F, 0x00, 0x00, 0x00, // size (little endian) + 0x02, 'a', 0x00, // entry: string + 0x0A, 0x00, 0x00, 0x00, // size of the string (little endian) + 'b', 'c' // the string ends early + }; + + json _; + CHECK_THROWS_WITH_AS(_ = json::from_bson(input), "[json.exception.parse_error.110] parse error at byte 14: syntax error while parsing BSON string: unexpected end of input", json::parse_error&); + CHECK(json::from_bson(input, true, false).is_discarded()); + } + SECTION("Some more complex document") { json const j = @@ -2174,6 +2230,25 @@ value = depth % 2 == 0 ? json{{"a", std::move(value)}, {"b", {1, "x"}}} : CHECK(output.empty()); } +#if !defined(JSON_NOEXCEPTION) + SECTION("U+0000 in a long key is found at any position") + { + // the key is searched eight characters at a time + for (const std::string& key : + { + std::string("abc\0defghijkl", 12), std::string("abcdefgh\0ijkl", 13), std::string("abcdefghijklmnop\0", 17), std::string("\0abcdefghijkl", 13), + std::string("abcdefghijklmno\0", 16) + }) + { + CAPTURE(key) + const json value = json::object({{key, 1}}); + std::vector output; + CHECK_THROWS_AS(json::to_bson(value, output), json::out_of_range&); + CHECK(output.empty()); + } + } +#endif + SECTION("a binary subtype that doesn't fit a byte is rejected before anything is written (#5675)") { // the offending value is nested, so this also covers that the check diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 930cbe7f5..c4d29543d 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -1809,6 +1809,30 @@ TEST_CASE("CBOR") CHECK(json::from_cbor(std::vector({0xa1, 0xff, 0x01}), true, false).is_discarded()); } + SECTION("tag in front of a key that is cut off by the end of the input") + { + // 0xD8: the tag number is in the next byte, which is missing + const std::vector input = {0xA1, 0xD8}; + json _; + for (const auto handler : + { + json::cbor_tag_handler_t::ignore, json::cbor_tag_handler_t::store + }) + { + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input, true, true, handler), "[json.exception.parse_error.110] parse error at byte 3: syntax error while parsing CBOR number: unexpected end of input", json::parse_error&); + CHECK(json::from_cbor(input, true, false, handler).is_discarded()); + } + } + + SECTION("chunk of an indefinite-length string that is cut off by the end of the input") + { + // the second byte announces a chunk of three bytes, but two follow + const std::vector input = {0x7F, 0x63, 'a', 'b'}; + json _; + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 5: syntax error while parsing CBOR string: unexpected end of input", json::parse_error&); + CHECK(json::from_cbor(input, true, false).is_discarded()); + } + SECTION("tag in front of a key") { const auto ignore = json::cbor_tag_handler_t::ignore; @@ -2520,6 +2544,34 @@ TEST_CASE("CBOR arrays of scalars from contiguous and non-contiguous input") } } +TEST_CASE("CBOR arrays of one element from contiguous and non-contiguous input") +{ + // contiguous input leaves an array of one element to the general path + // rather than to the loop that reads arrays of scalars; both inputs must + // still give the same events, value, or error for the complete document + // and for every truncation of it, with enough bytes after each array that + // the loop could have been entered + const json j = json::parse(R"({"a": [[[[1.5]]]], "b": [7], "c": [{"x": 2.5}], "d": ["s"], "e": [[1, 2]], "f": [null], "g": 1.25})"); + const std::vector bytes = json::to_cbor(j); + + CHECK(json::from_cbor(bytes) == j); + + for (std::size_t length = 0; length <= bytes.size(); ++length) + { + CAPTURE(length) + const std::vector contiguous(bytes.begin(), bytes.begin() + static_cast(length)); + const std::list bytewise(contiguous.begin(), contiguous.end()); + + CHECK(json::from_cbor(contiguous, true, false).dump() == json::from_cbor(bytewise.begin(), bytewise.end(), true, false).dump()); + + SaxEventLogger from_contiguous; + SaxEventLogger from_bytewise; + CHECK(json::sax_parse(contiguous, &from_contiguous, json::input_format_t::cbor, true, false) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::cbor, true, false)); + CHECK(from_contiguous.events == from_bytewise.events); + } +} + TEST_CASE("CBOR indefinite-length strings do not recurse per chunk") { // Reading an indefinite-length string or byte array used to call itself diff --git a/tests/src/unit-class_lexer.cpp b/tests/src/unit-class_lexer.cpp index 2926ef88b..616999896 100644 --- a/tests/src/unit-class_lexer.cpp +++ b/tests/src/unit-class_lexer.cpp @@ -2092,6 +2092,12 @@ TEST_CASE("UTF-8 validators accept the same sequences") } }; + // an ASCII byte is a sequence of its own + for (unsigned lead = 0; lead < 0x80; ++lead) + { + CHECK(nlohmann::detail::utf8_shape(static_cast(lead)).length == 1); + } + for (unsigned lead = 0x80; lead <= 0xFF; ++lead) { const auto shape = nlohmann::detail::utf8_shape(static_cast(lead)); @@ -2157,3 +2163,73 @@ TEST_CASE("UTF-8 validators accept the same sequences") } } } + +TEST_CASE("float_bigint") +{ + // the big integer of the exact comparison of a float token with the + // midpoint between two floats (digit_comparison()) + using nlohmann::detail::float_bigint; + + SECTION("zero") + { + // shifting zero keeps it zero, whatever the distance + float_bigint zero(0); + zero.shift_left(0); + zero.shift_left(5); + zero.shift_left(64); + CHECK(zero.compare(float_bigint(0)) == 0); + CHECK(zero.compare(float_bigint(1)) == -1); + CHECK(float_bigint(1).compare(zero) == 1); + } + + SECTION("compare numbers of different lengths") + { + // the number with more 32-bit limbs is the greater one + const float_bigint small(0xFFFFFFFFu); + const float_bigint large(std::uint64_t{1} << 32u); + CHECK(small.compare(large) == -1); + CHECK(large.compare(small) == 1); + CHECK(large.compare(large) == 0); + + // a shift can add a limb + float_bigint shifted(1); + shifted.shift_left(32); + CHECK(shifted.compare(large) == 0); + shifted.shift_left(1); + CHECK(shifted.compare(large) == 1); + CHECK(large.compare(shifted) == -1); + } + + SECTION("compare numbers of the same length") + { + CHECK(float_bigint(2).compare(float_bigint(3)) == -1); + CHECK(float_bigint(3).compare(float_bigint(2)) == 1); + CHECK(float_bigint(3).compare(float_bigint(3)) == 0); + CHECK(float_bigint((std::uint64_t{3} << 32u) | 1u).compare(float_bigint((std::uint64_t{3} << 32u) | 2u)) == -1); + } +} + +TEST_CASE("serialization of short strings") +{ + // the serializer takes strings of up to 32 characters that need no + // escaping by a table lookup of its own where the scan is not vectorized + // (this file is built that way, too, with JSON_NO_SIMD) + for (const std::string& s : + { + std::string(), std::string("a"), std::string("hello"), std::string("0123456789abcdef"), std::string("0123456789abcdef0123456789abcdef"), + std::string("0123456789abcdef0123456789abcdef0") + }) + { + CAPTURE(s) + const json j = s; + const std::string quoted = "\"" + s + "\""; // NOLINT(performance-inefficient-string-concatenation) + CHECK(j.dump() == quoted); + CHECK(j.dump(-1, ' ', true) == quoted); + CHECK(json({{s, s}}).dump() == "{" + quoted + ":" + quoted + "}"); // NOLINT(performance-inefficient-string-concatenation) + } + + // a character that needs an escape + CHECK(json("a\"b").dump() == "\"a\\\"b\""); + CHECK(json("caf\xC3\xA9").dump() == "\"caf\xC3\xA9\""); + CHECK(json("caf\xC3\xA9").dump(-1, ' ', true) == "\"caf\\u00e9\""); +} diff --git a/tests/src/unit-conversions1.cpp b/tests/src/unit-conversions1.cpp index 3d39e10c9..888979440 100644 --- a/tests/src/unit-conversions1.cpp +++ b/tests/src/unit-conversions1.cpp @@ -1260,6 +1260,44 @@ TEST_CASE("value conversion") #undef JSON_HAS_CPP_17 #endif +TEST_CASE("get() of a boolean (#5988)") +{ + // a boolean converts to 0/1 for every arithmetic type except number_integer_t, number_unsigned_t and + // number_float_t, whose dedicated from_json() overloads throw + const json t = true; + const json f = false; + + SECTION("0 and 1") + { + // long and unsigned long are left out: they are std::int64_t and std::uint64_t on some platforms + CHECK(t.get() == true); + CHECK(f.get() == false); + CHECK(t.get() == 1); + CHECK(t.get() == 1); + CHECK(t.get() == 1); + CHECK(t.get() == 1); // NOLINT(google-runtime-int) + CHECK(t.get() == 1); // NOLINT(google-runtime-int) + CHECK(t.get() == 1); + CHECK(t.get() == 1); + CHECK(t.get() == 1.0F); + CHECK(t.get() == 1.0L); + CHECK(f.get() == 0); + CHECK(f.get() == 0.0F); + } + +#if !defined(JSON_NOEXCEPTION) + SECTION("number_integer_t, number_unsigned_t and number_float_t throw") + { + CHECK_THROWS_WITH_AS(t.get(), "[json.exception.type_error.302] type must be number, but is boolean", json::type_error&); + CHECK_THROWS_WITH_AS(t.get(), "[json.exception.type_error.302] type must be number, but is boolean", json::type_error&); + CHECK_THROWS_WITH_AS(t.get(), "[json.exception.type_error.302] type must be number, but is boolean", json::type_error&); + CHECK_THROWS_WITH_AS(f.get(), "[json.exception.type_error.302] type must be number, but is boolean", json::type_error&); + CHECK_THROWS_WITH_AS(f.get(), "[json.exception.type_error.302] type must be number, but is boolean", json::type_error&); + CHECK_THROWS_WITH_AS(f.get(), "[json.exception.type_error.302] type must be number, but is boolean", json::type_error&); + } +#endif +} + #ifdef JSON_HAS_CPP_14 #undef JSON_HAS_CPP_14 #endif diff --git a/tests/src/unit-custom-array-type.cpp b/tests/src/unit-custom-array-type.cpp index e602b4716..65285eb54 100644 --- a/tests/src/unit-custom-array-type.cpp +++ b/tests/src/unit-custom-array-type.cpp @@ -210,3 +210,23 @@ TEST_CASE("regression test #5476 - array type without reserve()") CHECK(json::from_cbor(truncated, true, false).is_discarded()); } } + +TEST_CASE("array type with std::deque nested beyond the recursion limit") +{ + // the binary writers finish a value nested deeper than + // recursion_depth_limit() with an explicit stack of frames; with the + // iterators of std::deque, these frames are copied and moved through + // their own special members + deque_json j = 1; + for (int i = 0; i < 200; ++i) + { + deque_json outer = deque_json::array(); + outer.push_back(std::move(j)); + j = std::move(outer); + } + + // 200 arrays of one element each (0x81) around the integer 1 + std::vector expected(200, 0x81); + expected.push_back(0x01); + CHECK(deque_json::to_cbor(j) == expected); +} diff --git a/tests/src/unit-custom-base-class.cpp b/tests/src/unit-custom-base-class.cpp index 673c7ddbb..71698d041 100644 --- a/tests/src/unit-custom-base-class.cpp +++ b/tests/src/unit-custom-base-class.cpp @@ -7,6 +7,7 @@ // SPDX-License-Identifier: MIT #include +#include #include #include #include @@ -495,3 +496,92 @@ TEST_CASE("Regression test for issue #5674 - copy construction must not require CHECK(*c == 1); } } + +// counts the constructions and destructions of the base class (#5995) +struct counting_base +{ + static std::size_t constructions; + static std::size_t destructions; + + counting_base() noexcept + { + ++constructions; + } + counting_base(const counting_base& /*unused*/) noexcept + { + ++constructions; + } + counting_base(counting_base&& /*unused*/) noexcept + { + ++constructions; + } + counting_base& operator=(const counting_base& /*unused*/) noexcept = default; + counting_base& operator=(counting_base&& /*unused*/) noexcept = default; + ~counting_base() + { + ++destructions; + } + + static void reset() noexcept + { + constructions = 0; + destructions = 0; + } +}; + +std::size_t counting_base::constructions = 0; +std::size_t counting_base::destructions = 0; + +using json_with_counting_base = nlohmann::json::with_base_class_t; + +TEST_CASE("destroying a value does not construct the custom base class") +{ + // regression test for #5995: destroying an array or object used a + // default-constructed basic_json as a stand-in, which constructed a base + // class that belonged to no value (and could allocate in the noexcept + // destructor); now every destroyed value destroys exactly its own base + // object, and none is constructed + const auto check_destroy = [](std::unique_ptr j, std::size_t values) + { + counting_base::reset(); + j = nullptr; + CHECK(counting_base::constructions == 0); + CHECK(counting_base::destructions == values); + }; + + SECTION("scalar") + { + check_destroy(std::unique_ptr(new json_with_counting_base(42)), 1); + } + + SECTION("array and object") + { + check_destroy(std::unique_ptr(new json_with_counting_base(json_with_counting_base::parse(R"({"a":[1,2,3],"b":{"c":true}})"))), 7); + check_destroy(std::unique_ptr(new json_with_counting_base(json_with_counting_base::array())), 1); + } + + SECTION("nested deeper than the recursion depth limit") + { + // alternating arrays and objects, so both kinds of container are + // walked by destroy_walk() + constexpr std::size_t depth = 1000; + std::unique_ptr j(new json_with_counting_base(json_with_counting_base::array({1, 2}))); + for (std::size_t i = 0; i < depth; ++i) + { + json_with_counting_base outer = (i % 2 == 0) ? json_with_counting_base::object() : json_with_counting_base::array(); + if (outer.is_object()) + { + outer["x"] = std::move(*j); + outer["y"] = nullptr; + } + else + { + outer.push_back(std::move(*j)); + outer.push_back(true); + } + *j = std::move(outer); + } + // the innermost array with its two elements, and two values per level + check_destroy(std::move(j), 3 + (2 * depth)); + } +} diff --git a/tests/src/unit-custom-key-type-full.cpp b/tests/src/unit-custom-key-type-full.cpp index b72bd7ae0..69a052c9f 100644 --- a/tests/src/unit-custom-key-type-full.cpp +++ b/tests/src/unit-custom-key-type-full.cpp @@ -49,3 +49,24 @@ TEST_CASE("custom object key types: MessagePack (json_full)") { custom_key_test::test_msgpack(); } + +TEST_CASE("custom object key types: patch (json_full)") +{ + // the members of a patch are found by lookup, as their keys are not std::string + const json_full j = json_full::parse(R"({"a":1,"b":[1,2],"c":{"d":2}})"); + const json_full patch = json_full::parse(R"([ + {"op": "add", "path": "/e", "value": 5}, + {"op": "remove", "path": "/a"}, + {"op": "replace", "path": "/c/d", "value": 3}, + {"op": "test", "path": "/c/d", "value": 3}, + {"op": "move", "from": "/e", "path": "/f"}, + {"op": "copy", "from": "/f", "path": "/g"}, + {"op": "add", "path": "/b/-", "value": 7} + ])"); + + CHECK(j.patch(patch).dump() == R"({"b":[1,2,7],"c":{"d":3},"f":5,"g":5})"); + + // a patch with a missing member is rejected + CHECK_THROWS_AS(j.patch(json_full::parse(R"([{"op": "add", "path": "/e"}])")), json_full::parse_error&); + CHECK_THROWS_AS(j.patch(json_full::parse(R"([{"op": "test", "path": "/a", "value": 2}])")), json_full::other_error&); +} diff --git a/tests/src/unit-custom-number-types.cpp b/tests/src/unit-custom-number-types.cpp index e324417db..58353d67e 100644 --- a/tests/src/unit-custom-number-types.cpp +++ b/tests/src/unit-custom-number-types.cpp @@ -337,6 +337,52 @@ TEST_CASE("custom number types: integers beyond 64 bits") CHECK(wide_json::to_bson(fit_members) == nlohmann::json::to_bson(fit_members64)); } + SECTION("MessagePack integers of every width") + { + // an integer of a number type wider than 64 bits is written with the + // smallest MessagePack integer that holds it, as for 64-bit types + const wide_json widths = wide_json::array( + { + make_int(-33), make_int(-100), make_int(-128), make_int(-129), make_int(-32768), make_int(-32769), make_int(-2147483648LL), make_int(-2147483649LL), + make_int(128), make_int(255), make_int(256), make_int(300), make_int(65535), make_int(65536), make_int(70000), make_int(4294967295LL), make_int(4294967296LL), + make_uint(128), make_uint(255), make_uint(256), make_uint(300), make_uint(65535), make_uint(65536), make_uint(70000), make_uint(4294967295ULL), make_uint(4294967296ULL) + }); + const nlohmann::json widths64 = nlohmann::json::array( + { + -33, -100, -128, -129, -32768, -32769, -2147483648LL, -2147483649LL, + 128, 255, 256, 300, 65535, 65536, 70000, 4294967295LL, 4294967296LL, + 128u, 255u, 256u, 300u, 65535u, 65536u, 70000u, 4294967295ULL, 4294967296ULL + }); + CHECK(wide_json::to_msgpack(widths) == nlohmann::json::to_msgpack(widths64)); + + // as single values, too + CHECK(wide_json::to_msgpack(make_int(-100)) == std::vector({0xD0, 0x9C})); + CHECK(wide_json::to_msgpack(make_int(300)) == std::vector({0xCD, 0x01, 0x2C})); + CHECK(wide_json::to_msgpack(make_int(70000)) == std::vector({0xCE, 0x00, 0x01, 0x11, 0x70})); + CHECK(wide_json::to_msgpack(make_uint(300)) == std::vector({0xCD, 0x01, 0x2C})); + CHECK(wide_json::to_msgpack(make_uint(70000)) == std::vector({0xCE, 0x00, 0x01, 0x11, 0x70})); + } + + SECTION("dump") + { + // beyond 64 bits, the digits are counted before they are written + CHECK(make_int(0).dump() == "0"); + CHECK(make_int(two_64).dump() == "18446744073709551616"); + CHECK(make_int(-two_64).dump() == "-18446744073709551616"); + CHECK(make_int(two_100).dump() == "1267650600228229401496703205376"); + CHECK(make_int(-two_100).dump() == "-1267650600228229401496703205376"); + // powers of ten, where the count of digits ends on a single digit or two + const int128 ten_20 = static_cast(100000000000ULL) * static_cast(1000000000ULL); + CHECK(make_int(ten_20).dump() == "100000000000000000000"); + CHECK(make_int(-ten_20).dump() == "-100000000000000000000"); + CHECK(make_int(ten_20 * 10).dump() == "1000000000000000000000"); + CHECK(make_int(ten_20 * 100).dump() == "10000000000000000000000"); + CHECK(make_int(ten_20 * 1000).dump() == "100000000000000000000000"); + CHECK(make_uint(static_cast(ten_20) * 10u).dump() == "1000000000000000000000"); + CHECK(make_uint((std::numeric_limits::max)()).dump() == "340282366920938463463374607431768211455"); + CHECK(make_int((std::numeric_limits::min)()).dump() == "-170141183460469231731687303715884105728"); + } + SECTION("BJData ND-array") { // an ND-array element beyond 64 bits does not fit any dtype, so the @@ -417,3 +463,23 @@ TEST_CASE("custom number types") } } + +TEST_CASE("custom number types: 32-bit integers in the binary formats") +{ + // integers narrower than 64 bits are widened (a signed one with its sign) + // before the CBOR, MessagePack, and UBJSON writers pick their encoding + using json32 = nlohmann::json::with_integers_t; + json32 values = json32::array(); + values.push_back(std::uint32_t{0}); + values.push_back(std::uint32_t{200}); + values.push_back(std::uint32_t{4294967295u}); + values.push_back(std::int32_t{-1}); + values.push_back(std::int32_t{-2147483647 - 1}); + REQUIRE(values[2].is_number_unsigned()); + REQUIRE(values[4].is_number_integer()); + + const nlohmann::json values64 = {0u, 200u, 4294967295u, -1, -2147483648LL}; + CHECK(json32::to_cbor(values) == nlohmann::json::to_cbor(values64)); + CHECK(json32::to_msgpack(values) == nlohmann::json::to_msgpack(values64)); + CHECK(json32::to_ubjson(values) == nlohmann::json::to_ubjson(values64)); +} diff --git a/tests/src/unit-custom-object-key-type.cpp b/tests/src/unit-custom-object-key-type.cpp index 9c64ebce0..9149bbfb6 100644 --- a/tests/src/unit-custom-object-key-type.cpp +++ b/tests/src/unit-custom-object-key-type.cpp @@ -11,8 +11,13 @@ #include #include +#include +#include +#include #include +#include #include +#include #include "custom_object_key_type.hpp" @@ -125,3 +130,163 @@ TEST_CASE("MessagePack supports custom object key types nested deeper than the r CHECK(encoded == nlohmann::json::to_msgpack(expected)); CHECK(nlohmann::json::from_msgpack(encoded) == expected); } + +namespace +{ +// A key type that cannot be constructed from a std::string, so that a value +// with such keys cannot be converted from a nlohmann::json by the range +// constructor of its object type: the serializer converts the object instead. +class strict_key +{ + public: + strict_key(const char* value) + : m_value(value) + {} + + strict_key(const std::string& /*value*/) = delete; + +#if JSON_DIAGNOSTICS + // Required by JSON_DIAGNOSTICS, which reads object keys through data() + // when building the path of an exception. + const char* data() const noexcept + { + return m_value.data(); + } +#endif + + friend bool operator<(const strict_key& lhs, const strict_key& rhs) + { + return lhs.m_value < rhs.m_value; + } + + private: + std::string m_value; +}; + +strict_key make_strict_key(const std::string& key) +{ + return {key.c_str()}; +} + +template +class strict_object + : public std::map < + strict_key, + Value, + std::less, // NOLINT(modernize-use-transparent-functors) + typename std::allocator_traits::template rebind_alloc < + std::pair>> +{ + public: + strict_object() = default; +}; + +template +struct strict_serializer : nlohmann::adl_serializer {}; + +using strict_json = nlohmann::basic_json < strict_object, std::vector, std::string, bool, std::int64_t, + std::uint64_t, double, std::allocator, strict_serializer, std::vector>; + +// the members of an object are converted one by one, their keys by make_strict_key() +template<> +struct strict_serializer +{ + static void to_json(strict_json& j, const nlohmann::json::object_t& object) + { + strict_json::object_t converted; + for (const auto& member : object) + { + converted.emplace(make_strict_key(member.first), strict_json(member.second)); + } + j = strict_json(std::move(converted)); + } +}; +} // namespace + +TEST_CASE("converting a value with keys that cannot be made from a std::string nested deeper than the recursion depth limit") +{ + static_assert(!std::is_constructible::value, "the keys cannot be made from a std::string"); + + // below detail::recursion_depth_limit(), objects and arrays are converted + // by the serializers instead of the range constructors + const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 10; + + nlohmann::json value = 1; + for (std::size_t i = 0; i < depth; ++i) + { + if (i % 2 == 0) + { + value = nlohmann::json::object({{"a", std::move(value)}, {"b", true}}); + } + else + { + value = nlohmann::json::array({std::move(value), "sibling"}); + } + } + + // direct-initialization: a copy-initialization makes GCC 4.8 instantiate the + // copy constructor, which needs a range constructor strict_object lacks + const strict_json converted(value); + + // at() needs to make a std::string of a key for its error message + const auto member = [](const strict_json & object, const char* name) -> const strict_json & + { + const auto it = object.find(strict_key(name)); + REQUIRE(it != object.end()); + return *it; + }; + + // walk down the nesting levels without recursion + const strict_json* current = &converted; + for (std::size_t i = depth; i > 0; --i) + { + if (i % 2 == 1) + { + REQUIRE(current->is_object()); + CHECK(current->size() == 2); + CHECK(member(*current, "b").get()); + current = &member(*current, "a"); + } + else + { + REQUIRE(current->is_array()); + CHECK(current->size() == 2); + CHECK(current->at(1).get() == "sibling"); + current = ¤t->at(0); + } + } + CHECK(current->get() == 1); +} + +namespace +{ +// shorter keys come first +struct shortlex_less +{ + bool operator()(const std::string& lhs, const std::string& rhs) const + { + return lhs.size() != rhs.size() ? lhs.size() < rhs.size() : lhs < rhs; + } +}; + +template +using shortlex_map = std::map; + +using shortlex_json = nlohmann::json::with_object_t; +} // namespace + +TEST_CASE("unflatten with an object type that does not sort the keys alphabetically") +{ + // the keys of /a come before and after one of /b: unlike in an object + // sorted alphabetically, the values below a prefix are not found one + // after the other, so the value of /a must be looked up again + shortlex_json flat = shortlex_json::object(); + flat["/a/p"] = 1; + flat["/b/qq"] = 2; + flat["/a/pqr"] = 3; + REQUIRE(flat.begin().key() == "/a/p"); + REQUIRE((++flat.begin()).key() == "/b/qq"); + + CHECK(flat.unflatten().dump() == R"({"a":{"p":1,"pqr":3},"b":{"qq":2}})"); + CHECK(flat.unflatten().flatten() == flat); +} diff --git a/tests/src/unit-deserialization.cpp b/tests/src/unit-deserialization.cpp index e82685311..5b8f086fd 100644 --- a/tests/src/unit-deserialization.cpp +++ b/tests/src/unit-deserialization.cpp @@ -1195,6 +1195,28 @@ TEST_CASE("deserialization") #endif } +#ifndef JSON_TEST_DEPRECATED_FUNCTIONS_DELETED +TEST_CASE("deserialization from a pointer and a length (deprecated)") +{ + // the unit test build disables -Wdeprecated-declarations (see + // cmake/clang_flags.cmake) + const std::string text = R"({"foo": [1, 2, 3]})"; + const std::string invalid = R"({"foo": )"; + + SECTION("parse") + { + const json j = json::parse({text.data(), text.size()}); + CHECK(j == json({{"foo", {1, 2, 3}}})); + } + + SECTION("accept") + { + CHECK(json::accept({text.data(), text.size()})); + CHECK_FALSE(json::accept({invalid.data(), invalid.size()})); + } +} +#endif + // select the types to test - char8_t is only available since C++20 if and only // if __cpp_char8_t is defined. #define TYPE_LIST(...) __VA_ARGS__ diff --git a/tests/src/unit-diagnostic-positions.cpp b/tests/src/unit-diagnostic-positions.cpp index 9e0625115..85284d9d4 100644 --- a/tests/src/unit-diagnostic-positions.cpp +++ b/tests/src/unit-diagnostic-positions.cpp @@ -193,6 +193,28 @@ TEST_CASE("Better diagnostics with positions") check_conversion(300); } + SECTION("exception about the root value (#5987)") + { + // the root has no path, but still has a byte range + const json root = json::parse("true"); + const json nested = json::parse("[1, true]"); + const json constructed = true; +#if JSON_DIAGNOSTICS + CHECK_THROWS_WITH_AS(root.get(), + "[json.exception.type_error.302] (bytes 0-4) type must be string, but is boolean", json::type_error); + CHECK_THROWS_WITH_AS(nested[1].get(), + "[json.exception.type_error.302] (/1) (bytes 4-8) type must be string, but is boolean", json::type_error); +#else + CHECK_THROWS_WITH_AS(root.get(), + "[json.exception.type_error.302] (bytes 0-4) type must be string, but is boolean", json::type_error); + CHECK_THROWS_WITH_AS(nested[1].get(), + "[json.exception.type_error.302] (bytes 4-8) type must be string, but is boolean", json::type_error); +#endif + // a value without positions gets no range and no stray whitespace + CHECK_THROWS_WITH_AS(constructed.get(), + "[json.exception.type_error.302] type must be string, but is boolean", json::type_error); + } + SECTION("JSON patch add to primitive parent (#4292)") { // the JSON Patch "add" target /foo/bar/baz has a string parent @@ -243,3 +265,14 @@ TEST_CASE("values read from a binary format have no positions") check_no_positions(j); } } + +TEST_CASE("comparing a value-initialized iterator throws invalid_iterator.212") +{ + // such an iterator has no value to take the byte positions from; the + // comparison used to dereference the null pointer instead of throwing + json j = {1, 2}; + const json::iterator it{}; + CHECK_THROWS_WITH_AS(it == j.begin(), "[json.exception.invalid_iterator.212] cannot compare iterators of different containers", json::invalid_iterator); + CHECK_THROWS_WITH_AS(it < j.begin(), "[json.exception.invalid_iterator.212] cannot compare iterators of different containers", json::invalid_iterator); + CHECK_THROWS_WITH_AS(j.begin() == it, "[json.exception.invalid_iterator.212] cannot compare iterators of different containers", json::invalid_iterator); +} diff --git a/tests/src/unit-diagnostics.cpp b/tests/src/unit-diagnostics.cpp index 504a95322..2987d348c 100644 --- a/tests/src/unit-diagnostics.cpp +++ b/tests/src/unit-diagnostics.cpp @@ -868,3 +868,23 @@ TEST_CASE("Parent pointers stay consistent for object types not known to keep th obj["1"].push_back(42); CHECK_THROWS_WITH_AS(obj["1"].at(5), "[json.exception.out_of_range.401] (/1) array index 5 is out of range", relocating_json::out_of_range&); } + +TEST_CASE("patch_inplace() with a patch that is stored inside the patched value") +{ + // the patch is modified-while-iterated if it is not copied first: it is a + // descendant of the value it is applied to, which only the parent pointers + // (JSON_DIAGNOSTICS) reveal + json j = + { + {"patch", {{{"op", "add"}, {"path", "/x"}, {"value", 1}}, {{"op", "remove"}, {"path", "/patch/0"}}}}, + {"y", 2} + }; + + j.patch_inplace(j["patch"]); + + CHECK(j.size() == 3); + CHECK(j["x"] == 1); + CHECK(j["y"] == 2); + CHECK(j["patch"].size() == 1); + CHECK(j["patch"][0]["op"] == "remove"); +} diff --git a/tests/src/unit-file_input.cpp b/tests/src/unit-file_input.cpp index e88b6f4d0..ce190dfed 100644 --- a/tests/src/unit-file_input.cpp +++ b/tests/src/unit-file_input.cpp @@ -49,6 +49,14 @@ file_ptr open_with(const char* name, const std::string& content) return {std::fopen(name, "rb"), &std::fclose}; } +// the smallest size setvbuf() accepts for _IOFBF: the Microsoft C runtime +// rejects sizes below 2 as an invalid parameter, which ends the process +#if defined(_WIN32) + constexpr int min_full_buffer = 2; +#else + constexpr int min_full_buffer = 1; +#endif + // The stdio lock is recursive, so whether parsing released it can only be // seen from another thread: ftrylockfile() fails there while this thread // holds the lock. ftrylockfile() is POSIX; elsewhere, the check is skipped. @@ -225,6 +233,131 @@ TEST_CASE("parsing from a FILE*") check_unlocked(f.get()); } +#if !defined(JSON_NOEXCEPTION) + SECTION("tokens across the edges of the file's buffer") + { + // the adapter reads the buffer of the FILE directly where it can (see + // JSON_FILE_WINDOW): with buffers of 1 to 70 bytes and none, every + // token crosses an edge somewhere; the results, the error messages + // and their positions must be those of the string + const std::vector documents = + { + input, + R"({"long": "a string that is longer than the buffer of the file, with \"escapes\" and \u00e4", "n": [123456789012345678901, -0.000123e-45, 1.5E+300, 0, -0]})", + "[\n \"indented\",\n \"caf\xc3\xa9 \xe2\x82\xac \xf0\x9f\x98\x80\",\n {\n \"key\": 1\n }\n]", + R"({"a": [1, 2, 3], "b": "unterminated)", + R"([1, 2, 3, 1e999])", + R"({"a": "invalid escape \x"})", + "[\"invalid UTF-8 \xc3\x28\"]", + "[1, 2, 3] trailing", + R"([truex])", + }; + for (const auto& document : documents) + { + std::string expected; + try + { + expected = json::parse(document).dump(); + } + catch (const json::exception& e) + { + expected = e.what(); + } + const bool expected_accept = json::accept(document); + for (int size = 0; size <= 70; ++size) + { + if (size != 0 && size < min_full_buffer) + { + continue; + } + CAPTURE(document) + CAPTURE(size) + std::vector buffer(static_cast(size) + 1); + for (int pass = 0; pass < 2; ++pass) + { + const file_ptr f = open_with(name, document); + REQUIRE(f != nullptr); + // size 0: unbuffered + REQUIRE(std::setvbuf(f.get(), size == 0 ? nullptr : buffer.data(), size == 0 ? _IONBF : _IOFBF, static_cast(size)) == 0); + if (pass == 0) + { + std::string actual; + try + { + actual = json::parse(f.get()).dump(); + } + catch (const json::exception& e) + { + actual = e.what(); + } + CHECK(actual == expected); + } + else + { + CHECK(json::accept(f.get()) == expected_accept); + } + check_unlocked(f.get()); + } + } + } + } +#endif + + // the C standard guarantees only one pushed-back character (and the + // Windows C runtime fails further calls to ungetc()), so this is limited + // to the C libraries that keep them in a separate buffer +#if !defined(JSON_NOEXCEPTION) && ((defined(__GLIBC__) && !defined(__UCLIBC__)) || defined(__APPLE__)) + SECTION("characters pushed back with ungetc() before parsing") + { + const file_ptr f = open_with(name, "2, 3]"); + REQUIRE(f != nullptr); + REQUIRE(std::ungetc(',', f.get()) == ','); + REQUIRE(std::ungetc('1', f.get()) == '1'); + REQUIRE(std::ungetc('[', f.get()) == '['); + CHECK(json::parse(f.get()) == json({1, 2, 3})); + } +#endif + + SECTION("SAX events across the edges of the file's buffer") + { + const std::string document = R"({"key": "a value that is longer than the buffer", "n": [1, -2.5e3, true, null]})"; + std::vector expected; + { + const file_ptr f = open_with(name, document); + REQUIRE(f != nullptr); + position_reading_sax sax(f.get()); + CHECK(json::sax_parse(f.get(), &sax)); + expected = sax.positions(); + } + for (int size = min_full_buffer; size <= 20; ++size) + { + CAPTURE(size) + std::vector buffer(static_cast(size)); + const file_ptr f = open_with(name, document); + REQUIRE(f != nullptr); + REQUIRE(std::setvbuf(f.get(), buffer.data(), _IOFBF, buffer.size()) == 0); + position_reading_sax sax(f.get()); + CHECK(json::sax_parse(f.get(), &sax)); + // the events happen at the same positions of the file + CHECK(sax.positions() == expected); + } + } + + SECTION("binary strings across the edges of the file's buffer") + { + const json value = {{"a string that is longer than the buffer of the file", "and a value that is longer, too"}, {"n", {1, 2}}}; + const std::vector cbor = json::to_cbor(value); + for (int size = min_full_buffer; size <= 20; ++size) + { + CAPTURE(size) + std::vector buffer(static_cast(size)); + const file_ptr f = open_with(name, std::string(reinterpret_cast(cbor.data()), cbor.size())); + REQUIRE(f != nullptr); + REQUIRE(std::setvbuf(f.get(), buffer.data(), _IOFBF, buffer.size()) == 0); + CHECK(json::from_cbor(f.get()) == value); + } + } + SECTION("binary formats read from a FILE*") { const json j = {{"string", std::string(300, 'x')}, {"array", {1, 2, 3}}, {"number", 1.5}}; diff --git a/tests/src/unit-iterators2.cpp b/tests/src/unit-iterators2.cpp index 5bc738e21..67a5752e4 100644 --- a/tests/src/unit-iterators2.cpp +++ b/tests/src/unit-iterators2.cpp @@ -18,6 +18,8 @@ #include using nlohmann::json; +#include + #if JSON_HAS_RANGES #include #include @@ -665,6 +667,23 @@ TEST_CASE("iterators 2") } } + SECTION("reverse iterators of a const value") + { + const json j_array = {1, 2, 3}; + const json j_object = {{"one", 1}, {"two", 2}}; + + // rbegin() and rend() of a const value are crbegin() and crend() + CHECK(j_array.rbegin() == j_array.crbegin()); + CHECK(j_array.rend() == j_array.crend()); + CHECK(*j_array.rbegin() == 3); + CHECK(std::distance(j_array.rbegin(), j_array.rend()) == 3); + + CHECK(j_object.rbegin() == j_object.crbegin()); + CHECK(j_object.rend() == j_object.crend()); + CHECK(j_object.rbegin().key() == "two"); + CHECK(std::distance(j_object.rbegin(), j_object.rend()) == 2); + } + SECTION("reverse iterator arithmetic") { json j_object = {{"one", 1}, {"two", 2}, {"three", 3}}; diff --git a/tests/src/unit-json_patch.cpp b/tests/src/unit-json_patch.cpp index 83df15350..37c6e29d9 100644 --- a/tests/src/unit-json_patch.cpp +++ b/tests/src/unit-json_patch.cpp @@ -14,9 +14,11 @@ using nlohmann::json; using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) #endif +#include #include #include #include +#include #include #include "make_test_data_available.hpp" @@ -2453,3 +2455,99 @@ TEST_CASE("JSON patch - diff behavior documented in the API reference") CHECK(json::diff(json(nullptr), json::array()) == R"([{"op": "replace", "path": "", "value": []}])"_json); } } + +TEST_CASE("JSON patch - operations that are left to json_pointer") +{ + // the usual operations on a plain pointer are applied without building a + // json_pointer; the following are not plain, and some are errors + SECTION("a path without a slash") + { + const json j = {{"a", 1}}; + + CHECK_THROWS_AS(j.patch(R"([{"op": "add", "path": "abc", "value": 1}])"_json), json::parse_error&); + CHECK_THROWS_AS(j.patch(R"([{"op": "remove", "path": "abc"}])"_json), json::parse_error&); + } + + SECTION("remove with an escaped reference token") + { + const json j = {{"a/b", 1}, {"c~d", 2}, {"e", 3}}; + + CHECK(j.patch(R"([{"op": "remove", "path": "/a~1b"}])"_json) == json({{"c~d", 2}, {"e", 3}})); + CHECK(j.patch(R"([{"op": "remove", "path": "/c~0d"}])"_json) == json({{"a/b", 1}, {"e", 3}})); + CHECK_THROWS_AS(j.patch(R"([{"op": "remove", "path": "/missing~1key"}])"_json), json::out_of_range&); + } + + SECTION("a path through a primitive value") + { + const json j = {{"a", 1}}; + + CHECK_THROWS_AS(j.patch(R"([{"op": "test", "path": "/a/b", "value": 1}])"_json), json::other_error&); + CHECK_THROWS_AS(j.patch(R"([{"op": "replace", "path": "/a/b", "value": 1}])"_json), json::out_of_range&); + } +} + +TEST_CASE("JSON patch - diff of objects with many members") +{ + // more members than diff() keeps without allocating + for (const std::size_t members : + { + 16u, 17u, 40u + }) + { + CAPTURE(members) + + json source = json::object(); + json target = json::object(); + for (std::size_t i = 0; i < members; ++i) + { + const std::string key = "key" + std::to_string(i); + source[key] = i; + // every third member differs + target[key] = (i % 3 == 0) ? i + 1 : i; + } + // the same keys, a removed and an added one + source["removed"] = true; + target["added"] = false; + + const json patch = json::diff(source, target); + CHECK(source.patch(patch) == target); + CHECK(patch.size() == ((members + 2) / 3) + 2); + + const nlohmann::ordered_json ordered_source = source; + const nlohmann::ordered_json ordered_target = target; + CHECK(ordered_source.patch(nlohmann::ordered_json::diff(ordered_source, ordered_target)) == ordered_target); + } +} + +TEST_CASE("JSON patch - diff of leaf values below the bound of the recursive descent") +{ + // the values at the innermost level are diffed without recursion: equal + // leaves yield nothing, leaves of different types and different leaves + // are replaced + const std::size_t depth = 200; + + json source = {1, "text", 3, {{"x", 1}, {"y", 2}}, true, nullptr}; + json target = {1, 2, 4, {{"x", 1}, {"y", 3}}, true, nullptr}; + for (std::size_t i = 0; i < depth; ++i) + { + source = json::array({std::move(source)}); + target = json::array({std::move(target)}); + } + + const json patch = json::diff(source, target); + CHECK(patch.size() == 3); + CHECK(source.patch(patch) == target); + + // the same with objects as the outer levels + json object_source = {{"equal", 1}, {"type", "text"}, {"value", 3}}; + json object_target = {{"equal", 1}, {"type", 2}, {"value", 4}}; + for (std::size_t i = 0; i < depth; ++i) + { + object_source = json::object({{"a", std::move(object_source)}}); + object_target = json::object({{"a", std::move(object_target)}}); + } + + const json object_patch = json::diff(object_source, object_target); + CHECK(object_patch.size() == 2); + CHECK(object_source.patch(object_patch) == object_target); +} diff --git a/tests/src/unit-json_pointer.cpp b/tests/src/unit-json_pointer.cpp index 0e225b86e..8afb52f48 100644 --- a/tests/src/unit-json_pointer.cpp +++ b/tests/src/unit-json_pointer.cpp @@ -106,6 +106,12 @@ TEST_CASE("JSON pointers") CHECK_THROWS_WITH_AS(p.pop_back(), "[json.exception.out_of_range.405] JSON pointer has no parent", json::out_of_range&); + // the top of a pointer is its first reference token + const json::json_pointer nonempty("/foo/bar/0"); + CHECK(nonempty.top() == json::json_pointer("/foo")); + CHECK(nonempty.top().to_string() == "/foo"); + CHECK(json::json_pointer("/a~1b").top() == json::json_pointer("/a~1b")); + SECTION("array index error") { json v = {1, 2, 3, 4}; @@ -697,6 +703,16 @@ TEST_CASE("JSON pointers") CHECK_THROWS_WITH_AS(j_error.unflatten(), "[json.exception.type_error.313] invalid value to unflatten", json::type_error&); + // error for an invalid escape sequence in a key + CHECK_THROWS_WITH_AS(json({{"/a~2", 1}}).unflatten(), + "[json.exception.parse_error.108] parse error: escape character '~' must be followed with '0' or '1'", json::parse_error&); + CHECK_THROWS_WITH_AS(json({{"/a/b", 1}, {"/c~", 1}}).unflatten(), + "[json.exception.parse_error.108] parse error: escape character '~' must be followed with '0' or '1'", json::parse_error&); + + // error for an element of an array that is also a container + CHECK_THROWS_WITH_AS(json({{"/a/0", 1}, {"/a/0/x", 2}}).unflatten(), + "[json.exception.type_error.313] invalid value to unflatten", json::type_error&); + // explicit roundtrip check CHECK(j.flatten().unflatten() == j); diff --git a/tests/src/unit-json_view.cpp b/tests/src/unit-json_view.cpp index c823ef9e2..65180ef55 100644 --- a/tests/src/unit-json_view.cpp +++ b/tests/src/unit-json_view.cpp @@ -31,9 +31,11 @@ using json_view_test::has_duplicate_keys; #include #include #include +#include #include #include #include +#include #include #include #include @@ -98,6 +100,71 @@ template using bool_conversion_t = decltype(static_cast(std::declval())); #if !defined(JSON_NOEXCEPTION) +// an allocator that never has room +template +struct failing_allocator +{ + using value_type = T; + // GCC 4.8's std::map takes these from the allocator instead of from + // std::allocator_traits + using pointer = T*; + using const_pointer = const T*; + using reference = T&; + using const_reference = const T&; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + + template + struct rebind + { + using other = failing_allocator; + }; + + failing_allocator() = default; + + template + explicit failing_allocator(const failing_allocator& /*other*/) noexcept {} + + [[noreturn]] T* allocate(std::size_t /*n*/) // NOLINT(readability-convert-member-functions-to-static) + { + throw std::bad_alloc(); + } + + void deallocate(T* /*p*/, std::size_t /*n*/) noexcept {} // NOLINT(readability-convert-member-functions-to-static) + + // GCC 4.8's std::map calls these on the allocator directly instead of + // going through std::allocator_traits + std::size_t max_size() const noexcept // NOLINT(readability-convert-member-functions-to-static) + { + return (std::numeric_limits::max)() / sizeof(T); + } + + template + void construct(U* p, Args&& ... args) // NOLINT(readability-convert-member-functions-to-static) + { + ::new (static_cast(p)) U(std::forward(args)...); + } + + template + void destroy(U* p) noexcept // NOLINT(readability-convert-member-functions-to-static) + { + static_cast(p); // unused for trivially destructible types + p->~U(); + } + + template + bool operator==(const failing_allocator& /*other*/) const noexcept + { + return true; + } + + template + bool operator!=(const failing_allocator& /*other*/) const noexcept + { + return false; + } +}; + // the exception parse() throws for a text, or "" if it accepts it std::string parse_exception(const std::string& text, bool comments = false, bool trailing_commas = false) { @@ -235,7 +302,9 @@ TEST_CASE("json_view") { "", " ", "[", "]", "{", "[1,]", "{\"a\":1,}", "[1 2]", "{\"a\" 1}", "{1:2}", "tru", "nul", "fals", "truex", "-", "01", "1.", ".5", "1e", "\"", "\"abc", "\"\\x\"", "\"\\u12\"", "\"\\ud800\"", "\"\\udc00\"", "\"\x01\"", "\"\xff\"", "\"\xc3\"", "[1]x", "/", "/*", "[\n 1,\n x\n]", // NOLINT(modernize-raw-string-literal) - "1e400", "-1e400", "[1.7976931348623159e308]", "{\"a\":\n{\"b\": [1, 2,\n 3 x]}}" + "1e400", "-1e400", "[1.7976931348623159e308]", "{\"a\":\n{\"b\": [1, 2,\n 3 x]}}", + // (an error after every kind of value: the error is found by replaying the parser) + R"([null, true, -1, 1, 1.5, "s", {"a": 1} 2])", R"({"a": null, "b": -1, "c": 1.5, "d": "s", "e": {}, 1})" }) { CAPTURE(text) @@ -368,6 +437,10 @@ TEST_CASE("json_view") CHECK(from_stream.root().materialize() == expected); const std::list list(text.begin(), text.end()); CHECK(json_document::parse(list.begin(), list.end()).owns_source()); + // (an input that is not valid) + const std::list bad_list{'[', '1', ','}; + json_document from_bad_list; + CHECK_THROWS_AS(from_bad_list = json_document::parse(bad_list.begin(), bad_list.end()), json::parse_error&); CHECK(materialized(list.begin(), list.end()) == expected); // iterator pairs: pointers are borrowed, and so are contiguous library @@ -1511,6 +1584,17 @@ TEST_CASE("json_view values") const json_document dup = json_document::parse(R"({"a":1,"a":2})"); CHECK((dup.root().get>() == std::map {{"a", 2}})); + // a map of views from something that is not an object + using view_map = std::map; + const json_document not_an_object = json_document::parse("[1,2]"); + CHECK_THROWS_WITH_AS(not_an_object.root().get(), json_view_test::message("type_error.302", "", 0, 5, "type must be object, but is array").c_str(), json::type_error&); +#if !defined(JSON_NOEXCEPTION) + // (and one that cannot allocate its nodes) + using failing_map = std::map, failing_allocator>>; // NOLINT(modernize-use-transparent-functors) + const json_document one_member = json_document::parse(R"({"a":1})"); + CHECK_THROWS_AS(one_member.root().get(), std::bad_alloc&); +#endif + const json_view invalid{}; CHECK_THROWS_WITH_AS(invalid.get(), "[json.exception.type_error.302] type must be number, but is discarded", json::type_error&); CHECK(invalid.get().is_discarded()); @@ -1551,6 +1635,85 @@ TEST_CASE("json_view values") } } +namespace +{ +// get() of a boolean: the same value or the same exception in basic_json and in the view (#5988) +template +void check_boolean_get_parity() +{ + for (const bool b : + { + true, false + }) + { + const std::string text = b ? "true" : "false"; + const json j = json::parse(text); + const json_document d = json_document::parse(text); + CAPTURE(text) +#if !defined(JSON_NOEXCEPTION) + T dom_value {}; + T view_value{}; + const std::string dom_error = json_view_test::exception_of_call([&] { dom_value = j.get(); }); + const std::string view_error = json_view_test::exception_of_call([&] { view_value = d.root().template get(); }); + CHECK(dom_error == view_error); + if (dom_error.empty()) + { + CHECK(dom_value == view_value); + CHECK(view_value == static_cast(b)); + } +#else + // without exceptions, the combinations that throw (number_integer_t, number_unsigned_t, number_float_t) + // cannot be run: JSON_THROW aborts + const bool throws = std::is_same::value + || std::is_same::value + || std::is_same::value; + if (!throws) + { + CHECK(j.get() == d.root().template get()); + } +#endif + } +} +} // namespace + +TEST_CASE("json_view get() of a boolean (#5988)") +{ + check_boolean_get_parity(); + check_boolean_get_parity(); + check_boolean_get_parity(); + check_boolean_get_parity(); + check_boolean_get_parity(); // NOLINT(google-runtime-int) + check_boolean_get_parity(); // NOLINT(google-runtime-int) + check_boolean_get_parity(); + check_boolean_get_parity(); + check_boolean_get_parity(); // NOLINT(google-runtime-int) + check_boolean_get_parity(); // NOLINT(google-runtime-int) + check_boolean_get_parity(); // NOLINT(google-runtime-int) + check_boolean_get_parity(); // NOLINT(google-runtime-int) + check_boolean_get_parity(); + check_boolean_get_parity(); + check_boolean_get_parity(); + check_boolean_get_parity(); + check_boolean_get_parity(); + +#if !defined(JSON_NOEXCEPTION) + SECTION("the exceptions of the exact number types") + { + const json_document d = json_document::parse("true"); + CHECK_THROWS_WITH_AS(d.root().get(), json_view_test::message("type_error.302", "", 0, 4, "type must be number, but is boolean").c_str(), json::type_error&); + CHECK_THROWS_WITH_AS(d.root().get(), json_view_test::message("type_error.302", "", 0, 4, "type must be number, but is boolean").c_str(), json::type_error&); + CHECK_THROWS_WITH_AS(d.root().get(), json_view_test::message("type_error.302", "", 0, 4, "type must be number, but is boolean").c_str(), json::type_error&); + CHECK(d.root().get() == 1); + // get_to() and value() go through get() + json::number_float_t x = 0; + CHECK_THROWS_AS(d.root().get_to(x), json::type_error&); + const json_document o = json_document::parse("{\"b\":true}"); + CHECK_THROWS_AS(static_cast(o.root().value("b", 0.0)), json::type_error&); + CHECK(o.root().value("b", 0) == 1); + } +#endif +} + TEST_CASE("json_view JSON pointers") { SECTION("every value of generated documents") diff --git a/tests/src/unit-json_view_compare_values.cpp b/tests/src/unit-json_view_compare_values.cpp new file mode 100644 index 000000000..132978d8d --- /dev/null +++ b/tests/src/unit-json_view_compare_values.cpp @@ -0,0 +1,83 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; +using nlohmann::json_document; + +#include +#include +#include + +// These tests were split off unit-json_view_compare.cpp, whose object file got +// too large for the MinGW assembler ("file too big"). + +namespace +{ +// an object type that does not enumerate its members in ascending order of keys +template +using descending_object = std::map, Allocator>; +} // namespace + +TEST_CASE("json_view comparison of differing values") +{ + SECTION("values that differ in their first bytes, or are written differently") + { + const auto same = [](const char* x, const char* y) + { + const json_document dx = json_document::parse(x); + const json_document dy = json_document::parse(y); + const bool equal = dx.root() == dy.root(); + CHECK(equal == (json::parse(x) == json::parse(y))); + CHECK(equal == (dx.root() == json::parse(y))); + const nlohmann::json_editable_document ex = nlohmann::json_editable_document::parse(x); + CHECK(equal == (ex.root() == dy.root())); + return equal; + }; + // booleans + CHECK(!same("true", "false")); + CHECK(!same("[true]", "[false]")); + CHECK(!same(R"({"a":false,"b":[1]})", R"({"a":true,"b":[1]})")); + CHECK(same(R"({"a":true,"b":[1]})", R"({"a":true,"b":[1]})")); + // strings and keys of equal length that differ in the first eight bytes (they are compared in words) + CHECK(!same(R"("abcdefghijklmnopqrstuvwxyz")", R"("Xbcdefghijklmnopqrstuvwxyz")")); + CHECK(!same(R"(["abcdefghijklmnopq",1])", R"(["abcdefghXjklmnopq",1])")); + CHECK(!same(R"({"abcdefghijklmnopq":1})", R"({"Xbcdefghijklmnopq":1})")); + CHECK(!same(R"({"abcdefghijklmnopq":[1]})", R"({"abcdefghijklmnopX":[1]})")); + // floats of one layout and different tokens are the same value + CHECK(same("[1.5e0, 2]", "[1.5E0, 2]")); + CHECK(same("[1.5e0, 2]", "[1.5e+0, 2]")); + CHECK(same("[1.50, 2]", "[1.5, 2]")); + CHECK(!same("[1.5e0, 2]", "[1.6e0, 2]")); + CHECK(!same("[1.5e0, 2]", "[1.5E1, 2]")); + CHECK(same(R"({"a":1.25e0,"b":[3]})", R"({"a":1.25E0,"b":[3]})")); + } + + SECTION("a basic_json whose members are not sorted") + { + // (an object type that is not an ordered map, and does not keep its members sorted either) + using descending_json = nlohmann::basic_json; + using descending_document = nlohmann::basic_json_document; + std::string text = "{"; + for (char key = 'a'; key <= 'p'; ++key) + { + text += (key != 'a' ? ",\"" : "\"") + std::string(1, key) + "\":[" + std::to_string(key) + "]"; + } + text += "}"; + const descending_json j = descending_json::parse(text); + const descending_document d = descending_document::parse(text); + CHECK(d.root() == j); + CHECK(j == d.root()); + descending_json other = j; + other["k"] = 0; + CHECK(d.root() != other); + CHECK(other != d.root()); + } +} diff --git a/tests/src/unit-json_view_dump.cpp b/tests/src/unit-json_view_dump.cpp index be1ec48ad..fc6501f62 100644 --- a/tests/src/unit-json_view_dump.cpp +++ b/tests/src/unit-json_view_dump.cpp @@ -101,6 +101,10 @@ TEST_CASE("json_view dump") CHECK(d.root().dump(0, ' ', false, json_view::number_format::source) == "[\n1.50,\n1E2,\n-0,\n-0.0,\n123456789012345678901234567890,\n18446744073709551615,\n-9223372036854775808,\n0.1,\n1e-7,\n5e-324\n]"); CHECK(d.root().dump(-1, ' ', true, json_view::number_format::source) == "[1.50,1E2,-0,-0.0,123456789012345678901234567890,18446744073709551615,-9223372036854775808,0.1,1e-7,5e-324]"); + CHECK(d.root().dump(0, ' ', true, json_view::number_format::source) == d.root().dump(0, ' ', false, json_view::number_format::source)); + const json_document escaped = json_document::parse("[\"\xC3\xA9\", 1.50, {\"\xF0\x9F\x98\x80\": 1E2}]"); + CHECK(escaped.root().dump(1, ' ', true, json_view::number_format::source) == "[\n \"\\u00e9\",\n 1.50,\n {\n \"\\ud83d\\ude00\": 1E2\n }\n]"); + // float tokens of up to 17 significant digits in every spelling: those // of at most 15 digits are written from their digits, the others // through the conversion; both as dump() writes them @@ -232,6 +236,29 @@ TEST_CASE("json_view dump") CHECK(editable.root().dump() == expected.dump()); } + // tokens of 16 and 17 digits with few digits after the point, and with a value of 0, which are + // converted by the general algorithm; padded, as a token near the end of the text is not read in this way + { + const std::string padding(40, ' '); + for (const char* token : + { + "123456789012.3456", "-123456789012.3456", "1234567890123.456", "12345678901234.56", "123456789012345.6", "1234567890123456.0", + "0.0000000000000000", "-0.0000000000000000", "0.000000000000000", "1234567890123.4567", "99999999999999.999" + }) + { + CAPTURE(token) + const std::string padded = std::string("[") + token + padding + "]"; + const json_document padded_document = json_document::parse(padded); + const json padded_json = json::parse(padded); + CHECK(padded_document.root()[0].get() == padded_json[0].get()); + CHECK(padded_document.root().dump() == padded_json.dump()); + // a float document converts the tokens as well + using json_float = nlohmann::basic_json; + const nlohmann::basic_json_document float_doc = nlohmann::basic_json_document::parse(padded); + CHECK(float_doc.root()[0].get() == json_float::parse(padded)[0].get()); + } + } + // random doubles, written as parse() and dump() would std::mt19937_64 rng(1170); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) std::string many = "["; diff --git a/tests/src/unit-json_view_dump_escape.cpp b/tests/src/unit-json_view_dump_escape.cpp index 3def62df6..2716f76d4 100644 --- a/tests/src/unit-json_view_dump_escape.cpp +++ b/tests/src/unit-json_view_dump_escape.cpp @@ -214,6 +214,24 @@ TEST_CASE("json_view dump: escaping, nesting, and output buffers") CHECK(d.root()["a"].dump(-1, ' ', true).size() == 2 + (5000 * 6)); } + SECTION("the estimate for a value that is followed by many decoded strings") + { + // (the end of the value in the source is not known when the nodes after it have decoded strings) + std::string text = "[7"; + for (int i = 0; i < 20; ++i) + { + text += R"(,"\n")"; + } + text += ",8]"; + const json_document d = json_document::parse(text); + CHECK(d.root()[0].dump() == "7"); + CHECK(d.root()[1].dump() == R"("\n")"); + CHECK(d.root().dump() == json::parse(text).dump()); + // the same with a container + const json_document nested = json_document::parse(R"([[7],)" + text.substr(3)); + CHECK(nested.root()[0].dump() == "[7]"); + } + SECTION("ensure_ascii with strings of the source, decoded strings, and edited strings") { // plain ASCII, non-ASCII, escaped (decoded), and mixed strings, as values and as keys; @@ -259,6 +277,28 @@ TEST_CASE("json_view dump: escaping, nesting, and output buffers") } } + SECTION("the length of the run that ensure_ascii copies") + { + // (dump() only asks for strings with a byte to escape, so a run that ends the string is checked here) + for (const std::size_t length : + { + std::size_t{0}, std::size_t{1}, std::size_t{15}, std::size_t{16}, std::size_t{17}, std::size_t{31}, std::size_t{32}, std::size_t{48}, std::size_t{50} + }) + { + CAPTURE(length) + // a run of printable characters, followed by a byte that is not copied and more text + std::string text(length, 'a'); + text += "\"abcdefghijklmnopqrstuvwxyz"; + const auto* const bytes = reinterpret_cast(text.data()); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const auto* const end = bytes + text.size(); + CHECK(nlohmann::detail::view::ascii_copyable_run(bytes, length, end) == length); + CHECK(nlohmann::detail::view::ascii_copyable_run(bytes, length + 1, end) == length); + CHECK(nlohmann::detail::view::ascii_copyable_run(bytes, length + 5, end) == length); + // the end of the buffer right after the run + CHECK(nlohmann::detail::view::ascii_copyable_run(bytes, length, bytes + length) == length); + } + } + SECTION("ensure_ascii with DEL or a non-ASCII character at every position") { // the check whether a string of the source has no byte >= 0x7F reads diff --git a/tests/src/unit-json_view_edit.cpp b/tests/src/unit-json_view_edit.cpp index 2cc3d4701..db6d3054b 100644 --- a/tests/src/unit-json_view_edit.cpp +++ b/tests/src/unit-json_view_edit.cpp @@ -547,6 +547,10 @@ TEST_CASE("json_view edits: errors") CHECK_THROWS_WITH_AS(d.set(json::json_pointer("/a/01"), 1), "[json.exception.parse_error.106] parse error: array index '01' must not begin with '0'", json::parse_error&); CHECK_THROWS_WITH_AS(d.set(root, json_editable_view()), "[json.exception.type_error.302] type must be a value, but is discarded", json::type_error&); CHECK_THROWS_WITH_AS(d.set(root, json::binary({1, 2})), "[json.exception.type_error.319] cannot store a binary value in a json_document", json::type_error&); + CHECK_THROWS_WITH_AS(d.set(root, json(json::value_t::discarded)), "[json.exception.type_error.302] type must be a value, but is discarded", json::type_error&); + // (an index that does not fit the document, in a null: basic_json would pad it with 4 Gi nulls) + CHECK_THROWS_AS(d.set(json::json_pointer("/z/4294967295"), 1), json::out_of_range&); + CHECK(root["z"].is_null()); #if !defined(JSON_NOEXCEPTION) // invalid UTF-8 is rejected when it enters the document, with the error @@ -572,6 +576,19 @@ TEST_CASE("json_view edits: errors") static_cast(other); } +TEST_CASE("json_view edits: a container that was added is replaced by a scalar") +{ + // the view of an element that push_back() added is that of the new value itself, not of + // the slot that links to it: replacing it by a scalar has to find the container it is in + json_editable_document d = json_editable_document::parse(R"({"a": [1]})"); + d.push_back(d.root()["a"], json::array({2, 3})); + d.push_back(d.root()["a"], json::object({{"k", json::array({4, 5})}})); + d.set(d.root()["a"][1], 6); + d.set(d.root()["a"][2]["k"], "s"); + CHECK(d.root().dump() == R"({"a":[1,6,{"k":"s"}]})"); + CHECK(d.root().materialize() == json::parse(R"({"a":[1,6,{"k":"s"}]})")); +} + #if !defined(JSON_NOEXCEPTION) TEST_CASE("json_view edits: the size of a text arena") { diff --git a/tests/src/unit-json_view_from_json.cpp b/tests/src/unit-json_view_from_json.cpp index 824585a56..50c310cb6 100644 --- a/tests/src/unit-json_view_from_json.cpp +++ b/tests/src/unit-json_view_from_json.cpp @@ -277,6 +277,17 @@ TEST_CASE("json_view from_json") CHECK(true); } + SECTION("a map with enum keys from a view that is not an object") + { + // an object is read with the keys as the enum's names; any other + // value is read as an array of pairs, like a map of any other key type + const json_document d = json_document::parse(R"({"pairs": [["red", 1], ["green", 2]], "number": 1})"); + const json j = d.root().materialize(); + const json_view v = d.root(); + CHECK(same>(v, j, "pairs")); + CHECK((v["pairs"].get>() == std::map {{color::red, 1}, {color::green, 2}})); + } + SECTION("std types: the same values as from the materialized value") { const json_document d = json_document::parse(R"({"ints": [3, 1, 2, 1], "strings": ["b", "a", "b"], "pairs": [[1, "x"], [2, "y"]], diff --git a/tests/src/unit-json_view_image.cpp b/tests/src/unit-json_view_image.cpp index 5e392559d..499f85342 100644 --- a/tests/src/unit-json_view_image.cpp +++ b/tests/src/unit-json_view_image.cpp @@ -819,6 +819,13 @@ TEST_CASE("json_view images: errors") static_cast(d); }) == prefix + "too short"); + // an image that is moved into the document is checked as well + CHECK(exception_of([&] + { + const json_document d = json_document::load(std::vector(10, 0)); + static_cast(d); + }) == prefix + "too short"); + std::vector bad = image; bad[0] = 'X'; CHECK(load_result(bad, image_check::full) == prefix + "unknown format"); diff --git a/tests/src/unit-json_view_large.cpp b/tests/src/unit-json_view_large.cpp index 5ad65c812..b5f58386c 100644 --- a/tests/src/unit-json_view_large.cpp +++ b/tests/src/unit-json_view_large.cpp @@ -333,6 +333,62 @@ TEST_CASE("json_view large objects") CHECK(without_collisions.memory_usage() >= with_collisions.memory_usage() + (table_size * sizeof(std::uint32_t))); } + SECTION("a lookup of a missing key stops at the end of the probe window") + { + // Every slot of the first half of the table holds a key, each at its home slot, so + // that the table is built; a key that is not in the object but whose home slot is + // the first slot finds a full window of slots with other keys. + constexpr std::size_t members = 128; + constexpr std::size_t table_size = 256; // the table size for 128 members: the next power of two >= 256 + for (const std::size_t prefix_length : + { + std::size_t{0}, std::size_t{20} + }) + { + CAPTURE(prefix_length) + std::vector keys(members); + std::vector taken(members, false); + std::string missing; + std::size_t found = 0; + for (std::uint64_t counter = 0; found < members || missing.empty(); ++counter) + { + std::string key(prefix_length, 'p'); + std::uint64_t x = counter; + for (std::size_t i = 0; i < 8; ++i, x /= 26) + { + key += static_cast('a' + (x % 26)); + } + const std::size_t slot = nlohmann::detail::conditional_static_cast(nlohmann::detail::view::key_hash(key.data(), key.size())) & (table_size - 1); + if (slot < members && !taken[slot]) + { + taken[slot] = true; + keys[slot] = key; + ++found; + } + else if (slot == 0 && missing.empty()) + { + missing = key; + } + } + std::string text = "{"; + for (std::size_t i = 0; i < members; ++i) + { + text += (i != 0 ? ",\"" : "\"") + keys[i] + "\":" + std::to_string(i); + } + text += "}"; + const json_document d = json_document::parse(text); + const json_view v = d.root(); + for (std::size_t i = 0; i < members; ++i) + { + CAPTURE(i) + CHECK(v[keys[i]].get() == i); + } + CHECK(!v.contains(missing)); + CHECK(v.find(missing) == v.end()); + CHECK(v.count(missing) == 0); + } + } + SECTION("keys that differ in their last bytes are spread over the table") { // The hash must spread the usual families of keys (numbered names, hex diff --git a/tests/src/unit-json_view_positions.cpp b/tests/src/unit-json_view_positions.cpp index 966c4b46a..6fa1fa660 100644 --- a/tests/src/unit-json_view_positions.cpp +++ b/tests/src/unit-json_view_positions.cpp @@ -20,6 +20,7 @@ using nlohmann::ordered_json; using nlohmann::json_document; using nlohmann::json_editable_document; +#include #include #include #include @@ -300,6 +301,27 @@ TEST_CASE("json_view materialize() source positions") static_cast(j.at("list").at(1).get()); })); } + + SECTION("exception about the root value (#5987)") + { + const json_document root = json_document::parse("true"); + const json_document nested = json_document::parse("[1, true]"); + CHECK(json_view_test::exception_of_call([&] + { + static_cast(root.root().get()); + }) == "[json.exception.type_error.302] (bytes 0-4) type must be string, but is boolean"); +#if JSON_DIAGNOSTICS + CHECK(json_view_test::exception_of_call([&] + { + static_cast(nested.root()[1].get()); + }) == "[json.exception.type_error.302] (/1) (bytes 4-8) type must be string, but is boolean"); +#else + CHECK(json_view_test::exception_of_call([&] + { + static_cast(nested.root()[1].get()); + }) == "[json.exception.type_error.302] (bytes 4-8) type must be string, but is boolean"); +#endif + } #endif SECTION("edited documents") @@ -427,6 +449,31 @@ TEST_CASE("json_view materialize() source positions") CHECK(same_positions(m[std::string("a") + '\n'][2], j[std::string("a") + '\n'][2])); } + SECTION("a damaged text: the end of a container is not found") + { + // (the text of an image is not checked with image_check::none) + const std::string array_text = "[1,2]"; + const std::vector original = json_document::parse(array_text).save(); + const auto at = std::search(original.begin(), original.end(), array_text.begin(), array_text.end()); + REQUIRE(at != original.end()); + const auto closing = static_cast(at - original.begin()) + array_text.size() - 1; + // not a closing bracket, a slash that starts no comment, and the end of the text + for (const char replacement : + { + 'x', '/', ' ' + }) + { + CAPTURE(replacement) + std::vector damaged = original; + damaged[closing] = static_cast(replacement); + const json_document d = json_document::load(damaged, json_document::image_check::none); + const json m = d.root().materialize(); + CHECK(range(m) == "none"); + CHECK(range(m[0]) == "1-2"); + CHECK(range(m[1]) == "3-4"); + } + } + #if !defined(JSON_NOEXCEPTION) SECTION("a decoded string's position must be within the text") { @@ -461,6 +508,7 @@ TEST_CASE("json_view exceptions name the path and the bytes of the value") { "null", "true", "42", "\"s\"", "[]", "{}", "[1, [2, [3, []]]]", R"({"a": {"b": [1, {"c": null}]}, "d": "e"})", R"({"a/b": {"~": [1]}, "\u0041\n": {"x": "\t"}, "": [{}]})", + R"({"a\u0000b": [1, "x"], "c": {"d\u0000": 2}})", // (the path of a key ends at a null byte, as in basic_json) R"([1 /* ] */, {"a": [] // } , "b": {} } , ])", "\n\t[\r\n 1 ,\n {\n \"a\" : [ ]\n }\n]\n", @@ -512,6 +560,49 @@ TEST_CASE("json_view exceptions name the path and the bytes of the value") CHECK(exception_of_call([&] { static_cast(d.root()["a"][2]["n"][0].get()); }) == expected); } + SECTION("views of values that are no longer part of the document, and of no value") + { + const std::string expected_string = "[json.exception.type_error.302] type must be number, but is string"; + static_cast(expected_string); + // (a value that is not found has no path, and with it no bytes) + { + json_editable_document d = json_editable_document::parse(R"(["x", ["y"]])"); + const auto removed = d.root()[0]; + d.erase(d.root(), 0); +#if JSON_DIAGNOSTICS + CHECK(exception_of_call([&] { static_cast(removed.get()); }) == expected_string); +#endif + // (the root is a scalar now: the value cannot be in it) + d.set(d.root(), 5); +#if JSON_DIAGNOSTICS + CHECK(exception_of_call([&] { static_cast(removed.get()); }) == expected_string); +#endif + CHECK(d.root().get() == 5); + } + { + json_editable_document d = json_editable_document::parse(R"(["x"])"); + const auto removed = d.root()[0]; + d.set(d.root(), 5); +#if JSON_DIAGNOSTICS + CHECK(exception_of_call([&] { static_cast(removed.get()); }) == expected_string); +#endif + } + // a view of no value has no context + CHECK(exception_of_call([&] { static_cast(nlohmann::json_view().get()); }) == "[json.exception.type_error.302] type must be number, but is discarded"); + } + + SECTION("an array whose last element was replaced by a new container has no bytes") + { + json_editable_document d = json_editable_document::parse(R"({"a": [1, [2]], "b": 1})"); + d.set(d.root()["a"][1], json::array({3})); + std::string expected = "[json.exception.type_error.302] "; +#if JSON_DIAGNOSTICS + expected += "(/a) "; +#endif + expected += "type must be number, but is array"; + CHECK(exception_of_call([&] { static_cast(d.root()["a"].get()); }) == expected); + } + SECTION("deep documents with siblings after every nested container") { // the search for the path pushes and pops one frame per level (the @@ -551,12 +642,8 @@ TEST_CASE("json_view exceptions name the path and the bytes of the value") expected_zz += "(bytes 12-35) key 'zz' not found"; CHECK(exception_of_call([&] { static_cast(d.root().at(json::json_pointer("/people/0/zz"))); }) == expected_zz); - // the root has no path, and with JSON_DIAGNOSTICS then no bytes either (as basic_json) - std::string expected_missing = "[json.exception.out_of_range.403] "; -#if !JSON_DIAGNOSTICS - expected_missing += "(bytes 0-79) "; -#endif - expected_missing += "key 'missing' not found"; + // the root has no path, but still its bytes (#5987) + const std::string expected_missing = "[json.exception.out_of_range.403] (bytes 0-79) key 'missing' not found"; CHECK(exception_of_call([&] { static_cast(d.root().at("missing")); }) == expected_missing); } diff --git a/tests/src/unit-large_json.cpp b/tests/src/unit-large_json.cpp index ed988c242..e8cc0ed42 100644 --- a/tests/src/unit-large_json.cpp +++ b/tests/src/unit-large_json.cpp @@ -12,7 +12,10 @@ using nlohmann::json; #include +#include +#include #include +#include TEST_CASE("tests on very large JSONs") { @@ -140,6 +143,56 @@ TEST_CASE("tests on deeply nested JSONs") } } + SECTION("leaf values of every type below the bound of the recursive descent") + { + // the elements and members of the containers below the bound are + // copied one by one, whatever their type + const json leaves = json::array( + { + nullptr, true, false, -1, std::uint64_t{1}, 1.5, "text", json::binary({1, 2, 3}) + }); + const json members = + { + {"null", nullptr}, {"boolean", true}, {"integer", -1}, {"unsigned", std::uint64_t{1}}, + {"float", 1.5}, {"string", "text"}, {"binary", json::binary({1, 2, 3})} + }; + + for (const bool objects : + { + false, true + }) + { + CAPTURE(objects) + + json j = objects ? members : leaves; + for (std::size_t i = 0; i < 300; ++i) + { + json wrapper = objects ? json::object() : json::array(); + if (objects) + { + wrapper["a"] = std::move(j); + } + else + { + wrapper.push_back(std::move(j)); + } + j = std::move(wrapper); + } + + const json copy(j); // NOLINT(performance-unnecessary-copy-initialization): the copy is what is tested + + const json* current = © + for (std::size_t i = 0; i < 300; ++i) + { + current = objects ? ¤t->at("a") : ¤t->front(); + } + + // the innermost value is shallow: it is safe to compare + CHECK(*current == (objects ? members : leaves)); + CHECK(current->size() == (objects ? members.size() : leaves.size())); + } + } + SECTION("a value that is deep in one place only") { json j = json::object(); diff --git a/tests/src/unit-locale-cpp.cpp b/tests/src/unit-locale-cpp.cpp index b665b6531..10143326a 100644 --- a/tests/src/unit-locale-cpp.cpp +++ b/tests/src/unit-locale-cpp.cpp @@ -397,6 +397,14 @@ TEST_CASE("locale with a multi-byte decimal point") // a value the locale-independent paths convert is not affected CHECK(json::parse("12.5") == 12.5); + + // dump() of a long double that takes the snprintf path replaces the + // whole decimal point, not only its first byte (#5993) + CHECK(long_double_json(1.5L).dump() == "1.5"); + CHECK(long_double_json(2.0L).dump() == "2.0"); + CHECK(long_double_json(-0.25L).dump() == "-0.25"); + CHECK(long_double_json::parse(long_double_json(1.5e-7L).dump()) == long_double_json(1.5e-7L)); + CHECK(long_double_json::parse(ld.dump()) == expected_long_double); } if (!tested) { @@ -495,3 +503,206 @@ TEST_CASE("locale changes during a single dump() (#5709 item 3)") // fix #5597 made on the parser side CHECK(buf.data == "[\"" + padding + "\",1234.5]"); } + +namespace +{ +// a token whose first data() call switches LC_NUMERIC, so that the locale +// changes between convert_float_locale_aware() looking up the decimal point +// and strtod reading the token (the race of #5198, made deterministic) +struct locale_switching_token +{ + using value_type = char; + + locale_switching_token(std::string text_, std::string switch_to_) + : text(std::move(text_)), switch_to(std::move(switch_to_)) + {} + + std::size_t size() const + { + return text.size(); + } + + char& operator[](std::size_t i) + { + return text[i]; + } + + char* data() + { + if (!switched) + { + switched = std::setlocale(LC_NUMERIC, switch_to.c_str()) != nullptr; + } + return &text[0]; // NOLINT(readability-container-data-pointer): std::string::data() is const before C++17 + } + + std::string text; + std::string switch_to; + bool switched = false; +}; + +// set LC_NUMERIC to the first of @a names that is usable and has a decimal +// point of @a min_size bytes at least (and at most @a max_size) +bool use_decimal_point_locale(const std::vector& names, std::size_t min_size, std::size_t max_size, const char*& used) +{ + for (const char* name : names) + { + if (std::setlocale(LC_NUMERIC, name) == nullptr) + { + continue; + } + const std::size_t size = std::string(std::localeconv()->decimal_point).size(); + if (size >= min_size && size <= max_size) + { + used = name; + return true; + } + } + return false; +} + +// convert a token with convert_float_locale_aware(); the token must be given +// back as it was +template +FloatType convert_locale_aware(const std::string& text) +{ + std::string token = text; + const std::size_t dot = token.find('.'); + FloatType value{}; + nlohmann::detail::convert_float_locale_aware(token, dot, value); + CHECK(token == text); + return value; +} + +template +void check_locale_aware_conversions() +{ + CHECK(convert_locale_aware("3.5") == FloatType(3.5)); + CHECK(convert_locale_aware("-0.25") == FloatType(-0.25)); + CHECK(convert_locale_aware("1.25e2") == FloatType(125)); + // no decimal point at all + CHECK(convert_locale_aware("12e1") == FloatType(120)); + CHECK(convert_locale_aware("7") == FloatType(7)); +} +} // namespace + +TEST_CASE("convert_float_locale_aware") +{ + const std::vector single_byte_names = {"de_DE.UTF-8", "de_DE", "fr_FR.UTF-8", "fr_FR", "ru_RU.UTF-8"}; + const std::vector multi_byte_names = {"ar_EG.UTF-8", "ar_SA.UTF-8", "fa_IR.UTF-8", "ps_AF.UTF-8", "ar_EG", "fa_IR"}; + const char* used = nullptr; + + SECTION("decimal point '.'") + { + REQUIRE(std::setlocale(LC_NUMERIC, "C") != nullptr); + check_locale_aware_conversions(); + check_locale_aware_conversions(); + check_locale_aware_conversions(); + } + + SECTION("single-byte decimal point other than '.'") + { + if (use_decimal_point_locale(single_byte_names, 1, 1, used) && std::localeconv()->decimal_point[0] != '.') + { + CAPTURE(used) + check_locale_aware_conversions(); + check_locale_aware_conversions(); + check_locale_aware_conversions(); + } + else + { + MESSAGE("no locale with a single-byte decimal point other than '.' is usable"); + } + } + + SECTION("multi-byte decimal point") + { + if (use_decimal_point_locale(multi_byte_names, 2, 8, used)) + { + CAPTURE(used) + check_locale_aware_conversions(); + check_locale_aware_conversions(); + check_locale_aware_conversions(); + } + else + { + MESSAGE("no locale with a multi-byte decimal point is usable"); + } + } + + SECTION("a token that is not converted completely") + { + // the locale does not change, so there is nothing to retry: the + // value up to the garbage is kept + REQUIRE(std::setlocale(LC_NUMERIC, "C") != nullptr); + std::string token = "1.5x"; + double value = 0; + nlohmann::detail::convert_float_locale_aware(token, token.find('.'), value); + CHECK(value == 1.5); + CHECK(token == "1.5x"); + } + + SECTION("the locale changes before the conversion") + { + if (!use_decimal_point_locale(single_byte_names, 1, 1, used) || std::localeconv()->decimal_point[0] == '.') + { + MESSAGE("no locale with a single-byte decimal point other than '.' is usable"); + } + else + { + const std::string comma_locale = used; + REQUIRE(std::setlocale(LC_NUMERIC, "C") != nullptr); + + // "C" -> comma locale: strtod stops at the '.', so the conversion is repeated + { + locale_switching_token token("1.5", comma_locale); + double value = 0; + nlohmann::detail::convert_float_locale_aware(token, 1, value); + if (token.switched) + { + CHECK(value == 1.5); + CHECK(token.text == "1.5"); + } + } + + // the same for the other floating-point types + REQUIRE(std::setlocale(LC_NUMERIC, "C") != nullptr); + { + locale_switching_token token("2.25", comma_locale); + float value = 0; + nlohmann::detail::convert_float_locale_aware(token, 1, value); + if (token.switched) + { + CHECK(value == 2.25f); + } + } + REQUIRE(std::setlocale(LC_NUMERIC, "C") != nullptr); + { + locale_switching_token token("2.25", comma_locale); + long double value = 0; + nlohmann::detail::convert_float_locale_aware(token, 1, value); + if (token.switched) + { + CHECK(value == 2.25L); + } + } + + // comma locale -> multi-byte decimal point + const char* multi_byte = nullptr; + if (use_decimal_point_locale(multi_byte_names, 2, 8, multi_byte)) + { + REQUIRE(std::setlocale(LC_NUMERIC, comma_locale.c_str()) != nullptr); + locale_switching_token token("1.5", multi_byte); + double value = 0; + nlohmann::detail::convert_float_locale_aware(token, 1, value); + if (token.switched) + { + CHECK(value == 1.5); + CHECK(token.text == "1.5"); + } + } + } + } + + CHECK(std::setlocale(LC_NUMERIC, "C") != nullptr); +} diff --git a/tests/src/unit-msgpack-cpp17.cpp b/tests/src/unit-msgpack-cpp17.cpp index 6ae4d6d56..8b2349b90 100644 --- a/tests/src/unit-msgpack-cpp17.cpp +++ b/tests/src/unit-msgpack-cpp17.cpp @@ -18,6 +18,7 @@ using nlohmann::json; #ifdef JSON_HAS_CPP_17 #include +#include #include // Test suite for verifying MessagePack handling with std::byte input @@ -96,4 +97,34 @@ TEST_CASE("MessagePack with std::byte") } } } + +// JSON text (not MessagePack) as std::byte input: the error position is +// computed from the bytes consumed so far +TEST_CASE("JSON text with std::byte") +{ + const auto to_bytes = [](const std::string & text) + { + std::vector bytes; + for (const char c : text) + { + bytes.push_back(static_cast(c)); + } + return bytes; + }; + + SECTION("valid text") + { + const json j = json::parse(to_bytes("[1,\n2]")); + CHECK(j == json({1, 2})); + CHECK(json::accept(to_bytes("[1,\n2]"))); + } + + SECTION("invalid text") + { + CHECK_FALSE(json::accept(to_bytes("[1,\n2,"))); + json _; + CHECK_THROWS_WITH_AS(_ = json::parse(to_bytes("[1,\n2,")), + "[json.exception.parse_error.101] parse error at line 2, column 3: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal", json::parse_error&); + } +} #endif diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index 8fe399b58..f24df4b5a 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -2238,3 +2238,31 @@ TEST_CASE("MessagePack arrays of scalars from contiguous and non-contiguous inpu == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::msgpack)); } } + +TEST_CASE("MessagePack arrays of one element from contiguous and non-contiguous input") +{ + // contiguous input leaves an array of one element to the general path + // rather than to the loop that reads arrays of scalars; both inputs must + // still give the same events, value, or error for the complete document + // and for every truncation of it, with enough bytes after each array that + // the loop could have been entered + const json j = json::parse(R"({"a": [[[[1.5]]]], "b": [7], "c": [{"x": 2.5}], "d": ["s"], "e": [[1, 2]], "f": [null], "g": 1.25})"); + const std::vector bytes = json::to_msgpack(j); + + CHECK(json::from_msgpack(bytes) == j); + + for (std::size_t length = 0; length <= bytes.size(); ++length) + { + CAPTURE(length) + const std::vector contiguous(bytes.begin(), bytes.begin() + static_cast(length)); + const std::list bytewise(contiguous.begin(), contiguous.end()); + + CHECK(json::from_msgpack(contiguous, true, false).dump() == json::from_msgpack(bytewise.begin(), bytewise.end(), true, false).dump()); + + SaxEventLogger from_contiguous; + SaxEventLogger from_bytewise; + CHECK(json::sax_parse(contiguous, &from_contiguous, json::input_format_t::msgpack, true, false) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::msgpack, true, false)); + CHECK(from_contiguous.events == from_bytewise.events); + } +} diff --git a/tests/src/unit-ordered_json.cpp b/tests/src/unit-ordered_json.cpp index c328d8aea..4d894e649 100644 --- a/tests/src/unit-ordered_json.cpp +++ b/tests/src/unit-ordered_json.cpp @@ -16,6 +16,7 @@ using nlohmann::ordered_json; #include #include #include +#include TEST_CASE("ordered_json") { @@ -515,3 +516,89 @@ TEST_CASE("parsing random trees of arrays and objects") CHECK(json::parse(text).dump() == expected.dump()); } } + +TEST_CASE("object_key_index") +{ + // the hash table of the DOM parser finds the position of a key in an + // ordered_json object; the parser creates it for exactly 64 elements, but + // it is correct for any number of elements it is given + using nlohmann::detail::object_key_index; + const std::size_t not_found = object_key_index::npos; + + for (const std::size_t initial : + { + std::size_t{0}, std::size_t{10}, std::size_t{64}, std::size_t{65}, std::size_t{300} + }) + { + CAPTURE(initial) + + // the hashes are not distinct on purpose: elements with the same + // hash are told apart by the caller + const auto hash_of = [](std::size_t position) -> std::size_t + { + // truncated to 32 bits where std::size_t is that small + return static_cast((position % 50) * 0x9E3779B97F4A7C15ull); + }; + + std::vector hashes; + hashes.reserve(initial); + for (std::size_t i = 0; i < initial; ++i) + { + hashes.push_back(hash_of(i)); + } + object_key_index index(std::move(hashes)); + + // the elements that are there + for (std::size_t i = 0; i < initial; ++i) + { + CHECK(index.find(hash_of(i), [i](std::size_t position) + { + return position == i; + }) == i); + } + CHECK(index.find(hash_of(1), [](std::size_t /*position*/) + { + return false; + }) == not_found); + + // elements that are added later, until the table grows + for (std::size_t i = initial; i < initial + 600; ++i) + { + index.add(hash_of(i)); + CHECK(index.find(hash_of(i), [i](std::size_t position) + { + return position == i; + }) == i); + } + CHECK(index.find(hash_of(initial), [initial](std::size_t position) + { + return position == initial; + }) == initial); + } +} + +TEST_CASE("JSON pointers on ordered_json") +{ + using pointer = ordered_json::json_pointer; + + SECTION("operator[] inserts a missing key at the end") + { + ordered_json j = {{"z", 1}, {"x", {{"y", 2}}}}; + + j[pointer("/a")] = 3; + j[pointer("/x/b")] = 4; + + CHECK(j == ordered_json({{"z", 1}, {"x", {{"y", 2}, {"b", 4}}}, {"a", 3}})); + CHECK(j.dump() == R"({"z":1,"x":{"y":2,"b":4},"a":3})"); + } + + SECTION("unflatten of a flattened object whose keys are not sorted") + { + // the prefixes /a and /c denote arrays; a key of /a follows one of /c + const ordered_json flat = {{"/a/0/p", 1}, {"/c/0/x", 2}, {"/a/0/q", 3}, {"/a/1/p", 4}}; + const ordered_json expected = {{"a", {{{"p", 1}, {"q", 3}}, {{"p", 4}}}}, {"c", {{{"x", 2}}}}}; + + CHECK(flat.unflatten() == expected); + CHECK(expected.flatten().unflatten() == expected); + } +} diff --git a/tests/src/unit-regression1.cpp b/tests/src/unit-regression1.cpp index b98f2b6f3..367f52dfa 100644 --- a/tests/src/unit-regression1.cpp +++ b/tests/src/unit-regression1.cpp @@ -23,11 +23,13 @@ using nlohmann::json; using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) #endif +#include #include #include #include #include #include +#include #include "make_test_data_available.hpp" #include "test_utils.hpp" @@ -1519,6 +1521,76 @@ TEST_CASE("regression tests 1") } } + SECTION("operator[](json_pointer) continues with the member that emplace() found") + { + // see the section above: the lookup does not find "k", but emplace() + // finds "x" and the pointer continues with the member of "x". Here + // the member is a container, so the pointer can go on below it. Only + // the outermost object may be a copy (with restarted timestamps): the + // value of "x" is moved in, so that its own lookups work. + const my_json orig = my_json::parse(R"({"x":null})"); + const auto make = [&orig]() + { + my_json::object_t top = orig.get_ref(); + top.at("x") = my_json::parse(R"({"a":{"b":1},"n":null,"arr":[10,20],"objs":[{"m":5},{"m":6}]})"); + return my_json(std::move(top)); // NOLINT(modernize-return-braced-init-list): {top} would be an array + }; + + SECTION("existing members") + { + // emplace() registers the key "k" in the object, so that every + // pointer needs a new object to reach the member of "x" again + const std::array, 4> pointers = {{{"/k/a/b", "1"}, {"/k/arr/1", "20"}, {"/k/n", "null"}, {"/k/objs/1/m", "6"}}}; + for (const auto& pointer : pointers) + { + CAPTURE(pointer.first) + my_json j = make(); + CHECK(j[my_json::json_pointer(pointer.first)].dump() == pointer.second); + CHECK(j.size() == 1); + CHECK(j["x"].size() == 4); + } + } + + SECTION("a missing member of an object") + { + my_json j = make(); +#ifdef JSON_NO_THREAD_LOCAL + // (set by the user, or by the library for clang with MinGW) + // the objects below the outermost one are not copied by their + // object_t here, but member by member, which keeps their + // timestamps correct: "c" is not mistaken for "b" and is inserted + CHECK(j[my_json::json_pointer("/k/a/c/d")].is_null()); + CHECK(j["x"]["a"].dump() == R"({"b":1,"c":{"d":null}})"); +#else + // the timestamp of "c" is that of "b" again, so that emplace() + // finds the member "b", a number, which has no member "d" + CHECK_THROWS_AS(j[my_json::json_pointer("/k/a/c/d")], my_json::out_of_range&); + CHECK(j["x"]["a"].dump() == R"({"b":1})"); +#endif + } + + SECTION("a null value is converted to an object") + { + my_json j = make(); + j[my_json::json_pointer("/k/n/c")] = 3; + CHECK(j["x"]["n"].dump() == R"({"c":3})"); + } + + SECTION("the end of an array") + { + my_json j = make(); + j[my_json::json_pointer("/k/arr/-")] = 30; + CHECK(j["x"]["arr"].dump() == "[10,20,30]"); + } + + SECTION("an index past the end of an array") + { + my_json j = make(); + j[my_json::json_pointer("/k/arr/3")] = 40; + CHECK(j["x"]["arr"].dump() == "[10,20,null,40]"); + } + } + SECTION("issue #977 - Assigning between different json types") { foo_json lj = ns::foo{3}; diff --git a/tests/src/unit-regression2.cpp b/tests/src/unit-regression2.cpp index c7566c799..f7bd557e8 100644 --- a/tests/src/unit-regression2.cpp +++ b/tests/src/unit-regression2.cpp @@ -441,6 +441,10 @@ TEST_CASE("regression tests 2") float_json j2 = {1000.0, 2000.0, 3000.0}; CHECK(float_json::from_ubjson(float_json::to_ubjson(j2, true, true)) == j2); + + // half-precision floats of both signs are converted to the float type + CHECK(float_json::from_cbor(std::vector {0xF9, 0xC0, 0x00}) == -2.0f); + CHECK(float_json::from_cbor(std::vector {0xF9, 0x3C, 0x00}) == 1.0f); } SECTION("issue #1045 - Using STL algorithms with JSON containers with expected results?") @@ -1005,7 +1009,7 @@ TEST_CASE("regression test - excessive binary container size honors allow_except // allow_exceptions=true (the default) must still throw exactly as before. // The exact message text is not checked here: on platforms where // std::size_t is 32-bit, the CBOR reader's own length-narrowing check - // (get_cbor_container_size(), unrelated to this fix) intercepts a + // (get_container_size(), unrelated to this fix) intercepts a // declared length of 2^63 before it ever reaches the check this test // targets, with different (but equally valid, and already correct) // wording -- see unit-cbor.cpp for coverage of that message. diff --git a/tests/src/unit-stream-bulk-scan.cpp b/tests/src/unit-stream-bulk-scan.cpp index d61ce4bd0..f6869847d 100644 --- a/tests/src/unit-stream-bulk-scan.cpp +++ b/tests/src/unit-stream-bulk-scan.cpp @@ -19,11 +19,13 @@ using nlohmann::json; #include +#include #include #include #include #include #include +#include #include #include #include @@ -93,6 +95,34 @@ class chunked_streambuf : public std::streambuf std::size_t m_unbuffered; }; +#if TEST_STREAM_BULK_SCAN +// A streambuf that claims a get area of several gigabytes, of which only the +// first bytes exist. It is only used to move the read position of the adapter, +// never to read beyond them, so that skipping more than gbump() can bump at +// once can be tested without allocating that much. +class oversized_get_area_streambuf : public std::streambuf +{ + public: + explicit oversized_get_area_streambuf(std::size_t claimed_size) + { + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + setg(m_data.data(), m_data.data(), m_data.data() + claimed_size); + } + + // Give the get area back its real size, so that the adapter may read the + // character at the read position when it is destroyed (with + // JSON_PRECISE_STREAM_POSITION, it consumes the last character returned). + void restore_get_area() + { + setg(m_data.data(), m_data.data(), m_data.data() + m_data.size()); + } + + private: + std::array m_data{{'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'}}; +}; + +#endif + // how a test reads from the stream enum class read_mode { @@ -433,6 +463,39 @@ TEST_CASE("parsing from a std::istream scans the get area in bulk") ia.copy_token(token, false); CHECK(token.empty()); } + +#if SIZE_MAX > UINT32_MAX + // a get area larger than an int needs a 64-bit std::size_t (checked by the + // preprocessor, as a constant condition triggers MSVC's C4127) + SECTION("skipping more characters than gbump() takes at once") + { + // gbump() takes an int, but a get area can be larger + const std::size_t claimed = std::size_t{3} << 30u; + oversized_get_area_streambuf buf(claimed); + if (static_cast(buf.in_avail()) != claimed) + { + // a standard library whose streambuf stores the size of the get + // area in an int (MSVC) + MESSAGE("a get area larger than an int is not supported"); + } + else + { + std::istream is(&buf); + nlohmann::detail::input_stream_adapter ia(is); + CHECK(ia.get_character() == 'a'); + CHECK(ia.bulk_remaining() == claimed - 1); + + const std::size_t skipped = static_cast((std::numeric_limits::max)()) + 10; + ia.bulk_skip(skipped); + CHECK(ia.bulk_remaining() == claimed - 1 - skipped); + + ia.bulk_skip(ia.bulk_remaining()); + CHECK(ia.bulk_remaining() == 0); + + buf.restore_get_area(); + } + } +#endif #endif SECTION("SAX parsing from a stream") diff --git a/tests/src/unit-to_chars.cpp b/tests/src/unit-to_chars.cpp index f8863a5a9..3a5a8841d 100644 --- a/tests/src/unit-to_chars.cpp +++ b/tests/src/unit-to_chars.cpp @@ -928,3 +928,49 @@ TEST_CASE("short decimals") } } } + +TEST_CASE("decimals of up to 17 digits") +{ + // write_decimal() lays out digits * 10^exp as format_buffer() does; to_chars() + // calls it for the doubles whose digits write_shortest() does not convert at + // once, and the digits may be as many as 17 + const auto written = [](std::uint64_t digits, int exp) -> std::string + { + std::array buf{}; // (up to 41 bytes are written) + char* const end = nlohmann::detail::dtoa_impl::write_decimal(buf.data(), digits, exp); + return {buf.data(), end}; + }; + + SECTION("fixed notation") + { + CHECK(written(1, 0) == "1.0"); + CHECK(written(1234, -2) == "12.34"); + CHECK(written(123, 2) == "12300.0"); + CHECK(written(50000000000u, -1) == "5000000000.0"); + CHECK(written(100000000u, -8) == "1.0"); + CHECK(written(12345678u, -12) == "1.2345678e-05"); + CHECK(written(1234, 11) == "123400000000000.0"); + // the point moves left of the first digit + CHECK(written(5, -3) == "0.005"); + CHECK(written(12345678901234567u, -20) == "0.00012345678901234567"); + } + + SECTION("exponent notation") + { + CHECK(written(5, -5) == "5e-05"); + CHECK(written(1234, 12) == "1.234e+15"); + CHECK(written(5, -324) == "5e-324"); + } + + SECTION("17 digits") + { + CHECK(written(12345678901234567u, -16) == "1.2345678901234567"); + CHECK(written(12345678901234567u, -19) == "0.0012345678901234567"); + CHECK(written(12345678901234567u, 5) == "1.2345678901234567e+21"); + CHECK(written(99999999999999999u, -17) == "0.99999999999999999"); + CHECK(written(10000000000000001u, -16) == "1.0000000000000001"); + // trailing zeros are not written + CHECK(written(10000000000000000u, -17) == "0.1"); + CHECK(written(12000000000000000u, -15) == "12.0"); + } +} diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index 04983da1c..b4fdc0997 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -1673,6 +1673,31 @@ TEST_CASE("UBJSON") return true; }); CHECK_THROWS_AS(scp2.start_array((std::numeric_limits::max)() - 1), json::out_of_range&); + + // without exceptions, the error is reported to the caller + json l; + nlohmann::detail::json_sax_dom_callback_parser scp3(l, [](int /*unused*/, json::parse_event_t /*unused*/, const json& /*unused*/) noexcept + { + return true; + }, false); + CHECK_FALSE(scp3.start_array((std::numeric_limits::max)() - 1)); + CHECK(scp3.is_errored()); + } + + SECTION("array with a known size, read without exceptions") + { + std::vector const v_ubjson = {'[', '#', 'i', 2, 'i', 1, 'i', 2}; + json j; + nlohmann::detail::json_sax_dom_parser sax(j, false); + CHECK(json::sax_parse(v_ubjson, &sax, json::input_format_t::ubjson)); + CHECK(j == json({1, 2})); + + // the readers reject a size this large before they announce + // it, so it can only reach start_array() directly + json k; + nlohmann::detail::json_sax_dom_parser sax2(k, false); + CHECK_FALSE(sax2.start_array((std::numeric_limits::max)() - 1)); + CHECK(sax2.is_errored()); } } } @@ -2360,6 +2385,18 @@ TEST_CASE("UBJSON optimized arrays of a valueless type are bounded") } } + SECTION("a count equal to SIZE_MAX is rejected") + { + // the reader marks a container without a count by SIZE_MAX, so a + // count of that value must not be read as "no count"; UBJSON's + // largest count type (int64) can only reach it where size_t is 32 bits +#if SIZE_MAX == 0xffffffff + const std::vector input = {'[', '#', 'L', 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 'i', 0x01, ']'}; + CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.out_of_range.408] syntax error while parsing UBJSON size: integer value overflow", json::out_of_range&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); +#endif + } + SECTION("ordinary counts are unaffected") { CHECK(json::from_ubjson(std::vector({'[', '$', 'Z', '#', 'i', 3})) == json({nullptr, nullptr, nullptr}));