diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 1afed61c3..f1231966a 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -159,6 +159,10 @@ compiler workflows (Ubuntu, macOS, Windows); the documentation and license check [`tools/check_path_filters`](https://github.com/nlohmann/json/tree/develop/tools/check_path_filters) for the list of skipped files and what to do when a compiler job starts reading one of them. +Most compiler jobs run only once a maintainer adds the label "🚀 ready to merge" to a pull request; until then, a +smaller set of jobs runs on every push. See +[How CI runs](https://json.nlohmann.me/community/quality_assurance/#how-ci-runs) for the sets and when they run. + ### Amalgamate the source code The single-header files diff --git a/.github/workflows/repository_checks.yml b/.github/workflows/repository_checks.yml index bc19a9891..b4e45f22a 100644 --- a/.github/workflows/repository_checks.yml +++ b/.github/workflows/repository_checks.yml @@ -2,7 +2,7 @@ name: Repository checks # Checks that concern files the compiler workflows (ubuntu.yml, macos.yml, # windows.yml) skip with their path filter: the documentation, the license -# information, and the path filter itself. This workflow has no path filter. +# information, and the CI configuration itself. This workflow has no path filter. on: push: branches: @@ -20,7 +20,7 @@ concurrency: cancel-in-progress: true jobs: - check_path_filters: + check_ci_configuration: runs-on: ubuntu-26.04 steps: - name: Harden Runner @@ -33,6 +33,11 @@ jobs: persist-credentials: false - name: Check the path filters of the compiler workflows run: python3 tools/check_path_filters/check_path_filters.py + - name: Check the job sets of the compiler workflows and their documentation + run: | + python3 -m venv venv + venv/bin/pip install --quiet -r tools/ci_jobs/requirements.txt + venv/bin/python tools/ci_jobs/ci_jobs.py ci_reuse_compliance: runs-on: ubuntu-26.04 diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 2cde95f03..066f5f68b 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -7,6 +7,8 @@ on: - master - release/* pull_request: + # labeled: the "🚀 ready to merge" label starts the full set (see below) + types: [opened, synchronize, reopened, labeled] # Skip pull requests that change only files no job here reads. The list is checked by # tools/check_path_filters/check_path_filters.py (repository_checks.yml); change both together. paths: @@ -26,17 +28,41 @@ on: # built by the swiftpm job (macos.yml) - 'docs/mkdocs/docs/integration/swift/**' - '.github/workflows/ubuntu.yml' + schedule: + # the nightly set (see below), on develop + - cron: '0 2 * * *' workflow_dispatch: permissions: contents: read +# A run triggered by any other label gets a group of its own, so it does not +# cancel the run of the last push; its jobs are skipped. concurrency: - group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.event_name == 'schedule' && 'nightly' || github.ref || github.run_id }}${{ github.event.action == 'labeled' && github.event.label.name != '🚀 ready to merge' && format('-{0}', github.run_id) || '' }} cancel-in-progress: true +# The jobs are split into three sets (see "How CI runs" in +# docs/mkdocs/docs/community/quality_assurance.md): +# +# - PR set: every pull request push and every push; chosen because these jobs +# caught 92 % of the failing commits in Sep/Oct 2026. +# - full set: after the PR set passed, on pushes and on pull requests with the +# "🚀 ready to merge" label; every remaining compiler, version and standard. +# - nightly set: slow tools and rare compilers, nightly on develop, on pushes +# to master and release/*, and when started by hand. +# +# A matrix whose entries belong to different sets is split into one job per +# set; the jobs share their steps through YAML anchors. tools/ci_jobs checks +# that every job belongs to exactly one set. + jobs: + ########################################################################### + # PR set + ########################################################################### + ci_test_gcc: + if: &pr_set ${{ github.event_name != 'schedule' && (github.event.action != 'labeled' || github.event.label.name == '🚀 ready to merge') }} runs-on: ubuntu-latest container: gcc:latest steps: @@ -51,6 +77,7 @@ jobs: run: cmake --build build --target ci_test_gcc ci_meson_install: + if: *pr_set runs-on: ubuntu-26.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -95,7 +122,342 @@ jobs: cmake -S tests/cmake_import/project -B build-import-split -DCMAKE_PREFIX_PATH=${{ github.workspace }}/install-split cmake --build build-import-split + ci_static_analysis_ubuntu: + if: *pr_set + runs-on: ubuntu-26.04 + strategy: + matrix: + target: [ci_cpplint] + steps: &static_analysis_ubuntu_steps + - name: Harden Runner + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: audit + + - name: Install Valgrind + if: matrix.target == 'ci_test_valgrind' + run: sudo apt-get update ; sudo apt-get install -y valgrind + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Get latest CMake and ninja + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + - name: Run CMake + run: cmake -S . -B build -DJSON_CI=On + - name: Build + run: cmake --build build --target ${{ matrix.target }} + + ci_static_analysis_clang: + if: *pr_set + runs-on: ubuntu-latest + container: silkeh/clang:22 + strategy: + matrix: + target: [ci_test_clang, ci_clang_tidy, ci_test_clang_sanitizer, ci_single_binaries] + steps: &static_analysis_clang_steps + - name: Install git, clang-tools, iwyu (ci_single_binaries), and unzip + run: apt-get update ; apt-get install -y git clang-tools iwyu unzip + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Get latest CMake and ninja + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + - name: Run CMake + run: cmake -S . -B build -DJSON_CI=On + - name: Build + run: cmake --build build --target ${{ matrix.target }} + + ci_cmake_options: + if: *pr_set + runs-on: ubuntu-latest + container: ubuntu:24.04 + strategy: + matrix: + target: [ci_test_diagnostics, ci_test_diagnostic_positions, ci_test_noexceptions, ci_test_noglobaludls] + steps: &cmake_options_steps + - name: Install build-essential + run: apt-get update ; apt-get install -y build-essential unzip wget git + # the CMake 3.5.0 release binary that ci_cmake_flags downloads links + # against libidn.so.11, which Ubuntu 24.04 no longer ships + - name: Install libidn11 for CMake 3.5.0 + if: matrix.target == 'ci_cmake_flags' + run: | + wget -q http://archive.ubuntu.com/ubuntu/pool/main/libi/libidn/libidn11_1.33-2.2ubuntu2_amd64.deb + dpkg -i libidn11_1.33-2.2ubuntu2_amd64.deb + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Get latest CMake and ninja + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + - name: Run CMake + run: cmake -S . -B build -DJSON_CI=On + - name: Build + run: cmake --build build --target ${{ matrix.target }} + + ci_test_compilers_gcc_old: + if: *pr_set + runs-on: ubuntu-latest + strategy: + matrix: + compiler: ['4.8'] + # official gcc:4.8/4.9/5/6 images fail to check out code (too old for + # actions/checkout); install the old compilers on top of official ubuntu:20.04 + # instead, mirroring what the (now retired) custom json-ci image did. + container: ubuntu:20.04 + steps: &compilers_gcc_old_steps + - name: Install g++-${{ matrix.compiler }} + run: | + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y --no-install-recommends software-properties-common ca-certificates gnupg make git + # add-apt-repository resolves the PPA through the Launchpad API, + # which intermittently times out or fails the team lookup (the plain + # "deb ..." sources below never hit Launchpad and never flake). + # Retry with backoff so a transient Launchpad blip does not fail CI. + for attempt in 1 2 3 4 5; do + add-apt-repository -y ppa:ubuntu-toolchain-r/test && break + echo "::warning::add-apt-repository ppa:ubuntu-toolchain-r/test failed (attempt ${attempt}/5); retrying" + sleep $((attempt * 10)) + done + apt-add-repository -y "deb http://archive.ubuntu.com/ubuntu/ bionic main" + apt-add-repository -y "deb http://archive.ubuntu.com/ubuntu/ bionic universe" + apt-add-repository -y "deb http://archive.ubuntu.com/ubuntu/ xenial main" + apt-add-repository -y "deb http://archive.ubuntu.com/ubuntu/ xenial universe" + apt-add-repository -y "deb http://archive.ubuntu.com/ubuntu/ xenial-updates main" + apt-add-repository -y "deb http://archive.ubuntu.com/ubuntu/ xenial-updates universe" + apt-get update + apt-get install -y --no-install-recommends g++-${{ matrix.compiler }} + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Get latest CMake and ninja + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + - name: Run CMake + run: CXX=g++-${{ matrix.compiler }} cmake -S . -B build -DJSON_CI=On + - name: Build + run: cmake --build build --target ci_test_compiler_g++-${{ matrix.compiler }} + + ci_test_compilers_clang: + if: *pr_set + runs-on: ubuntu-latest + strategy: + matrix: + compiler: ['3.4'] + container: silkeh/clang:${{ matrix.compiler }} + steps: &compilers_clang_steps + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Get latest CMake and ninja + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + - name: Disable filesystem support (std::filesystem is unusable) for clang 7 / 8 / 9 / 10 + run: echo "JSON_FORCED_GLOBAL_COMPILE_OPTIONS=-DJSON_HAS_FILESYSTEM=0;-DJSON_HAS_EXPERIMENTAL_FILESYSTEM=0" >> "$GITHUB_ENV" + if: ${{ matrix.compiler == '7' || matrix.compiler == '8' || matrix.compiler == '9' || matrix.compiler == '10' }} + - name: Run CMake + run: cmake -S . -B build -DJSON_CI=On + - name: Build + run: cmake --build build --target ci_test_compiler_default + + ci_test_standards_gcc: + if: *pr_set + runs-on: ubuntu-latest + container: gcc:latest + strategy: + matrix: + standard: [26] + steps: &standards_gcc_steps + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Get latest CMake and ninja + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + - name: Run CMake + run: cmake -S . -B build -DJSON_CI=On + - name: Build + run: cmake --build build --target ci_test_gcc_cxx${{ matrix.standard }} + + ci_test_standards_clang: + if: *pr_set + runs-on: ubuntu-latest + container: silkeh/clang:latest + strategy: + matrix: + standard: [26] + stdlib: [libcxx] + steps: &standards_clang_steps + - name: Install git and unzip + run: apt-get update ; apt-get install -y git unzip + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Get latest CMake and ninja + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + - name: Run CMake + run: cmake -S . -B build -DJSON_CI=On + - name: Build with libc++ + run: cmake --build build --target ci_test_clang_libcxx_cxx${{ matrix.standard }} + if: ${{ matrix.stdlib == 'libcxx' }} + - name: Build with libstdc++ + run: cmake --build build --target ci_test_clang_cxx${{ matrix.standard }} + if: ${{ matrix.stdlib == 'libstdcxx' }} + + ci_emscripten: + if: *pr_set + runs-on: ubuntu-26.04 + steps: + - name: Harden Runner + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: audit + + - name: Install emscripten + uses: mymindstorm/setup-emsdk@4528d102f7230f0e7b276855c01ea1159be0e984 # v16 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Get latest CMake and ninja + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + - name: Run CMake + run: cmake -S . -B build -DCMAKE_TOOLCHAIN_FILE=$EMSDK/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -GNinja + - name: Build + run: cmake --build build + + ########################################################################### + # full set: starts after the PR set passed + ########################################################################### + + ci_static_analysis_ubuntu_full: + needs: &pr_set_jobs [ci_test_gcc, ci_meson_install, ci_static_analysis_ubuntu, ci_static_analysis_clang, ci_cmake_options, ci_test_compilers_gcc_old, ci_test_compilers_clang, ci_test_standards_gcc, ci_test_standards_clang, ci_emscripten] + if: &full_set ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, '🚀 ready to merge') }} + runs-on: ubuntu-26.04 + strategy: + matrix: + target: [ci_test_single_header, ci_cppcheck, ci_reproducible_tests, ci_non_git_tests, ci_offline_testdata] + steps: *static_analysis_ubuntu_steps + + ci_static_analysis_clang_full: + needs: *pr_set_jobs + if: *full_set + runs-on: ubuntu-latest + container: silkeh/clang:22 + strategy: + matrix: + target: [ci_clang_analyze] + steps: *static_analysis_clang_steps + + ci_cmake_options_full: + needs: *pr_set_jobs + if: *full_set + runs-on: ubuntu-latest + container: ubuntu:24.04 + strategy: + matrix: + target: [ci_cmake_flags, ci_test_noimplicitconversions, ci_test_legacycomparison, ci_test_disableenumserialization, ci_test_disabletuplereferenceconversion, ci_test_skiplibraryversioncheck, ci_test_simdutf, ci_test_strict_nul_handling, ci_test_delete_deprecated_functions, ci_test_no_thread_local] + steps: *cmake_options_steps + + ci_test_compilers_gcc_old_full: + needs: *pr_set_jobs + if: *full_set + runs-on: ubuntu-latest + strategy: + matrix: + compiler: ['4.9', '5', '6'] + container: ubuntu:20.04 + steps: *compilers_gcc_old_steps + + ci_test_compilers_gcc_full: + needs: *pr_set_jobs + if: *full_set + runs-on: ubuntu-latest + strategy: + matrix: + # older GCC docker images (4, 5, 6) fail to check out code + compiler: ['7', '8', '9', '10', '11', '12', '13', '14', '15', '16', 'latest'] + container: gcc:${{ matrix.compiler }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Get latest CMake and ninja + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + - name: Run CMake + run: cmake -S . -B build -DJSON_CI=On + - name: Build + run: cmake --build build --target ci_test_compiler_default + + ci_test_compilers_clang_full: + needs: *pr_set_jobs + if: *full_set + runs-on: ubuntu-latest + strategy: + matrix: + compiler: ['3.5', '3.6', '3.7', '3.8', '3.9', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15-bullseye', '16', '17', '18', '19', '20', '21', '22', 'latest'] + container: silkeh/clang:${{ matrix.compiler }} + steps: *compilers_clang_steps + + ci_test_standards_gcc_full: + needs: *pr_set_jobs + if: *full_set + runs-on: ubuntu-latest + container: gcc:latest + strategy: + matrix: + standard: [11, 14, 17, 20, 23] + steps: *standards_gcc_steps + + ci_test_standards_clang_full: + needs: *pr_set_jobs + if: *full_set + runs-on: ubuntu-latest + container: silkeh/clang:latest + strategy: + matrix: + standard: [11, 14, 17, 20, 23, 26] + stdlib: [libcxx, libstdcxx] + exclude: + # in the PR set + - standard: 26 + stdlib: libcxx + steps: *standards_clang_steps + + ci_module_cpp20: + needs: *pr_set_jobs + if: *full_set + strategy: + matrix: + container: ['gcc:latest', 'silkeh/clang:latest'] + runs-on: ubuntu-latest + container: ${{ matrix.container }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The module test uses `import std;`, which needs CMake's experimental + # import-std support. Its opt-in token is CMake-version-specific, so pin + # CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt. + - name: Get pinned CMake and ninja + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 + with: + cmakeVersion: 4.3.4 + # Clang: the std library module is provided by libc++ (the image's libstdc++ + # ships none), and the image's libc++ module manifest has a broken relative + # path — repoint it at the real module sources. + - name: Use libc++ and fix its module manifest path (Clang) + if: matrix.container == 'silkeh/clang:latest' + run: | + echo "CXXFLAGS=-stdlib=libc++" >> "$GITHUB_ENV" + mkdir -p /usr/lib/share && ln -sf /usr/lib/llvm-*/share/libc++ /usr/lib/share/libc++ + - name: Run CMake + run: cmake -S . -B build -DJSON_CI=On + - name: Build + run: cmake --build build --target ci_module_cpp20 + + ########################################################################### + # nightly set + ########################################################################### + ci_infer: + if: &nightly_set ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref != 'refs/heads/develop') }} runs-on: ubuntu-26.04 steps: - name: Harden Runner @@ -124,76 +486,16 @@ jobs: path: ${{ github.workspace }}/build/build_infer/infer-out/report.txt if-no-files-found: ignore - ci_static_analysis_ubuntu: + ci_static_analysis_ubuntu_nightly: + if: *nightly_set runs-on: ubuntu-26.04 strategy: matrix: - target: [ci_test_single_header, ci_cppcheck, ci_cpplint, ci_reproducible_tests, ci_non_git_tests, ci_offline_testdata, ci_test_valgrind] - steps: - - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 - with: - egress-policy: audit - - - name: Install Valgrind - if: matrix.target == 'ci_test_valgrind' - run: sudo apt-get update ; sudo apt-get install -y valgrind - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - - name: Run CMake - run: cmake -S . -B build -DJSON_CI=On - - name: Build - run: cmake --build build --target ${{ matrix.target }} - - ci_static_analysis_clang: - runs-on: ubuntu-latest - container: silkeh/clang:22 - strategy: - matrix: - target: [ci_test_clang, ci_clang_tidy, ci_test_clang_sanitizer, ci_clang_analyze, ci_single_binaries] - steps: - - name: Install git, clang-tools, iwyu (ci_single_binaries), and unzip - run: apt-get update ; apt-get install -y git clang-tools iwyu unzip - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - - name: Run CMake - run: cmake -S . -B build -DJSON_CI=On - - name: Build - run: cmake --build build --target ${{ matrix.target }} - - ci_cmake_options: - runs-on: ubuntu-latest - container: ubuntu:24.04 - strategy: - matrix: - target: [ci_cmake_flags, ci_test_diagnostics, ci_test_diagnostic_positions, ci_test_noexceptions, ci_test_noimplicitconversions, ci_test_legacycomparison, ci_test_noglobaludls, ci_test_disableenumserialization, ci_test_disabletuplereferenceconversion, ci_test_skiplibraryversioncheck, ci_test_simdutf, ci_test_strict_nul_handling, ci_test_delete_deprecated_functions, ci_test_no_thread_local] - steps: - - name: Install build-essential - run: apt-get update ; apt-get install -y build-essential unzip wget git - # the CMake 3.5.0 release binary that ci_cmake_flags downloads links - # against libidn.so.11, which Ubuntu 24.04 no longer ships - - name: Install libidn11 for CMake 3.5.0 - if: matrix.target == 'ci_cmake_flags' - run: | - wget -q http://archive.ubuntu.com/ubuntu/pool/main/libi/libidn/libidn11_1.33-2.2ubuntu2_amd64.deb - dpkg -i libidn11_1.33-2.2ubuntu2_amd64.deb - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - - name: Run CMake - run: cmake -S . -B build -DJSON_CI=On - - name: Build - run: cmake --build build --target ${{ matrix.target }} + target: [ci_test_valgrind] + steps: *static_analysis_ubuntu_steps ci_test_coverage: + if: *nightly_set runs-on: ubuntu-26.04 steps: - name: Harden Runner @@ -227,128 +529,8 @@ jobs: path-to-lcov: ${{ github.workspace }}/build/json.info.filtered.noexcept fail-on-error: false - ci_test_compilers_gcc_old: - runs-on: ubuntu-latest - strategy: - matrix: - compiler: ['4.8', '4.9', '5', '6'] - # official gcc:4.8/4.9/5/6 images fail to check out code (too old for - # actions/checkout); install the old compilers on top of official ubuntu:20.04 - # instead, mirroring what the (now retired) custom json-ci image did. - container: ubuntu:20.04 - steps: - - name: Install g++-${{ matrix.compiler }} - run: | - export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y --no-install-recommends software-properties-common ca-certificates gnupg make git - # add-apt-repository resolves the PPA through the Launchpad API, - # which intermittently times out or fails the team lookup (the plain - # "deb ..." sources below never hit Launchpad and never flake). - # Retry with backoff so a transient Launchpad blip does not fail CI. - for attempt in 1 2 3 4 5; do - add-apt-repository -y ppa:ubuntu-toolchain-r/test && break - echo "::warning::add-apt-repository ppa:ubuntu-toolchain-r/test failed (attempt ${attempt}/5); retrying" - sleep $((attempt * 10)) - done - apt-add-repository -y "deb http://archive.ubuntu.com/ubuntu/ bionic main" - apt-add-repository -y "deb http://archive.ubuntu.com/ubuntu/ bionic universe" - apt-add-repository -y "deb http://archive.ubuntu.com/ubuntu/ xenial main" - apt-add-repository -y "deb http://archive.ubuntu.com/ubuntu/ xenial universe" - apt-add-repository -y "deb http://archive.ubuntu.com/ubuntu/ xenial-updates main" - apt-add-repository -y "deb http://archive.ubuntu.com/ubuntu/ xenial-updates universe" - apt-get update - apt-get install -y --no-install-recommends g++-${{ matrix.compiler }} - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - - name: Run CMake - run: CXX=g++-${{ matrix.compiler }} cmake -S . -B build -DJSON_CI=On - - name: Build - run: cmake --build build --target ci_test_compiler_g++-${{ matrix.compiler }} - - ci_test_compilers_gcc: - runs-on: ubuntu-latest - strategy: - matrix: - # older GCC docker images (4, 5, 6) fail to check out code - compiler: ['7', '8', '9', '10', '11', '12', '13', '14', '15', '16', 'latest'] - container: gcc:${{ matrix.compiler }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - - name: Run CMake - run: cmake -S . -B build -DJSON_CI=On - - name: Build - run: cmake --build build --target ci_test_compiler_default - - ci_test_compilers_clang: - runs-on: ubuntu-latest - strategy: - matrix: - compiler: ['3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15-bullseye', '16', '17', '18', '19', '20', '21', '22', 'latest'] - container: silkeh/clang:${{ matrix.compiler }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - - name: Disable filesystem support (std::filesystem is unusable) for clang 7 / 8 / 9 / 10 - run: echo "JSON_FORCED_GLOBAL_COMPILE_OPTIONS=-DJSON_HAS_FILESYSTEM=0;-DJSON_HAS_EXPERIMENTAL_FILESYSTEM=0" >> "$GITHUB_ENV" - if: ${{ matrix.compiler == '7' || matrix.compiler == '8' || matrix.compiler == '9' || matrix.compiler == '10' }} - - name: Run CMake - run: cmake -S . -B build -DJSON_CI=On - - name: Build - run: cmake --build build --target ci_test_compiler_default - - ci_test_standards_gcc: - runs-on: ubuntu-latest - container: gcc:latest - strategy: - matrix: - standard: [11, 14, 17, 20, 23, 26] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - - name: Run CMake - run: cmake -S . -B build -DJSON_CI=On - - name: Build - run: cmake --build build --target ci_test_gcc_cxx${{ matrix.standard }} - - ci_test_standards_clang: - runs-on: ubuntu-latest - container: silkeh/clang:latest - strategy: - matrix: - standard: [11, 14, 17, 20, 23, 26] - stdlib: [libcxx, libstdcxx] - steps: - - name: Install git and unzip - run: apt-get update ; apt-get install -y git unzip - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - - name: Run CMake - run: cmake -S . -B build -DJSON_CI=On - - name: Build with libc++ - run: cmake --build build --target ci_test_clang_libcxx_cxx${{ matrix.standard }} - if: ${{ matrix.stdlib == 'libcxx' }} - - name: Build with libstdc++ - run: cmake --build build --target ci_test_clang_cxx${{ matrix.standard }} - if: ${{ matrix.stdlib == 'libstdcxx' }} - ci_cuda_example: + if: *nightly_set runs-on: ubuntu-latest strategy: fail-fast: false @@ -371,37 +553,8 @@ jobs: - name: Build run: cmake --build build --target ci_cuda_example - ci_module_cpp20: - strategy: - matrix: - container: ['gcc:latest', 'silkeh/clang:latest'] - runs-on: ubuntu-latest - container: ${{ matrix.container }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - # The module test uses `import std;`, which needs CMake's experimental - # import-std support. Its opt-in token is CMake-version-specific, so pin - # CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt. - - name: Get pinned CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - with: - cmakeVersion: 4.3.4 - # Clang: the std library module is provided by libc++ (the image's libstdc++ - # ships none), and the image's libc++ module manifest has a broken relative - # path — repoint it at the real module sources. - - name: Use libc++ and fix its module manifest path (Clang) - if: matrix.container == 'silkeh/clang:latest' - run: | - echo "CXXFLAGS=-stdlib=libc++" >> "$GITHUB_ENV" - mkdir -p /usr/lib/share && ln -sf /usr/lib/llvm-*/share/libc++ /usr/lib/share/libc++ - - name: Run CMake - run: cmake -S . -B build -DJSON_CI=On - - name: Build - run: cmake --build build --target ci_module_cpp20 - ci_icpc: + if: *nightly_set runs-on: ubuntu-latest # Intel discontinued the classic icc/icpc compiler in oneAPI 2024.0; this is # Intel's own last officially published image that still includes it. @@ -421,6 +574,7 @@ jobs: run: cmake --build build --target ci_icpc ci_icpx: + if: *nightly_set runs-on: ubuntu-latest container: intel/oneapi-hpckit:latest steps: @@ -435,6 +589,7 @@ jobs: run: cmake --build build --target ci_icpx ci_nvhpc: + if: *nightly_set runs-on: ubuntu-latest container: nvcr.io/nvidia/nvhpc:25.5-devel-cuda12.9-ubuntu22.04 steps: @@ -448,29 +603,10 @@ jobs: - name: Build run: cmake --build build --target ci_nvhpc - ci_emscripten: - runs-on: ubuntu-26.04 - steps: - - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 - with: - egress-policy: audit - - - name: Install emscripten - uses: mymindstorm/setup-emsdk@4528d102f7230f0e7b276855c01ea1159be0e984 # v16 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - name: Get latest CMake and ninja - uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - - name: Run CMake - run: cmake -S . -B build -DCMAKE_TOOLCHAIN_FILE=$EMSDK/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -GNinja - - name: Build - run: cmake --build build - # Native Linux ARM64 runner with the latest GCC. The ubuntu-24.04-arm label is # only available for public repositories. ci_test_arm64: + if: *nightly_set runs-on: ubuntu-24.04-arm container: gcc:latest steps: @@ -485,3 +621,33 @@ jobs: run: cmake -S . -B build -DJSON_CI=On - name: Build run: cmake --build build --target ci_test_compiler_default + + # A failure of a scheduled run opens an issue, or comments on the open one. + nightly_report: + needs: [ci_infer, ci_static_analysis_ubuntu_nightly, ci_test_coverage, ci_cuda_example, ci_icpc, ci_icpx, ci_nvhpc, ci_test_arm64] + if: ${{ always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure') }} + runs-on: ubuntu-26.04 + permissions: + issues: write + steps: + - name: Harden Runner + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: audit + + - name: Open or update the nightly failure issue + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + WORKFLOW: ${{ github.workflow }} + SHA: ${{ github.sha }} + run: | + title="Nightly CI failure (${WORKFLOW})" + body="The nightly run of ${WORKFLOW} on ${SHA:0:10} failed: ${RUN_URL}" + number=$(gh issue list --state open --search "in:title \"${title}\"" --json number,title --jq "map(select(.title == \"${title}\")) | .[0].number // empty") + if [ -n "$number" ]; then + gh issue comment "$number" --body "$body" + else + gh issue create --title "$title" --body "$body" + fi diff --git a/docs/mkdocs/docs/community/quality_assurance.md b/docs/mkdocs/docs/community/quality_assurance.md index 61ffdff85..4d5bba619 100644 --- a/docs/mkdocs/docs/community/quality_assurance.md +++ b/docs/mkdocs/docs/community/quality_assurance.md @@ -13,6 +13,72 @@ describe each of them.

The checks grouped by the stage at which they run, from push or pull request through the merge into develop to a release. The badge on each card says whether the check runs on every push or PR, on a schedule, or continuously.

+## How CI runs + +Not every job runs for every change. The compiler jobs on Ubuntu are split into three sets, so a pull request gets fast +feedback and the long jobs run when they matter: + +| Set | When it runs | What it contains | +|-------------|-------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| PR set | every push to a pull request, and every push to `develop`, `master`, and `release/*` | GCC and Clang with all warnings, Clang-Tidy, sanitizers, IWYU, cpplint, the oldest and newest GCC and Clang, common CMake options, Meson, Emscripten; with the checks that always run, they would have caught at least 92 % of the failing commits of September and October 2026 | +| full set | after the PR set passed: on pull requests with the label :rocket: ready to merge, and on every push | every other compiler, version, C++ standard, and CMake option, the Clang Static Analyzer, Cppcheck, and the C++20 module tests | +| nightly set | every night on `develop`, on every push to `master` and `release/*`, and when started by hand | Infer, Valgrind, coverage, the Intel and NVIDIA compilers, CUDA, and ARM64; a failure opens an issue | + +The jobs on macOS and Windows, the documentation and license checks, and the amalgamation check run on every push and +pull request. A pull request that changes only the documentation, Markdown files, GitHub metadata, or license files +skips the compiler workflows (see [`tools/check_path_filters`](https://github.com/nlohmann/json/tree/develop/tools/check_path_filters)). + +A maintainer adds the label :rocket: ready to merge when a pull request is ready; the full set then runs, and again on +every later push to that pull request. A pull request is merged only when the full set passed. + +??? abstract "Jobs of the compiler workflows and their sets" + + The table is generated by [`tools/ci_jobs`](https://github.com/nlohmann/json/tree/develop/tools/ci_jobs), which + also checks in CI that it is up to date and that every job belongs to exactly one set. + + + | Set | Workflow | Job | Configurations | + |-----|----------|-----|----------------| + | PR set | ubuntu.yml | `ci_test_gcc` | - | + | PR set | ubuntu.yml | `ci_meson_install` | - | + | PR set | ubuntu.yml | `ci_static_analysis_ubuntu` | ci_cpplint | + | PR set | ubuntu.yml | `ci_static_analysis_clang` | ci_test_clang; ci_clang_tidy; ci_test_clang_sanitizer; ci_single_binaries | + | PR set | ubuntu.yml | `ci_cmake_options` | ci_test_diagnostics; ci_test_diagnostic_positions; ci_test_noexceptions; ci_test_noglobaludls | + | PR set | ubuntu.yml | `ci_test_compilers_gcc_old` | 4.8 | + | PR set | ubuntu.yml | `ci_test_compilers_clang` | 3.4 | + | PR set | ubuntu.yml | `ci_test_standards_gcc` | 26 | + | PR set | ubuntu.yml | `ci_test_standards_clang` | 26, libcxx | + | PR set | ubuntu.yml | `ci_emscripten` | - | + | full set | ubuntu.yml | `ci_static_analysis_ubuntu_full` | ci_test_single_header; ci_cppcheck; ci_reproducible_tests; ci_non_git_tests; ci_offline_testdata | + | full set | ubuntu.yml | `ci_static_analysis_clang_full` | ci_clang_analyze | + | full set | ubuntu.yml | `ci_cmake_options_full` | ci_cmake_flags; ci_test_noimplicitconversions; ci_test_legacycomparison; ci_test_disableenumserialization; ci_test_disabletuplereferenceconversion; ci_test_skiplibraryversioncheck; ci_test_simdutf; ci_test_strict_nul_handling; ci_test_delete_deprecated_functions; ci_test_no_thread_local | + | full set | ubuntu.yml | `ci_test_compilers_gcc_old_full` | 4.9; 5; 6 | + | full set | ubuntu.yml | `ci_test_compilers_gcc_full` | 7; 8; 9; 10; 11; 12; 13; 14; 15; 16; latest | + | full set | ubuntu.yml | `ci_test_compilers_clang_full` | 3.5; 3.6; 3.7; 3.8; 3.9; 4; 5; 6; 7; 8; 9; 10; 11; 12; 13; 14; 15-bullseye; 16; 17; 18; 19; 20; 21; 22; latest | + | full set | ubuntu.yml | `ci_test_standards_gcc_full` | 11; 14; 17; 20; 23 | + | full set | ubuntu.yml | `ci_test_standards_clang_full` | 11, libcxx; 11, libstdcxx; 14, libcxx; 14, libstdcxx; 17, libcxx; 17, libstdcxx; 20, libcxx; 20, libstdcxx; 23, libcxx; 23, libstdcxx; 26, libstdcxx | + | full set | ubuntu.yml | `ci_module_cpp20` | gcc:latest; silkeh/clang:latest | + | nightly set | ubuntu.yml | `ci_infer` | - | + | nightly set | ubuntu.yml | `ci_static_analysis_ubuntu_nightly` | ci_test_valgrind | + | nightly set | ubuntu.yml | `ci_test_coverage` | - | + | nightly set | ubuntu.yml | `ci_cuda_example` | 11.8.0; 12.1.1; 12.6.3 | + | nightly set | ubuntu.yml | `ci_icpc` | - | + | nightly set | ubuntu.yml | `ci_icpx` | - | + | nightly set | ubuntu.yml | `ci_nvhpc` | - | + | nightly set | ubuntu.yml | `ci_test_arm64` | - | + | every push and PR | macos.yml | `macos-15` | 16.0; 16.1; 16.2; 16.3; 16.4; 26.0.1; 26.1.1; 26.2; 26.3 | + | every push and PR | macos.yml | `macos-26` | 26.4.1; 26.5; 26.6 | + | every push and PR | macos.yml | `xcode_standards` | 11; 14; 17; 20; 23; 26 | + | every push and PR | macos.yml | `swiftpm` | - | + | every push and PR | windows.yml | `mingw` | x64; x86 | + | every push and PR | windows.yml | `msvc` | Debug, Win32, default; Debug, Win32, latest; Debug, x64, default; Debug, x64, latest; Release, Win32, default; Release, Win32, latest; Release, x64, default; Release, x64, latest | + | every push and PR | windows.yml | `msvc-vs2026` | Debug, Win32, default; Debug, Win32, latest; Debug, x64, default; Debug, x64, latest; Release, Win32, default; Release, Win32, latest; Release, x64, default; Release, x64, latest | + | every push and PR | windows.yml | `msvc-arm64` | Debug; Release | + | every push and PR | windows.yml | `clang` | 11.0.1; 12.0.1; 13.0.1; 14.0.6; 15.0.7; 16.0.6; 18.1.8; 19.1.7; 20.1.8 | + | every push and PR | windows.yml | `clang-cl` | Win32; x64 | + | every push and PR | windows.yml | `ci_module_cpp20` | - | + + ## C++ language compliance and compiler compatibility !!! success "Requirement: Compiler support" diff --git a/docs/mkdocs/docs/images/qa_pipeline.svg b/docs/mkdocs/docs/images/qa_pipeline.svg index 4bf54e1e2..86bc7a730 100644 --- a/docs/mkdocs/docs/images/qa_pipeline.svg +++ b/docs/mkdocs/docs/images/qa_pipeline.svg @@ -1,26 +1,31 @@ - + Quality assurance pipeline -Every push or pull request starts these checks in parallel: Build matrix (every push / PR): 50+ compilers: G​CC, Clang, M​SVC, AppleClang, Intel, NVIDIA, ... C++11, 14, 17, 20, 23, and 26 libc++ and libstdc++ warnings: Clang -Weverything, G​CC 300+ warnings; Unit tests (doctest) (every push / PR): every public A​PI function every exception id and message all Unicode code points single header and split headers custom number, string, array, and object types; Coverage (every push / PR): unit tests cover all lines of the code base; Static analysis (every push / PR, scheduled): Clang-Tidy, Cppcheck, Infer Clang Static Analyzer (52 checkers), I​WYU, Codacy CodeQL, Semgrep, Flawfinder (also scheduled); Dynamic analysis (every push / PR): runtime assertions Valgrind (Memcheck) address and undefined behavior sanitizers, integer overflow, nullability violations; Fuzzing (every PR, continuous): CIFuzz on every pull request OSS-Fuzz: billions of inputs every crash fixed with a reproducing unit test; Style (every push / PR): Artistic Style (astyle), enforced in the C​I cpplint with 61 rules; Docs and examples (every push / PR, scheduled): every documentation example is tested; changed output fails site built with warnings as errors; links checked regularly (scheduled); Integration, supply chain (every push / PR, scheduled): CMake 3.5, 3.31.6, and 4.0.0 Meson, Swift Package Manager dependency review on every PR Dependabot daily, OpenSSF Scorecard, R​EUSE. When they pass, the change is merged into develop; a release is made only when no vulnerability alert is open. +A push or pull request starts these checks; the badges say when each runs: Build matrix (every push / PR, before merge): 50+ compilers: G​CC, Clang, M​SVC, AppleClang, Intel, NVIDIA, ... C++11, 14, 17, 20, 23, and 26 libc++ and libstdc++ warnings: Clang -Weverything, G​CC 300+ warnings a subset on every push and PR, all before merge; Intel, NVIDIA, C​UDA, and ARM64 nightly; Unit tests (doctest) (every push / PR): every public A​PI function every exception id and message all Unicode code points single header and split headers custom number, string, array, and object types; Coverage (nightly): unit tests cover all lines of the code base; Static analysis (every push / PR, scheduled): Clang-Tidy, Cppcheck, Infer Clang Static Analyzer (52 checkers), I​WYU, Codacy CodeQL, Semgrep, Flawfinder (also scheduled); Infer nightly; Dynamic analysis (every push / PR, nightly): runtime assertions Valgrind (Memcheck), nightly address and undefined behavior sanitizers, integer overflow, nullability violations; Fuzzing (every PR, continuous): CIFuzz on every pull request OSS-Fuzz: billions of inputs every crash fixed with a reproducing unit test; Style (every push / PR): Artistic Style (astyle), enforced in the C​I cpplint with 61 rules; Docs and examples (every push / PR, scheduled): every documentation example is tested; changed output fails site built with warnings as errors; links checked regularly (scheduled); Integration, supply chain (every push / PR, scheduled): CMake 3.5, 3.31.6, and 4.0.0 Meson, Swift Package Manager dependency review on every PR Dependabot daily, OpenSSF Scorecard, R​EUSE. When they pass, the change is merged into develop; a release is made only when no vulnerability alert is open. - + push or pull request checks, run in parallel on GitHub Actions and AppVeyor - + Build matrix every push / PR + +before merge 50+ compilers: G​CC, Clang, M​SVC, AppleClang, Intel, NVIDIA, ... C++11, 14, 17, 20, 23, and 26 libc++ and libstdc++ warnings: Clang -Weverything, G​CC 300+ warnings +a subset on every push and PR, +all before merge; Intel, NVIDIA, +C​UDA, and ARM64 nightly - + Unit tests (doctest) every push / PR @@ -32,209 +37,218 @@ and object types - + Coverage - -every push / PR + +nightly unit tests cover all lines of the code base - -Static analysis - -every push / PR - -scheduled -Clang-Tidy, Cppcheck, Infer -Clang Static Analyzer (52 -checkers), I​WYU, Codacy -CodeQL, Semgrep, Flawfinder -(also scheduled) + +Static analysis + +every push / PR + +scheduled +Clang-Tidy, Cppcheck, Infer +Clang Static Analyzer (52 +checkers), I​WYU, Codacy +CodeQL, Semgrep, Flawfinder +(also scheduled); Infer nightly - -Dynamic analysis - -every push / PR -runtime assertions -Valgrind (Memcheck) -address and undefined behavior -sanitizers, integer overflow, -nullability violations + +Dynamic analysis + +every push / PR + +nightly +runtime assertions +Valgrind (Memcheck), nightly +address and undefined behavior +sanitizers, integer overflow, +nullability violations - -Fuzzing - -every PR - -continuous -CIFuzz on every pull request -OSS-Fuzz: billions of inputs -every crash fixed with a -reproducing unit test + +Fuzzing + +every PR + +continuous +CIFuzz on every pull request +OSS-Fuzz: billions of inputs +every crash fixed with a +reproducing unit test - -Style - -every push / PR -Artistic Style (astyle), -enforced in the C​I -cpplint with 61 rules + +Style + +every push / PR +Artistic Style (astyle), +enforced in the C​I +cpplint with 61 rules - -Docs and examples - -every push / PR - -scheduled -every documentation example is -tested; changed output fails -site built with warnings -as errors; links checked -regularly (scheduled) + +Docs and examples + +every push / PR + +scheduled +every documentation example is +tested; changed output fails +site built with warnings +as errors; links checked +regularly (scheduled) - -Integration, supply chain - -every push / PR - -scheduled -CMake 3.5, 3.31.6, and 4.0.0 -Meson, Swift Package Manager -dependency review on every PR -Dependabot daily, OpenSSF -Scorecard, R​EUSE + +Integration, supply chain + +every push / PR + +scheduled +CMake 3.5, 3.31.6, and 4.0.0 +Meson, Swift Package Manager +dependency review on every PR +Dependabot daily, OpenSSF +Scorecard, R​EUSE - - -merge into develop - - -release -no open vulnerability alert + + +merge into develop + + +release +no open vulnerability alert - + Quality assurance pipeline -Every push or pull request starts these checks in parallel: Build matrix (every push / PR): 50+ compilers: G​CC, Clang, M​SVC, AppleClang, Intel, NVIDIA, ... C++11, 14, 17, 20, 23, and 26 libc++ and libstdc++ warnings: Clang -Weverything, G​CC 300+ warnings; Unit tests (doctest) (every push / PR): every public A​PI function every exception id and message all Unicode code points single header and split headers custom number, string, array, and object types; Coverage (every push / PR): unit tests cover all lines of the code base; Static analysis (every push / PR, scheduled): Clang-Tidy, Cppcheck, Infer Clang Static Analyzer (52 checkers), I​WYU, Codacy CodeQL, Semgrep, Flawfinder (also scheduled); Dynamic analysis (every push / PR): runtime assertions Valgrind (Memcheck) address and undefined behavior sanitizers, integer overflow, nullability violations; Fuzzing (every PR, continuous): CIFuzz on every pull request OSS-Fuzz: billions of inputs every crash fixed with a reproducing unit test; Style (every push / PR): Artistic Style (astyle), enforced in the C​I cpplint with 61 rules; Docs and examples (every push / PR, scheduled): every documentation example is tested; changed output fails site built with warnings as errors; links checked regularly (scheduled); Integration, supply chain (every push / PR, scheduled): CMake 3.5, 3.31.6, and 4.0.0 Meson, Swift Package Manager dependency review on every PR Dependabot daily, OpenSSF Scorecard, R​EUSE. When they pass, the change is merged into develop; a release is made only when no vulnerability alert is open. +A push or pull request starts these checks; the badges say when each runs: Build matrix (every push / PR, before merge): 50+ compilers: G​CC, Clang, M​SVC, AppleClang, Intel, NVIDIA, ... C++11, 14, 17, 20, 23, and 26 libc++ and libstdc++ warnings: Clang -Weverything, G​CC 300+ warnings a subset on every push and PR, all before merge; Intel, NVIDIA, C​UDA, and ARM64 nightly; Unit tests (doctest) (every push / PR): every public A​PI function every exception id and message all Unicode code points single header and split headers custom number, string, array, and object types; Coverage (nightly): unit tests cover all lines of the code base; Static analysis (every push / PR, scheduled): Clang-Tidy, Cppcheck, Infer Clang Static Analyzer (52 checkers), I​WYU, Codacy CodeQL, Semgrep, Flawfinder (also scheduled); Infer nightly; Dynamic analysis (every push / PR, nightly): runtime assertions Valgrind (Memcheck), nightly address and undefined behavior sanitizers, integer overflow, nullability violations; Fuzzing (every PR, continuous): CIFuzz on every pull request OSS-Fuzz: billions of inputs every crash fixed with a reproducing unit test; Style (every push / PR): Artistic Style (astyle), enforced in the C​I cpplint with 61 rules; Docs and examples (every push / PR, scheduled): every documentation example is tested; changed output fails site built with warnings as errors; links checked regularly (scheduled); Integration, supply chain (every push / PR, scheduled): CMake 3.5, 3.31.6, and 4.0.0 Meson, Swift Package Manager dependency review on every PR Dependabot daily, OpenSSF Scorecard, R​EUSE. When they pass, the change is merged into develop; a release is made only when no vulnerability alert is open. - + push or pull request checks, run in parallel - + Build matrix every push / PR + +before merge 50+ compilers: G​CC, Clang, M​SVC, AppleClang, Intel, NVIDIA, ... C++11, 14, 17, 20, 23, and 26 libc++ and libstdc++ warnings: Clang -Weverything, G​CC 300+ warnings +a subset on every push and PR, +all before merge; Intel, NVIDIA, +C​UDA, and ARM64 nightly - -Unit tests (doctest) - -every push / PR -every public A​PI function -every exception id and message -all Unicode code points -single header and split headers -custom number, string, array, -and object types + +Unit tests (doctest) + +every push / PR +every public A​PI function +every exception id and message +all Unicode code points +single header and split headers +custom number, string, array, +and object types - -Coverage - -every push / PR -unit tests cover all lines -of the code base + +Coverage + +nightly +unit tests cover all lines +of the code base - -Static analysis - -every push / PR - -scheduled -Clang-Tidy, Cppcheck, Infer -Clang Static Analyzer (52 -checkers), I​WYU, Codacy -CodeQL, Semgrep, Flawfinder -(also scheduled) + +Static analysis + +every push / PR + +scheduled +Clang-Tidy, Cppcheck, Infer +Clang Static Analyzer (52 +checkers), I​WYU, Codacy +CodeQL, Semgrep, Flawfinder +(also scheduled); Infer nightly - -Dynamic analysis - -every push / PR -runtime assertions -Valgrind (Memcheck) -address and undefined behavior -sanitizers, integer overflow, -nullability violations + +Dynamic analysis + +every push / PR + +nightly +runtime assertions +Valgrind (Memcheck), nightly +address and undefined behavior +sanitizers, integer overflow, +nullability violations - -Fuzzing - -every PR - -continuous -CIFuzz on every pull request -OSS-Fuzz: billions of inputs -every crash fixed with a -reproducing unit test + +Fuzzing + +every PR + +continuous +CIFuzz on every pull request +OSS-Fuzz: billions of inputs +every crash fixed with a +reproducing unit test - -Style - -every push / PR -Artistic Style (astyle), -enforced in the C​I -cpplint with 61 rules + +Style + +every push / PR +Artistic Style (astyle), +enforced in the C​I +cpplint with 61 rules - -Docs and examples - -every push / PR - -scheduled -every documentation example is -tested; changed output fails -site built with warnings -as errors; links checked -regularly (scheduled) + +Docs and examples + +every push / PR + +scheduled +every documentation example is +tested; changed output fails +site built with warnings +as errors; links checked +regularly (scheduled) - -Integration, supply chain - -every push / PR - -scheduled -CMake 3.5, 3.31.6, and 4.0.0 -Meson, Swift Package Manager -dependency review on every PR -Dependabot daily, OpenSSF -Scorecard, R​EUSE + +Integration, supply chain + +every push / PR + +scheduled +CMake 3.5, 3.31.6, and 4.0.0 +Meson, Swift Package Manager +dependency review on every PR +Dependabot daily, OpenSSF +Scorecard, R​EUSE - - -merge into develop - - -release -no open vulnerability alert + + +merge into develop + + +release +no open vulnerability alert diff --git a/docs/mkdocs/scripts/figures/qa_pipeline.py b/docs/mkdocs/scripts/figures/qa_pipeline.py index b706e76a4..e09614302 100644 --- a/docs/mkdocs/scripts/figures/qa_pipeline.py +++ b/docs/mkdocs/scripts/figures/qa_pipeline.py @@ -22,19 +22,24 @@ P = "nj-exceptions-" TRIGGERS = { "push": ("every push / PR", "push"), "pr": ("every PR", "push"), + "merge": ("before merge", "push"), + "nightly": ("nightly", "sched"), "sched": ("scheduled", "sched"), "cont": ("continuous", "cont"), } # title, triggers, lines (at most 31 characters, so they fit a card of the three-column drawing) LANES = [ - ("Build matrix", ["push"], [ + ("Build matrix", ["push", "merge"], [ "50+ compilers: GCC, Clang, MSVC,", "AppleClang, Intel, NVIDIA, ...", "C++11, 14, 17, 20, 23, and 26", "libc++ and libstdc++", "warnings: Clang -Weverything,", "GCC 300+ warnings", + "a subset on every push and PR,", + "all before merge; Intel, NVIDIA,", + "CUDA, and ARM64 nightly", ]), ("Unit tests (doctest)", ["push"], [ "every public API function", @@ -44,7 +49,7 @@ LANES = [ "custom number, string, array,", "and object types", ]), - ("Coverage", ["push"], [ + ("Coverage", ["nightly"], [ "unit tests cover all lines", "of the code base", ]), @@ -53,11 +58,11 @@ LANES = [ "Clang Static Analyzer (52", "checkers), IWYU, Codacy", "CodeQL, Semgrep, Flawfinder", - "(also scheduled)", + "(also scheduled); Infer nightly", ]), - ("Dynamic analysis", ["push"], [ + ("Dynamic analysis", ["push", "nightly"], [ "runtime assertions", - "Valgrind (Memcheck)", + "Valgrind (Memcheck), nightly", "address and undefined behavior", "sanitizers, integer overflow,", "nullability violations", @@ -178,7 +183,7 @@ def drawing(W, cols, suffix): defs = ('' '' % mid) tid, did = "nj-qa-title-" + suffix, "nj-qa-desc-" + suffix - desc = ("Every push or pull request starts these checks in parallel: " + + desc = ("A push or pull request starts these checks; the badges say when each runs: " + "; ".join("%s (%s): %s" % (t, ", ".join(TRIGGERS[k][0] for k in trig), " ".join(lines)) for t, trig, lines in LANES) + ". When they pass, the change is merged into develop; a release is made only when no vulnerability alert is open.") out = ['' diff --git a/tools/check_path_filters/check_path_filters.py b/tools/check_path_filters/check_path_filters.py index 6a00ae508..633cd1a09 100755 --- a/tools/check_path_filters/check_path_filters.py +++ b/tools/check_path_filters/check_path_filters.py @@ -69,6 +69,10 @@ KNOWN_REFERENCES = [ ('tests/src/make_test_data_available.hpp', 'README.md', 'a file of the external test data repository'), ('tools/check_build_options/check_build_options.py', 'docs/', 'builds the path of package_managers.md, which ' 'PATHS includes'), + ('tools/ci_jobs/ci_jobs.py', 'docs/mkdocs/docs/community/quality_assurance.md', + 'checks the job table; runs in repository_checks.yml, which has no path filter'), + ('tools/ci_jobs/ci_jobs.py', '.github/workflows/repository_checks.yml', 'read by --compare, a manual check'), + ('tools/ci_jobs/ci_jobs.py', '.github/workflows/check_amalgamation.yml', 'read by --compare, a manual check'), ] diff --git a/tools/ci_jobs/README.md b/tools/ci_jobs/README.md new file mode 100644 index 000000000..83390eda8 --- /dev/null +++ b/tools/ci_jobs/README.md @@ -0,0 +1,27 @@ +# ci_jobs + +The compiler jobs of [`ubuntu.yml`](../../.github/workflows/ubuntu.yml) are split into three sets, chosen by each job's +`if:` condition (see [How CI runs](https://json.nlohmann.me/community/quality_assurance/#how-ci-runs)): + +- **PR set**: every push to a pull request, and every push to `develop`, `master`, and `release/*`; +- **full set**: after the PR set passed, on pull requests with the label "🚀 ready to merge" and on every push; +- **nightly set**: every night on `develop`, on every push to `master` and `release/*`, and when started by hand. + +A matrix whose entries belong to different sets is split into one job per set: the full-set job gets the suffix +`_full`, the nightly-set job `_nightly`, and both share their steps with the PR-set job through YAML anchors. + +[`ci_jobs.py`](ci_jobs.py) checks that + +- every job of `ubuntu.yml` has the `if:` condition of exactly one set (the conditions are in `SETS`), +- every full-set job waits for (`needs:`) exactly the PR set, and `nightly_report` for exactly the nightly set, and +- the job table in [`quality_assurance.md`](../../docs/mkdocs/docs/community/quality_assurance.md) is up to date. + +```shell +python3 -m venv venv && venv/bin/pip install -r tools/ci_jobs/requirements.txt +venv/bin/python tools/ci_jobs/ci_jobs.py # check +venv/bin/python tools/ci_jobs/ci_jobs.py --markdown # print the job table for the documentation +venv/bin/python tools/ci_jobs/ci_jobs.py --compare origin/develop # list configurations added or removed since develop +``` + +Use `--compare` when changing the workflows: a configuration that disappears fails it, so no compiler, version, or +standard is dropped by accident when jobs are split, merged, or moved between sets. diff --git a/tools/ci_jobs/ci_jobs.py b/tools/ci_jobs/ci_jobs.py new file mode 100755 index 000000000..faa8ee741 --- /dev/null +++ b/tools/ci_jobs/ci_jobs.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""List and check the jobs of the compiler workflows and the set each job belongs to. + +The jobs of a tiered workflow (ubuntu.yml) belong to one of three sets, chosen +by the job's `if:` condition: + +- PR set: every pull request push and every push, +- full set: after the PR set passed, on pushes and on pull requests with the + "🚀 ready to merge" label, +- nightly set: nightly on develop, on pushes to master and release/*, and when + started by hand. + +The jobs of the other workflows run on every push and pull request. + +The script checks that every job of a tiered workflow belongs to exactly one +set, that the full set waits for the whole PR set, that the nightly report +waits for the whole nightly set, and that the job table in +docs/mkdocs/docs/community/quality_assurance.md is up to date. + + --markdown print the job table for the documentation + --compare REF list the configurations (job and matrix values) added or + removed since the git revision REF; fails if one was removed +""" + +import argparse +import collections +import itertools +import os +import re +import subprocess # nosec B404 +import sys + +import yaml + +REPO_ROOT = os.path.normpath(os.path.join(sys.path[0], '..', '..')) +DOCS = 'docs/mkdocs/docs/community/quality_assurance.md' +BEGIN, END = '', '' + +WORKFLOWS = ['.github/workflows/ubuntu.yml', '.github/workflows/macos.yml', '.github/workflows/windows.yml'] +TIERED = ['.github/workflows/ubuntu.yml'] +# --compare also counts the jobs of these workflows, so a job moved between workflows is not a removed one +COMPARED = WORKFLOWS + ['.github/workflows/repository_checks.yml', '.github/workflows/check_amalgamation.yml'] + +LABEL = '🚀 ready to merge' +SETS = collections.OrderedDict([ + ('PR set', "${{ github.event_name != 'schedule' && (github.event.action != 'labeled' || " + "github.event.label.name == '" + LABEL + "') }}"), + ('full set', "${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || " + "contains(github.event.pull_request.labels.*.name, '" + LABEL + "') }}"), + ('nightly set', "${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || " + "(github.event_name == 'push' && github.ref != 'refs/heads/develop') }}"), +]) +UNTIERED = 'every push and PR' +REPORT_JOB = 'nightly_report' + +# a job split by set keeps its name and gets one of these suffixes +SUFFIXES = {'full set': '_full', 'nightly set': '_nightly'} + + +def read(root, path, ref=None): + if ref is None: + with open(os.path.join(root, path), encoding='utf-8') as f: + return f.read() + # fixed command; ref is a git revision given on the command line + return subprocess.check_output(['git', 'show', '{}:{}'.format(ref, path)], cwd=root, # nosec B603, B607 + stderr=subprocess.DEVNULL).decode('utf-8') + + +def add_include(combos, keys, inc): + """Apply one include entry: extend the matching configurations, or add it as a new one.""" + matching = [c for c in combos if all(c[k] == v for k, v in inc.items() if k in keys)] + for c in matching: + c.update({k: v for k, v in inc.items() if k not in keys}) + if not matching: + combos.append(dict(inc)) + + +def expand(matrix): + """Return the configurations of a job's strategy.matrix like GitHub Actions does.""" + if not matrix: + return [{}] + keys = [k for k in matrix if k not in ('include', 'exclude')] + combos = [dict(zip(keys, values)) for values in itertools.product(*(matrix[k] for k in keys))] if keys else [] + for ex in matrix.get('exclude') or []: + combos = [c for c in combos if any(c.get(k) != v for k, v in ex.items())] + for inc in matrix.get('include') or []: + add_include(combos, keys, inc) + return combos + + +def jobs_of(root, workflow, ref=None): + """Yield (job id, set, list of configurations) for every job of a workflow.""" + data = yaml.safe_load(read(root, workflow, ref)) + for job_id, job in data['jobs'].items(): + condition = job.get('if') + job_set = UNTIERED if condition is None else next((s for s, c in SETS.items() if c == condition), None) + matrix = (job.get('strategy') or {}).get('matrix') + yield job_id, job_set, job, expand(matrix) + + +def base_id(job_id, job_set): + suffix = SUFFIXES.get(job_set) + return job_id[:-len(suffix)] if suffix and job_id.endswith(suffix) else job_id + + +def label(config): + return ', '.join(str(v) for k, v in config.items() if k != 'os' or len(config) == 1) + + +def configurations(root, ref=None): + """Return a Counter of (job name without set suffix, configuration) over the compared workflows.""" + result = collections.Counter() + for workflow in COMPARED: + try: + jobs = list(jobs_of(root, workflow, ref)) + except (OSError, subprocess.CalledProcessError): + continue # the workflow does not exist in this revision + for job_id, job_set, _, configs in jobs: + if job_id == REPORT_JOB: + continue + for c in configs: + result[(base_id(job_id, job_set), tuple(sorted((k, str(v)) for k, v in c.items())))] += 1 + return result + + +def markdown(root): + rows = [] + order = list(SETS) + [UNTIERED] + for workflow in WORKFLOWS: + for job_id, job_set, _, configs in jobs_of(root, workflow): + if job_id == REPORT_JOB: + continue + labels = [label(c) for c in configs if c] + rows.append((order.index(job_set) if job_set in order else len(order), os.path.basename(workflow), job_id, + job_set or 'no set', '; '.join(labels) or '-')) + rows.sort(key=lambda r: (r[0], r[1])) + lines = ['| Set | Workflow | Job | Configurations |', '|-----|----------|-----|----------------|'] + lines += ['| {} | {} | `{}` | {} |'.format(s, w, j, c) for _, w, j, s, c in rows] + return lines + + +def check_sets(workflow, jobs): + """Return the errors in the sets of a tiered workflow's jobs and the jobs of each set.""" + errors = [] + by_set = collections.defaultdict(set) + for job_id, job_set, _, _ in jobs: + if job_id == REPORT_JOB: + continue + if job_set in SETS: + by_set[job_set].add(job_id) + else: + errors.append('{}: job {} has an if: condition of no set; use one of the conditions in ' + 'tools/ci_jobs/ci_jobs.py'.format(workflow, job_id)) + return errors, by_set + + +def check_needs(workflow, jobs, by_set): + """Return the errors in the needs: of a tiered workflow's jobs.""" + required = {'full set': by_set['PR set'], 'PR set': set(), 'nightly set': set()} + errors = [] + for job_id, job_set, job, _ in jobs: + needs = job.get('needs') or [] + needs = {needs} if isinstance(needs, str) else set(needs) + expected = by_set['nightly set'] if job_id == REPORT_JOB else required.get(job_set, needs) + if needs != expected: + errors.append('{}: job {} must need exactly: {}'.format(workflow, job_id, ', '.join(sorted(expected)) or 'no job')) + return errors + + +def check_docs(root): + """Return an error if the job table in the documentation is not up to date.""" + text = read(root, DOCS) + block = re.search(re.escape(BEGIN) + r'\n(.*?)\n[ \t]*' + re.escape(END), text, re.S) + if not block: + return ['{}: markers {} and {} not found'.format(DOCS, BEGIN, END)] + indent = re.match(r'[ \t]*', block.group(1)).group(0) + if block.group(1) != '\n'.join(indent + line for line in markdown(root)): + return ['{}: the job table is out of date; replace it with the output of ' + 'python3 tools/ci_jobs/ci_jobs.py --markdown'.format(DOCS)] + return [] + + +def check(root): + errors = [] + for workflow in TIERED: + jobs = list(jobs_of(root, workflow)) + set_errors, by_set = check_sets(workflow, jobs) + errors += set_errors + check_needs(workflow, jobs, by_set) + errors += check_docs(root) + + for e in errors: + print(e) + if not errors: + counts = collections.Counter() + for workflow in WORKFLOWS: + for job_id, job_set, _, configs in jobs_of(root, workflow): + if job_id != REPORT_JOB: + counts[job_set] += len(configs) + print('CI jobs OK: ' + ', '.join('{} {}'.format(n, s) for s, n in counts.items())) + return 1 if errors else 0 + + +def compare(root, ref): + before, after = configurations(root, ref), configurations(root) + removed, added = before - after, after - before + for title, diff in (('removed', removed), ('added', added)): + for job, config in sorted(diff.elements()): + print('{}: {} ({})'.format(title, job, ', '.join('{}={}'.format(k, v) for k, v in config))) + print('{} configurations before, {} after; {} removed, {} added'.format( + sum(before.values()), sum(after.values()), sum(removed.values()), sum(added.values()))) + return 1 if removed else 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument('--root', default=REPO_ROOT, help='repository root') + group = parser.add_mutually_exclusive_group() + group.add_argument('--markdown', action='store_true', help='print the job table for the documentation') + group.add_argument('--compare', metavar='REF', help='list the configurations added or removed since REF') + args = parser.parse_args() + if args.markdown: + print('\n'.join(markdown(args.root))) + return 0 + if args.compare: + return compare(args.root, args.compare) + return check(args.root) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/ci_jobs/requirements.txt b/tools/ci_jobs/requirements.txt new file mode 100644 index 000000000..f62ce0c56 --- /dev/null +++ b/tools/ci_jobs/requirements.txt @@ -0,0 +1 @@ +PyYAML==6.0.3