Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4b538d80f6 | ||
|
|
7a4e6b7a5d | ||
|
|
fd37b04813 | ||
|
|
bd442d5d07 | ||
|
|
eae3c8217b | ||
|
|
960c234cc6 | ||
|
|
b6e24d19a9 |
No files matched your search
@@ -154,6 +154,11 @@ and each page's title (H1) and declaration by
|
||||
[`docs/docset/generate_docset.py`](https://github.com/nlohmann/json/blob/develop/docs/docset/generate_docset.py);
|
||||
`make build` reports API pages that cannot be classified.
|
||||
|
||||
A pull request that changes only the documentation, Markdown files, GitHub metadata, or license files skips the
|
||||
compiler workflows (Ubuntu, macOS, Windows); the documentation and license checks still run. See
|
||||
[`tools/check_path_filters`](https://github.com/nlohmann/json/tree/develop/tools/check_path_filters) for the list of
|
||||
skipped files and what to do when a compiler job starts reading one of them.
|
||||
|
||||
### Amalgamate the source code
|
||||
|
||||
The single-header files
|
||||
|
||||
@@ -7,6 +7,25 @@ on:
|
||||
- master
|
||||
- release/*
|
||||
pull_request:
|
||||
# Skip pull requests that change only files no job here reads. The list is checked by
|
||||
# tools/check_path_filters/check_path_filters.py (repository_checks.yml); change both together.
|
||||
paths:
|
||||
- '**'
|
||||
- '!docs/**'
|
||||
- '!**.md'
|
||||
- '!.github/*'
|
||||
- '!.github/ISSUE_TEMPLATE/**'
|
||||
- '!.github/external_ci/**'
|
||||
- '!.github/workflows/**'
|
||||
- '!.reuse/**'
|
||||
- '!LICENSES/**'
|
||||
- '!CITATION.cff'
|
||||
- '!REUSE.toml'
|
||||
# read by tools/check_build_options in the ci_meson_install job (ubuntu.yml)
|
||||
- 'docs/mkdocs/docs/integration/package_managers.md'
|
||||
# built by the swiftpm job (macos.yml)
|
||||
- 'docs/mkdocs/docs/integration/swift/**'
|
||||
- '.github/workflows/macos.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
name: Repository checks
|
||||
|
||||
# Checks that concern files the compiler workflows (ubuntu.yml, macos.yml,
|
||||
# windows.yml) skip with their path filter: the documentation, the license
|
||||
# information, and the path filter itself. This workflow has no path filter.
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- develop
|
||||
- master
|
||||
- release/*
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
check_path_filters:
|
||||
runs-on: ubuntu-26.04
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Check the path filters of the compiler workflows
|
||||
run: python3 tools/check_path_filters/check_path_filters.py
|
||||
|
||||
ci_reuse_compliance:
|
||||
runs-on: ubuntu-26.04
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Get latest CMake and ninja
|
||||
uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
|
||||
- name: Run CMake
|
||||
run: cmake -S . -B build -DJSON_CI=On
|
||||
- name: Build
|
||||
run: cmake --build build --target ci_reuse_compliance
|
||||
|
||||
ci_test_documentation:
|
||||
runs-on: ubuntu-26.04
|
||||
strategy:
|
||||
matrix:
|
||||
target: [ci_test_examples, ci_test_build_documentation, ci_test_documentation_mermaid]
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
# the git-revision-date-localized plugin needs the history; a shallow clone makes the strict build fail
|
||||
fetch-depth: ${{ matrix.target == 'ci_test_build_documentation' && '0' || '1' }}
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
if: matrix.target == 'ci_test_documentation_mermaid'
|
||||
with:
|
||||
node-version: 24
|
||||
- name: Run CMake
|
||||
run: cmake -S . -B build -DJSON_CI=On
|
||||
- name: Build
|
||||
run: cmake --build build --target ${{ matrix.target }}
|
||||
@@ -7,6 +7,25 @@ on:
|
||||
- master
|
||||
- release/*
|
||||
pull_request:
|
||||
# Skip pull requests that change only files no job here reads. The list is checked by
|
||||
# tools/check_path_filters/check_path_filters.py (repository_checks.yml); change both together.
|
||||
paths:
|
||||
- '**'
|
||||
- '!docs/**'
|
||||
- '!**.md'
|
||||
- '!.github/*'
|
||||
- '!.github/ISSUE_TEMPLATE/**'
|
||||
- '!.github/external_ci/**'
|
||||
- '!.github/workflows/**'
|
||||
- '!.reuse/**'
|
||||
- '!LICENSES/**'
|
||||
- '!CITATION.cff'
|
||||
- '!REUSE.toml'
|
||||
# read by tools/check_build_options in the ci_meson_install job (ubuntu.yml)
|
||||
- 'docs/mkdocs/docs/integration/package_managers.md'
|
||||
# built by the swiftpm job (macos.yml)
|
||||
- 'docs/mkdocs/docs/integration/swift/**'
|
||||
- '.github/workflows/ubuntu.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
@@ -109,7 +128,7 @@ jobs:
|
||||
runs-on: ubuntu-26.04
|
||||
strategy:
|
||||
matrix:
|
||||
target: [ci_test_single_header, ci_cppcheck, ci_cpplint, ci_reproducible_tests, ci_non_git_tests, ci_offline_testdata, ci_reuse_compliance, ci_test_valgrind]
|
||||
target: [ci_test_single_header, ci_cppcheck, ci_cpplint, ci_reproducible_tests, ci_non_git_tests, ci_offline_testdata, ci_test_valgrind]
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2
|
||||
@@ -449,31 +468,6 @@ jobs:
|
||||
- name: Build
|
||||
run: cmake --build build
|
||||
|
||||
ci_test_documentation:
|
||||
runs-on: ubuntu-26.04
|
||||
strategy:
|
||||
matrix:
|
||||
target: [ci_test_examples, ci_test_build_documentation, ci_test_documentation_mermaid]
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
# the git-revision-date-localized plugin needs the history; a shallow clone makes the strict build fail
|
||||
fetch-depth: ${{ matrix.target == 'ci_test_build_documentation' && '0' || '1' }}
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
if: matrix.target == 'ci_test_documentation_mermaid'
|
||||
with:
|
||||
node-version: 24
|
||||
- name: Run CMake
|
||||
run: cmake -S . -B build -DJSON_CI=On
|
||||
- name: Build
|
||||
run: cmake --build build --target ${{ matrix.target }}
|
||||
|
||||
# Native Linux ARM64 runner with the latest GCC. The ubuntu-24.04-arm label is
|
||||
# only available for public repositories.
|
||||
ci_test_arm64:
|
||||
|
||||
@@ -7,6 +7,25 @@ on:
|
||||
- master
|
||||
- release/*
|
||||
pull_request:
|
||||
# Skip pull requests that change only files no job here reads. The list is checked by
|
||||
# tools/check_path_filters/check_path_filters.py (repository_checks.yml); change both together.
|
||||
paths:
|
||||
- '**'
|
||||
- '!docs/**'
|
||||
- '!**.md'
|
||||
- '!.github/*'
|
||||
- '!.github/ISSUE_TEMPLATE/**'
|
||||
- '!.github/external_ci/**'
|
||||
- '!.github/workflows/**'
|
||||
- '!.reuse/**'
|
||||
- '!LICENSES/**'
|
||||
- '!CITATION.cff'
|
||||
- '!REUSE.toml'
|
||||
# read by tools/check_build_options in the ci_meson_install job (ubuntu.yml)
|
||||
- 'docs/mkdocs/docs/integration/package_managers.md'
|
||||
# built by the swiftpm job (macos.yml)
|
||||
- 'docs/mkdocs/docs/integration/swift/**'
|
||||
- '.github/workflows/windows.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -465,17 +465,15 @@ class binary_reader
|
||||
}
|
||||
|
||||
const std::size_t element_type_parse_position = chars_read;
|
||||
// an array's elements are named "0", "1", ... in the wire format,
|
||||
// and those names are not passed on: they are skipped unless
|
||||
// error_handler_t::strict must reject ill-formed UTF-8 in them
|
||||
const bool is_object = container_stack.back().is_object;
|
||||
key.clear();
|
||||
if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key, is_object || error_handler == error_handler_t::strict)))
|
||||
if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (is_object && !sax->key(key))
|
||||
// an array's elements are named "0", "1", ... in the wire format,
|
||||
// and those names are not passed on
|
||||
if (container_stack.back().is_object && !sax->key(key))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -491,18 +489,14 @@ class binary_reader
|
||||
@brief Parses a C-style string from the BSON input.
|
||||
@param[in,out] result A reference to the string variable where the read
|
||||
string is to be stored.
|
||||
@param[in] store whether the characters are needed; if not (the names of
|
||||
an array's elements), they are skipped and @a result may
|
||||
be left unchanged, and they are not checked against
|
||||
@ref error_handler
|
||||
@return `true` if the \\x00-byte indicating the end of the string was
|
||||
encountered before the EOF; false` indicates an unexpected EOF.
|
||||
*/
|
||||
bool get_bson_cstr(string_t& result, const bool store = true)
|
||||
bool get_bson_cstr(string_t& result)
|
||||
{
|
||||
if (get_bson_cstr_bulk(result, store, std::integral_constant<bool, bulk_scan> {}))
|
||||
if (get_bson_cstr_bulk(result, std::integral_constant<bool, bulk_scan> {}))
|
||||
{
|
||||
return !store || check_string_utf8(result, "key");
|
||||
return check_string_utf8(result, "key");
|
||||
}
|
||||
|
||||
auto out = std::back_inserter(result);
|
||||
@@ -515,7 +509,7 @@ class binary_reader
|
||||
}
|
||||
if (current == 0x00)
|
||||
{
|
||||
return !store || check_string_utf8(result, "key");
|
||||
return check_string_utf8(result, "key");
|
||||
}
|
||||
*out++ = static_cast<typename string_t::value_type>(current);
|
||||
}
|
||||
@@ -525,11 +519,10 @@ class binary_reader
|
||||
@brief read a C-style string from contiguous input in one step
|
||||
|
||||
@param[in,out] result the string to append to
|
||||
@param[in] store whether to append the characters to @a result
|
||||
@return whether the string was read; if the input has no \\x00-byte, nothing
|
||||
is read, and @ref get_bson_cstr reports the end of the input
|
||||
*/
|
||||
bool get_bson_cstr_bulk(string_t& result, const bool store, std::true_type /*bulk*/)
|
||||
bool get_bson_cstr_bulk(string_t& result, std::true_type /*bulk*/)
|
||||
{
|
||||
const std::size_t remaining = ia.bulk_remaining();
|
||||
if (remaining == 0)
|
||||
@@ -537,15 +530,18 @@ class binary_reader
|
||||
return false;
|
||||
}
|
||||
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
||||
const std::size_t length = find_zero_byte(data, remaining);
|
||||
// a plain loop rather than std::memchr: most keys are short (array
|
||||
// indices are keys, too), and the call would cost more than it saves
|
||||
std::size_t length = 0;
|
||||
while (length < remaining && data[length] != 0x00)
|
||||
{
|
||||
++length;
|
||||
}
|
||||
if (length == remaining)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if (store)
|
||||
{
|
||||
result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
|
||||
}
|
||||
result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
|
||||
// consume the string and its \x00-byte, as the byte-wise path does
|
||||
ia.bulk_skip(length + 1);
|
||||
chars_read += length + 1;
|
||||
@@ -554,7 +550,7 @@ class binary_reader
|
||||
}
|
||||
|
||||
/// input that is not contiguous: C-style strings are read byte by byte
|
||||
bool get_bson_cstr_bulk(string_t& /*result*/, const bool /*store*/, std::false_type /*bulk*/) const noexcept
|
||||
bool get_bson_cstr_bulk(string_t& /*result*/, std::false_type /*bulk*/) const noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -90,34 +90,6 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t find_string_special(const unsigned char* d
|
||||
return n;
|
||||
}
|
||||
|
||||
// return the index of the first 0x00 byte in [data, data+n), or n if there is
|
||||
// none; scans 8 bytes at a time (BSON's C-style strings)
|
||||
JSON_HEDLEY_ALWAYS_INLINE std::size_t find_zero_byte(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
constexpr std::uint64_t ones = 0x0101010101010101ull;
|
||||
constexpr std::uint64_t high = 0x8080808080808080ull;
|
||||
std::size_t i = 0;
|
||||
for (; i + 8 <= n; i += 8)
|
||||
{
|
||||
const std::uint64_t v = read_eight_bytes(data + i);
|
||||
const std::uint64_t zero = (v - ones) & ~v & high;
|
||||
if (zero != 0)
|
||||
{
|
||||
// the lowest flagged byte is the first zero byte (see
|
||||
// find_string_special())
|
||||
return i + (static_cast<std::size_t>(count_trailing_zeros(zero)) / 8);
|
||||
}
|
||||
}
|
||||
for (; i < n; ++i)
|
||||
{
|
||||
if (data[i] == 0x00)
|
||||
{
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
// classify a byte as one the serializer must NOT copy verbatim when
|
||||
// ensure_ascii is requested: the closing quote, an escape, a control character
|
||||
// (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else -
|
||||
|
||||
@@ -766,6 +766,9 @@ class basic_json_view
|
||||
template<typename, typename> friend class detail::view::view_side;
|
||||
friend iterator;
|
||||
|
||||
/// whether the view is one of an editable document
|
||||
static constexpr bool is_editable = Editable;
|
||||
|
||||
basic_json_view(const document_data* d, const node* n) noexcept
|
||||
: m_doc(d), m_node(n)
|
||||
{}
|
||||
|
||||
Generated
+18
-50
@@ -12458,34 +12458,6 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t find_string_special(const unsigned char* d
|
||||
return n;
|
||||
}
|
||||
|
||||
// return the index of the first 0x00 byte in [data, data+n), or n if there is
|
||||
// none; scans 8 bytes at a time (BSON's C-style strings)
|
||||
JSON_HEDLEY_ALWAYS_INLINE std::size_t find_zero_byte(const unsigned char* data, std::size_t n) noexcept
|
||||
{
|
||||
constexpr std::uint64_t ones = 0x0101010101010101ull;
|
||||
constexpr std::uint64_t high = 0x8080808080808080ull;
|
||||
std::size_t i = 0;
|
||||
for (; i + 8 <= n; i += 8)
|
||||
{
|
||||
const std::uint64_t v = read_eight_bytes(data + i);
|
||||
const std::uint64_t zero = (v - ones) & ~v & high;
|
||||
if (zero != 0)
|
||||
{
|
||||
// the lowest flagged byte is the first zero byte (see
|
||||
// find_string_special())
|
||||
return i + (static_cast<std::size_t>(count_trailing_zeros(zero)) / 8);
|
||||
}
|
||||
}
|
||||
for (; i < n; ++i)
|
||||
{
|
||||
if (data[i] == 0x00)
|
||||
{
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
// classify a byte as one the serializer must NOT copy verbatim when
|
||||
// ensure_ascii is requested: the closing quote, an escape, a control character
|
||||
// (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else -
|
||||
@@ -18269,17 +18241,15 @@ class binary_reader
|
||||
}
|
||||
|
||||
const std::size_t element_type_parse_position = chars_read;
|
||||
// an array's elements are named "0", "1", ... in the wire format,
|
||||
// and those names are not passed on: they are skipped unless
|
||||
// error_handler_t::strict must reject ill-formed UTF-8 in them
|
||||
const bool is_object = container_stack.back().is_object;
|
||||
key.clear();
|
||||
if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key, is_object || error_handler == error_handler_t::strict)))
|
||||
if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (is_object && !sax->key(key))
|
||||
// an array's elements are named "0", "1", ... in the wire format,
|
||||
// and those names are not passed on
|
||||
if (container_stack.back().is_object && !sax->key(key))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -18295,18 +18265,14 @@ class binary_reader
|
||||
@brief Parses a C-style string from the BSON input.
|
||||
@param[in,out] result A reference to the string variable where the read
|
||||
string is to be stored.
|
||||
@param[in] store whether the characters are needed; if not (the names of
|
||||
an array's elements), they are skipped and @a result may
|
||||
be left unchanged, and they are not checked against
|
||||
@ref error_handler
|
||||
@return `true` if the \\x00-byte indicating the end of the string was
|
||||
encountered before the EOF; false` indicates an unexpected EOF.
|
||||
*/
|
||||
bool get_bson_cstr(string_t& result, const bool store = true)
|
||||
bool get_bson_cstr(string_t& result)
|
||||
{
|
||||
if (get_bson_cstr_bulk(result, store, std::integral_constant<bool, bulk_scan> {}))
|
||||
if (get_bson_cstr_bulk(result, std::integral_constant<bool, bulk_scan> {}))
|
||||
{
|
||||
return !store || check_string_utf8(result, "key");
|
||||
return check_string_utf8(result, "key");
|
||||
}
|
||||
|
||||
auto out = std::back_inserter(result);
|
||||
@@ -18319,7 +18285,7 @@ class binary_reader
|
||||
}
|
||||
if (current == 0x00)
|
||||
{
|
||||
return !store || check_string_utf8(result, "key");
|
||||
return check_string_utf8(result, "key");
|
||||
}
|
||||
*out++ = static_cast<typename string_t::value_type>(current);
|
||||
}
|
||||
@@ -18329,11 +18295,10 @@ class binary_reader
|
||||
@brief read a C-style string from contiguous input in one step
|
||||
|
||||
@param[in,out] result the string to append to
|
||||
@param[in] store whether to append the characters to @a result
|
||||
@return whether the string was read; if the input has no \\x00-byte, nothing
|
||||
is read, and @ref get_bson_cstr reports the end of the input
|
||||
*/
|
||||
bool get_bson_cstr_bulk(string_t& result, const bool store, std::true_type /*bulk*/)
|
||||
bool get_bson_cstr_bulk(string_t& result, std::true_type /*bulk*/)
|
||||
{
|
||||
const std::size_t remaining = ia.bulk_remaining();
|
||||
if (remaining == 0)
|
||||
@@ -18341,15 +18306,18 @@ class binary_reader
|
||||
return false;
|
||||
}
|
||||
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
|
||||
const std::size_t length = find_zero_byte(data, remaining);
|
||||
// a plain loop rather than std::memchr: most keys are short (array
|
||||
// indices are keys, too), and the call would cost more than it saves
|
||||
std::size_t length = 0;
|
||||
while (length < remaining && data[length] != 0x00)
|
||||
{
|
||||
++length;
|
||||
}
|
||||
if (length == remaining)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if (store)
|
||||
{
|
||||
result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
|
||||
}
|
||||
result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
|
||||
// consume the string and its \x00-byte, as the byte-wise path does
|
||||
ia.bulk_skip(length + 1);
|
||||
chars_read += length + 1;
|
||||
@@ -18358,7 +18326,7 @@ class binary_reader
|
||||
}
|
||||
|
||||
/// input that is not contiguous: C-style strings are read byte by byte
|
||||
bool get_bson_cstr_bulk(string_t& /*result*/, const bool /*store*/, std::false_type /*bulk*/) const noexcept
|
||||
bool get_bson_cstr_bulk(string_t& /*result*/, std::false_type /*bulk*/) const noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
Generated
+3
@@ -9807,6 +9807,9 @@ class basic_json_view
|
||||
template<typename, typename> friend class detail::view::view_side;
|
||||
friend iterator;
|
||||
|
||||
/// whether the view is one of an editable document
|
||||
static constexpr bool is_editable = Editable;
|
||||
|
||||
basic_json_view(const document_data* d, const node* n) noexcept
|
||||
: m_doc(d), m_node(n)
|
||||
{}
|
||||
|
||||
+2
-35
@@ -1164,13 +1164,7 @@ TEST_CASE("BSON keys from contiguous and stream input")
|
||||
// contiguous input reads a key up to its \x00-byte in one step, a stream
|
||||
// reads it byte by byte; both must give the same value or error for the
|
||||
// complete document and for every truncation of it
|
||||
json j = {{"", true}, {"k", {1, 2, 3}}, {std::string(40, 'x'), {{"nested key", "value"}}}};
|
||||
// keys and array element names ("0" to "11") that end at every position of
|
||||
// an 8-byte word, which contiguous input scans at a time
|
||||
for (std::size_t length = 1; length <= 17; ++length)
|
||||
{
|
||||
j[std::string(length, static_cast<char>('a' + length))] = json::array({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, {11}});
|
||||
}
|
||||
const json j = {{"", true}, {"k", {1, 2, 3}}, {std::string(40, 'x'), {{"nested key", "value"}}}};
|
||||
const std::vector<std::uint8_t> bson = json::to_bson(j);
|
||||
CHECK(json::from_bson(bson) == j);
|
||||
|
||||
@@ -1190,7 +1184,7 @@ TEST_CASE("BSON keys from contiguous and stream input")
|
||||
}
|
||||
try
|
||||
{
|
||||
std::istringstream stream(std::string(reinterpret_cast<const char*>(input.data()), input.size()));
|
||||
std::istringstream stream(std::string(input.begin(), input.end()));
|
||||
from_stream = json::from_bson(stream).dump();
|
||||
}
|
||||
catch (const json::parse_error& e)
|
||||
@@ -1200,33 +1194,6 @@ TEST_CASE("BSON keys from contiguous and stream input")
|
||||
CHECK(from_vector == from_stream);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("BSON array element names")
|
||||
{
|
||||
// an array's element names are not part of the value, so they are not
|
||||
// passed on; error_handler_t::strict still rejects ill-formed UTF-8 in them
|
||||
const std::vector<std::uint8_t> bson =
|
||||
{
|
||||
0x1D, 0x00, 0x00, 0x00, // size (little endian)
|
||||
0x04, 'a', 0x00, // entry: array "a"
|
||||
0x15, 0x00, 0x00, 0x00, // size (little endian)
|
||||
0x08, 0xFF, 0x00, 0x01, // entry: boolean named "\xFF", true
|
||||
0x08, '1', '2', '3', '4', '5', '6', '7', '8', '9', 0x00, 0x00, // entry: boolean named "123456789", false
|
||||
0x00, // end of the array
|
||||
0x00 // end of the document
|
||||
};
|
||||
const json expected = {{"a", {true, false}}};
|
||||
|
||||
CHECK(json::from_bson(bson) == expected);
|
||||
CHECK(json::from_bson(bson, true, true, json::error_handler_t::replace) == expected);
|
||||
CHECK(json::from_bson(bson, true, true, json::error_handler_t::ignore) == expected);
|
||||
|
||||
json _;
|
||||
CHECK_THROWS_WITH_AS(_ = json::from_bson(bson, true, true, json::error_handler_t::strict), "[json.exception.parse_error.113] parse error at byte 14: syntax error while parsing BSON key: invalid string: ill-formed UTF-8 byte", json::parse_error&);
|
||||
|
||||
std::istringstream stream(std::string(reinterpret_cast<const char*>(bson.data()), bson.size()));
|
||||
CHECK(json::from_bson(stream) == expected);
|
||||
}
|
||||
#endif
|
||||
|
||||
TEST_CASE("Negative size of binary value")
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
# check_path_filters
|
||||
|
||||
The compiler workflows [`ubuntu.yml`](../../.github/workflows/ubuntu.yml),
|
||||
[`macos.yml`](../../.github/workflows/macos.yml), and [`windows.yml`](../../.github/workflows/windows.yml) skip pull
|
||||
requests that change only files no compiler job reads: the documentation, Markdown files, GitHub metadata, other
|
||||
workflows, and the license files. Pushes to `develop`, `master`, and `release/*` always run them. The documentation,
|
||||
license, and path-filter checks run in [`repository_checks.yml`](../../.github/workflows/repository_checks.yml), which
|
||||
has no path filter.
|
||||
|
||||
The filter starts with `'**'` and excludes these files explicitly, so a new file runs the compiler workflows unless it
|
||||
is excluded on purpose. [`check_path_filters.py`](check_path_filters.py) holds the filter (`PATHS`) and checks that
|
||||
|
||||
- each of the three workflows has exactly this filter, with its own workflow file included,
|
||||
- the CI targets that read excluded files (`UNFILTERED_TARGETS`) do not run in these workflows, and
|
||||
- no file the filter keeps (sources, tests, build files, tools, workflows) mentions an excluded file or directory
|
||||
outside a comment line, unless the mention is listed in `KNOWN_REFERENCES` with a reason.
|
||||
|
||||
When the check reports a mention, decide whether a compiler job reads that file:
|
||||
|
||||
- If it does, include the file in `PATHS` after the exclusions (as `package_managers.md` and the Swift example are) and
|
||||
copy the new filter into the three workflows.
|
||||
- If it does not, add the mention to `KNOWN_REFERENCES` with the reason.
|
||||
|
||||
Run the check with
|
||||
|
||||
```shell
|
||||
python3 tools/check_path_filters/check_path_filters.py
|
||||
```
|
||||
|
||||
To see which workflows a pull request would run, pass the changed files:
|
||||
|
||||
```shell
|
||||
python3 tools/check_path_filters/check_path_filters.py --changed $(git diff --name-only origin/develop)
|
||||
```
|
||||
+269
@@ -0,0 +1,269 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check the path filters of the compiler workflows.
|
||||
|
||||
ubuntu.yml, macos.yml and windows.yml skip pull requests that change only files
|
||||
no compiler job reads (documentation, Markdown files, GitHub metadata, license
|
||||
files). The filter starts with '**' and excludes these files explicitly, so a
|
||||
new file runs the workflows unless it is excluded on purpose.
|
||||
|
||||
The script checks that
|
||||
|
||||
- each of the three workflows has exactly the filter defined in PATHS below,
|
||||
with its own workflow file included, and
|
||||
- no tracked file that the filter keeps (sources, tests, build files, tools,
|
||||
workflows) mentions a file the filter excludes, unless the mention is listed
|
||||
in KNOWN_REFERENCES with the reason why it does not matter.
|
||||
|
||||
A mention found this way must either be added to PATHS as an included path
|
||||
(the file is read by a compiler job) or to KNOWN_REFERENCES (it is not).
|
||||
|
||||
With --changed FILE..., the script prints instead which workflows a pull
|
||||
request changing these files would run.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import subprocess # nosec B404
|
||||
import sys
|
||||
|
||||
REPO_ROOT = os.path.normpath(os.path.join(sys.path[0], '..', '..'))
|
||||
SELF = 'tools/check_path_filters/check_path_filters.py'
|
||||
|
||||
WORKFLOWS = ['.github/workflows/ubuntu.yml', '.github/workflows/macos.yml', '.github/workflows/windows.yml']
|
||||
|
||||
# The pull_request paths filter of the workflows above; '{workflow}' is replaced
|
||||
# with the workflow's own file. GitHub applies the patterns in order: a file
|
||||
# matching a later pattern is included ('pattern') or excluded ('!pattern').
|
||||
PATHS = [
|
||||
'**',
|
||||
'!docs/**',
|
||||
'!**.md',
|
||||
'!.github/*',
|
||||
'!.github/ISSUE_TEMPLATE/**',
|
||||
'!.github/external_ci/**',
|
||||
'!.github/workflows/**',
|
||||
'!.reuse/**',
|
||||
'!LICENSES/**',
|
||||
'!CITATION.cff',
|
||||
'!REUSE.toml',
|
||||
# read by tools/check_build_options in the ci_meson_install job (ubuntu.yml)
|
||||
'docs/mkdocs/docs/integration/package_managers.md',
|
||||
# built by the swiftpm job (macos.yml)
|
||||
'docs/mkdocs/docs/integration/swift/**',
|
||||
'{workflow}',
|
||||
]
|
||||
|
||||
# Mentions of excluded files that no job of the three workflows depends on:
|
||||
# (file that mentions it, the mention, reason)
|
||||
KNOWN_REFERENCES = [
|
||||
('cmake/ci.cmake', 'docs/', 'ci_test_examples; runs in repository_checks.yml, which has no path filter'),
|
||||
('cmake/ci.cmake', 'docs/mkdocs/', 'ci_test_build_documentation and ci_test_documentation_mermaid; they run in '
|
||||
'repository_checks.yml, which has no path filter'),
|
||||
('cmake/ci.cmake', 'docs/mkdocs/docs/examples/', 'indentation check of ci_test_amalgamation; the same check runs in '
|
||||
'check_amalgamation.yml, which has no path filter'),
|
||||
('Makefile', 'docs/mkdocs/docs/examples/', 'make pretty; runs in check_amalgamation.yml, which has no path filter'),
|
||||
('Makefile', 'ChangeLog.md', 'make ChangeLog.md writes the file; no CI job runs it'),
|
||||
('Makefile', 'tests/fuzzing.md', 'help text'),
|
||||
('Makefile', 'tools/generate_natvis/README.md', 'help text'),
|
||||
('tests/src/make_test_data_available.hpp', 'README.md', 'a file of the external test data repository'),
|
||||
('tools/check_build_options/check_build_options.py', 'docs/', 'builds the path of package_managers.md, which '
|
||||
'PATHS includes'),
|
||||
]
|
||||
|
||||
|
||||
# CI targets that read excluded files; they must run in a workflow without a path filter
|
||||
UNFILTERED_TARGETS = ['ci_test_examples', 'ci_test_build_documentation', 'ci_test_documentation_mermaid',
|
||||
'ci_test_amalgamation', 'ci_reuse_compliance']
|
||||
|
||||
|
||||
def git_files(root):
|
||||
# fixed command, no user input; includes new files not yet added, so a local run
|
||||
# sees what CI sees after the commit
|
||||
out = subprocess.check_output(['git', 'ls-files', '-z', '--cached', '--others', '--exclude-standard'], cwd=root) # nosec B603, B607
|
||||
return [f for f in out.decode('utf-8').split('\0') if f]
|
||||
|
||||
|
||||
def glob_to_regex(pattern):
|
||||
"""Translate a GitHub Actions path pattern to a regular expression."""
|
||||
regex = ''
|
||||
i = 0
|
||||
while i < len(pattern):
|
||||
if pattern.startswith('**', i):
|
||||
regex += '.*'
|
||||
i += 2
|
||||
elif pattern[i] == '*':
|
||||
regex += '[^/]*'
|
||||
i += 1
|
||||
elif pattern[i] == '?':
|
||||
regex += '[^/]'
|
||||
i += 1
|
||||
else:
|
||||
regex += re.escape(pattern[i])
|
||||
i += 1
|
||||
return re.compile(regex + r'\Z')
|
||||
|
||||
|
||||
def compile_filter(paths, workflow):
|
||||
return [(p.startswith('!'), glob_to_regex(p.lstrip('!').replace('{workflow}', workflow))) for p in paths]
|
||||
|
||||
|
||||
def included(compiled, path):
|
||||
result = False
|
||||
for negated, regex in compiled:
|
||||
if regex.match(path):
|
||||
result = not negated
|
||||
return result
|
||||
|
||||
|
||||
def read_filter(root, workflow):
|
||||
"""Return the pull_request paths list of a workflow file."""
|
||||
with open(os.path.join(root, workflow), encoding='utf-8') as f:
|
||||
lines = f.read().splitlines()
|
||||
paths = None
|
||||
in_pull_request = False
|
||||
for line in lines:
|
||||
if re.match(r'^ pull_request:', line):
|
||||
in_pull_request = True
|
||||
continue
|
||||
if in_pull_request:
|
||||
if re.match(r'^ \S', line) or re.match(r'^\S', line):
|
||||
break
|
||||
if re.match(r'^ paths:\s*$', line):
|
||||
paths = []
|
||||
continue
|
||||
m = re.match(r"^ - '(.*)'\s*(#.*)?$", line)
|
||||
if paths is not None and m:
|
||||
paths.append(m.group(1))
|
||||
elif paths is not None and line.strip() and not line.strip().startswith('#'):
|
||||
break
|
||||
return paths
|
||||
|
||||
|
||||
TOKEN = re.compile(r'(["\'/]?)([\w.-]+(?:/[\w.-]+)*)')
|
||||
|
||||
|
||||
COMMENT_LINE = re.compile(r'^[ \t]*(//|#|/?\*|--).*$', re.MULTILINE)
|
||||
|
||||
|
||||
class Mentions:
|
||||
"""Finds the words and paths in a text that name an excluded file or directory."""
|
||||
|
||||
def __init__(self, files, excluded, is_excluded):
|
||||
self.files = set(files)
|
||||
self.excluded = excluded
|
||||
self.dirs = {'/'.join(f.split('/')[:n]) for f in files for n in range(1, f.count('/') + 1)}
|
||||
# an excluded directory: a new file in it would be excluded
|
||||
self.excluded_dirs = {d for d in self.dirs if is_excluded(d + '/new_file')}
|
||||
# base names that only excluded files have, e.g. 'ChangeLog.md' but not 'CMakeLists.txt'
|
||||
self.names = {os.path.basename(f) for f in excluded} - {os.path.basename(f) for f in files if f not in excluded}
|
||||
|
||||
def find(self, text):
|
||||
"""Yield (offset, mention) for every word or path in text that names an excluded file or directory."""
|
||||
for m in TOKEN.finditer(text):
|
||||
parts = m.group(2).rstrip('/').split('/')
|
||||
# a single word names a directory only in a path context, e.g. 'docs' in os.path.join('docs', ...)
|
||||
path_context = len(parts) > 1 or m.group(1)
|
||||
for i in range(len(parts)):
|
||||
path = '/'.join(parts[i:])
|
||||
if path in self.files:
|
||||
if path in self.excluded:
|
||||
yield m.start(2), path
|
||||
break
|
||||
if path in self.dirs and path_context:
|
||||
if path in self.excluded_dirs:
|
||||
yield m.start(2), path + '/'
|
||||
break
|
||||
else:
|
||||
if parts[-1] in self.names and '.' in parts[-1]:
|
||||
yield m.start(2), parts[-1]
|
||||
|
||||
|
||||
def known(path, mention):
|
||||
return any(path == file and mention == known_mention for file, known_mention, _ in KNOWN_REFERENCES)
|
||||
|
||||
|
||||
def check_workflows(root):
|
||||
"""Return the errors in the filters of the workflows and in the targets they run."""
|
||||
errors = []
|
||||
for workflow in WORKFLOWS:
|
||||
expected = [p.replace('{workflow}', workflow) for p in PATHS]
|
||||
actual = read_filter(root, workflow)
|
||||
if actual != expected:
|
||||
errors.append('{}: the pull_request paths filter differs from tools/check_path_filters/check_path_filters.py\n'
|
||||
' expected: {}\n actual: {}'.format(workflow, expected, actual))
|
||||
with open(os.path.join(root, workflow), encoding='utf-8') as f:
|
||||
text = f.read()
|
||||
errors += ['{}: runs {}, which reads excluded files; run it in repository_checks.yml'.format(workflow, target)
|
||||
for target in UNFILTERED_TARGETS if re.search(r'\b' + target + r'\b', text)]
|
||||
return errors
|
||||
|
||||
|
||||
def scanned_text(root, path):
|
||||
"""Return the text of a kept file without comment lines and without the filter itself, or None to skip it."""
|
||||
# third-party code and Markdown files read nothing; this script lists the excluded files
|
||||
if path.startswith('tests/thirdparty/') or path.endswith('.md') or path == SELF:
|
||||
return None
|
||||
try:
|
||||
with open(os.path.join(root, path), encoding='utf-8') as f:
|
||||
# a mention in a comment line reads nothing; keep the line numbers
|
||||
text = COMMENT_LINE.sub('', f.read())
|
||||
except (UnicodeDecodeError, OSError):
|
||||
return None
|
||||
if path in WORKFLOWS:
|
||||
# the filter itself lists the excluded files
|
||||
for p in PATHS:
|
||||
text = text.replace(" - '{}'".format(p.replace('{workflow}', path)), '')
|
||||
return text
|
||||
|
||||
|
||||
def check_mentions(root, files, excluded, finder):
|
||||
"""Return an error for every mention of an excluded file in a kept file that KNOWN_REFERENCES does not list."""
|
||||
errors = []
|
||||
for path in sorted(set(files) - excluded):
|
||||
text = scanned_text(root, path)
|
||||
reported = set()
|
||||
for offset, mention in finder.find(text or ''):
|
||||
if mention not in reported and not known(path, mention):
|
||||
reported.add(mention)
|
||||
line = text.count('\n', 0, offset) + 1
|
||||
errors.append('{}:{}: mentions excluded {}; include it in PATHS if a compiler job reads it, '
|
||||
'or add it to KNOWN_REFERENCES'.format(path, line, mention))
|
||||
return errors
|
||||
|
||||
|
||||
def check(root):
|
||||
files = git_files(root)
|
||||
union = [compile_filter(PATHS, w) for w in WORKFLOWS]
|
||||
|
||||
def is_excluded(path):
|
||||
return not any(included(c, path) for c in union)
|
||||
|
||||
excluded = {f for f in files if is_excluded(f)}
|
||||
errors = check_workflows(root) + check_mentions(root, files, excluded, Mentions(files, excluded, is_excluded))
|
||||
for e in errors:
|
||||
print(e)
|
||||
if not errors:
|
||||
print('path filters OK: {} of {} tracked files skip the compiler workflows'.format(len(excluded), len(files)))
|
||||
return 1 if errors else 0
|
||||
|
||||
|
||||
def explain(changed):
|
||||
for workflow in WORKFLOWS:
|
||||
compiled = compile_filter(PATHS, workflow)
|
||||
hits = [f for f in changed if included(compiled, f)]
|
||||
print('{}: {}'.format(workflow, 'runs ({})'.format(', '.join(hits[:3]) + (' ...' if len(hits) > 3 else ''))
|
||||
if hits else 'skipped'))
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument('--root', default=REPO_ROOT, help='repository root')
|
||||
parser.add_argument('--changed', nargs='+', metavar='FILE', help='show which workflows these changed files run')
|
||||
args = parser.parse_args()
|
||||
return explain(args.changed) if args.changed else check(args.root)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user