Compare commits

..
Author SHA1 Message Date
Niels Lohmann 4b538d80f6 Merge branch 'develop' into ci/path-filters
Keep ci_test_documentation in repository_checks.yml and apply the
harden-runner 2.22.2 bump from #5962 there.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-30 15:35:39 +01:00
Niels Lohmann 7a4e6b7a5d Merge develop into ci/path-filters
Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-28 22:09:53 +01:00
Niels Lohmann fd37b04813 Merge origin/develop into ci/path-filters
Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-28 20:32:10 +01:00
Niels Lohmann bd442d5d07 Merge develop
Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-28 07:11:43 +01:00
Niels Lohmann eae3c8217b Split the checks of check_path_filters.py into smaller functions
Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-27 19:39:20 +01:00
Niels Lohmann 960c234cc6 Do not report the excluded files that check_path_filters.py lists itself
Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-27 18:58:36 +01:00
Niels Lohmann b6e24d19a9 Skip the compiler workflows for pull requests that change no build input
ubuntu.yml, macos.yml and windows.yml run about 155 jobs on every pull
request, including those that change only the documentation. They now
have a pull_request path filter: it starts with '**' and excludes the
documentation, Markdown files, GitHub metadata, other workflows and the
license files, so a new file runs them unless it is excluded on purpose.
Pushes to develop, master and release/* are not filtered.

The documentation jobs and the REUSE check move from ubuntu.yml to the
new repository_checks.yml, which has no path filter and also runs
tools/check_path_filters. That check keeps the three filters identical
and fails when a file the compiler jobs keep mentions an excluded file.
It found two: tools/check_build_options reads the Meson section of
package_managers.md, and the swiftpm job builds the Swift example from
the documentation; both are included in the filter.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-27 18:35:03 +01:00
14 changed files with 540 additions and 244 deletions

No files matched your search

+5
View File
@@ -154,6 +154,11 @@ and each page's title (H1) and declaration by
[`docs/docset/generate_docset.py`](https://github.com/nlohmann/json/blob/develop/docs/docset/generate_docset.py);
`make build` reports API pages that cannot be classified.
A pull request that changes only the documentation, Markdown files, GitHub metadata, or license files skips the
compiler workflows (Ubuntu, macOS, Windows); the documentation and license checks still run. See
[`tools/check_path_filters`](https://github.com/nlohmann/json/tree/develop/tools/check_path_filters) for the list of
skipped files and what to do when a compiler job starts reading one of them.
### Amalgamate the source code
The single-header files
+19
View File
@@ -7,6 +7,25 @@ on:
- master
- release/*
pull_request:
# Skip pull requests that change only files no job here reads. The list is checked by
# tools/check_path_filters/check_path_filters.py (repository_checks.yml); change both together.
paths:
- '**'
- '!docs/**'
- '!**.md'
- '!.github/*'
- '!.github/ISSUE_TEMPLATE/**'
- '!.github/external_ci/**'
- '!.github/workflows/**'
- '!.reuse/**'
- '!LICENSES/**'
- '!CITATION.cff'
- '!REUSE.toml'
# read by tools/check_build_options in the ci_meson_install job (ubuntu.yml)
- 'docs/mkdocs/docs/integration/package_managers.md'
# built by the swiftpm job (macos.yml)
- 'docs/mkdocs/docs/integration/swift/**'
- '.github/workflows/macos.yml'
workflow_dispatch:
concurrency:
+78
View File
@@ -0,0 +1,78 @@
name: Repository checks
# Checks that concern files the compiler workflows (ubuntu.yml, macos.yml,
# windows.yml) skip with their path filter: the documentation, the license
# information, and the path filter itself. This workflow has no path filter.
on:
push:
branches:
- develop
- master
- release/*
pull_request:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
cancel-in-progress: true
jobs:
check_path_filters:
runs-on: ubuntu-26.04
steps:
- name: Harden Runner
uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2
with:
egress-policy: audit
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Check the path filters of the compiler workflows
run: python3 tools/check_path_filters/check_path_filters.py
ci_reuse_compliance:
runs-on: ubuntu-26.04
steps:
- name: Harden Runner
uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2
with:
egress-policy: audit
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Get latest CMake and ninja
uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
- name: Run CMake
run: cmake -S . -B build -DJSON_CI=On
- name: Build
run: cmake --build build --target ci_reuse_compliance
ci_test_documentation:
runs-on: ubuntu-26.04
strategy:
matrix:
target: [ci_test_examples, ci_test_build_documentation, ci_test_documentation_mermaid]
steps:
- name: Harden Runner
uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2
with:
egress-policy: audit
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
# the git-revision-date-localized plugin needs the history; a shallow clone makes the strict build fail
fetch-depth: ${{ matrix.target == 'ci_test_build_documentation' && '0' || '1' }}
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
if: matrix.target == 'ci_test_documentation_mermaid'
with:
node-version: 24
- name: Run CMake
run: cmake -S . -B build -DJSON_CI=On
- name: Build
run: cmake --build build --target ${{ matrix.target }}
+20 -26
View File
@@ -7,6 +7,25 @@ on:
- master
- release/*
pull_request:
# Skip pull requests that change only files no job here reads. The list is checked by
# tools/check_path_filters/check_path_filters.py (repository_checks.yml); change both together.
paths:
- '**'
- '!docs/**'
- '!**.md'
- '!.github/*'
- '!.github/ISSUE_TEMPLATE/**'
- '!.github/external_ci/**'
- '!.github/workflows/**'
- '!.reuse/**'
- '!LICENSES/**'
- '!CITATION.cff'
- '!REUSE.toml'
# read by tools/check_build_options in the ci_meson_install job (ubuntu.yml)
- 'docs/mkdocs/docs/integration/package_managers.md'
# built by the swiftpm job (macos.yml)
- 'docs/mkdocs/docs/integration/swift/**'
- '.github/workflows/ubuntu.yml'
workflow_dispatch:
permissions:
@@ -109,7 +128,7 @@ jobs:
runs-on: ubuntu-26.04
strategy:
matrix:
target: [ci_test_single_header, ci_cppcheck, ci_cpplint, ci_reproducible_tests, ci_non_git_tests, ci_offline_testdata, ci_reuse_compliance, ci_test_valgrind]
target: [ci_test_single_header, ci_cppcheck, ci_cpplint, ci_reproducible_tests, ci_non_git_tests, ci_offline_testdata, ci_test_valgrind]
steps:
- name: Harden Runner
uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2
@@ -449,31 +468,6 @@ jobs:
- name: Build
run: cmake --build build
ci_test_documentation:
runs-on: ubuntu-26.04
strategy:
matrix:
target: [ci_test_examples, ci_test_build_documentation, ci_test_documentation_mermaid]
steps:
- name: Harden Runner
uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2
with:
egress-policy: audit
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
# the git-revision-date-localized plugin needs the history; a shallow clone makes the strict build fail
fetch-depth: ${{ matrix.target == 'ci_test_build_documentation' && '0' || '1' }}
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
if: matrix.target == 'ci_test_documentation_mermaid'
with:
node-version: 24
- name: Run CMake
run: cmake -S . -B build -DJSON_CI=On
- name: Build
run: cmake --build build --target ${{ matrix.target }}
# Native Linux ARM64 runner with the latest GCC. The ubuntu-24.04-arm label is
# only available for public repositories.
ci_test_arm64:
+19
View File
@@ -7,6 +7,25 @@ on:
- master
- release/*
pull_request:
# Skip pull requests that change only files no job here reads. The list is checked by
# tools/check_path_filters/check_path_filters.py (repository_checks.yml); change both together.
paths:
- '**'
- '!docs/**'
- '!**.md'
- '!.github/*'
- '!.github/ISSUE_TEMPLATE/**'
- '!.github/external_ci/**'
- '!.github/workflows/**'
- '!.reuse/**'
- '!LICENSES/**'
- '!CITATION.cff'
- '!REUSE.toml'
# read by tools/check_build_options in the ci_meson_install job (ubuntu.yml)
- 'docs/mkdocs/docs/integration/package_managers.md'
# built by the swiftpm job (macos.yml)
- 'docs/mkdocs/docs/integration/swift/**'
- '.github/workflows/windows.yml'
workflow_dispatch:
permissions:
+41 -83
View File
@@ -268,10 +268,10 @@ class lexer : public lexer_base<BasicJsonType>
public:
using token_type = typename lexer_base<BasicJsonType>::token_type;
explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_values_ = false) noexcept
explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept
: ia(std::move(adapter))
, ignore_comments(ignore_comments_)
, discard_values(discard_values_)
, discard_number_values(discard_number_values_)
{
// the token string of an error before the first token is everything
// read so far
@@ -388,8 +388,6 @@ class lexer : public lexer_base<BasicJsonType>
checks if it is inside the range. If a violation was detected, set up an
error message and return false. Otherwise, return true.
@param[in] keep whether the bytes are added to token_buffer (false for
accept(), see scan())
@param[in] ranges list of integers; interpreted as list of pairs of
inclusive lower and upper bound, respectively
@@ -398,18 +396,17 @@ class lexer : public lexer_base<BasicJsonType>
@return true if and only if no range violation was detected
*/
template<bool Keep>
bool next_byte_in_range(std::integral_constant<bool, Keep> keep, std::initializer_list<char_int_type> ranges)
bool next_byte_in_range(std::initializer_list<char_int_type> ranges)
{
JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6);
add_string_char(keep, current);
add(current);
for (auto range = ranges.begin(); range != ranges.end(); ++range)
{
get();
if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions)
{
add_string_char(keep, current);
add(current);
}
else
{
@@ -422,16 +419,14 @@ class lexer : public lexer_base<BasicJsonType>
}
/// contiguous input: bulk-append the run of ordinary characters and complete
/// well-formed UTF-8 sequences starting at the current read position (only
/// skip it if @a keep is false), then
/// well-formed UTF-8 sequences starting at the current read position, then
/// read the first byte that needs individual handling (the closing quote, an
/// escape, a control character, or an ill-formed UTF-8 byte) like get().
/// That byte is taken from the buffer that was just scanned. Through get(),
/// Apple Clang merged this read with the other get() calls of the scan
/// function, which load the read position again right after it was stored
/// (3% of json::accept() on jeopardy.json on Apple M1).
template<bool Keep>
char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant<bool, Keep> keep)
char_int_type get_after_string_run(std::true_type /*bulk*/)
{
// a pending unget must be consumed through the normal path first
if (next_unget)
@@ -449,7 +444,7 @@ class lexer : public lexer_base<BasicJsonType>
const std::size_t pos = lexer_string_bulk_run(data, remaining);
if (pos != 0)
{
append_string_run(keep, data, pos);
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), pos);
bulk_consume(reinterpret_cast<const char*>(data), pos);
}
if (pos == remaining)
@@ -493,8 +488,7 @@ class lexer : public lexer_base<BasicJsonType>
}
/// streaming input: no bulk fast path
template<bool Keep>
char_int_type get_after_string_run(std::false_type /*bulk*/, std::integral_constant<bool, Keep> /*keep*/)
char_int_type get_after_string_run(std::false_type /*bulk*/)
{
return get();
}
@@ -508,18 +502,13 @@ class lexer : public lexer_base<BasicJsonType>
may contain \0 bytes), and token_buffer.size() is the number of bytes in the
string.
@param[in] keep whether the string is copied into token_buffer; if false
(accept(), see scan()), it is only checked, with the same
errors, and token_buffer stays empty
@return token_type::value_string if string could be successfully scanned,
token_type::parse_error otherwise
@note In case of errors, variable error_message contains a textual
description.
*/
template<bool Keep>
token_type scan_string(std::integral_constant<bool, Keep> keep)
token_type scan_string()
{
// reset token_buffer (ignore opening quote)
reset();
@@ -531,7 +520,7 @@ class lexer : public lexer_base<BasicJsonType>
{
// bulk-consume ordinary characters from contiguous input, then
// get the next character, the one that ends the run
switch (get_after_string_run(std::integral_constant<bool, bulk_scan>(), keep))
switch (get_after_string_run(std::integral_constant<bool, bulk_scan>()))
{
// end of file while parsing the string
case char_traits<char_type>::eof():
@@ -553,35 +542,35 @@ class lexer : public lexer_base<BasicJsonType>
{
// quotation mark
case '\"':
add_string_char(keep, '\"');
add('\"');
break;
// reverse solidus
case '\\':
add_string_char(keep, '\\');
add('\\');
break;
// solidus
case '/':
add_string_char(keep, '/');
add('/');
break;
// backspace
case 'b':
add_string_char(keep, '\b');
add('\b');
break;
// form feed
case 'f':
add_string_char(keep, '\f');
add('\f');
break;
// line feed
case 'n':
add_string_char(keep, '\n');
add('\n');
break;
// carriage return
case 'r':
add_string_char(keep, '\r');
add('\r');
break;
// tab
case 't':
add_string_char(keep, '\t');
add('\t');
break;
// unicode escapes
@@ -649,7 +638,10 @@ class lexer : public lexer_base<BasicJsonType>
JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF);
// translate codepoint into bytes
add_codepoint(keep, codepoint);
encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte)
{
add(static_cast<char_int_type>(byte));
});
break;
}
@@ -832,7 +824,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0x7E:
case 0x7F:
{
add_string_char(keep, current);
add(current);
break;
}
@@ -868,7 +860,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xDE:
case 0xDF:
{
if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range(keep, {0x80, 0xBF})))
if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF})))
{
return token_type::parse_error;
}
@@ -878,7 +870,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+0800..U+0FFF: bytes E0 A0..BF 80..BF
case 0xE0:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0xA0, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -902,7 +894,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xEE:
case 0xEF:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -912,7 +904,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+D000..U+D7FF: bytes ED 80..9F 80..BF
case 0xED:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x9F, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -922,7 +914,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
case 0xF0:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -934,7 +926,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xF2:
case 0xF3:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -944,7 +936,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
case 0xF4:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -1435,7 +1427,7 @@ scan_number_done:
token_type convert_number(token_type number_type, std::size_t mantissa_end)
{
// accept() only needs to know whether the input is valid, so it sets
// discard_values (see json.hpp), and an integer token whose
// discard_number_values (see json.hpp), and an integer token whose
// digit count shows that it fits is reported without calling
// convert_integer(). A number with up to 18 digits always fits into
// both std::uint64_t and std::int64_t (18 nines is about 1e18, below
@@ -1450,7 +1442,7 @@ scan_number_done:
// json_sax_acceptor, whose number callbacks discard their argument and
// return true, and the parser rejects neither integers nor finite
// floats. value_unsigned/value_integer are left unset here, so a caller
// that reads the converted value must not set discard_values.
// that reads the converted value must not set discard_number_values.
//
// On contiguous input, scan_number_bulk_contiguous() converts integer
// tokens itself and does not pass them to this function, unless
@@ -1458,7 +1450,7 @@ scan_number_done:
// reached for input without bulk access (e.g. streams), with
// JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous()
// falls back to scan_number().
if (discard_values)
if (discard_number_values)
{
constexpr std::size_t safe_digit_count = 18;
if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count)
@@ -1709,7 +1701,7 @@ scan_number_done:
// w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a
// value that underflows is a finite number as well. Only a token that
// may overflow has to be converted to find out.
if (JSON_HEDLEY_UNLIKELY(discard_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
{
value_float = 0;
return true;
@@ -2041,34 +2033,6 @@ scan_number_done:
token_buffer.push_back(static_cast<typename string_t::value_type>(c));
}
/// add a character of a string token; accept() checks the string but does
/// not keep its value (@a keep is false)
void add_string_char(std::true_type /*keep*/, char_int_type c)
{
token_buffer.push_back(static_cast<typename string_t::value_type>(c));
}
static void add_string_char(std::false_type /*keep*/, char_int_type /*c*/) noexcept {}
/// add the UTF-8 bytes of a code point from a \\u escape to a string token
void add_codepoint(std::true_type /*keep*/, int codepoint)
{
encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte)
{
add(static_cast<char_int_type>(byte));
});
}
static void add_codepoint(std::false_type /*keep*/, int /*codepoint*/) noexcept {}
/// append a run of ordinary string characters (see get_after_string_run())
void append_string_run(std::true_type /*keep*/, const unsigned char* data, std::size_t n)
{
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), n);
}
static void append_string_run(std::false_type /*keep*/, const unsigned char* /*data*/, std::size_t /*n*/) noexcept {}
public:
/////////////////////
// value getters
@@ -2463,11 +2427,7 @@ scan_number_done:
// string
case '\"':
// accept() (discard_values) checks strings without copying them
// into token_buffer. The choice is made once per string: a
// check inside the string loop made number-heavy documents up
// to 3 % slower (mesh.json, Apple M1)
return discard_values ? scan_string(std::false_type {}) : scan_string(std::true_type {});
return scan_string();
// number
case '-':
@@ -2552,13 +2512,11 @@ scan_number_done:
std::size_t decimal_point_position = std::string::npos;
/// whether the caller only needs the token types and never looks at the
/// values of number and string tokens; set only by accept(), which parses
/// through json_sax_acceptor. When set, convert_number() skips converting
/// integer tokens whose digit count guarantees that they fit into 64 bits
/// (see there), floats are converted only when they may overflow (see
/// convert_accumulated_float()), and strings are checked without being
/// copied into token_buffer (see scan())
const bool discard_values = false;
/// converted numeric values; set only by accept(), which parses through
/// json_sax_acceptor. When set, convert_number() skips converting integer
/// tokens whose digit count guarantees that they fit into 64 bits (see
/// there)
const bool discard_number_values = false;
};
} // namespace detail
+2 -2
View File
@@ -89,9 +89,9 @@ class parser
const bool allow_exceptions_ = true,
const bool ignore_comments = false,
const bool ignore_trailing_commas_ = false,
const bool discard_values_ = false)
const bool discard_number_values_ = false)
: callback(std::move(cb))
, m_lexer(std::move(adapter), ignore_comments, discard_values_)
, m_lexer(std::move(adapter), ignore_comments, discard_number_values_)
, allow_exceptions(allow_exceptions_)
, ignore_trailing_commas(ignore_trailing_commas_)
{
+2 -2
View File
@@ -184,11 +184,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
const bool allow_exceptions = true,
const bool ignore_comments = false,
const bool ignore_trailing_commas = false,
const bool discard_values = false
const bool discard_number_values = false
)
{
return ::nlohmann::detail::parser<basic_json, InputAdapterType>(std::move(adapter),
std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_values);
std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values);
}
private:
+3
View File
@@ -766,6 +766,9 @@ class basic_json_view
template<typename, typename> friend class detail::view::view_side;
friend iterator;
/// whether the view is one of an editable document
static constexpr bool is_editable = Editable;
basic_json_view(const document_data* d, const node* n) noexcept
: m_doc(d), m_node(n)
{}
+45 -87
View File
@@ -13124,10 +13124,10 @@ class lexer : public lexer_base<BasicJsonType>
public:
using token_type = typename lexer_base<BasicJsonType>::token_type;
explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_values_ = false) noexcept
explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept
: ia(std::move(adapter))
, ignore_comments(ignore_comments_)
, discard_values(discard_values_)
, discard_number_values(discard_number_values_)
{
// the token string of an error before the first token is everything
// read so far
@@ -13244,8 +13244,6 @@ class lexer : public lexer_base<BasicJsonType>
checks if it is inside the range. If a violation was detected, set up an
error message and return false. Otherwise, return true.
@param[in] keep whether the bytes are added to token_buffer (false for
accept(), see scan())
@param[in] ranges list of integers; interpreted as list of pairs of
inclusive lower and upper bound, respectively
@@ -13254,18 +13252,17 @@ class lexer : public lexer_base<BasicJsonType>
@return true if and only if no range violation was detected
*/
template<bool Keep>
bool next_byte_in_range(std::integral_constant<bool, Keep> keep, std::initializer_list<char_int_type> ranges)
bool next_byte_in_range(std::initializer_list<char_int_type> ranges)
{
JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6);
add_string_char(keep, current);
add(current);
for (auto range = ranges.begin(); range != ranges.end(); ++range)
{
get();
if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions)
{
add_string_char(keep, current);
add(current);
}
else
{
@@ -13278,16 +13275,14 @@ class lexer : public lexer_base<BasicJsonType>
}
/// contiguous input: bulk-append the run of ordinary characters and complete
/// well-formed UTF-8 sequences starting at the current read position (only
/// skip it if @a keep is false), then
/// well-formed UTF-8 sequences starting at the current read position, then
/// read the first byte that needs individual handling (the closing quote, an
/// escape, a control character, or an ill-formed UTF-8 byte) like get().
/// That byte is taken from the buffer that was just scanned. Through get(),
/// Apple Clang merged this read with the other get() calls of the scan
/// function, which load the read position again right after it was stored
/// (3% of json::accept() on jeopardy.json on Apple M1).
template<bool Keep>
char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant<bool, Keep> keep)
char_int_type get_after_string_run(std::true_type /*bulk*/)
{
// a pending unget must be consumed through the normal path first
if (next_unget)
@@ -13305,7 +13300,7 @@ class lexer : public lexer_base<BasicJsonType>
const std::size_t pos = lexer_string_bulk_run(data, remaining);
if (pos != 0)
{
append_string_run(keep, data, pos);
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), pos);
bulk_consume(reinterpret_cast<const char*>(data), pos);
}
if (pos == remaining)
@@ -13349,8 +13344,7 @@ class lexer : public lexer_base<BasicJsonType>
}
/// streaming input: no bulk fast path
template<bool Keep>
char_int_type get_after_string_run(std::false_type /*bulk*/, std::integral_constant<bool, Keep> /*keep*/)
char_int_type get_after_string_run(std::false_type /*bulk*/)
{
return get();
}
@@ -13364,18 +13358,13 @@ class lexer : public lexer_base<BasicJsonType>
may contain \0 bytes), and token_buffer.size() is the number of bytes in the
string.
@param[in] keep whether the string is copied into token_buffer; if false
(accept(), see scan()), it is only checked, with the same
errors, and token_buffer stays empty
@return token_type::value_string if string could be successfully scanned,
token_type::parse_error otherwise
@note In case of errors, variable error_message contains a textual
description.
*/
template<bool Keep>
token_type scan_string(std::integral_constant<bool, Keep> keep)
token_type scan_string()
{
// reset token_buffer (ignore opening quote)
reset();
@@ -13387,7 +13376,7 @@ class lexer : public lexer_base<BasicJsonType>
{
// bulk-consume ordinary characters from contiguous input, then
// get the next character, the one that ends the run
switch (get_after_string_run(std::integral_constant<bool, bulk_scan>(), keep))
switch (get_after_string_run(std::integral_constant<bool, bulk_scan>()))
{
// end of file while parsing the string
case char_traits<char_type>::eof():
@@ -13409,35 +13398,35 @@ class lexer : public lexer_base<BasicJsonType>
{
// quotation mark
case '\"':
add_string_char(keep, '\"');
add('\"');
break;
// reverse solidus
case '\\':
add_string_char(keep, '\\');
add('\\');
break;
// solidus
case '/':
add_string_char(keep, '/');
add('/');
break;
// backspace
case 'b':
add_string_char(keep, '\b');
add('\b');
break;
// form feed
case 'f':
add_string_char(keep, '\f');
add('\f');
break;
// line feed
case 'n':
add_string_char(keep, '\n');
add('\n');
break;
// carriage return
case 'r':
add_string_char(keep, '\r');
add('\r');
break;
// tab
case 't':
add_string_char(keep, '\t');
add('\t');
break;
// unicode escapes
@@ -13505,7 +13494,10 @@ class lexer : public lexer_base<BasicJsonType>
JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF);
// translate codepoint into bytes
add_codepoint(keep, codepoint);
encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte)
{
add(static_cast<char_int_type>(byte));
});
break;
}
@@ -13688,7 +13680,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0x7E:
case 0x7F:
{
add_string_char(keep, current);
add(current);
break;
}
@@ -13724,7 +13716,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xDE:
case 0xDF:
{
if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range(keep, {0x80, 0xBF})))
if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF})))
{
return token_type::parse_error;
}
@@ -13734,7 +13726,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+0800..U+0FFF: bytes E0 A0..BF 80..BF
case 0xE0:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0xA0, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -13758,7 +13750,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xEE:
case 0xEF:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -13768,7 +13760,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+D000..U+D7FF: bytes ED 80..9F 80..BF
case 0xED:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x9F, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -13778,7 +13770,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
case 0xF0:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -13790,7 +13782,7 @@ class lexer : public lexer_base<BasicJsonType>
case 0xF2:
case 0xF3:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -13800,7 +13792,7 @@ class lexer : public lexer_base<BasicJsonType>
// U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
case 0xF4:
{
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
{
return token_type::parse_error;
}
@@ -14291,7 +14283,7 @@ scan_number_done:
token_type convert_number(token_type number_type, std::size_t mantissa_end)
{
// accept() only needs to know whether the input is valid, so it sets
// discard_values (see json.hpp), and an integer token whose
// discard_number_values (see json.hpp), and an integer token whose
// digit count shows that it fits is reported without calling
// convert_integer(). A number with up to 18 digits always fits into
// both std::uint64_t and std::int64_t (18 nines is about 1e18, below
@@ -14306,7 +14298,7 @@ scan_number_done:
// json_sax_acceptor, whose number callbacks discard their argument and
// return true, and the parser rejects neither integers nor finite
// floats. value_unsigned/value_integer are left unset here, so a caller
// that reads the converted value must not set discard_values.
// that reads the converted value must not set discard_number_values.
//
// On contiguous input, scan_number_bulk_contiguous() converts integer
// tokens itself and does not pass them to this function, unless
@@ -14314,7 +14306,7 @@ scan_number_done:
// reached for input without bulk access (e.g. streams), with
// JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous()
// falls back to scan_number().
if (discard_values)
if (discard_number_values)
{
constexpr std::size_t safe_digit_count = 18;
if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count)
@@ -14565,7 +14557,7 @@ scan_number_done:
// w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a
// value that underflows is a finite number as well. Only a token that
// may overflow has to be converted to find out.
if (JSON_HEDLEY_UNLIKELY(discard_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
{
value_float = 0;
return true;
@@ -14897,34 +14889,6 @@ scan_number_done:
token_buffer.push_back(static_cast<typename string_t::value_type>(c));
}
/// add a character of a string token; accept() checks the string but does
/// not keep its value (@a keep is false)
void add_string_char(std::true_type /*keep*/, char_int_type c)
{
token_buffer.push_back(static_cast<typename string_t::value_type>(c));
}
static void add_string_char(std::false_type /*keep*/, char_int_type /*c*/) noexcept {}
/// add the UTF-8 bytes of a code point from a \\u escape to a string token
void add_codepoint(std::true_type /*keep*/, int codepoint)
{
encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte)
{
add(static_cast<char_int_type>(byte));
});
}
static void add_codepoint(std::false_type /*keep*/, int /*codepoint*/) noexcept {}
/// append a run of ordinary string characters (see get_after_string_run())
void append_string_run(std::true_type /*keep*/, const unsigned char* data, std::size_t n)
{
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), n);
}
static void append_string_run(std::false_type /*keep*/, const unsigned char* /*data*/, std::size_t /*n*/) noexcept {}
public:
/////////////////////
// value getters
@@ -15319,11 +15283,7 @@ scan_number_done:
// string
case '\"':
// accept() (discard_values) checks strings without copying them
// into token_buffer. The choice is made once per string: a
// check inside the string loop made number-heavy documents up
// to 3 % slower (mesh.json, Apple M1)
return discard_values ? scan_string(std::false_type {}) : scan_string(std::true_type {});
return scan_string();
// number
case '-':
@@ -15408,13 +15368,11 @@ scan_number_done:
std::size_t decimal_point_position = std::string::npos;
/// whether the caller only needs the token types and never looks at the
/// values of number and string tokens; set only by accept(), which parses
/// through json_sax_acceptor. When set, convert_number() skips converting
/// integer tokens whose digit count guarantees that they fit into 64 bits
/// (see there), floats are converted only when they may overflow (see
/// convert_accumulated_float()), and strings are checked without being
/// copied into token_buffer (see scan())
const bool discard_values = false;
/// converted numeric values; set only by accept(), which parses through
/// json_sax_acceptor. When set, convert_number() skips converting integer
/// tokens whose digit count guarantees that they fit into 64 bits (see
/// there)
const bool discard_number_values = false;
};
} // namespace detail
@@ -22623,9 +22581,9 @@ class parser
const bool allow_exceptions_ = true,
const bool ignore_comments = false,
const bool ignore_trailing_commas_ = false,
const bool discard_values_ = false)
const bool discard_number_values_ = false)
: callback(std::move(cb))
, m_lexer(std::move(adapter), ignore_comments, discard_values_)
, m_lexer(std::move(adapter), ignore_comments, discard_number_values_)
, allow_exceptions(allow_exceptions_)
, ignore_trailing_commas(ignore_trailing_commas_)
{
@@ -37022,11 +36980,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
const bool allow_exceptions = true,
const bool ignore_comments = false,
const bool ignore_trailing_commas = false,
const bool discard_values = false
const bool discard_number_values = false
)
{
return ::nlohmann::detail::parser<basic_json, InputAdapterType>(std::move(adapter),
std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_values);
std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values);
}
private:
+3
View File
@@ -9807,6 +9807,9 @@ class basic_json_view
template<typename, typename> friend class detail::view::view_side;
friend iterator;
/// whether the view is one of an editable document
static constexpr bool is_editable = Editable;
basic_json_view(const document_data* d, const node* n) noexcept
: m_doc(d), m_node(n)
{}
-44
View File
@@ -2982,50 +2982,6 @@ TEST_CASE("accept() agrees with parse() on floats around the overflow limit")
}
}
TEST_CASE("accept() agrees with parse() on strings, which it checks without copying them")
{
// accept() scans strings and object keys with a separate instantiation of
// the string scanner that does not copy them into the token buffer; it
// must accept and reject exactly what parse() does
const std::vector<std::string> pieces =
{
"", "abc", "\\\"\\\\\\/\\b\\f\\n\\r\\t", "\\u0041\\u00e4\\u20AC", "\\ud83d\\ude00", "\\uD83D\\uDE00",
"\\ud800", "\\udc00", "\\ud800\\u0041", "\\ud800x", "\\udbff\\udfff", "\\u12", "\\u12g4", "\\x", "\\",
"\xc3\xa4", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xf4\x8f\xbf\xbf", "\xc0\xaf", "\xed\xa0\x80", "\xf4\x90\x80\x80",
"\xff", "\xe2\x82", "\xc3", "\x01", "\x1f", "\x7f", "\n", std::string(1, '\0')
};
for (std::size_t run = 0; run <= 70; ++run)
{
for (const std::string& piece : pieces)
{
// a run of ordinary characters puts the piece at every offset of the bulk scanners
const std::string str = std::string(run, 'a') + piece;
for (const std::string& text :
{
"\"" + str + "\"", "[\"" + str + "\"]", "{\"" + str + "\":1}", "{\"k\":\"" + str + "\",\"" + str + "\":[\"" + str + "\"]}",
"[\"" + str
})
{
CAPTURE(text)
for (int options = 0; options < 4; ++options)
{
const bool ignore_comments = (options & 1) != 0;
const bool ignore_trailing_commas = (options & 2) != 0;
const bool parses = !json::parse(text, nullptr, false, ignore_comments, ignore_trailing_commas).is_discarded();
CHECK(json::accept(text, ignore_comments, ignore_trailing_commas) == parses);
std::istringstream stream(text);
CHECK(json::accept(stream, ignore_comments, ignore_trailing_commas) == parses);
const std::list<char> chars(text.begin(), text.end());
CHECK(json::accept(chars.begin(), chars.end(), ignore_comments, ignore_trailing_commas) == parses);
}
}
}
}
}
namespace
{
// a DOM parser that reports parse errors without throwing, as in the sax_no_exception example
+34
View File
@@ -0,0 +1,34 @@
# check_path_filters
The compiler workflows [`ubuntu.yml`](../../.github/workflows/ubuntu.yml),
[`macos.yml`](../../.github/workflows/macos.yml), and [`windows.yml`](../../.github/workflows/windows.yml) skip pull
requests that change only files no compiler job reads: the documentation, Markdown files, GitHub metadata, other
workflows, and the license files. Pushes to `develop`, `master`, and `release/*` always run them. The documentation,
license, and path-filter checks run in [`repository_checks.yml`](../../.github/workflows/repository_checks.yml), which
has no path filter.
The filter starts with `'**'` and excludes these files explicitly, so a new file runs the compiler workflows unless it
is excluded on purpose. [`check_path_filters.py`](check_path_filters.py) holds the filter (`PATHS`) and checks that
- each of the three workflows has exactly this filter, with its own workflow file included,
- the CI targets that read excluded files (`UNFILTERED_TARGETS`) do not run in these workflows, and
- no file the filter keeps (sources, tests, build files, tools, workflows) mentions an excluded file or directory
outside a comment line, unless the mention is listed in `KNOWN_REFERENCES` with a reason.
When the check reports a mention, decide whether a compiler job reads that file:
- If it does, include the file in `PATHS` after the exclusions (as `package_managers.md` and the Swift example are) and
copy the new filter into the three workflows.
- If it does not, add the mention to `KNOWN_REFERENCES` with the reason.
Run the check with
```shell
python3 tools/check_path_filters/check_path_filters.py
```
To see which workflows a pull request would run, pass the changed files:
```shell
python3 tools/check_path_filters/check_path_filters.py --changed $(git diff --name-only origin/develop)
```
+269
View File
@@ -0,0 +1,269 @@
#!/usr/bin/env python3
"""Check the path filters of the compiler workflows.
ubuntu.yml, macos.yml and windows.yml skip pull requests that change only files
no compiler job reads (documentation, Markdown files, GitHub metadata, license
files). The filter starts with '**' and excludes these files explicitly, so a
new file runs the workflows unless it is excluded on purpose.
The script checks that
- each of the three workflows has exactly the filter defined in PATHS below,
with its own workflow file included, and
- no tracked file that the filter keeps (sources, tests, build files, tools,
workflows) mentions a file the filter excludes, unless the mention is listed
in KNOWN_REFERENCES with the reason why it does not matter.
A mention found this way must either be added to PATHS as an included path
(the file is read by a compiler job) or to KNOWN_REFERENCES (it is not).
With --changed FILE..., the script prints instead which workflows a pull
request changing these files would run.
"""
import argparse
import os
import re
import subprocess # nosec B404
import sys
REPO_ROOT = os.path.normpath(os.path.join(sys.path[0], '..', '..'))
SELF = 'tools/check_path_filters/check_path_filters.py'
WORKFLOWS = ['.github/workflows/ubuntu.yml', '.github/workflows/macos.yml', '.github/workflows/windows.yml']
# The pull_request paths filter of the workflows above; '{workflow}' is replaced
# with the workflow's own file. GitHub applies the patterns in order: a file
# matching a later pattern is included ('pattern') or excluded ('!pattern').
PATHS = [
'**',
'!docs/**',
'!**.md',
'!.github/*',
'!.github/ISSUE_TEMPLATE/**',
'!.github/external_ci/**',
'!.github/workflows/**',
'!.reuse/**',
'!LICENSES/**',
'!CITATION.cff',
'!REUSE.toml',
# read by tools/check_build_options in the ci_meson_install job (ubuntu.yml)
'docs/mkdocs/docs/integration/package_managers.md',
# built by the swiftpm job (macos.yml)
'docs/mkdocs/docs/integration/swift/**',
'{workflow}',
]
# Mentions of excluded files that no job of the three workflows depends on:
# (file that mentions it, the mention, reason)
KNOWN_REFERENCES = [
('cmake/ci.cmake', 'docs/', 'ci_test_examples; runs in repository_checks.yml, which has no path filter'),
('cmake/ci.cmake', 'docs/mkdocs/', 'ci_test_build_documentation and ci_test_documentation_mermaid; they run in '
'repository_checks.yml, which has no path filter'),
('cmake/ci.cmake', 'docs/mkdocs/docs/examples/', 'indentation check of ci_test_amalgamation; the same check runs in '
'check_amalgamation.yml, which has no path filter'),
('Makefile', 'docs/mkdocs/docs/examples/', 'make pretty; runs in check_amalgamation.yml, which has no path filter'),
('Makefile', 'ChangeLog.md', 'make ChangeLog.md writes the file; no CI job runs it'),
('Makefile', 'tests/fuzzing.md', 'help text'),
('Makefile', 'tools/generate_natvis/README.md', 'help text'),
('tests/src/make_test_data_available.hpp', 'README.md', 'a file of the external test data repository'),
('tools/check_build_options/check_build_options.py', 'docs/', 'builds the path of package_managers.md, which '
'PATHS includes'),
]
# CI targets that read excluded files; they must run in a workflow without a path filter
UNFILTERED_TARGETS = ['ci_test_examples', 'ci_test_build_documentation', 'ci_test_documentation_mermaid',
'ci_test_amalgamation', 'ci_reuse_compliance']
def git_files(root):
# fixed command, no user input; includes new files not yet added, so a local run
# sees what CI sees after the commit
out = subprocess.check_output(['git', 'ls-files', '-z', '--cached', '--others', '--exclude-standard'], cwd=root) # nosec B603, B607
return [f for f in out.decode('utf-8').split('\0') if f]
def glob_to_regex(pattern):
"""Translate a GitHub Actions path pattern to a regular expression."""
regex = ''
i = 0
while i < len(pattern):
if pattern.startswith('**', i):
regex += '.*'
i += 2
elif pattern[i] == '*':
regex += '[^/]*'
i += 1
elif pattern[i] == '?':
regex += '[^/]'
i += 1
else:
regex += re.escape(pattern[i])
i += 1
return re.compile(regex + r'\Z')
def compile_filter(paths, workflow):
return [(p.startswith('!'), glob_to_regex(p.lstrip('!').replace('{workflow}', workflow))) for p in paths]
def included(compiled, path):
result = False
for negated, regex in compiled:
if regex.match(path):
result = not negated
return result
def read_filter(root, workflow):
"""Return the pull_request paths list of a workflow file."""
with open(os.path.join(root, workflow), encoding='utf-8') as f:
lines = f.read().splitlines()
paths = None
in_pull_request = False
for line in lines:
if re.match(r'^ pull_request:', line):
in_pull_request = True
continue
if in_pull_request:
if re.match(r'^ \S', line) or re.match(r'^\S', line):
break
if re.match(r'^ paths:\s*$', line):
paths = []
continue
m = re.match(r"^ - '(.*)'\s*(#.*)?$", line)
if paths is not None and m:
paths.append(m.group(1))
elif paths is not None and line.strip() and not line.strip().startswith('#'):
break
return paths
TOKEN = re.compile(r'(["\'/]?)([\w.-]+(?:/[\w.-]+)*)')
COMMENT_LINE = re.compile(r'^[ \t]*(//|#|/?\*|--).*$', re.MULTILINE)
class Mentions:
"""Finds the words and paths in a text that name an excluded file or directory."""
def __init__(self, files, excluded, is_excluded):
self.files = set(files)
self.excluded = excluded
self.dirs = {'/'.join(f.split('/')[:n]) for f in files for n in range(1, f.count('/') + 1)}
# an excluded directory: a new file in it would be excluded
self.excluded_dirs = {d for d in self.dirs if is_excluded(d + '/new_file')}
# base names that only excluded files have, e.g. 'ChangeLog.md' but not 'CMakeLists.txt'
self.names = {os.path.basename(f) for f in excluded} - {os.path.basename(f) for f in files if f not in excluded}
def find(self, text):
"""Yield (offset, mention) for every word or path in text that names an excluded file or directory."""
for m in TOKEN.finditer(text):
parts = m.group(2).rstrip('/').split('/')
# a single word names a directory only in a path context, e.g. 'docs' in os.path.join('docs', ...)
path_context = len(parts) > 1 or m.group(1)
for i in range(len(parts)):
path = '/'.join(parts[i:])
if path in self.files:
if path in self.excluded:
yield m.start(2), path
break
if path in self.dirs and path_context:
if path in self.excluded_dirs:
yield m.start(2), path + '/'
break
else:
if parts[-1] in self.names and '.' in parts[-1]:
yield m.start(2), parts[-1]
def known(path, mention):
return any(path == file and mention == known_mention for file, known_mention, _ in KNOWN_REFERENCES)
def check_workflows(root):
"""Return the errors in the filters of the workflows and in the targets they run."""
errors = []
for workflow in WORKFLOWS:
expected = [p.replace('{workflow}', workflow) for p in PATHS]
actual = read_filter(root, workflow)
if actual != expected:
errors.append('{}: the pull_request paths filter differs from tools/check_path_filters/check_path_filters.py\n'
' expected: {}\n actual: {}'.format(workflow, expected, actual))
with open(os.path.join(root, workflow), encoding='utf-8') as f:
text = f.read()
errors += ['{}: runs {}, which reads excluded files; run it in repository_checks.yml'.format(workflow, target)
for target in UNFILTERED_TARGETS if re.search(r'\b' + target + r'\b', text)]
return errors
def scanned_text(root, path):
"""Return the text of a kept file without comment lines and without the filter itself, or None to skip it."""
# third-party code and Markdown files read nothing; this script lists the excluded files
if path.startswith('tests/thirdparty/') or path.endswith('.md') or path == SELF:
return None
try:
with open(os.path.join(root, path), encoding='utf-8') as f:
# a mention in a comment line reads nothing; keep the line numbers
text = COMMENT_LINE.sub('', f.read())
except (UnicodeDecodeError, OSError):
return None
if path in WORKFLOWS:
# the filter itself lists the excluded files
for p in PATHS:
text = text.replace(" - '{}'".format(p.replace('{workflow}', path)), '')
return text
def check_mentions(root, files, excluded, finder):
"""Return an error for every mention of an excluded file in a kept file that KNOWN_REFERENCES does not list."""
errors = []
for path in sorted(set(files) - excluded):
text = scanned_text(root, path)
reported = set()
for offset, mention in finder.find(text or ''):
if mention not in reported and not known(path, mention):
reported.add(mention)
line = text.count('\n', 0, offset) + 1
errors.append('{}:{}: mentions excluded {}; include it in PATHS if a compiler job reads it, '
'or add it to KNOWN_REFERENCES'.format(path, line, mention))
return errors
def check(root):
files = git_files(root)
union = [compile_filter(PATHS, w) for w in WORKFLOWS]
def is_excluded(path):
return not any(included(c, path) for c in union)
excluded = {f for f in files if is_excluded(f)}
errors = check_workflows(root) + check_mentions(root, files, excluded, Mentions(files, excluded, is_excluded))
for e in errors:
print(e)
if not errors:
print('path filters OK: {} of {} tracked files skip the compiler workflows'.format(len(excluded), len(files)))
return 1 if errors else 0
def explain(changed):
for workflow in WORKFLOWS:
compiled = compile_filter(PATHS, workflow)
hits = [f for f in changed if included(compiled, f)]
print('{}: {}'.format(workflow, 'runs ({})'.format(', '.join(hits[:3]) + (' ...' if len(hits) > 3 else ''))
if hits else 'skipped'))
return 0
def main():
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument('--root', default=REPO_ROOT, help='repository root')
parser.add_argument('--changed', nargs='+', metavar='FILE', help='show which workflows these changed files run')
args = parser.parse_args()
return explain(args.changed) if args.changed else check(args.root)
if __name__ == '__main__':
sys.exit(main())