diff --git a/.cspell.json b/.cspell.json index b29c863c..6d90d413 100644 --- a/.cspell.json +++ b/.cspell.json @@ -22,6 +22,8 @@ "ignorePaths": [ "**/*.rst_t", "**/.cspell.json", + "*.ico", + "*.rst_t", ".editorconfig", ".gitignore", ".gitpod.*", @@ -33,6 +35,7 @@ "docs/conf.py", "labels/*.toml", "pyproject.toml", + "pyrightconfig.json", "tox.ini", "typings" ], @@ -85,6 +88,7 @@ "noreply", "orcid", "pandoc", + "pathspec", "prebuilds", "precommit", "prereleased", diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0c2863d8..30c334d4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -78,8 +78,7 @@ repos: rev: v0.23.1 hooks: - id: toml-sort - args: - - --in-place + args: [--in-place] exclude: (?x)^(labels/.*\.toml)$ - repo: https://github.com/streetsidesoftware/cspell-cli diff --git a/pyproject.toml b/pyproject.toml index f8105147..64d368fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "html2text", "ini2toml", "nbformat", + "pathspec", "pip-tools", "ruamel.yaml", # better YAML dumping "tomlkit", # preserve original TOML formatting diff --git a/src/compwa_policy/check_dev_files/cspell.py b/src/compwa_policy/check_dev_files/cspell.py index 56dba85d..949173aa 100644 --- a/src/compwa_policy/check_dev_files/cspell.py +++ b/src/compwa_policy/check_dev_files/cspell.py @@ -8,12 +8,12 @@ import json import os -from glob import glob from typing import TYPE_CHECKING, Any, Iterable, Sequence from compwa_policy.errors import PrecommitError from compwa_policy.utilities import COMPWA_POLICY_DIR, CONFIG_PATH, rename_file, vscode from compwa_policy.utilities.executor import Executor +from compwa_policy.utilities.match import filter_patterns from compwa_policy.utilities.precommit.struct import Hook, Repo from compwa_policy.utilities.readme import add_badge, remove_badge @@ -157,9 +157,7 @@ def __get_expected_content(config: dict, section: str, *, extend: bool = False) return expected_section_content if isinstance(expected_section_content, list): if section == "ignorePaths": - expected_section_content = [ - p for p in expected_section_content if glob(p, recursive=True) - ] + expected_section_content = filter_patterns(expected_section_content) if not extend: return __sort_section(expected_section_content, section) expected_section_content_set = set(expected_section_content) diff --git a/src/compwa_policy/check_dev_files/github_labels.py b/src/compwa_policy/check_dev_files/github_labels.py index 08b9af8e..65e5fd2c 100644 --- a/src/compwa_policy/check_dev_files/github_labels.py +++ b/src/compwa_policy/check_dev_files/github_labels.py @@ -7,10 +7,12 @@ from __future__ import annotations import os -import pathlib +from functools import lru_cache +from pathlib import Path from compwa_policy.errors import PrecommitError from compwa_policy.utilities import CONFIG_PATH +from compwa_policy.utilities.match import filter_files __LABELS_CONFIG_FILE = "labels.toml" @@ -39,7 +41,7 @@ def main() -> None: raise PrecommitError(msg) -def _check_has_labels_requirement(path: pathlib.Path) -> bool: +def _check_has_labels_requirement(path: Path) -> bool: with open(path) as stream: lines = stream.readlines() for line in lines: @@ -49,12 +51,15 @@ def _check_has_labels_requirement(path: pathlib.Path) -> bool: return False -def _get_requirement_files() -> list[pathlib.Path]: - return [ - *pathlib.Path(".").glob("**/requirements*.in"), - *pathlib.Path(".").glob("**/requirements*.txt"), - *pathlib.Path(".").glob(str(CONFIG_PATH.setup_cfg)), +@lru_cache(maxsize=1) +def _get_requirement_files() -> list[Path]: + patterns = [ + "**/requirements*.in", + "**/requirements*.txt", + str(CONFIG_PATH.setup_cfg), ] + filenames = filter_files(patterns) + return [Path(file) for file in filenames] def _get_package_name(line: str) -> str: @@ -72,7 +77,7 @@ def _remove_all_labels_requirement() -> None: _remove_labels_requirement(file) -def _remove_labels_requirement(path: pathlib.Path) -> None: +def _remove_labels_requirement(path: Path) -> None: with open(path) as stream: original_lines = stream.readlines() with open(path, "w") as stream: diff --git a/src/compwa_policy/check_dev_files/readthedocs.py b/src/compwa_policy/check_dev_files/readthedocs.py index 9736b511..119ada65 100644 --- a/src/compwa_policy/check_dev_files/readthedocs.py +++ b/src/compwa_policy/check_dev_files/readthedocs.py @@ -7,7 +7,7 @@ from typing import IO, TYPE_CHECKING, cast from ruamel.yaml.comments import CommentedMap -from ruamel.yaml.scalarstring import DoubleQuotedScalarString +from ruamel.yaml.scalarstring import DoubleQuotedScalarString, LiteralScalarString from compwa_policy.errors import PrecommitError from compwa_policy.utilities import CONFIG_PATH, get_nested_dict @@ -95,7 +95,7 @@ def __get_install_steps(python_version: PythonVersion) -> list[str]: install_statement = f"{pip_install} -c {constraints_file} -e .[doc]" return [ "curl -LsSf https://astral.sh/uv/install.sh | sh", - install_statement, + LiteralScalarString(install_statement), ] diff --git a/src/compwa_policy/check_dev_files/toml.py b/src/compwa_policy/check_dev_files/toml.py index 97a0eb0b..7ae97be4 100644 --- a/src/compwa_policy/check_dev_files/toml.py +++ b/src/compwa_policy/check_dev_files/toml.py @@ -3,7 +3,6 @@ from __future__ import annotations import shutil -from glob import glob from pathlib import Path from typing import TYPE_CHECKING @@ -13,6 +12,7 @@ from compwa_policy.errors import PrecommitError from compwa_policy.utilities import COMPWA_POLICY_DIR, CONFIG_PATH, vscode from compwa_policy.utilities.executor import Executor +from compwa_policy.utilities.match import filter_patterns from compwa_policy.utilities.precommit.struct import Hook, Repo from compwa_policy.utilities.pyproject import ModifiablePyproject from compwa_policy.utilities.toml import to_toml_array @@ -72,16 +72,17 @@ def _update_tomlsort_hook(precommit: ModifiablePrecommit) -> None: rev="", hooks=[Hook(id="toml-sort", args=YAML(typ="rt").load("[--in-place]"))], ) - excludes = [] - if glob("labels/*.toml"): - excludes.append(r"labels/.*\.toml") - if glob("labels*.toml"): - excludes.append(r"labels.*\.toml") - if any(glob(f"**/{f}.toml", recursive=True) for f in ("Manifest", "Project")): - excludes.append(r".*(Manifest|Project)\.toml") + excludes = filter_patterns([ + "**/Manifest.toml", + "**/Project.toml", + "labels*.toml", + "labels/*.toml", + ]) if excludes: - excludes = sorted(excludes, key=str.lower) - expected_hook["hooks"][0]["exclude"] = "(?x)^(" + "|".join(excludes) + ")$" + regex_excludes = sorted(_to_regex(r) for r in excludes) + expected_hook["hooks"][0]["exclude"] = ( + "(?x)^(" + "|".join(regex_excludes) + ")$" + ) precommit.update_single_hook_repo(expected_hook) @@ -102,10 +103,11 @@ def _update_taplo_config() -> None: raise PrecommitError(msg) with open(template_path) as f: expected = tomlkit.load(f) - excludes: list[str] = [p for p in expected["exclude"] if glob(p, recursive=True)] # type: ignore[union-attr] + + excludes = filter_patterns(expected["exclude"]) # type:ignore[arg-type] if excludes: - excludes = sorted(excludes, key=str.lower) - expected["exclude"] = to_toml_array(excludes, enforce_multiline=True) + sorted_excludes = sorted(excludes, key=str.lower) + expected["exclude"] = to_toml_array(sorted_excludes, enforce_multiline=True) else: del expected["exclude"] with open(CONFIG_PATH.taplo) as f: @@ -133,3 +135,12 @@ def _update_vscode_extensions() -> None: with Executor() as do: do(vscode.add_extension_recommendation, "tamasfe.even-better-toml") do(vscode.remove_extension_recommendation, "bungcip.better-toml", unwanted=True) + + +def _to_regex(glob: str) -> str: + r"""Convert glob pattern to regex. + + >>> _to_regex("**/*.toml") + '.*/.*\\.toml' + """ + return glob.replace("**", "*").replace(".", r"\.").replace("*", r".*") diff --git a/src/compwa_policy/check_dev_files/update_pip_constraints.py b/src/compwa_policy/check_dev_files/update_pip_constraints.py index fb9d48f0..493ce69a 100644 --- a/src/compwa_policy/check_dev_files/update_pip_constraints.py +++ b/src/compwa_policy/check_dev_files/update_pip_constraints.py @@ -8,7 +8,6 @@ from __future__ import annotations import sys -from glob import glob from typing import TYPE_CHECKING from compwa_policy.check_dev_files.github_workflows import ( @@ -18,6 +17,7 @@ from compwa_policy.errors import PrecommitError from compwa_policy.utilities import COMPWA_POLICY_DIR, CONFIG_PATH from compwa_policy.utilities.executor import Executor +from compwa_policy.utilities.match import filter_patterns from compwa_policy.utilities.yaml import create_prettier_round_trip_yaml if TYPE_CHECKING: @@ -74,9 +74,9 @@ def overwrite_workflow(workflow_file: str) -> None: if frequency == "outsource": del expected_data["on"]["schedule"] else: - paths: list[str] = expected_data["on"]["pull_request"]["paths"] + paths = filter_patterns(expected_data["on"]["pull_request"]["paths"]) + expected_data["on"]["pull_request"]["paths"] = paths expected_data["on"]["schedule"][0]["cron"] = _to_cron_schedule(frequency) - expected_data["on"]["pull_request"]["paths"] = [p for p in paths if glob(p)] workflow_path = CONFIG_PATH.github_workflow_dir / workflow_file if not workflow_path.exists(): update_workflow(yaml, expected_data, workflow_path) diff --git a/src/compwa_policy/utilities/match.py b/src/compwa_policy/utilities/match.py new file mode 100644 index 00000000..ee03da85 --- /dev/null +++ b/src/compwa_policy/utilities/match.py @@ -0,0 +1,80 @@ +"""Functions for checking whether files exist on disk.""" + +from __future__ import annotations + +import subprocess # noqa: S404 + +from pathspec import PathSpec +from pathspec.patterns import GitWildMatchPattern + + +def filter_files(patterns: list[str], files: list[str] | None = None) -> list[str]: + """Filter filenames that match certain patterns. + + If :code:`files` is not supplied, get the files with :func:`git_ls_files`. + + >>> filter_files(["**/*.json", "**/*.txt"], ["a/b/file.json", "file.yaml"]) + ['a/b/file.json'] + """ + if files is None: + files = git_ls_files(untracked=True) + return [file for file in files if matches_patterns(file, patterns)] + + +def filter_patterns(patterns: list[str], files: list[str] | None = None) -> list[str]: + """Filter patterns that match files. + + If :code:`files` is not supplied, get the files with :func:`git_ls_files`. + + >>> filter_patterns(["**/*.json", "**/*.txt"], ["file.json", "file.yaml"]) + ['**/*.json'] + """ + if files is None: + files = git_ls_files(untracked=True) + return [pattern for pattern in patterns if matches_files(pattern, files)] + + +def git_ls_files(untracked: bool = False) -> list[str]: + """Get the tracked and untracked files, but excluding files in .gitignore.""" + output = subprocess.check_output([ # noqa: S603, S607 + "git", + "ls-files", + ]).decode("utf-8") + tracked_files = output.splitlines() + if untracked: + output = subprocess.check_output([ # noqa: S603, S607 + "git", + "ls-files", + "--others", + "--exclude-standard", + ]).decode("utf-8") + return tracked_files + output.splitlines() + return tracked_files + + +def matches_files(pattern: str, files: list[str]) -> bool: + """Use git wild-match patterns to match a filename. + + >>> matches_files("**/*.json", [".cspell.json"]) + True + >>> matches_files("**/*.json", ["some/random/path/.cspell.json"]) + True + >>> matches_files("*/*.json", ["some/random/path/.cspell.json"]) + False + """ + spec = PathSpec.from_lines(GitWildMatchPattern, [pattern]) + return any(spec.match_file(file) for file in files) + + +def matches_patterns(filename: str, patterns: list[str]) -> bool: + """Use git wild-match patterns to match a filename. + + >>> matches_patterns(".cspell.json", patterns=["**/*.json"]) + True + >>> matches_patterns("some/random/path/.cspell.json", patterns=["**/*.json"]) + True + >>> matches_patterns("some/random/path/.cspell.json", patterns=["*/*.json"]) + False + """ + spec = PathSpec.from_lines(GitWildMatchPattern, patterns) + return spec.match_file(filename) diff --git a/tests/check_dev_files/readthedocs/.readthedocs-good.yml b/tests/check_dev_files/readthedocs/.readthedocs-good.yml index fd564c0a..110567ea 100644 --- a/tests/check_dev_files/readthedocs/.readthedocs-good.yml +++ b/tests/check_dev_files/readthedocs/.readthedocs-good.yml @@ -6,4 +6,5 @@ build: jobs: post_install: - curl -LsSf https://astral.sh/uv/install.sh | sh - - /home/docs/.cargo/bin/uv pip install --system -e .[doc] + - |- + /home/docs/.cargo/bin/uv pip install --system -e .[doc]