From 2e0cc011549f87bebfbdaaee48dfe0b330d8715a Mon Sep 17 00:00:00 2001 From: Remco de Boer <29308176+redeboer@users.noreply.github.com> Date: Thu, 7 Mar 2024 12:43:40 +0100 Subject: [PATCH] BREAK: remove `pin-nb-requirements` hook (#308) --- .pre-commit-hooks.yaml | 8 - README.md | 1 - docs/index.md | 2 - pyproject.toml | 1 - src/compwa_policy/pin_nb_requirements.py | 196 ----------------------- 5 files changed, 208 deletions(-) delete mode 100644 src/compwa_policy/pin_nb_requirements.py diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 29d3bdef..131c0861 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -25,14 +25,6 @@ language: python files: ^setup.cfg$ -- id: pin-nb-requirements - name: Check whether notebook contains a pip install line - description: Specify which packages to install specifically in order to run this notebook. - entry: pin-nb-requirements - language: python - types: - - jupyter - - id: set-nb-cells name: Add or update default cells in a Jupyter notebook description: > diff --git a/README.md b/README.md index cd4ecf99..c5572072 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,6 @@ repos: - id: check-dev-files - id: fix-nbformat-version - id: format-setup-cfg - - id: pin-nb-requirements - id: set-nb-cells ``` diff --git a/docs/index.md b/docs/index.md index e9cd7425..f84c3866 100644 --- a/docs/index.md +++ b/docs/index.md @@ -20,7 +20,6 @@ repos: - id: colab-toc-visible - id: fix-nbformat-version - id: format-setup-cfg - - id: pin-nb-requirements - id: set-nb-cells ``` @@ -38,7 +37,6 @@ The **ComPWA/policy** repository provides the following hooks: - {mod}`colab-toc-visible <.colab_toc_visible>` - {mod}`fix-nbformat-version <.fix_nbformat_version>` - {mod}`format-setup-cfg <.format_setup_cfg>` -- {mod}`pin-nb-requirements <.pin_nb_requirements>` - {mod}`set-nb-cells <.set_nb_cells>` ```{toctree} diff --git a/pyproject.toml b/pyproject.toml index 326753d5..dd508e17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,6 @@ check-dev-files = "compwa_policy.check_dev_files:main" colab-toc-visible = "compwa_policy.colab_toc_visible:main" fix-nbformat-version = "compwa_policy.fix_nbformat_version:main" format-setup-cfg = "compwa_policy.format_setup_cfg:main" -pin-nb-requirements = "compwa_policy.pin_nb_requirements:main" self-check = "compwa_policy.self_check:main" set-nb-cells = "compwa_policy.set_nb_cells:main" diff --git a/src/compwa_policy/pin_nb_requirements.py b/src/compwa_policy/pin_nb_requirements.py deleted file mode 100644 index a63d2bb2..00000000 --- a/src/compwa_policy/pin_nb_requirements.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Enforce adding a pip install statement in the notebook. - -In the `compwa.github.io repo `_, notebooks -should specify which package versions should be used to run the notebook. This -hook checks whether a notebook has such install statements and whether they -comply with the expected formatting. -""" - -from __future__ import annotations - -import argparse -import re -import sys -from functools import lru_cache -from textwrap import dedent -from typing import Sequence - -import nbformat -from nbformat import NotebookNode - -from compwa_policy.utilities.executor import Executor -from compwa_policy.utilities.notebook import load_notebook - -from .errors import PrecommitError - -__EXPECTED_PIP_INSTALL_LINE = "%pip install -q" - - -def check_pinned_requirements(filename: str) -> None: - notebook = load_notebook(filename) - if not __has_python_kernel(notebook): - return - for cell_id, cell in enumerate(notebook["cells"]): - if cell["cell_type"] != "code": - continue - source = __to_oneline(cell["source"]) - pip_requirements = extract_pip_requirements(source) - if pip_requirements is None: - continue - executor = Executor() - executor(_check_pip_requirements, filename, pip_requirements) - executor(_format_pip_requirements, filename, source, notebook, cell_id) - executor(_update_metadata, filename, cell["metadata"], notebook) - executor.finalize() - return - msg = ( - f'Notebook "{filename}" does not contain a pip install cell of the form' - f" {__EXPECTED_PIP_INSTALL_LINE} some-package==0.1.0 package2==3.2" - ) - raise PrecommitError(msg) - - -def __has_python_kernel(notebook: dict) -> bool: - # cspell:ignore kernelspec - metadata = notebook.get("metadata", {}) - kernel_specification = metadata.get("kernelspec", {}) - kernel_language = kernel_specification.get("language", "") - return "python" in kernel_language - - -@lru_cache(maxsize=1) -def __to_oneline(source: str) -> str: - src_lines = source.split("\n") - return "".join(s.rstrip().rstrip("\\") for s in src_lines) - - -@lru_cache(maxsize=1) -def extract_pip_requirements(source: str) -> list[str] | None: - r"""Check if the source in a cell is a pip install statement. - - >>> extract_pip_requirements("Not a pip install statement") - >>> extract_pip_requirements("pip install") - [] - >>> extract_pip_requirements("pip3 install attrs") - ['attrs'] - >>> extract_pip_requirements("pip3 install -q attrs") - ['attrs'] - >>> extract_pip_requirements("pip3 install attrs &> /dev/null") - ['attrs'] - >>> extract_pip_requirements("%pip install attrs numpy==1.24.4 ") - ['attrs', 'numpy==1.24.4'] - >>> extract_pip_requirements("!python3 -mpip install sympy") - ['sympy'] - >>> extract_pip_requirements(''' - ... python3 -m pip install \ - ... attrs numpy \ - ... sympy \ - ... tensorflow - ... ''') - ['attrs', 'numpy', 'sympy', 'tensorflow'] - """ - # cspell:ignore mpip - matches = re.match( - r"[%\!]?\s*(python3?\s+-m\s*)?pip3?\s+install\s*(-q)?(.*?)(&?>\s*/dev/null)?$", - __to_oneline(source).strip(), - ) - if matches is None: - return None - packages = matches.group(3).split(" ") - packages = [p.strip() for p in packages] - return [p for p in packages if p] - - -def _check_pip_requirements(filename: str, requirements: list[str]) -> None: - if len(requirements) == 0: - msg = f'At least one dependency required in install cell of "{filename}"' - raise PrecommitError(msg) - for req in requirements: - req = req.strip() - if not req: - continue - if "git+" in req: - continue - unpinned_requirements = [] - for req in requirements: - if req.startswith("git+"): - continue - if any(equal_sign in req for equal_sign in ["==", "~="]): - continue - package = req.split("<")[0].split(">")[0].strip() - unpinned_requirements.append(package) - if unpinned_requirements: - msg = ( - f'Install cell in notebook "{filename}" contains requirements without' - "pinning (== or ~=):" - ) - for req in unpinned_requirements: - msg += f"\n - {req}" - msg_unformatted = f""" - Get the currently installed versions with: - - python3 -m pip freeze | grep -iE '{"|".join(sorted(unpinned_requirements))}' - """ - msg += dedent(msg_unformatted) - raise PrecommitError(msg) - - -def _format_pip_requirements( - filename: str, install_statement: str, notebook: NotebookNode, cell_id: int -) -> None: - requirements = extract_pip_requirements(install_statement) - if requirements is None: - return - git_requirements = {r for r in requirements if r.startswith("git+")} - pip_requirements = set(requirements) - git_requirements - pip_requirements = {r.lower().replace("_", "-") for r in pip_requirements} - sorted_requirements = sorted(pip_requirements) + sorted(git_requirements) - expected = f"{__EXPECTED_PIP_INSTALL_LINE} {' '.join(sorted_requirements)}" - if install_statement != expected: - notebook["cells"][cell_id]["source"] = expected - nbformat.write(notebook, filename) - msg = f'Ordered and formatted pip install cell in "{filename}"' - raise PrecommitError(msg) - - -def _update_metadata(filename: str, metadata: dict, notebook: NotebookNode) -> None: - updated_metadata = False - jupyter_metadata = metadata.get("jupyter") - if jupyter_metadata is not None and jupyter_metadata.get("source_hidden"): - if len(jupyter_metadata) == 1: - metadata.pop("jupyter") - else: - jupyter_metadata.pop("source_hidden") - updated_metadata = True - tags = set(metadata.get("tags", [])) - expected_tags = {"remove-cell"} - if expected_tags != tags: - metadata["tags"] = sorted(expected_tags) - updated_metadata = True - if updated_metadata: - nbformat.write(notebook, filename) - msg = f'Updated metadata of pip install cell in notebook "{filename}"' - raise PrecommitError(msg) - - -def main(argv: Sequence[str] | None = None) -> int: - parser = argparse.ArgumentParser(__doc__) - parser.add_argument("filenames", nargs="*", help="Filenames to check.") - args = parser.parse_args(argv) - - errors: list[PrecommitError] = [] - for filename in args.filenames: - try: - check_pinned_requirements(filename) - except PrecommitError as exception: - errors.append(exception) - if errors: - for error in errors: - error_msg = "\n ".join(error.args) - print(error_msg) # noqa: T201 - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main())