Merge pull request #1964 from hauntsaninja/static-pyproj

Parse pyproject.toml statically where possible

piptools/build.py

@@ -14,6 +14,13 @@
 import pyproject_hooks
 from pip._internal.req import InstallRequirement
 from pip._internal.req.constructors import install_req_from_line, parse_req_from_line
+from pip._vendor.packaging.markers import Marker
+from pip._vendor.packaging.requirements import Requirement
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
 
 PYPROJECT_TOML = "pyproject.toml"
 
@@ -32,23 +39,95 @@ def get_all(self, name: str, failobj: None = None) -> list[Any] | None: ...
         def get_all(self, name: str, failobj: _T) -> list[Any] | _T: ...
 
 
+@dataclass
+class StaticProjectMetadata:
+    extras: tuple[str, ...]
+    requirements: tuple[InstallRequirement, ...]
+
+
 @dataclass
 class ProjectMetadata:
     extras: tuple[str, ...]
     requirements: tuple[InstallRequirement, ...]
     build_requirements: tuple[InstallRequirement, ...]
 
 
+def maybe_statically_parse_project_metadata(
+    src_file: pathlib.Path,
+) -> StaticProjectMetadata | None:
+    """
+    Return the metadata for a project, if it can be statically parsed from ``pyproject.toml``.
+
+    This function is typically significantly faster than invoking a build backend.
+    Returns None if the project metadata cannot be statically parsed.
+    """
+    if src_file.name != PYPROJECT_TOML:
+        return None
+
+    try:
+        with open(src_file, "rb") as f:
+            pyproject_contents = tomllib.load(f)
+    except tomllib.TOMLDecodeError:
+        return None
+
+    # Not valid PEP 621 metadata
+    if (
+        "project" not in pyproject_contents
+        or "name" not in pyproject_contents["project"]
+    ):
+        return None
+
+    project_table = pyproject_contents["project"]
+
+    # Dynamic dependencies require build backend invocation
+    dynamic = project_table.get("dynamic", [])
+    if "dependencies" in dynamic or "optional-dependencies" in dynamic:
+        return None
+
+    package_name = project_table["name"]
+    comes_from = f"{package_name} ({src_file})"
+
+    extras = project_table.get("optional-dependencies", {}).keys()
+    install_requirements = [
+        InstallRequirement(Requirement(req), comes_from)
+        for req in project_table.get("dependencies", [])
+    ]
+    for extra, reqs in (
+        pyproject_contents.get("project", {}).get("optional-dependencies", {}).items()
+    ):
+        for req in reqs:
+            requirement = Requirement(req)
+            if requirement.name == package_name:
+                # Similar to logic for handling self-referential requirements
+                # from _prepare_requirements
+                requirement.url = src_file.parent.as_uri()
+            # Note we don't need to modify `requirement` to include this extra
+            marker = Marker(f"extra == '{extra}'")
+            install_requirements.append(
+                InstallRequirement(requirement, comes_from, markers=marker)
+            )
+
+    return StaticProjectMetadata(
+        extras=tuple(extras),
+        requirements=tuple(install_requirements),
+    )
+
+
 def build_project_metadata(
     src_file: pathlib.Path,
     build_targets: tuple[str, ...],
     *,
+    attempt_static_parse: bool,
     isolated: bool,
     quiet: bool,
-) -> ProjectMetadata:
+) -> ProjectMetadata | StaticProjectMetadata:
     """
     Return the metadata for a project.
 
+    First, optionally attempt to determine the metadata statically from the
+    ``pyproject.toml`` file. This will not work if build_targets are specified,
+    since we cannot determine build requirements statically.
+
     Uses the ``prepare_metadata_for_build_wheel`` hook for the wheel metadata
     if available, otherwise ``build_wheel``.
 
@@ -58,12 +137,25 @@ def build_project_metadata(
     :param src_file: Project source file
     :param build_targets: A tuple of build targets to get the dependencies
                                 of (``sdist`` or ``wheel`` or ``editable``).
+    :param attempt_static_parse: Whether to attempt to statically parse the
+                                 project metadata from ``pyproject.toml``.
+                                 Cannot be used with ``build_targets``.
     :param isolated: Whether to run invoke the backend in the current
                      environment or to create an isolated one and invoke it
                      there.
     :param quiet: Whether to suppress the output of subprocesses.
     """
 
+    if attempt_static_parse:
+        if build_targets:
+            raise ValueError(
+                "Cannot execute the PEP 517 optional get_requires_for_build* "
+                "hooks statically, as build requirements are requested"
+            )
+        project_metadata = maybe_statically_parse_project_metadata(src_file)
+        if project_metadata is not None:
+            return project_metadata
+
     src_dir = src_file.parent
     with _create_project_builder(src_dir, isolated=isolated, quiet=quiet) as builder:
         metadata = _build_project_wheel_metadata(builder)

piptools/scripts/compile.py

@@ -16,7 +16,7 @@
 from pip._internal.utils.misc import redact_auth_from_url
 
 from .._compat import parse_requirements
-from ..build import build_project_metadata
+from ..build import ProjectMetadata, build_project_metadata
 from ..cache import DependencyCache
 from ..exceptions import NoCandidateFound, PipToolsError
 from ..logging import log
@@ -365,6 +365,7 @@ def cli(
                 metadata = build_project_metadata(
                     src_file=Path(src_file),
                     build_targets=build_deps_targets,
+                    attempt_static_parse=not bool(build_deps_targets),
                     isolated=build_isolation,
                     quiet=log.verbosity <= 0,
                 )
@@ -378,6 +379,7 @@ def cli(
                 if all_extras:
                     extras += metadata.extras
             if build_deps_targets:
+                assert isinstance(metadata, ProjectMetadata)
                 constraints.extend(metadata.build_requirements)
         else:
             constraints.extend(

tests/test_build.py

@@ -5,7 +5,12 @@
 
 import pytest
 
-from piptools.build import build_project_metadata
+from piptools.build import (
+    ProjectMetadata,
+    StaticProjectMetadata,
+    build_project_metadata,
+    maybe_statically_parse_project_metadata,
+)
 from tests.constants import PACKAGES_PATH
 
 
@@ -25,8 +30,9 @@ def test_build_project_metadata_resolved_correct_build_dependencies(
     shutil.copytree(src_pkg_path, tmp_path, dirs_exist_ok=True)
     src_file = tmp_path / "setup.py"
     metadata = build_project_metadata(
-        src_file, ("editable",), isolated=True, quiet=False
+        src_file, ("editable",), attempt_static_parse=False, isolated=True, quiet=False
     )
+    assert isinstance(metadata, ProjectMetadata)
     build_requirements = sorted(r.name for r in metadata.build_requirements)
     assert build_requirements == [
         "fake_dynamic_build_dep_for_all",
@@ -35,3 +41,98 @@ def test_build_project_metadata_resolved_correct_build_dependencies(
         "setuptools",
         "wheel",
     ]
+
+
+def test_build_project_metadata_static(tmp_path):
+    """Test static parsing branch of build_project_metadata"""
+    src_pkg_path = pathlib.Path(PACKAGES_PATH) / "small_fake_with_pyproject"
+    shutil.copytree(src_pkg_path, tmp_path, dirs_exist_ok=True)
+    src_file = tmp_path / "pyproject.toml"
+    metadata = build_project_metadata(
+        src_file, (), attempt_static_parse=True, isolated=True, quiet=False
+    )
+    assert isinstance(metadata, StaticProjectMetadata)
+    requirements = [(r.name, r.extras, str(r.markers)) for r in metadata.requirements]
+    requirements.sort(key=lambda x: x[0])
+    assert requirements == [
+        ("fake_direct_extra_runtime_dep", {"with_its_own_extra"}, 'extra == "x"'),
+        ("fake_direct_runtime_dep", set(), "None"),
+    ]
+    assert metadata.extras == ("x",)
+
+
+def test_build_project_metadata_raises_error(tmp_path):
+    src_pkg_path = pathlib.Path(PACKAGES_PATH) / "small_fake_with_build_deps"
+    shutil.copytree(src_pkg_path, tmp_path, dirs_exist_ok=True)
+    src_file = tmp_path / "setup.py"
+    with pytest.raises(
+        ValueError, match="Cannot execute the PEP 517 optional.* hooks statically"
+    ):
+        build_project_metadata(
+            src_file,
+            ("editable",),
+            attempt_static_parse=True,
+            isolated=True,
+            quiet=False,
+        )
+
+
+def test_static_parse_valid(tmp_path):
+    src_file = tmp_path / "pyproject.toml"
+
+    valid = """
+[project]
+name = "foo"
+version = "0.1.0"
+dependencies = ["bar>=1"]
+[project.optional-dependencies]
+baz = ["qux[extra]"]
+"""
+    src_file.write_text(valid)
+    metadata = maybe_statically_parse_project_metadata(src_file)
+    assert isinstance(metadata, StaticProjectMetadata)
+    assert [str(r.req) for r in metadata.requirements] == ["bar>=1", "qux[extra]"]
+    assert metadata.extras == ("baz",)
+
+
+def test_static_parse_invalid(tmp_path):
+    src_file = tmp_path / "pyproject.toml"
+
+    invalid_toml = """this is not valid toml"""
+    src_file.write_text(invalid_toml)
+    assert maybe_statically_parse_project_metadata(src_file) is None
+
+    no_pep621 = """
+[build-system]
+requires = ["setuptools"]
+"""
+    src_file.write_text(no_pep621)
+    assert maybe_statically_parse_project_metadata(src_file) is None
+
+    invalid_pep621 = """
+[project]
+# no name
+version = "0.1.0"
+"""
+    src_file.write_text(invalid_pep621)
+    assert maybe_statically_parse_project_metadata(src_file) is None
+
+    dynamic_deps = """
+[project]
+name = "foo"
+dynamic = ["dependencies"]
+"""
+    src_file.write_text(dynamic_deps)
+    assert maybe_statically_parse_project_metadata(src_file) is None
+
+    dynamic_optional_deps = """
+[project]
+name = "foo"
+dynamic = ["optional-dependencies"]
+"""
+    src_file.write_text(dynamic_optional_deps)
+    assert maybe_statically_parse_project_metadata(src_file) is None
+
+    src_file = tmp_path / "setup.py"
+    src_file.write_text("print('hello')")
+    assert maybe_statically_parse_project_metadata(src_file) is None

tests/test_cli_compile.py

@@ -3330,7 +3330,7 @@ def test_pass_pip_cache_to_pip_args(tmpdir, runner, current_resolver):
 
 
 @backtracking_resolver_only
-def test_compile_recursive_extras(runner, tmp_path, current_resolver):
+def test_compile_recursive_extras_static(runner, tmp_path, current_resolver):
     (tmp_path / "pyproject.toml").write_text(
         dedent(
             """
@@ -3364,8 +3364,66 @@ def test_compile_recursive_extras(runner, tmp_path, current_resolver):
 small-fake-a==0.2
 small-fake-b==0.3
 """
-    assert out.exit_code == 0
-    assert expected == out.stdout
+    try:
+        assert out.exit_code == 0
+        assert expected == out.stdout
+    except Exception:  # pragma: no cover
+        print(out.stdout)
+        print(out.stderr)
+        raise
+
+
+@backtracking_resolver_only
+def test_compile_recursive_extras_build_targets(runner, tmp_path, current_resolver):
+    (tmp_path / "pyproject.toml").write_text(
+        dedent(
+            """
+            [project]
+            name = "foo"
+            version = "0.0.1"
+            dependencies = ["small-fake-a"]
+            [project.optional-dependencies]
+            footest = ["small-fake-b"]
+            dev = ["foo[footest]"]
+            """
+        )
+    )
+    (tmp_path / "constraints.txt").write_text("wheel<0.43")
+    out = runner.invoke(
+        cli,
+        [
+            "--no-build-isolation",
+            "--no-header",
+            "--no-annotate",
+            "--no-emit-options",
+            "--extra",
+            "dev",
+            "--build-deps-for",
+            "wheel",
+            "--find-links",
+            os.fspath(MINIMAL_WHEELS_PATH),
+            os.fspath(tmp_path / "pyproject.toml"),
+            "--constraint",
+            os.fspath(tmp_path / "constraints.txt"),
+            "--output-file",
+            "-",
+        ],
+    )
+    expected = rf"""foo[footest] @ {tmp_path.as_uri()}
+small-fake-a==0.2
+small-fake-b==0.3
+wheel==0.42.0
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
+"""
+    try:
+        assert out.exit_code == 0
+        assert expected == out.stdout
+    except Exception:  # pragma: no cover
+        print(out.stdout)
+        print(out.stderr)
+        raise
 
 
 def test_config_option(pip_conf, runner, tmp_path, make_config_file):

tests/test_data/packages/small_fake_with_pyproject/pyproject.toml

@@ -0,0 +1,8 @@
+[project]
+name="small_fake_with_pyproject"
+version=0.1
+dependencies=[
+    "fake_direct_runtime_dep",
+]
+[project.optional-dependencies]
+x = ["fake_direct_extra_runtime_dep[with_its_own_extra]"]