DM-42226: Write python package metadata

.github/workflows/build.yaml

@@ -11,16 +11,15 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-        pyversion: ["3.10", "3.11", "3.12"]
+        pyversion: ["3.11", "3.12", "3.13"]
 
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           python-version: ${{ matrix.pyversion }}
-          auto-update-conda: true
           channels: conda-forge,defaults
           miniforge-variant: Miniforge3
           use-mamba: true
@@ -37,7 +36,7 @@ jobs:
         shell: bash -l {0}
         run: |
           mamba install -y -q \
-            pytest pytest-xdist pytest-openfiles pytest-cov pytest-session2file
+            pytest pytest-xdist pytest-cov pytest-session2file
 
       - name: List installed packages
         shell: bash -l {0}
@@ -50,8 +49,10 @@ jobs:
         run: |
           setup -k -r .
           scons -j2
+          python -c 'import importlib.metadata as M; print(M.version("lsst.sconsUtils"))'
 
       - name: Upload coverage to codecov
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v4
         with:
-          file: tests/.tests/pytest-sconsUtils.xml-cov-sconsUtils.xml
+          files: tests/.tests/pytest-sconsUtils.xml-cov-sconsUtils.xml
+          token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -23,3 +23,4 @@ tests/.tests
 tests/testFailedTests/python
 tests/testFailedTests/tests/.tests
 .coverage
+python/*.dist-info/

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: check-toml
       - id: check-yaml
@@ -9,7 +9,7 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/psf/black
-    rev: 24.3.0
+    rev: 24.10.0
     hooks:
       - id: black
         # It is recommended to specify the latest version of Python
@@ -24,6 +24,6 @@ repos:
         name: isort (python)
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.3.3
+    rev: v0.8.0
     hooks:
       - id: ruff

python/lsst/sconsUtils/builders.py

@@ -7,13 +7,14 @@
 import os
 import re
 import shlex
+from stat import ST_MODE
 
 import SCons.Script
 from SCons.Script.SConscript import SConsEnvironment
 
 from . import state
 from .installation import determineVersion, getFingerprint
-from .utils import memberOf
+from .utils import memberOf, whichPython
 
 
 @memberOf(SConsEnvironment)
@@ -544,27 +545,32 @@ def Doxygen(self, config, **kwargs):
     return builder(self, config)
 
 
-@memberOf(SConsEnvironment)
-def VersionModule(self, filename, versionString=None):
+def _get_version_string(versionString):
     if versionString is None:
         for n in ("git", "hg", "svn"):
             if os.path.isdir(f".{n}"):
                 versionString = n
 
         if not versionString:
             versionString = "git"
+    return versionString
 
-    def calcMd5(filename):
-        try:
-            import hashlib
 
-            md5 = hashlib.md5(open(filename, "rb").read()).hexdigest()
-        except OSError:
-            md5 = None
+def _calcMd5(filename):
+    try:
+        import hashlib
+
+        md5 = hashlib.md5(open(filename, "rb").read()).hexdigest()
+    except OSError:
+        md5 = None
 
-        return md5
+    return md5
 
-    oldMd5 = calcMd5(filename)
+
+@memberOf(SConsEnvironment)
+def VersionModule(self, filename, versionString=None):
+    versionString = _get_version_string(versionString)
+    oldMd5 = _calcMd5(filename)
 
     def makeVersionModule(target, source, env):
         try:
@@ -629,10 +635,162 @@ def makeVersionModule(target, source, env):
                 outFile.write(f'    "{n}",\n')
             outFile.write(")\n")
 
-        if calcMd5(target[0].abspath) != oldMd5:  # only print if something's changed
+        if _calcMd5(target[0].abspath) != oldMd5:  # only print if something's changed
             state.log.info(f'makeVersionModule(["{target[0]}"], [])')
 
     result = self.Command(filename, [], self.Action(makeVersionModule, strfunction=lambda *args: None))
 
     self.AlwaysBuild(result)
     return result
+
+
+@memberOf(SConsEnvironment)
+def PackageInfo(self, pythonDir, versionString=None):
+    versionString = _get_version_string(versionString)
+
+    if not os.path.exists(pythonDir):
+        return
+
+    # Some information can come from the pyproject file.
+    toml_metadata = {}
+    if os.path.exists("pyproject.toml"):
+        import tomllib
+
+        with open("pyproject.toml", "rb") as fd:
+            toml_metadata = tomllib.load(fd)
+
+    pythonPackageName = ""
+    if "project" in toml_metadata and "name" in toml_metadata["project"]:
+        pythonPackageName = toml_metadata["project"]["name"]
+    else:
+        if os.path.exists(os.path.join(pythonDir, "lsst")):
+            pythonPackageName = "lsst_" + state.env["packageName"]
+        else:
+            pythonPackageName = state.env["packageName"]
+        pythonPackageName = pythonPackageName.replace("_", "-")
+    # The directory name is required to use "_" instead of "-"
+    eggDir = os.path.join(pythonDir, f"{pythonPackageName.replace('-', '_')}.dist-info")
+    filename = os.path.join(eggDir, "METADATA")
+    oldMd5 = _calcMd5(filename)
+
+    def makePackageMetadata(target, source, env):
+        # Create the metadata file.
+        try:
+            version = determineVersion(state.env, versionString)
+        except RuntimeError:
+            version = "unknown"
+
+        os.makedirs(os.path.dirname(target[0].abspath), exist_ok=True)
+        with open(target[0].abspath, "w") as outFile:
+            print("Metadata-Version: 1.0", file=outFile)
+            print(f"Name: {pythonPackageName}", file=outFile)
+            print(f"Version: {version}", file=outFile)
+
+        if _calcMd5(target[0].abspath) != oldMd5:  # only print if something's changed
+            state.log.info(f'PackageInfo(["{target[0]}"], [])')
+
+    results = []
+    results.append(
+        self.Command(filename, [], self.Action(makePackageMetadata, strfunction=lambda *args: None))
+    )
+
+    # Create the entry points file if defined in the pyproject.toml file.
+    entryPoints = {}
+    if "project" in toml_metadata and "entry-points" in toml_metadata["project"]:
+        entryPoints = toml_metadata["project"]["entry-points"]
+
+    if entryPoints:
+        filename = os.path.join(eggDir, "entry_points.txt")
+        oldMd5 = _calcMd5(filename)
+
+    def makeEntryPoints(target, source, env):
+        # Make the entry points file as necessary.
+        if not entryPoints:
+            return
+        os.makedirs(os.path.dirname(target[0].abspath), exist_ok=True)
+
+        # Structure of entry points dict is something like:
+        # "entry-points": {
+        #   "butler.cli": {
+        #     "pipe_base": "lsst.pipe.base.cli:get_cli_subcommands"
+        #   }
+        # }
+        # Which becomes a file with:
+        # [butler.cli]
+        # pipe_base = lsst.pipe.base.cli:get_cli_subcommands
+        with open(target[0].abspath, "w") as fd:
+            for entryGroup in entryPoints:
+                print(f"[{entryGroup}]", file=fd)
+                for entryPoint, entryValue in entryPoints[entryGroup].items():
+                    print(f"{entryPoint} = {entryValue}", file=fd)
+
+        if _calcMd5(target[0].abspath) != oldMd5:  # only print if something's changed
+            state.log.info(f'PackageInfo(["{target[0]}"], [])')
+
+    if entryPoints:
+        results.append(
+            self.Command(filename, [], self.Action(makeEntryPoints, strfunction=lambda *args: None))
+        )
+
+    self.AlwaysBuild(results)
+    return results
+
+
+@memberOf(SConsEnvironment)
+def PythonScripts(self):
+    # Scripts are defined in the pyproject.toml file.
+    toml_metadata = {}
+    if os.path.exists("pyproject.toml"):
+        import tomllib
+
+        with open("pyproject.toml", "rb") as fd:
+            toml_metadata = tomllib.load(fd)
+
+    if not toml_metadata:
+        return []
+
+    scripts = {}
+    if "project" in toml_metadata and "scripts" in toml_metadata["project"]:
+        scripts = toml_metadata["project"]["scripts"]
+
+    def makePythonScript(target, source, env):
+        cmdfile = target[0].abspath
+        command = os.path.basename(cmdfile)
+        if command not in scripts:
+            return
+        os.makedirs(os.path.dirname(cmdfile), exist_ok=True)
+        package, func = scripts[command].split(":", maxsplit=1)
+        with open(cmdfile, "w") as fd:
+            # Follow setuptools convention and always change the shebang.
+            # Can not add noqa on Linux for long paths so do not add anywhere.
+            print(
+                rf"""#!{whichPython()}
+import sys
+from {package} import {func}
+if __name__ == '__main__':
+    sys.exit({func}())
+""",
+                file=fd,
+            )
+
+        # Ensure the bin/ file is executable
+        oldmode = os.stat(cmdfile)[ST_MODE] & 0o7777
+        newmode = (oldmode | 0o555) & 0o7777
+        if newmode != oldmode:
+            state.log.info(f"Changing mode of {cmdfile} from {oldmode} to {newmode}")
+            os.chmod(cmdfile, newmode)
+
+    results = []
+    for cmd, code in scripts.items():
+        filename = f"bin/{cmd}"
+
+        # Do not do anything if there is an equivalent target in bin.src
+        # that shebang would trigger.
+        if os.path.exists(f"bin.src/{cmd}"):
+            continue
+
+        results.append(
+            self.Command(filename, [], self.Action(makePythonScript, strfunction=lambda *args: None))
+        )
+
+    return results

python/lsst/sconsUtils/scripts.py

@@ -127,7 +127,7 @@ def initialize(
         disableCC : `bool`, optional
             Should the C++ compiler check be disabled? Disabling this checks
             allows a faster startup and permits building on systems that don't
-            meet the requirements for the C++ compilter (e.g., for
+            meet the requirements for the C++ compiler (e.g., for
             pure-python packages).
 
         Returns
@@ -145,13 +145,20 @@ def initialize(
         state.env.BuildETags()
         if cleanExt is None:
             cleanExt = r"*~ core core.[1-9]* *.so *.os *.o *.pyc *.pkgc"
-        state.env.CleanTree(cleanExt, "__pycache__ .pytest_cache")
+        state.env.CleanTree(cleanExt, "__pycache__ .pytest_cache *.dist-info")
         if versionModuleName is not None:
             try:
                 versionModuleName = versionModuleName % "/".join(packageName.split("_"))
             except TypeError:
                 pass
             state.targets["version"] = state.env.VersionModule(versionModuleName)
+        # Always attempt to write python package info into the python
+        # directory.
+        if os.path.exists("python"):
+            state.targets["pkginfo"] = state.env.PackageInfo("python")
+        # Python script generation does no harm since it will only do anything
+        # if there is a scripts entry in pyproject.toml.
+        state.targets["scripts"] = state.env.PythonScripts()
         scripts = []
         for root, dirs, files in os.walk("."):
             if "SConstruct" in files and root != ".":
@@ -233,7 +240,14 @@ def finish(defaultTargets=DEFAULT_TARGETS, subDirList=None, ignoreRegex=None):
         )
         if "version" in state.targets:
             state.env.Default(state.targets["version"])
-        state.env.Requires(state.targets["tests"], state.targets["version"])
+            state.env.Requires(state.targets["tests"], state.targets["version"])
+        if "pkginfo" in state.targets:
+            state.env.Default(state.targets["pkginfo"])
+            state.env.Requires(state.targets["tests"], state.targets["pkginfo"])
+        if "scripts" in state.targets:
+            state.env.Default(state.targets["scripts"])
+            state.env.Requires(state.targets["tests"], state.targets["scripts"])
+
         state.env.Decider("MD5-timestamp")  # if timestamps haven't changed, don't do MD5 checks
         #
         # Check if any of the tests failed by looking for *.failed files.

python/lsst/sconsUtils/state.py

@@ -49,6 +49,8 @@
     "include": [],
     "version": [],
     "shebang": [],
+    "pkginfo": [],
+    "scripts": [],
 }
 
 env = None