Upgrade dependencies (#585)

* Pin dependencies based on Python version
* Upgrade dependencies to latest version
* Improve CI to catch dependencies that build whl files

.github/actions/test/action.yml

@@ -63,15 +63,32 @@ runs:
     - name: Install Python dependencies
       run: |
         python3 -V
+
+        # inspect pip cache
         python3 -m pip freeze | sort
         python3 -m pip cache info || true
         python3 -m pip cache list || true
-        python3 -m pip install --upgrade --force pip wheel
-        python3 -m pip install --force -r python/requirements.txt
+
+        # remove cached built whl files
+        rm -rf "$(python3 -m pip cache info | grep "Locally built wheels location: " | cut -d ":" -f 2- | cut -d " " -f 2-)"
+        python3 -m pip cache list || true
+
+        # install dependencies
+        python_minor_version="$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')"
+        python3 -m pip install --force -r python/requirements-$python_minor_version.txt
         python3 -m pip install --force -r python/test/requirements.txt -c python/test/constraints.txt
+
+        # inspect pip cache
         python3 -m pip freeze | sort
         python3 -m pip cache info || true
         python3 -m pip cache list || true
+
+        # assert no whl files have been built
+        if [ python3 -m pip cache info ] && [ "$(python3 -m pip cache info | grep "Number of locally built wheels:")" != "Number of locally built wheels: 0" ]
+        then
+          echo "Dependency whl files have been built"
+          exit 1
+        fi
       shell: bash
 
     - name: Update expectation files

.github/upgrade-pip-packages.sh

@@ -2,12 +2,13 @@
 set -euo pipefail
 
 base="$(dirname "$0")"
+python_minor_version="$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')"
 
-pip install --upgrade --force pip==22.0.0
-pip install --upgrade --upgrade-strategy eager -r "$base/../python/requirements-direct.txt"
+pip install --upgrade --force pip==24.0.0
+pip install --upgrade --upgrade-strategy eager -r "$base/../python/requirements.txt"
 
 pip install pipdeptree
-pipdeptree --packages="$(sed -e "s/;.*//" -e "s/=.*//g" "$base/../python/requirements-direct.txt" | paste -s -d ,)" --freeze > "$base/../python/requirements.txt"
+pipdeptree --packages="$(sed -e "s/;.*//" -e "s/=.*//g" "$base/../python/requirements.txt" | paste -s -d ,)" --freeze > "$base/../python/requirements-$python_minor_version.txt"
 
-git diff "$base/../python/requirements.txt"
+git diff "$base/../python/requirements-$python_minor_version.txt"
 

.github/workflows/ci-cd.yml

@@ -20,14 +20,17 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Check requirements.txt against requirements-direct.txt
+      - name: Check requirements.txt
         run: |
-          (diff -w python/requirements-direct.txt python/requirements.txt || true) | (! grep -e "^<")
+          (diff -w <(grep -v "python_version >  '3.7'" python/requirements.txt | sed -e "s/;.*//") python/requirements-3.7.txt || true) | (! grep -e "^<")
+          (diff -w <(grep -v "python_version <= '3.7'" python/requirements.txt | sed -e "s/;.*//") python/requirements-post-3.7.txt || true) | (! grep -e "^<")
         shell: bash
       - name: Check for dependency updates
         continue-on-error: true
-        run:
-          .github/upgrade-pip-packages.sh
+        run: |
+          pip install tox
+          tox
+          git diff --exit-code
         shell: bash
 
   test-mac:

Dockerfile

@@ -11,7 +11,7 @@ LABEL com.github.actions.color="green"
 
 RUN apk add --no-cache --upgrade expat libuuid
 
-COPY python/requirements.txt /action/
+COPY python/requirements-post-3.7.txt /action/requirements.txt
 RUN apk add --no-cache build-base libffi-dev; \
     pip install --upgrade --force --no-cache-dir pip && \
     pip install --upgrade --force --no-cache-dir -r /action/requirements.txt; \

README.md

@@ -854,7 +854,7 @@ Self-hosted runners may require setting up a Python environment first:
     python-version: 3.8
 ```
 
-Start-up of the action is faster with `virtualenv` or `venv`, as well as `wheel` packages are installed.
+Start-up of the action is faster with `virtualenv` or `venv` package installed.
 
 ## Running as a composite action
 

composite/action.yml

@@ -181,7 +181,15 @@ runs:
 
         echo "Python that creates venv: $PYTHON_BIN"
         echo "PYTHON_BIN=$PYTHON_BIN" >> "$GITHUB_ENV"
-        echo "version=$("$PYTHON_BIN" -V)" >> "$GITHUB_OUTPUT"
+
+        PYTHON_VERSION="$($PYTHON_BIN -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')"
+        if [[ "$PYTHON_VERSION" == "3.7" ]]
+        then
+          echo "DEPENDENCIES_VERSION=3.7" >> "$GITHUB_ENV"
+        else
+          echo "DEPENDENCIES_VERSION=post-3.7" >> "$GITHUB_ENV"
+        fi
+        echo "version=$PYTHON_VERSION" >> "$GITHUB_OUTPUT"
 
         echo '::endgroup::'
       shell: bash
@@ -210,7 +218,7 @@ runs:
       continue-on-error: true
       with:
         path: ${{ steps.os.outputs.pip-cache }}
-        key: enricomi-publish-action-${{ runner.os }}-${{ runner.arch }}-pip-${{ steps.python.outputs.version }}-70a313922fdbeb7398ec60313d908b11
+        key: enricomi-publish-action-${{ runner.os }}-${{ runner.arch }}-pip-${{ steps.python.outputs.version }}-e594996205319a7990b3a4ec677d10a3
 
     - name: Create virtualenv
       id: venv
@@ -262,11 +270,7 @@ runs:
       run: |
         # Install Python dependencies
         echo '::group::Install Python dependencies'
-
-        # make sure wheel is installed, which improves installing our dependencies
-        "$PYTHON_VENV" -m pip install wheel
-        "$PYTHON_VENV" -m pip install -r "$GITHUB_ACTION_PATH/../python/requirements.txt"
-
+        "$PYTHON_VENV" -m pip install -r "$GITHUB_ACTION_PATH/../python/requirements-$DEPENDENCIES_VERSION.txt"
         echo '::endgroup::'
       shell: bash
 

linux/action.yml

@@ -175,7 +175,15 @@ runs:
 
         echo "Python that creates venv: $PYTHON_BIN"
         echo "PYTHON_BIN=$PYTHON_BIN" >> "$GITHUB_ENV"
-        echo "version=$("$PYTHON_BIN" -V)" >> "$GITHUB_OUTPUT"
+
+        PYTHON_VERSION="$($PYTHON_BIN -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')"
+        if [[ "$PYTHON_VERSION" == "3.7" ]]
+        then
+          echo "DEPENDENCIES_VERSION=3.7" >> "$GITHUB_ENV"
+        else
+          echo "DEPENDENCIES_VERSION=post-3.7" >> "$GITHUB_ENV"
+        fi
+        echo "version=$PYTHON_VERSION" >> "$GITHUB_OUTPUT"
 
         echo '::endgroup::'
       shell: bash
@@ -186,7 +194,7 @@ runs:
       continue-on-error: true
       with:
         path: '~/.cache/pip'
-        key: enricomi-publish-action-${{ runner.os }}-${{ runner.arch }}-pip-${{ steps.python.outputs.version }}-70a313922fdbeb7398ec60313d908b11
+        key: enricomi-publish-action-${{ runner.os }}-${{ runner.arch }}-pip-${{ steps.python.outputs.version }}-e594996205319a7990b3a4ec677d10a3
 
     - name: Create virtualenv
       id: venv
@@ -227,11 +235,7 @@ runs:
       run: |
         # Install Python dependencies
         echo '::group::Install Python dependencies'
-
-        # make sure wheel is installed, which improves installing our dependencies
-        "$PYTHON_VENV" -m pip install wheel
-        "$PYTHON_VENV" -m pip install -r "$GITHUB_ACTION_PATH/../python/requirements.txt"
-
+        "$PYTHON_VENV" -m pip install -r "$GITHUB_ACTION_PATH/../python/requirements-$DEPENDENCIES_VERSION.txt"
         echo '::endgroup::'
       shell: bash
 

macos/action.yml

@@ -175,7 +175,15 @@ runs:
 
         echo "Python that creates venv: $PYTHON_BIN"
         echo "PYTHON_BIN=$PYTHON_BIN" >> "$GITHUB_ENV"
-        echo "version=$("$PYTHON_BIN" -V)" >> "$GITHUB_OUTPUT"
+
+        PYTHON_VERSION="$($PYTHON_BIN -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')"
+        if [[ "$PYTHON_VERSION" == "3.7" ]]
+        then
+          echo "DEPENDENCIES_VERSION=3.7" >> "$GITHUB_ENV"
+        else
+          echo "DEPENDENCIES_VERSION=post-3.7" >> "$GITHUB_ENV"
+        fi
+        echo "version=$PYTHON_VERSION" >> "$GITHUB_OUTPUT"
 
         echo '::endgroup::'
       shell: bash
@@ -186,7 +194,7 @@ runs:
       continue-on-error: true
       with:
         path: '~/Library/Caches/pip'
-        key: enricomi-publish-action-${{ runner.os }}-${{ runner.arch }}-pip-${{ steps.python.outputs.version }}-70a313922fdbeb7398ec60313d908b11
+        key: enricomi-publish-action-${{ runner.os }}-${{ runner.arch }}-pip-${{ steps.python.outputs.version }}-e594996205319a7990b3a4ec677d10a3
 
     - name: Create virtualenv
       id: venv
@@ -227,11 +235,7 @@ runs:
       run: |
         # Install Python dependencies
         echo '::group::Install Python dependencies'
-
-        # make sure wheel is installed, which improves installing our dependencies
-        "$PYTHON_VENV" -m pip install wheel
-        "$PYTHON_VENV" -m pip install -r "$GITHUB_ACTION_PATH/../python/requirements.txt"
-
+        "$PYTHON_VENV" -m pip install -r "$GITHUB_ACTION_PATH/../python/requirements-$DEPENDENCIES_VERSION.txt"
         echo '::endgroup::'
       shell: bash
 

python/requirements-3.10.txt

@@ -0,0 +1,18 @@
+humanize==4.9.0
+junitparser==3.1.2
+lxml==5.1.0
+psutil==5.9.8
+PyGithub==2.2.0
+  Deprecated==1.2.14
+    wrapt==1.16.0
+  PyJWT==2.8.0
+  PyNaCl==1.5.0
+    cffi==1.16.0
+      pycparser==2.21
+  requests==2.31.0
+    certifi==2024.2.2
+    charset-normalizer==3.3.2
+    idna==3.6
+    urllib3==2.2.1
+  typing_extensions==4.10.0
+  urllib3==2.2.1

python/requirements-3.11.txt

@@ -0,0 +1,18 @@
+humanize==4.9.0
+junitparser==3.1.2
+lxml==5.1.0
+psutil==5.9.8
+PyGithub==2.2.0
+  Deprecated==1.2.14
+    wrapt==1.16.0
+  PyJWT==2.8.0
+  PyNaCl==1.5.0
+    cffi==1.16.0
+      pycparser==2.21
+  requests==2.31.0
+    certifi==2024.2.2
+    charset-normalizer==3.3.2
+    idna==3.6
+    urllib3==2.2.1
+  typing_extensions==4.10.0
+  urllib3==2.2.1

python/requirements-3.12.txt

@@ -0,0 +1,18 @@
+humanize==4.9.0
+junitparser==3.1.2
+lxml==5.1.0
+psutil==5.9.8
+PyGithub==2.2.0
+  Deprecated==1.2.14
+    wrapt==1.16.0
+  PyJWT==2.8.0
+  PyNaCl==1.5.0
+    cffi==1.16.0
+      pycparser==2.21
+  requests==2.31.0
+    certifi==2024.2.2
+    charset-normalizer==3.3.2
+    idna==3.6
+    urllib3==2.2.1
+  typing_extensions==4.10.0
+  urllib3==2.2.1

python/requirements-3.7.txt

@@ -0,0 +1,22 @@
+humanize==4.6.0
+  importlib-metadata==6.7.0
+    typing_extensions==4.7.1
+    zipp==3.15.0
+junitparser==3.1.2
+lxml==5.1.0
+psutil==5.9.8
+PyGithub==2.2.0
+  Deprecated==1.2.14
+    wrapt==1.16.0
+  PyJWT==2.8.0
+    typing_extensions==4.7.1
+  PyNaCl==1.5.0
+    cffi==1.15.1
+      pycparser==2.21
+  requests==2.31.0
+    certifi==2024.2.2
+    charset-normalizer==3.3.2
+    idna==3.6
+    urllib3==2.0.7
+  typing_extensions==4.7.1
+  urllib3==2.0.7

python/requirements-3.8.txt

@@ -0,0 +1,18 @@
+humanize==4.9.0
+junitparser==3.1.2
+lxml==5.1.0
+psutil==5.9.8
+PyGithub==2.2.0
+  Deprecated==1.2.14
+    wrapt==1.16.0
+  PyJWT==2.8.0
+  PyNaCl==1.5.0
+    cffi==1.16.0
+      pycparser==2.21
+  requests==2.31.0
+    certifi==2024.2.2
+    charset-normalizer==3.3.2
+    idna==3.6
+    urllib3==2.2.1
+  typing_extensions==4.10.0
+  urllib3==2.2.1

python/requirements-3.9.txt

@@ -0,0 +1,18 @@
+humanize==4.9.0
+junitparser==3.1.2
+lxml==5.1.0
+psutil==5.9.8
+PyGithub==2.2.0
+  Deprecated==1.2.14
+    wrapt==1.16.0
+  PyJWT==2.8.0
+  PyNaCl==1.5.0
+    cffi==1.16.0
+      pycparser==2.21
+  requests==2.31.0
+    certifi==2024.2.2
+    charset-normalizer==3.3.2
+    idna==3.6
+    urllib3==2.2.1
+  typing_extensions==4.10.0
+  urllib3==2.2.1

python/requirements-post-3.7.txt

@@ -0,0 +1,18 @@
+humanize==4.9.0
+junitparser==3.1.2
+lxml==5.1.0
+psutil==5.9.8
+PyGithub==2.2.0
+  Deprecated==1.2.14
+    wrapt==1.16.0
+  PyJWT==2.8.0
+  PyNaCl==1.5.0
+    cffi==1.16.0
+      pycparser==2.21
+  requests==2.31.0
+    certifi==2024.2.2
+    charset-normalizer==3.3.2
+    idna==3.6
+    urllib3==2.2.1
+  typing_extensions==4.10.0
+  urllib3==2.2.1

python/requirements.txt

@@ -1,25 +1,6 @@
-humanize==3.14.0
-junitparser==3.1.0
-  future==0.18.3
+humanize==4.6.0; python_version <= '3.7'
+humanize==4.9.0; python_version >  '3.7'
+junitparser==3.1.2
 lxml==5.1.0
-psutil==5.9.7
-PyGithub==2.1.1
-  Deprecated==1.2.14
-    wrapt==1.16.0
-  PyJWT==2.8.0
-  PyNaCl==1.5.0
-    # latest version that support Python 3.7
-    cffi==1.15.1
-      pycparser==2.21
-  python-dateutil==2.8.2
-    six==1.16.0
-  requests==2.31.0
-    certifi==2023.11.17
-    charset-normalizer==3.3.2
-    idna==3.6
-    # latest version that support Python 3.7
-    urllib3==2.0.7
-  # latest version that support Python 3.7
-  typing_extensions==4.7.1
-  # latest version that support Python 3.7
-  urllib3==2.0.7
+psutil==5.9.8
+PyGithub==2.2.0

python/test/constraints.txt

@@ -1,2 +0,0 @@
-# test_github.py fails with newer version
-Werkzeug<2.1.0

tox.ini

@@ -0,0 +1,7 @@
+[tox]
+envlist = py{37,38,39,310,311,312}
+
+[testenv]
+commands = .github/upgrade-pip-packages.sh
+allowlist_externals = .github/upgrade-pip-packages.sh
+