From 466ab0b314344b3e139d89f65e65980baedf587a Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Wed, 20 Mar 2024 14:11:41 +0100 Subject: [PATCH] Upgrade dependencies (#585) * Pin dependencies based on Python version * Upgrade dependencies to latest version * Improve CI to catch dependencies that build whl files --- .github/actions/test/action.yml | 21 +++++++++++++++++++-- .github/upgrade-pip-packages.sh | 9 +++++---- .github/workflows/ci-cd.yml | 11 +++++++---- Dockerfile | 2 +- README.md | 2 +- composite/action.yml | 18 +++++++++++------- linux/action.yml | 18 +++++++++++------- macos/action.yml | 18 +++++++++++------- python/requirements-3.10.txt | 18 ++++++++++++++++++ python/requirements-3.11.txt | 18 ++++++++++++++++++ python/requirements-3.12.txt | 18 ++++++++++++++++++ python/requirements-3.7.txt | 22 ++++++++++++++++++++++ python/requirements-3.8.txt | 18 ++++++++++++++++++ python/requirements-3.9.txt | 18 ++++++++++++++++++ python/requirements-direct.txt | 5 ----- python/requirements-post-3.7.txt | 18 ++++++++++++++++++ python/requirements.txt | 29 +++++------------------------ python/test/constraints.txt | 2 -- tox.ini | 7 +++++++ windows/action.yml | 18 +++++++++++------- windows/bash/action.yml | 18 +++++++++++------- 21 files changed, 230 insertions(+), 78 deletions(-) create mode 100644 python/requirements-3.10.txt create mode 100644 python/requirements-3.11.txt create mode 100644 python/requirements-3.12.txt create mode 100644 python/requirements-3.7.txt create mode 100644 python/requirements-3.8.txt create mode 100644 python/requirements-3.9.txt delete mode 100644 python/requirements-direct.txt create mode 100644 python/requirements-post-3.7.txt create mode 100644 tox.ini diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index 8759e441..708f975d 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -63,15 +63,32 @@ runs: - name: Install Python dependencies run: | python3 -V + + # inspect pip cache python3 -m pip freeze | sort python3 -m pip cache info || true python3 -m pip cache list || true - python3 -m pip install --upgrade --force pip wheel - python3 -m pip install --force -r python/requirements.txt + + # remove cached built whl files + rm -rf "$(python3 -m pip cache info | grep "Locally built wheels location: " | cut -d ":" -f 2- | cut -d " " -f 2-)" + python3 -m pip cache list || true + + # install dependencies + python_minor_version="$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')" + python3 -m pip install --force -r python/requirements-$python_minor_version.txt python3 -m pip install --force -r python/test/requirements.txt -c python/test/constraints.txt + + # inspect pip cache python3 -m pip freeze | sort python3 -m pip cache info || true python3 -m pip cache list || true + + # assert no whl files have been built + if [ python3 -m pip cache info ] && [ "$(python3 -m pip cache info | grep "Number of locally built wheels:")" != "Number of locally built wheels: 0" ] + then + echo "Dependency whl files have been built" + exit 1 + fi shell: bash - name: Update expectation files diff --git a/.github/upgrade-pip-packages.sh b/.github/upgrade-pip-packages.sh index bfcfddf4..e9c2f0d2 100755 --- a/.github/upgrade-pip-packages.sh +++ b/.github/upgrade-pip-packages.sh @@ -2,12 +2,13 @@ set -euo pipefail base="$(dirname "$0")" +python_minor_version="$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')" -pip install --upgrade --force pip==22.0.0 -pip install --upgrade --upgrade-strategy eager -r "$base/../python/requirements-direct.txt" +pip install --upgrade --force pip==24.0.0 +pip install --upgrade --upgrade-strategy eager -r "$base/../python/requirements.txt" pip install pipdeptree -pipdeptree --packages="$(sed -e "s/;.*//" -e "s/=.*//g" "$base/../python/requirements-direct.txt" | paste -s -d ,)" --freeze > "$base/../python/requirements.txt" +pipdeptree --packages="$(sed -e "s/;.*//" -e "s/=.*//g" "$base/../python/requirements.txt" | paste -s -d ,)" --freeze > "$base/../python/requirements-$python_minor_version.txt" -git diff "$base/../python/requirements.txt" +git diff "$base/../python/requirements-$python_minor_version.txt" diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index ff8f0e6c..3e4bcad5 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -20,14 +20,17 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - name: Check requirements.txt against requirements-direct.txt + - name: Check requirements.txt run: | - (diff -w python/requirements-direct.txt python/requirements.txt || true) | (! grep -e "^<") + (diff -w <(grep -v "python_version > '3.7'" python/requirements.txt | sed -e "s/;.*//") python/requirements-3.7.txt || true) | (! grep -e "^<") + (diff -w <(grep -v "python_version <= '3.7'" python/requirements.txt | sed -e "s/;.*//") python/requirements-post-3.7.txt || true) | (! grep -e "^<") shell: bash - name: Check for dependency updates continue-on-error: true - run: - .github/upgrade-pip-packages.sh + run: | + pip install tox + tox + git diff --exit-code shell: bash test-mac: diff --git a/Dockerfile b/Dockerfile index 455b6ede..dc544bda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ LABEL com.github.actions.color="green" RUN apk add --no-cache --upgrade expat libuuid -COPY python/requirements.txt /action/ +COPY python/requirements-post-3.7.txt /action/requirements.txt RUN apk add --no-cache build-base libffi-dev; \ pip install --upgrade --force --no-cache-dir pip && \ pip install --upgrade --force --no-cache-dir -r /action/requirements.txt; \ diff --git a/README.md b/README.md index 104b9589..8c9eb9ec 100644 --- a/README.md +++ b/README.md @@ -854,7 +854,7 @@ Self-hosted runners may require setting up a Python environment first: python-version: 3.8 ``` -Start-up of the action is faster with `virtualenv` or `venv`, as well as `wheel` packages are installed. +Start-up of the action is faster with `virtualenv` or `venv` package installed. ## Running as a composite action diff --git a/composite/action.yml b/composite/action.yml index e16c0ad5..a9da1709 100644 --- a/composite/action.yml +++ b/composite/action.yml @@ -181,7 +181,15 @@ runs: echo "Python that creates venv: $PYTHON_BIN" echo "PYTHON_BIN=$PYTHON_BIN" >> "$GITHUB_ENV" - echo "version=$("$PYTHON_BIN" -V)" >> "$GITHUB_OUTPUT" + + PYTHON_VERSION="$($PYTHON_BIN -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')" + if [[ "$PYTHON_VERSION" == "3.7" ]] + then + echo "DEPENDENCIES_VERSION=3.7" >> "$GITHUB_ENV" + else + echo "DEPENDENCIES_VERSION=post-3.7" >> "$GITHUB_ENV" + fi + echo "version=$PYTHON_VERSION" >> "$GITHUB_OUTPUT" echo '::endgroup::' shell: bash @@ -210,7 +218,7 @@ runs: continue-on-error: true with: path: ${{ steps.os.outputs.pip-cache }} - key: enricomi-publish-action-${{ runner.os }}-${{ runner.arch }}-pip-${{ steps.python.outputs.version }}-70a313922fdbeb7398ec60313d908b11 + key: enricomi-publish-action-${{ runner.os }}-${{ runner.arch }}-pip-${{ steps.python.outputs.version }}-e594996205319a7990b3a4ec677d10a3 - name: Create virtualenv id: venv @@ -262,11 +270,7 @@ runs: run: | # Install Python dependencies echo '::group::Install Python dependencies' - - # make sure wheel is installed, which improves installing our dependencies - "$PYTHON_VENV" -m pip install wheel - "$PYTHON_VENV" -m pip install -r "$GITHUB_ACTION_PATH/../python/requirements.txt" - + "$PYTHON_VENV" -m pip install -r "$GITHUB_ACTION_PATH/../python/requirements-$DEPENDENCIES_VERSION.txt" echo '::endgroup::' shell: bash diff --git a/linux/action.yml b/linux/action.yml index b57869a5..b0e5844c 100644 --- a/linux/action.yml +++ b/linux/action.yml @@ -175,7 +175,15 @@ runs: echo "Python that creates venv: $PYTHON_BIN" echo "PYTHON_BIN=$PYTHON_BIN" >> "$GITHUB_ENV" - echo "version=$("$PYTHON_BIN" -V)" >> "$GITHUB_OUTPUT" + + PYTHON_VERSION="$($PYTHON_BIN -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')" + if [[ "$PYTHON_VERSION" == "3.7" ]] + then + echo "DEPENDENCIES_VERSION=3.7" >> "$GITHUB_ENV" + else + echo "DEPENDENCIES_VERSION=post-3.7" >> "$GITHUB_ENV" + fi + echo "version=$PYTHON_VERSION" >> "$GITHUB_OUTPUT" echo '::endgroup::' shell: bash @@ -186,7 +194,7 @@ runs: continue-on-error: true with: path: '~/.cache/pip' - key: enricomi-publish-action-${{ runner.os }}-${{ runner.arch }}-pip-${{ steps.python.outputs.version }}-70a313922fdbeb7398ec60313d908b11 + key: enricomi-publish-action-${{ runner.os }}-${{ runner.arch }}-pip-${{ steps.python.outputs.version }}-e594996205319a7990b3a4ec677d10a3 - name: Create virtualenv id: venv @@ -227,11 +235,7 @@ runs: run: | # Install Python dependencies echo '::group::Install Python dependencies' - - # make sure wheel is installed, which improves installing our dependencies - "$PYTHON_VENV" -m pip install wheel - "$PYTHON_VENV" -m pip install -r "$GITHUB_ACTION_PATH/../python/requirements.txt" - + "$PYTHON_VENV" -m pip install -r "$GITHUB_ACTION_PATH/../python/requirements-$DEPENDENCIES_VERSION.txt" echo '::endgroup::' shell: bash diff --git a/macos/action.yml b/macos/action.yml index 439d7891..21cf5260 100644 --- a/macos/action.yml +++ b/macos/action.yml @@ -175,7 +175,15 @@ runs: echo "Python that creates venv: $PYTHON_BIN" echo "PYTHON_BIN=$PYTHON_BIN" >> "$GITHUB_ENV" - echo "version=$("$PYTHON_BIN" -V)" >> "$GITHUB_OUTPUT" + + PYTHON_VERSION="$($PYTHON_BIN -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')" + if [[ "$PYTHON_VERSION" == "3.7" ]] + then + echo "DEPENDENCIES_VERSION=3.7" >> "$GITHUB_ENV" + else + echo "DEPENDENCIES_VERSION=post-3.7" >> "$GITHUB_ENV" + fi + echo "version=$PYTHON_VERSION" >> "$GITHUB_OUTPUT" echo '::endgroup::' shell: bash @@ -186,7 +194,7 @@ runs: continue-on-error: true with: path: '~/Library/Caches/pip' - key: enricomi-publish-action-${{ runner.os }}-${{ runner.arch }}-pip-${{ steps.python.outputs.version }}-70a313922fdbeb7398ec60313d908b11 + key: enricomi-publish-action-${{ runner.os }}-${{ runner.arch }}-pip-${{ steps.python.outputs.version }}-e594996205319a7990b3a4ec677d10a3 - name: Create virtualenv id: venv @@ -227,11 +235,7 @@ runs: run: | # Install Python dependencies echo '::group::Install Python dependencies' - - # make sure wheel is installed, which improves installing our dependencies - "$PYTHON_VENV" -m pip install wheel - "$PYTHON_VENV" -m pip install -r "$GITHUB_ACTION_PATH/../python/requirements.txt" - + "$PYTHON_VENV" -m pip install -r "$GITHUB_ACTION_PATH/../python/requirements-$DEPENDENCIES_VERSION.txt" echo '::endgroup::' shell: bash diff --git a/python/requirements-3.10.txt b/python/requirements-3.10.txt new file mode 100644 index 00000000..cee1e9f3 --- /dev/null +++ b/python/requirements-3.10.txt @@ -0,0 +1,18 @@ +humanize==4.9.0 +junitparser==3.1.2 +lxml==5.1.0 +psutil==5.9.8 +PyGithub==2.2.0 + Deprecated==1.2.14 + wrapt==1.16.0 + PyJWT==2.8.0 + PyNaCl==1.5.0 + cffi==1.16.0 + pycparser==2.21 + requests==2.31.0 + certifi==2024.2.2 + charset-normalizer==3.3.2 + idna==3.6 + urllib3==2.2.1 + typing_extensions==4.10.0 + urllib3==2.2.1 diff --git a/python/requirements-3.11.txt b/python/requirements-3.11.txt new file mode 100644 index 00000000..cee1e9f3 --- /dev/null +++ b/python/requirements-3.11.txt @@ -0,0 +1,18 @@ +humanize==4.9.0 +junitparser==3.1.2 +lxml==5.1.0 +psutil==5.9.8 +PyGithub==2.2.0 + Deprecated==1.2.14 + wrapt==1.16.0 + PyJWT==2.8.0 + PyNaCl==1.5.0 + cffi==1.16.0 + pycparser==2.21 + requests==2.31.0 + certifi==2024.2.2 + charset-normalizer==3.3.2 + idna==3.6 + urllib3==2.2.1 + typing_extensions==4.10.0 + urllib3==2.2.1 diff --git a/python/requirements-3.12.txt b/python/requirements-3.12.txt new file mode 100644 index 00000000..cee1e9f3 --- /dev/null +++ b/python/requirements-3.12.txt @@ -0,0 +1,18 @@ +humanize==4.9.0 +junitparser==3.1.2 +lxml==5.1.0 +psutil==5.9.8 +PyGithub==2.2.0 + Deprecated==1.2.14 + wrapt==1.16.0 + PyJWT==2.8.0 + PyNaCl==1.5.0 + cffi==1.16.0 + pycparser==2.21 + requests==2.31.0 + certifi==2024.2.2 + charset-normalizer==3.3.2 + idna==3.6 + urllib3==2.2.1 + typing_extensions==4.10.0 + urllib3==2.2.1 diff --git a/python/requirements-3.7.txt b/python/requirements-3.7.txt new file mode 100644 index 00000000..e11b37c4 --- /dev/null +++ b/python/requirements-3.7.txt @@ -0,0 +1,22 @@ +humanize==4.6.0 + importlib-metadata==6.7.0 + typing_extensions==4.7.1 + zipp==3.15.0 +junitparser==3.1.2 +lxml==5.1.0 +psutil==5.9.8 +PyGithub==2.2.0 + Deprecated==1.2.14 + wrapt==1.16.0 + PyJWT==2.8.0 + typing_extensions==4.7.1 + PyNaCl==1.5.0 + cffi==1.15.1 + pycparser==2.21 + requests==2.31.0 + certifi==2024.2.2 + charset-normalizer==3.3.2 + idna==3.6 + urllib3==2.0.7 + typing_extensions==4.7.1 + urllib3==2.0.7 diff --git a/python/requirements-3.8.txt b/python/requirements-3.8.txt new file mode 100644 index 00000000..cee1e9f3 --- /dev/null +++ b/python/requirements-3.8.txt @@ -0,0 +1,18 @@ +humanize==4.9.0 +junitparser==3.1.2 +lxml==5.1.0 +psutil==5.9.8 +PyGithub==2.2.0 + Deprecated==1.2.14 + wrapt==1.16.0 + PyJWT==2.8.0 + PyNaCl==1.5.0 + cffi==1.16.0 + pycparser==2.21 + requests==2.31.0 + certifi==2024.2.2 + charset-normalizer==3.3.2 + idna==3.6 + urllib3==2.2.1 + typing_extensions==4.10.0 + urllib3==2.2.1 diff --git a/python/requirements-3.9.txt b/python/requirements-3.9.txt new file mode 100644 index 00000000..cee1e9f3 --- /dev/null +++ b/python/requirements-3.9.txt @@ -0,0 +1,18 @@ +humanize==4.9.0 +junitparser==3.1.2 +lxml==5.1.0 +psutil==5.9.8 +PyGithub==2.2.0 + Deprecated==1.2.14 + wrapt==1.16.0 + PyJWT==2.8.0 + PyNaCl==1.5.0 + cffi==1.16.0 + pycparser==2.21 + requests==2.31.0 + certifi==2024.2.2 + charset-normalizer==3.3.2 + idna==3.6 + urllib3==2.2.1 + typing_extensions==4.10.0 + urllib3==2.2.1 diff --git a/python/requirements-direct.txt b/python/requirements-direct.txt deleted file mode 100644 index b7d77eb9..00000000 --- a/python/requirements-direct.txt +++ /dev/null @@ -1,5 +0,0 @@ -humanize==3.14.0 -junitparser==3.1.0 -lxml==5.1.0 -psutil==5.9.7 -PyGithub==2.1.1 diff --git a/python/requirements-post-3.7.txt b/python/requirements-post-3.7.txt new file mode 100644 index 00000000..cee1e9f3 --- /dev/null +++ b/python/requirements-post-3.7.txt @@ -0,0 +1,18 @@ +humanize==4.9.0 +junitparser==3.1.2 +lxml==5.1.0 +psutil==5.9.8 +PyGithub==2.2.0 + Deprecated==1.2.14 + wrapt==1.16.0 + PyJWT==2.8.0 + PyNaCl==1.5.0 + cffi==1.16.0 + pycparser==2.21 + requests==2.31.0 + certifi==2024.2.2 + charset-normalizer==3.3.2 + idna==3.6 + urllib3==2.2.1 + typing_extensions==4.10.0 + urllib3==2.2.1 diff --git a/python/requirements.txt b/python/requirements.txt index b43209f1..da2e0001 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,25 +1,6 @@ -humanize==3.14.0 -junitparser==3.1.0 - future==0.18.3 +humanize==4.6.0; python_version <= '3.7' +humanize==4.9.0; python_version > '3.7' +junitparser==3.1.2 lxml==5.1.0 -psutil==5.9.7 -PyGithub==2.1.1 - Deprecated==1.2.14 - wrapt==1.16.0 - PyJWT==2.8.0 - PyNaCl==1.5.0 - # latest version that support Python 3.7 - cffi==1.15.1 - pycparser==2.21 - python-dateutil==2.8.2 - six==1.16.0 - requests==2.31.0 - certifi==2023.11.17 - charset-normalizer==3.3.2 - idna==3.6 - # latest version that support Python 3.7 - urllib3==2.0.7 - # latest version that support Python 3.7 - typing_extensions==4.7.1 - # latest version that support Python 3.7 - urllib3==2.0.7 +psutil==5.9.8 +PyGithub==2.2.0 diff --git a/python/test/constraints.txt b/python/test/constraints.txt index f42c5f7c..e69de29b 100644 --- a/python/test/constraints.txt +++ b/python/test/constraints.txt @@ -1,2 +0,0 @@ -# test_github.py fails with newer version -Werkzeug<2.1.0 \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..2c92a58a --- /dev/null +++ b/tox.ini @@ -0,0 +1,7 @@ +[tox] +envlist = py{37,38,39,310,311,312} + +[testenv] +commands = .github/upgrade-pip-packages.sh +allowlist_externals = .github/upgrade-pip-packages.sh + diff --git a/windows/action.yml b/windows/action.yml index 6199931d..cc8f43a2 100644 --- a/windows/action.yml +++ b/windows/action.yml @@ -173,9 +173,15 @@ runs: Write-Output "Python that creates venv: $PYTHON_BIN" "PYTHON_BIN=$PYTHON_BIN" | Out-File -FilePath $env:GITHUB_ENV -Append - $VERSION = Invoke-Expression -Command "& '$PYTHON_BIN' -V" - Write-Output "Python version: $VERSION" - "version=$VERSION" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + $PYTHON_VERSION = Invoke-Expression -Command "& '$PYTHON_BIN' -c 'import sys; print(f""{sys.version_info.major}.{sys.version_info.minor}"")'" + Write-Output "Python version: $PYTHON_VERSION" + + if ( $PYTHON_VERSION -eq "3.7" ) { + "DEPENDENCIES_VERSION=3.7" | Out-File -FilePath $env:GITHUB_ENV -Append + } else { + "DEPENDENCIES_VERSION=post-3.7" | Out-File -FilePath $env:GITHUB_ENV -Append + } + "version=$PYTHON_VERSION" | Out-File -FilePath $env:GITHUB_OUTPUT -Append } finally { Write-Output "::endgroup::" } @@ -187,7 +193,7 @@ runs: continue-on-error: true with: path: '~\AppData\Local\pip\Cache' - key: enricomi-publish-action-${{ runner.os }}-${{ runner.arch }}-pip-${{ steps.python.outputs.version }}-70a313922fdbeb7398ec60313d908b11 + key: enricomi-publish-action-${{ runner.os }}-${{ runner.arch }}-pip-${{ steps.python.outputs.version }}-e594996205319a7990b3a4ec677d10a3 - name: Create virtualenv id: venv @@ -229,9 +235,7 @@ runs: # Install Python dependencies Write-Output "::group::Install Python dependencies" try { - # make sure wheel is installed, which improves installing our dependencies - Invoke-Expression -Command "& '$env:PYTHON_VENV' -m pip install wheel" - Invoke-Expression -Command "& '$env:PYTHON_VENV' -m pip install -r '$env:GITHUB_ACTION_PATH\..\python\requirements.txt'" + Invoke-Expression -Command "& '$env:PYTHON_VENV' -m pip install -r '$env:GITHUB_ACTION_PATH\..\python\requirements-$env:DEPENDENCIES_VERSION.txt'" } finally { Write-Output "::endgroup::" } diff --git a/windows/bash/action.yml b/windows/bash/action.yml index 77925df4..e9a5974d 100644 --- a/windows/bash/action.yml +++ b/windows/bash/action.yml @@ -175,7 +175,15 @@ runs: echo "Python that creates venv: $PYTHON_BIN" echo "PYTHON_BIN=$PYTHON_BIN" >> "$GITHUB_ENV" - echo "version=$("$PYTHON_BIN" -V)" >> "$GITHUB_OUTPUT" + + PYTHON_VERSION="$($PYTHON_BIN -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')" + if [[ "$PYTHON_VERSION" == "3.7" ]] + then + echo "DEPENDENCIES_VERSION=3.7" >> "$GITHUB_ENV" + else + echo "DEPENDENCIES_VERSION=post-3.7" >> "$GITHUB_ENV" + fi + echo "version=$PYTHON_VERSION" >> "$GITHUB_OUTPUT" echo '::endgroup::' shell: bash @@ -186,7 +194,7 @@ runs: continue-on-error: true with: path: '~\AppData\Local\pip\Cache' - key: enricomi-publish-action-${{ runner.os }}-${{ runner.arch }}-pip-${{ steps.python.outputs.version }}-70a313922fdbeb7398ec60313d908b11 + key: enricomi-publish-action-${{ runner.os }}-${{ runner.arch }}-pip-${{ steps.python.outputs.version }}-e594996205319a7990b3a4ec677d10a3 - name: Create virtualenv id: venv @@ -231,11 +239,7 @@ runs: run: | # Install Python dependencies echo '::group::Install Python dependencies' - - # make sure wheel is installed, which improves installing our dependencies - "$PYTHON_VENV" -m pip install wheel - "$PYTHON_VENV" -m pip install -r "$GITHUB_ACTION_PATH/../../python/requirements.txt" - + "$PYTHON_VENV" -m pip install -r "$GITHUB_ACTION_PATH/../../python/requirements-$DEPENDENCIES_VERSION.txt" echo '::endgroup::' shell: bash