Unified integratoin tests

.github/workflows/_shared_integration_tests.yml

@@ -0,0 +1,103 @@
+on:
+  workflow_call:
+
+
+jobs:
+  detect-version-changed:
+    runs-on: ubuntu-20.04
+    outputs:
+      version_changed: ${{ steps.versions.outputs.version_changed }}
+      new_version: ${{ steps.versions.outputs.new_version }}
+      new_base_image_version: ${{ steps.versions.outputs.new_base_image_version }}
+      build_base_images: ${{ steps.versions.outputs.build_base_images }}
+      release_version: ${{ steps.versions.outputs.release_version }}
+      is_prerelease_version: ${{ steps.versions.outputs.is_prerelease_version }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # We need to use a different github token because GITHUB_TOKEN cannot trigger a workflow from another
+          token: ${{secrets.BASETENBOT_GITHUB_TOKEN}}
+          fetch-depth: 2
+      - uses: ./.github/actions/detect-versions/
+        id: versions
+
+  build-and-push-truss-base-images-if-needed:
+    needs: [detect-version-changed]
+    if: needs.detect-version-changed.outputs.build_base_images == 'true'
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python_version: ["3.8", "3.9", "3.10", "3.11"]
+        use_gpu: ["y", "n"]
+        job_type: ["server"]
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-python/
+      - run: poetry install
+      - shell: bash
+        run: |
+          poetry run bin/generate_base_images.py \
+            --use-gpu ${{ matrix.use_gpu }}  \
+            --python-version ${{ matrix.python_version }} \
+            --job-type ${{ matrix.job_type }} \
+            --version-tag ${{ needs.detect-version-changed.outputs.new_base_image_version }} \
+            --skip-login --push
+
+  integration-tests:
+    needs: [build-and-push-truss-base-images-if-needed]
+    if: ${{ !failure() && !cancelled() && (needs.build-and-push-truss-base-images-if-needed.result == 'success' || needs.build-and-push-truss-base-images-if-needed.result == 'skipped') }}
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        split_group: [ "1", "2", "3", "4", "5" ]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-python/
+      - run: poetry install
+      - run: poetry run pytest truss/tests --durations=0 -m 'integration' --splits 5 --group ${{ matrix.split_group }} -k "not test_requirements_pydantic[1]"
+
+  # Running `test_requirements_pydantic[1]` started failing when running together with the other tests.
+  # We could pin down the issue to the fact that the built image had v2 installed, despite v1 in `requirements.txt`
+  # The test passes locally and when running in its own env, suggesting that there is a bug in docker caching / hash
+  # computation that makes the test falsely run with the wrong version in the image. As a stop gap solution, separating
+  # this particular test in its own job made it pass again.
+  integration-tests-pydantic-v1:
+    needs: [build-and-push-truss-base-images-if-needed]
+    if: ${{ !failure() && !cancelled() && (needs.build-and-push-truss-base-images-if-needed.result == 'success' || needs.build-and-push-truss-base-images-if-needed.result == 'skipped') }}
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-python/
+      - run: poetry install
+      - run: poetry run pytest truss/tests/test_model_inference.py::test_requirements_pydantic[1]
+
+  chain-integration-tests:
+    needs: [build-and-push-truss-base-images-if-needed]
+    if: ${{ !failure() && !cancelled() && (needs.build-and-push-truss-base-images-if-needed.result == 'success' || needs.build-and-push-truss-base-images-if-needed.result == 'skipped') }}
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-python/
+      - run: poetry install
+      - run: poetry run pytest truss-chains/tests -s --log-cli-level=INFO  --durations=0 -m 'integration'
+
+  outputs:
+    version_changed: ${{ .outputs.version_changed }}
+    new_version: ${{ .outputs.new_version }}
+    new_base_image_version: ${{ .outputs.new_base_image_version }}
+    build_base_images: ${{ .outputs.build_base_images }}
+    release_version: ${{ .outputs.release_version }}
+    is_prerelease_version: ${{ .outputs.is_prerelease_version }}

.github/workflows/integration-tests.yml

@@ -8,91 +8,5 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
-  detect-version-changed:
-    runs-on: ubuntu-20.04
-    outputs:
-      version_changed: ${{ steps.versions.outputs.version_changed }}
-      new_version: ${{ steps.versions.outputs.new_version }}
-      new_base_image_version: ${{ steps.versions.outputs.new_base_image_version }}
-      build_base_images: ${{ steps.versions.outputs.build_base_images }}
-      release_version: ${{ steps.versions.outputs.release_version }}
-      is_prerelease_version: ${{ steps.versions.outputs.is_prerelease_version }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          # We need to use a different github token because GITHUB_TOKEN cannot trigger a workflow from another
-          token: ${{secrets.BASETENBOT_GITHUB_TOKEN}}
-          fetch-depth: 2
-      - uses: ./.github/actions/detect-versions/
-        id: versions
-  build-and-push-truss-base-images-if-needed:
-    needs: [detect-version-changed]
-    if: needs.detect-version-changed.outputs.build_base_images == 'true'
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python_version: ["3.8", "3.9", "3.10", "3.11"]
-        use_gpu: ["y", "n"]
-        job_type: ["server"]
-    steps:
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/setup-python/
-      - run: poetry install
-      - shell: bash
-        run: |
-          poetry run bin/generate_base_images.py \
-            --use-gpu ${{ matrix.use_gpu }}  \
-            --python-version ${{ matrix.python_version }} \
-            --job-type ${{ matrix.job_type }} \
-            --version-tag ${{ needs.detect-version-changed.outputs.new_base_image_version }} \
-            --skip-login --push
-
   integration-tests:
-    needs: [detect-version-changed, build-and-push-truss-base-images-if-needed]
-    if: ${{ !failure() && !cancelled() && (needs.build-and-push-truss-base-images-if-needed.result == 'success' || needs.build-and-push-truss-base-images-if-needed.result == 'skipped') }}
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        split_group: [ "1", "2", "3", "4", "5" ]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/setup-python/
-      - run: poetry install
-      - run: poetry run pytest truss/tests --durations=0 -m 'integration' --splits 5 --group ${{ matrix.split_group }} -k "not test_requirements_pydantic[1]"
-
-  # Running `test_requirements_pydantic[1]` started failing when running together with the other tests.
-  # We could pin down the issue to the fact that the built image had v2 installed, despite v1 in `requirements.txt`
-  # The test passes locally and when running in its own env, suggesting that there is a bug in docker caching / hash
-  # computation that makes the test falsely run with the wrong version in the image. As a stop gap solution, separating
-  # this particular test in its own job made it pass again.
-  integration-tests-pydantic-v1:
-    needs: [detect-version-changed, build-and-push-truss-base-images-if-needed]
-    if: ${{ !failure() && !cancelled() && (needs.build-and-push-truss-base-images-if-needed.result == 'success' || needs.build-and-push-truss-base-images-if-needed.result == 'skipped') }}
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/setup-python/
-      - run: poetry install
-      - run: poetry run pytest truss/tests/test_model_inference.py::test_requirements_pydantic[1]
-
-  chain-integration-tests:
-    needs: [detect-version-changed, build-and-push-truss-base-images-if-needed]
-    if: ${{ !failure() && !cancelled() && (needs.build-and-push-truss-base-images-if-needed.result == 'success' || needs.build-and-push-truss-base-images-if-needed.result == 'skipped') }}
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/setup-python/
-      - run: poetry install
-      - run: poetry run pytest truss-chains/tests -s --log-cli-level=INFO  --durations=0 -m 'integration'
+    uses: ./.github/workflows/_shared_integration_tests.yml

.github/workflows/main.yml

@@ -11,106 +11,20 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
-  detect-version-changed:
-    runs-on: ubuntu-20.04
-    outputs:
-      version_changed: ${{ steps.versions.outputs.version_changed }}
-      new_version: ${{ steps.versions.outputs.new_version }}
-      new_base_image_version: ${{ steps.versions.outputs.new_base_image_version }}
-      build_base_images: ${{ steps.versions.outputs.build_base_images }}
-      release_version: ${{ steps.versions.outputs.release_version }}
-      is_prerelease_version: ${{ steps.versions.outputs.is_prerelease_version }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          # We need to use a different github token because GITHUB_TOKEN cannot trigger a workflow from another
-          token: ${{secrets.BASETENBOT_GITHUB_TOKEN}}
-          fetch-depth: 2
-      - uses: ./.github/actions/detect-versions/
-        id: versions
-  build-and-push-truss-base-images-if-needed:
-    needs: [detect-version-changed]
-    if: needs.detect-version-changed.outputs.build_base_images == 'true'
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python_version: ["3.8", "3.9", "3.10", "3.11"]
-        use_gpu: ["y", "n"]
-        job_type: ["server"]
-    steps:
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/setup-python/
-      - run: poetry install
-      - shell: bash
-        run: |
-          poetry run bin/generate_base_images.py \
-            --use-gpu ${{ matrix.use_gpu }}  \
-            --python-version ${{ matrix.python_version }} \
-            --job-type ${{ matrix.job_type }} \
-            --version-tag ${{ needs.detect-version-changed.outputs.new_base_image_version }} \
-            --skip-login --push
-
   integration-tests:
-    needs: [detect-version-changed, build-and-push-truss-base-images-if-needed]
-    if: ${{ !failure() && !cancelled() && (needs.build-and-push-truss-base-images-if-needed.result == 'success' || needs.build-and-push-truss-base-images-if-needed.result == 'skipped') }}
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        split_group: [ "1", "2", "3", "4", "5" ]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/setup-python/
-      - run: poetry install
-      - run: poetry run pytest truss/tests --durations=0 -m 'integration' --splits 5 --group ${{ matrix.split_group }} -k "not test_requirements_pydantic[1]"
-
-  # Running `test_requirements_pydantic[1]` started failing when running together with the other tests.
-  # We could pin down the issue to the fact that the built image had v2 installed, despite v1 in `requirements.txt`
-  # The test passes locally and when running in its own env, suggesting that there is a bug in docker caching / hash
-  # computation that makes the test falsely run with the wrong version in the image. As a stop gap solution, separating
-  # this particular test in its own job made it pass again.
-  integration-tests-pydantic-v1:
-    needs: [detect-version-changed, build-and-push-truss-base-images-if-needed]
-    if: ${{ !failure() && !cancelled() && (needs.build-and-push-truss-base-images-if-needed.result == 'success' || needs.build-and-push-truss-base-images-if-needed.result == 'skipped') }}
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/setup-python/
-      - run: poetry install
-      - run: poetry run pytest truss/tests/test_model_inference.py::test_requirements_pydantic[1]
-
-  chain-integration-tests:
-    needs: [detect-version-changed, build-and-push-truss-base-images-if-needed]
-    if: ${{ !failure() && !cancelled() && (needs.build-and-push-truss-base-images-if-needed.result == 'success' || needs.build-and-push-truss-base-images-if-needed.result == 'skipped') }}
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/setup-python/
-      - run: poetry install
-      - run: poetry run pytest truss-chains/tests -s --log-cli-level=INFO  --durations=0 -m 'integration'
+    uses: ./.github/workflows/_shared_integration_tests.yml
 
   publish-to-pypi:
-    needs: [detect-version-changed]
-    if: ${{ !failure() && !cancelled() && needs.detect-version-changed.outputs.release_version == 'true' && needs.detect-version-changed.outputs.is_prerelease_version == 'true' }}
+    needs: [integration-tests]
+    if: ${{ !failure() && !cancelled() && needs.integration-tests.outputs.release_version == 'true' && needs.integration-tests.outputs.release_version.is_prerelease_version == 'true' }}
     runs-on: ubuntu-20.04
     steps:
       - name: "Git tag release"
         uses: actions/checkout@v4
         with:
           token: ${{secrets.BASETENBOT_GITHUB_TOKEN}}
       - run: |
-          NEW_VERSION=v${{ needs.detect-version-changed.outputs.new_version }}
+          NEW_VERSION=v${{ needs.integration-tests.outputs.new_version }}
           git config --global user.name "Github action"
           git config --global user.email "github.action@baseten.co"
 
@@ -131,11 +45,11 @@ jobs:
           artifacts: "dist/*"
           token: ${{ secrets.BASETENBOT_GITHUB_TOKEN }}
           draft: false
-          prerelease: ${{ needs.detect-version-changed.outputs.is_prerelease_version }}
+          prerelease: ${{ needs.integration-tests.outputs.is_prerelease_version }}
           generateReleaseNotes: true
           makeLatest: true
           skipIfReleaseExists: true
-          tag: "v${{ needs.detect-version-changed.outputs.new_version }}"
+          tag: "v${{ needs.integration-tests.outputs.new_version }}"
 
       - name: Publish to PyPI
         if: ${{ github.event_name != 'pull_request' }}

.github/workflows/pr.yml

@@ -2,6 +2,7 @@ name: PR - pre-commit
 
 on:
   pull_request:
+  workflow_dispatch:
 
 concurrency:
   group: pr-${{ github.ref_name }}
@@ -78,3 +79,9 @@ jobs:
           else
             echo "::notice file=truss-chains/examples/::Chains examples modified and ack found int PR description."
           fi
+
+  integration-tests:
+    if: ${{ github.event_name == 'workflow_dispatch' }}
+    uses: ./.github/workflows/_shared_integration_tests.yml
+    with:
+      build-and-push-truss-base-images-if-needed-result: "skipped"