Alter pipelines to matrix strategy (#299)

* feat: altered check pipelines * feat: altered build pipelines * chore: cleanup of python pipelines from black * fix: syntax of outputs * chore: missing checkout in prepare_env * feat: maybe even better building and tagging * feat: final version of build pipelines * fix: jobs names * fix: missing step id * fix: missing fail-fast
kouprlabs · Sep 4, 2024 · 8be7c16 · 8be7c16
1 parent 40f3183
commit 8be7c16
Show file tree

Hide file tree

Showing 18 changed files with 677 additions and 345 deletions.
diff --git a/.github/workflows/build-api.yml b/.github/workflows/build-api.yml
@@ -12,12 +12,6 @@ name: Build and Push voltaserve/api
 
 on:
   workflow_dispatch:
-    inputs:
-      branch:
-        required: false
-        default: ""
-        type: string
-        description: Override code checkout branch (e.g. "feature/branch")
   push:
     branches:
       - main
@@ -26,23 +20,24 @@ on:
     tags:
       - 'v*'
 
+env:
+  IMAGE: voltaserve/api
+
 jobs:
-  build_and_push:
+  build:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ "arm64", "amd64" ]
     steps:
-      - name: Prepare input branch
-        if: ${{ github.event.inputs.branch != '' }}
-        run: echo "branch=refs/heads/${{ github.event.inputs.branch }}" >> $GITHUB_ENV
-
       - name: Checkout Repository
         uses: actions/checkout@v4
-        with:
-          ref: ${{ env.branch || github.ref }}
 
       - name: Set Up QEMU
         uses: docker/setup-qemu-action@v3
         with:
-          platforms: arm64, amd64
+          platforms: ${{ matrix.platform }}
 
       - name: Set Up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -53,31 +48,71 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: Create tag list matching semver from executing tag
-        if: ${{ github.ref_type == 'tag' }}
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE }}
+          tags: |
+            type=ref,event=branch,enable=${{ github.ref_type != 'tag' }}
+            type=ref,event=tag
+            type=sha,enable=${{ github.ref_type != 'tag' }}
+            type=semver,pattern={{version}},enable=${{ github.ref_type == 'tag' }}
+            type=semver,pattern={{major}}.{{minor}},enable=${{ github.ref_type == 'tag' }}
+            type=semver,pattern={{major}},enable=${{ github.ref_type == 'tag' }}
+
+      - name: Build and Push Docker Image
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: ./api
+          platforms: ${{ matrix.platform }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Export digest
         run: |
-          name="voltaserve/api"
-          version="${{ github.ref_name }}"
-          version="${version#v}"
-          IFS='.' read -r -a parts <<< "$version"
-          TAGS="$name:$version"
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"          
 
-          for ((i=${#parts[@]}-1; i>0; i--)); do
-            tag=$(IFS='.'; echo "${parts[*]:0:$i}")
-            TAGS+=",$name:$tag"
-          done
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ matrix.platform }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    runs-on: ubuntu-latest
+    needs:
+      - build
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
 
-          TAGS+=",$name:latest"
-          echo "TAGS=$TAGS" >> $GITHUB_ENV
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
-      - name: Set the docker tag on runs other than tag push
-        if: ${{ github.ref_type != 'tag' }}
-        run: echo "TAGS=voltaserve/api:$(git rev-parse --short HEAD),voltaserve/api:main" >> $GITHUB_ENV
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE }}
 
-      - name: Build and Push Docker Image
-        uses: docker/build-push-action@v5
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
         with:
-          context: ./api
-          push: true
-          tags: ${{ env.TAGS }}
-          platforms: linux/amd64,linux/arm64
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.IMAGE }}@sha256:%s ' *)
diff --git a/.github/workflows/build-console.yml b/.github/workflows/build-console.yml
@@ -12,12 +12,6 @@ name: Build and Push voltaserve/console
 
 on:
   workflow_dispatch:
-    inputs:
-      branch:
-        required: false
-        default: ""
-        type: string
-        description: Override code checkout branch (e.g. "feature/branch")
   push:
     branches:
       - main
@@ -26,23 +20,24 @@ on:
     tags:
       - 'v*'
 
+env:
+  IMAGE: voltaserve/console
+
 jobs:
-  build_and_push:
+  build:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ "arm64", "amd64" ]
     steps:
-      - name: Prepare input branch
-        if: ${{ github.event.inputs.branch != '' }}
-        run: echo "branch=refs/heads/${{ github.event.inputs.branch }}" >> $GITHUB_ENV
-
       - name: Checkout Repository
         uses: actions/checkout@v4
-        with:
-          ref: ${{ env.branch || github.ref }}
 
       - name: Set Up QEMU
         uses: docker/setup-qemu-action@v3
         with:
-          platforms: arm64, amd64
+          platforms: ${{ matrix.platform }}
 
       - name: Set Up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -53,31 +48,71 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: Create tag list matching semver from executing tag
-        if: ${{ github.ref_type == 'tag' }}
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE }}
+          tags: |
+            type=ref,event=branch,enable=${{ github.ref_type != 'tag' }}
+            type=ref,event=tag
+            type=sha,enable=${{ github.ref_type != 'tag' }}
+            type=semver,pattern={{version}},enable=${{ github.ref_type == 'tag' }}
+            type=semver,pattern={{major}}.{{minor}},enable=${{ github.ref_type == 'tag' }}
+            type=semver,pattern={{major}},enable=${{ github.ref_type == 'tag' }}
+
+      - name: Build and Push Docker Image
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: ./console
+          platforms: ${{ matrix.platform }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Export digest
         run: |
-          name="voltaserve/console"
-          version="${{ github.ref_name }}"
-          version="${version#v}"
-          IFS='.' read -r -a parts <<< "$version"
-          TAGS="$name:$version"
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"          
 
-          for ((i=${#parts[@]}-1; i>0; i--)); do
-            tag=$(IFS='.'; echo "${parts[*]:0:$i}")
-            TAGS+=",$name:$tag"
-          done
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ matrix.platform }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    runs-on: ubuntu-latest
+    needs:
+      - build
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
 
-          TAGS+=",$name:latest"
-          echo "TAGS=$TAGS" >> $GITHUB_ENV
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
-      - name: Set the tag on workflow dispatch
-        if: ${{ github.ref_type != 'tag' }}
-        run: echo "TAGS=voltaserve/console:$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE }}
 
-      - name: Build and Push Docker Image
-        uses: docker/build-push-action@v5
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
         with:
-          context: ./console
-          push: true
-          tags: ${{ env.TAGS }}
-          platforms: linux/amd64,linux/arm64
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.IMAGE }}@sha256:%s ' *)