E2E #102
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E | |
| on: | |
| workflow_dispatch: | |
| env: | |
| COMMIT_NAME: Monkeynator | |
| COMMIT_EMAIL: monkeynator@enix.io | |
| jobs: | |
| build: | |
| name: Build test image | |
| runs-on: ubuntu-22.04 | |
| env: | |
| VERSION: ${{ github.run_id }} | |
| HARBOR_URL: "harbor.enix.io" | |
| HARBOR_REPO: "kube-image-keeper/kube-image-keeper" | |
| GHCR_IMAGE: "ghcr.io/enix/kube-image-keeper" | |
| QUAY_IMAGE: "quay.io/enix/kube-image-keeper" | |
| TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db:2" | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: repository | |
| - name: check-for-cc | |
| id: check-for-cc | |
| uses: webiny/action-conventional-commits@v1.3.0 | |
| - name: Run Trivy vulnerability scanner | |
| uses: aquasecurity/trivy-action@0.28.0 | |
| with: | |
| scan-type: 'fs' | |
| ignore-unfixed: true | |
| format: 'sarif' | |
| output: 'trivy-results.sarif' | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@v3 | |
| with: | |
| sarif_file: 'trivy-results.sarif' | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.HARBOR_URL }} | |
| username: ${{ secrets.HARBOR_USERNAME }} | |
| password: ${{ secrets.HARBOR_PASSWORD }} | |
| - name: Generate image metadata | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: | | |
| ${{ env.GHCR_IMAGE }} | |
| ${{ github.repository }} | |
| ${{ env.QUAY_IMAGE }} | |
| - name: Build container images | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: repository | |
| platforms: linux/amd64,linux/arm64 | |
| build-args: | | |
| "VERSION=${{ env.VERSION }}" | |
| "REVISION=${{ github.sha }}" | |
| push: true | |
| labels: ${{ steps.meta.outputs.labels }} | |
| tags: | | |
| ${{ env.HARBOR_URL }}/${{ env.HARBOR_REPO }}:${{ env.VERSION }} | |
| - name: Build alpine container images | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: repository | |
| platforms: linux/amd64,linux/arm64 | |
| build-args: | | |
| "VERSION=${{ env.VERSION }}" | |
| "REVISION=${{ github.sha }}" | |
| target: alpine | |
| push: true | |
| labels: ${{ steps.meta.outputs.labels }} | |
| tags: | | |
| ${{ env.HARBOR_URL }}/${{ env.HARBOR_REPO }}:${{ env.VERSION }}-alpine | |
| e2e_install: | |
| name: Tests e2e on K8s (Fresh install) | |
| needs: | |
| - build | |
| runs-on: ubuntu-22.04 | |
| env: | |
| VERSION: ${{ github.run_id }} | |
| HARBOR_IMAGE: "harbor.enix.io/kube-image-keeper/kube-image-keeper" | |
| HARBOR_REGISTRY: "harbor.enix.io" | |
| HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }} | |
| HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }} | |
| strategy: | |
| max-parallel: 6 | |
| matrix: | |
| k8sversion: ["v1.24.17", "v1.25.16", "v1.26.15", "v1.27.16", "v1.28.13", "v1.29.8", "v1.30.4", "v1.31.0"] | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| - name: Setup KinD | |
| uses: helm/kind-action@v1.10.0 | |
| with: | |
| node_image: kindest/node:${{ matrix.k8sversion }} | |
| - name: Run cert-manager installation | |
| run: | | |
| kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.yaml | |
| kubectl wait pods -n cert-manager -l app.kubernetes.io/instance=cert-manager --for condition=Ready --timeout=90s | |
| - name: Set up chart-testing | |
| uses: helm/chart-testing-action@v2.6.1 | |
| - name: Set up helm | |
| uses: azure/setup-helm@v4 | |
| with: | |
| version: '3.9.0' | |
| - name: Run chart-testing (lint) | |
| run: | | |
| set -euo pipefail | |
| ct lint \ | |
| --charts helm/kube-image-keeper \ | |
| --chart-repos bitnami=https://charts.bitnami.com/bitnami \ | |
| --chart-repos joxit=https://helm.joxit.dev \ | |
| --validate-maintainers=false --check-version-increment=false | |
| # Need wait for the next release with flash --skip-clean-up | |
| # - name: Run chart-testing (install) | |
| # run: | | |
| # set -euo pipefail | |
| # ct install \ | |
| # --charts helm/cache-registry \ | |
| # --helm-extra-set-args "--set controllers.image.tag=latest --set proxy.image.tag=latest" | |
| - name: Run helm (install) | |
| run : | | |
| set -euo pipefail | |
| kubectl create namespace kuik-system | |
| kubectl create secret docker-registry harbor-secret -n kuik-system --docker-server=${{ env.HARBOR_REGISTRY }} \ | |
| --docker-username="$HARBOR_USERNAME" --docker-password="$HARBOR_PASSWORD" | |
| helm upgrade --install kube-image-keeper -n kuik-system --create-namespace ./helm/kube-image-keeper \ | |
| --set controllers.image.tag=$VERSION --set proxy.image.tag=$VERSION \ | |
| --set controllers.image.repository=$HARBOR_IMAGE --set proxy.image.repository=$HARBOR_IMAGE \ | |
| --set controllers.imagePullSecrets[0].name=harbor-secret --set proxy.image.imagePullSecrets[0].name=harbor-secret --debug | |
| kubectl wait pods -n kuik-system -l app.kubernetes.io/instance=kube-image-keeper --for condition=Ready --timeout=90s | |
| helm history kube-image-keeper -n kuik-system | |
| - name: Deploy test container | |
| run: | | |
| set -euo pipefail | |
| kubectl create deploy nginx --image=nginx:stable-alpine --replicas=2 | |
| kubectl rollout status deploy nginx | |
| kubectl wait deployment nginx --for condition=Available=True --timeout=30s | |
| echo "kubectl get cachedimage" | |
| kubectl get cachedimages | |
| echo "kubectl get repository" | |
| kubectl get repository | |
| - name: Test cachedimage (CRD) | |
| run: | | |
| set -euo pipefail | |
| ## Check if our test image is cached | |
| if [ $(kubectl get cachedimages docker.io-library-nginx-stable-alpine -o json | jq ".status.isCached") ]; | |
| then | |
| if [ $(kubectl get cachedimages docker.io-library-nginx-stable-alpine -o json | jq ".status.usedBy.count") -eq 2 ]; | |
| then | |
| echo "Found cached image used by 2 pods" | |
| else | |
| echo "Error: pods count should be equal 2" | |
| exit 1 | |
| fi | |
| else | |
| echo "Error: image cached status is false" | |
| exit 1 | |
| fi | |
| - name: Test repository (CRD) | |
| run: | | |
| set -euo pipefail | |
| ## Check repository status | |
| if [ $(kubectl get repository docker.io-library-nginx -o json | jq '.status.phase') == '"Ready"' ] ; | |
| then | |
| echo "Found repository" | |
| else | |
| echo "Error: image repository status is not Ready" | |
| exit 1 | |
| fi | |
| - name: Test metrics endpoint | |
| run: | | |
| set -euo pipefail | |
| ## Check for kuik's components metrics | |
| for component in proxy controllers | |
| do | |
| echo "Testing $component metrics endpoint" | |
| for ip in $(kubectl get po -l "app.kubernetes.io/component=$component" -n kuik-system -o jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}') | |
| do | |
| attempts=0 | |
| success=false | |
| while [[ $attempts -lt 3 && $success == false ]] | |
| do | |
| response=$(kubectl run curl-pod --image=curlimages/curl --rm -ti --quiet --restart=Never -- curl -s -o /dev/null -w "%{http_code}\n" http://$ip:8080/metrics) | |
| if [[ -z "$response" ]]; then | |
| echo "No HTTP response received from $ip" | |
| elif [[ $response -ge 200 && $response -lt 300 ]]; then | |
| echo "HTTP status code $response is valid for $ip" | |
| success=true | |
| else | |
| echo "HTTP status code $response is not valid for $ip" | |
| fi | |
| attempts=$(( $attempts + 1 )) | |
| sleep 3 | |
| done | |
| if [[ $success == false ]]; then | |
| echo "Failed after 3 attempts for $ip" | |
| exit 1 | |
| fi | |
| done | |
| done | |
| e2e_upgrade: | |
| name: Tests e2e on K8s (Upgrade) | |
| needs: | |
| - build | |
| - e2e_install | |
| runs-on: ubuntu-22.04 | |
| env: | |
| VERSION: ${{ github.run_id }} | |
| HARBOR_IMAGE: "harbor.enix.io/kube-image-keeper/kube-image-keeper" | |
| HARBOR_REGISTRY: "harbor.enix.io" | |
| HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }} | |
| HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }} | |
| strategy: | |
| max-parallel: 6 | |
| matrix: | |
| k8sversion: ["v1.24.17", "v1.25.16", "v1.26.15", "v1.27.16", "v1.28.13", "v1.29.8", "v1.30.4", "v1.31.0"] | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| - name: Setup KinD | |
| uses: helm/kind-action@v1.10.0 | |
| with: | |
| node_image: kindest/node:${{ matrix.k8sversion }} | |
| - name: Run cert-manager installation | |
| run: | | |
| kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.yaml | |
| kubectl wait pods -n cert-manager -l app.kubernetes.io/instance=cert-manager --for condition=Ready --timeout=30s | |
| - name: Set up chart-testing | |
| uses: helm/chart-testing-action@v2.6.1 | |
| - name: Set up helm | |
| uses: azure/setup-helm@v4 | |
| with: | |
| version: '3.9.0' | |
| - name: Run chart-testing (lint) | |
| run: | | |
| set -euo pipefail | |
| ct lint \ | |
| --charts helm/kube-image-keeper \ | |
| --chart-repos bitnami=https://charts.bitnami.com/bitnami \ | |
| --chart-repos joxit=https://helm.joxit.dev \ | |
| --validate-maintainers=false --check-version-increment=false | |
| # Need wait for the next release with flash --skip-clean-up | |
| # - name: Run chart-testing (install) | |
| # run: | | |
| # set -euo pipefail | |
| # ct install \ | |
| # --charts helm/cache-registry \ | |
| # --helm-extra-set-args "--set controllers.image.tag=latest --set proxy.image.tag=latest" | |
| - name: Run helm (install latest release) | |
| run : | | |
| set -euo pipefail | |
| helm repo add enix https://charts.enix.io/ | |
| helm repo update | |
| helm upgrade --install kube-image-keeper -n kuik-system --create-namespace enix/kube-image-keeper --debug | |
| kubectl wait pods -n kuik-system -l app.kubernetes.io/instance=kube-image-keeper --for condition=Ready --timeout=30s | |
| kubectl get po -n kuik-system | |
| - name: Run helm (upgrade) | |
| run : | | |
| set -euo pipefail | |
| kubectl create secret docker-registry harbor-secret -n kuik-system --docker-server=${{ env.HARBOR_REGISTRY }} \ | |
| --docker-username="$HARBOR_USERNAME" --docker-password="$HARBOR_PASSWORD" | |
| helm upgrade --install kube-image-keeper -n kuik-system --create-namespace ./helm/kube-image-keeper \ | |
| --set controllers.image.tag=$VERSION --set proxy.image.tag=$VERSION \ | |
| --set controllers.image.repository=$HARBOR_IMAGE --set proxy.image.repository=$HARBOR_IMAGE \ | |
| --set controllers.imagePullSecrets[0].name=harbor-secret --set proxy.image.imagePullSecrets[0].name=harbor-secret --wait --debug | |
| kubectl rollout status deploy kube-image-keeper-controllers -n kuik-system | |
| kubectl rollout status ds kube-image-keeper-proxy -n kuik-system | |
| helm history kube-image-keeper -n kuik-system | |
| - name: Deploy test container | |
| run: | | |
| set -euo pipefail | |
| kubectl create deploy nginx --image=nginx:stable-alpine --replicas=2 | |
| kubectl rollout status deploy nginx | |
| kubectl wait deployment nginx --for condition=Available=True --timeout=30s | |
| echo "kubectl get cachedimage" | |
| kubectl get cachedimages | |
| echo "kubectl get repository" | |
| kubectl get repository | |
| - name: Test cachedimage (CRD) | |
| run: | | |
| set -euo pipefail | |
| ## Check if our test image is cached | |
| if [ $(kubectl get cachedimages docker.io-library-nginx-stable-alpine -o json | jq ".status.isCached") ]; | |
| then | |
| if [ $(kubectl get cachedimages docker.io-library-nginx-stable-alpine -o json | jq ".status.usedBy.count") -eq 2 ]; | |
| then | |
| echo "Found cached image used by 2 pods" | |
| else | |
| echo "Error: pods count should be equal 2" | |
| exit 1 | |
| fi | |
| else | |
| echo "Error: image cached status is false" | |
| exit 1 | |
| fi | |
| - name: Test repository (CRD) | |
| run: | | |
| set -euo pipefail | |
| ## Check repository status | |
| if [ $(kubectl get repository docker.io-library-nginx -o json | jq '.status.phase') == '"Ready"' ] ; | |
| then | |
| echo "Found repository" | |
| else | |
| echo "Error: image repository status is not Ready" | |
| exit 1 | |
| fi | |
| - name: Test metrics endpoint | |
| run: | | |
| set -euo pipefail | |
| ## Check for kuik's components metrics | |
| for component in proxy controllers | |
| do | |
| echo "Testing $component metrics endpoint" | |
| for ip in $(kubectl get po -l "app.kubernetes.io/component=$component" -n kuik-system -o jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}') | |
| do | |
| attempts=0 | |
| success=false | |
| while [[ $attempts -lt 3 && $success == false ]] | |
| do | |
| response=$(kubectl run curl-pod --image=curlimages/curl --rm -ti --quiet --restart=Never -- curl -s -o /dev/null -w "%{http_code}\n" http://$ip:8080/metrics) | |
| if [[ -z "$response" ]]; then | |
| echo "No HTTP response received from $ip" | |
| elif [[ $response -ge 200 && $response -lt 300 ]]; then | |
| echo "HTTP status code $response is valid for $ip" | |
| success=true | |
| else | |
| echo "HTTP status code $response is not valid for $ip" | |
| fi | |
| attempts=$(( $attempts + 1 )) | |
| sleep 3 | |
| done | |
| if [[ $success == false ]]; then | |
| echo "Failed after 3 attempts for $ip" | |
| exit 1 | |
| fi | |
| done | |
| done |