Refactored CICD to support multiplatform

.github/workflows/build_push_docker_hub.yaml

@@ -31,7 +31,6 @@ jobs:
           { name: data-producer, context: ./services/data-producer, file: ./services/data-producer/Dockerfile },
           { name: mlflow, context: ./services/mlflow, file: ./services/mlflow/Dockerfile },
           { name: airflow-spark, context: ./services/airflow, file: ./services/airflow/Dockerfile },
-          { name: ray, context: ./services/ray, file: ./services/ray/Dockerfile },
           { name: forecast-service, context: ./services/forecast-service, file: ./services/forecast-service/Dockerfile }
         ]
     steps:
@@ -60,8 +59,7 @@ jobs:
         with:
           context: ${{ matrix.image.context }}
           file: ${{ matrix.image.file }}
-          # platforms: linux/arm64
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: ariya23156/sfmlops-${{ matrix.image.name }}:latest
           labels: ${{ steps.meta.outputs.labels }}
@@ -72,4 +70,50 @@ jobs:
           build-args: |
             AIRFLOW_HOME=/opt/airflow
             MLFLOW_ARTIFACT_ROOT=/storage/mlruns
-            ARCH_TRAILING_IMG_NAME=
+
+  build_and_push_ray_images_only: # Job for building and pushing the images
+    name: Build and Push Ray Images
+    runs-on: ubuntu-latest
+    needs: shared_steps # Dependency on the shared steps
+    strategy:
+      matrix:
+        image: [
+          { name: ray, context: ./services/ray, file: ./services/ray/Dockerfile, platform: linux/arm64 },
+          { name: ray, context: ./services/ray, file: ./services/ray/Dockerfile, platform: linux/amd64 }
+        ]
+    steps:
+      - name: Download repository artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: repository
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Extract metadata (tags, labels) for ${{ matrix.image.name }} Docker
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ariya23156/sfmlops-${{ matrix.image.name }}
+
+      - name: Build and push ${{ matrix.image.name }} Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ${{ matrix.image.context }}
+          file: ${{ matrix.image.file }}
+          platforms: ${{ matrix.image.platform }}
+          push: true
+          tags: ariya23156/sfmlops-${{ matrix.image.name }}:latest-${{ matrix.image.platform }}
+          labels: ${{ steps.meta.outputs.labels }}
+          # Hardcoded all build-args here, couldn't find a proper way
+          # to put this into matrix
+          ## for amd64 (x86_64) please leave ARCH_TRAILING_IMG_NAME empty like
+          ## ARCH_TRAILING_IMG_NAME=
+          build-args: |
+            ARCH_TRAILING_IMG_NAME=${{ matrix.platform == 'linux/amd64' ? '' : '-aarch64' }}

README.md

@@ -72,7 +72,8 @@ Prerequisites: GKE Cluster (Standard cluster, *NOT* Autopilot), Artifact Registr
 2. Download your Service Account's JSON key
 3. Activate your service account: `gcloud auth activate-service-account --key-file=<path to the JSON key>`
 4. Connect local kubectl to cloud `gcloud container clusters get-credentials <GKE_CLUSTER_NAME> --zone <GKE_ZONE> --project <PROJECT_NAME>`
-5. Create a namespace for airflow in prior `kubectl create namespace airflow` because our main chart contains 1 manifest pointing to 'airflow' namespace instead of 'mlops' like the rest. On local, it works pretty fine but on GCP it seems like it cannot auto create 'airflow' namespace during helm install mlops namespace.
+5. Create a namespace for airflow in prior `kubectl create namespace airflow` because our main chart contains 1 manifest `spark-streaming-pvc` pointing to 'airflow' namespace instead of 'mlops' like the rest. On local, it works pretty fine but on GCP it seems like it cannot auto create 'airflow' namespace during helm install mlops namespace.
+6. Follow the steps in **With Kubernetes/Helm (Local cluster)** section
 
 ## Cleanup steps
 1. `helm uninstall sfmlops-helm -n mlops`

sfmlops-helm/templates/data-producer-deployment.yaml

@@ -15,6 +15,7 @@ spec:
       containers:
         - name: data-producer
           image: ariya23156/sfmlops-data-producer
+          imagePullPolicy: Always
           resources:
             requests:
               cpu: 100m

sfmlops-helm/templates/forecast-service-deployment-service.yaml

@@ -19,6 +19,7 @@ spec:
       containers:
         - name: forecast-service
           image: ariya23156/sfmlops-forecast-service
+          imagePullPolicy: Always
           ports:
             - containerPort: {{ .Values.forecastServicePort }}
           volumeMounts:

sfmlops-helm/templates/training-service-deployment-service.yaml

@@ -15,6 +15,7 @@ spec:
       containers:
         - name: training-service
           image: ariya23156/sfmlops-training-service
+          imagePullPolicy: Always
           ports:
             - containerPort: {{ .Values.trainingServicePort }}
           envFrom:

sfmlops-helm/templates/web-ui-deployment-service.yaml

@@ -15,6 +15,7 @@ spec:
       containers:
         - name: web-ui
           image: ariya23156/sfmlops-web-ui
+          imagePullPolicy: Always
           ports:
             - containerPort: {{ .Values.webUiPort }}
           livenessProbe: