All images building successfully for aarch64 latest state

Geogouz · May 31, 2024 · 5025e17 · 5025e17
1 parent 5b67b56
commit 5025e17
Show file tree

Hide file tree

Showing 12 changed files with 532 additions and 74 deletions.
diff --git a/.github/workflows/docker-auto-build.yml b/.github/workflows/docker-auto-build.yml
@@ -1,8 +1,6 @@
 name: docker-auto-build
 
 on:
-  release: 
-    types: [created]
   workflow_dispatch:
 
 jobs:
@@ -11,22 +9,22 @@ jobs:
     strategy:
       matrix:
         context: [
-          'clients, prepend-rucio, prepend-release, push-tagged, push-latest', 
-          'server, prepend-rucio, prepend-release, push-tagged, push-latest, push-alma9-latest',
-          'daemons, prepend-rucio, prepend-release, push-tagged, push-latest, push-alma9-latest',
-          'dev, prepend-rucio, prepend-release, push-tagged, push-latest, push-alma9-latest',
-          'ui, prepend-rucio, prepend-release, push-tagged, push-latest',
-          'webui, prepend-rucio, prepend-release, push-tagged, push-latest',
-          'init, prepend-rucio, prepend-release, push-tagged, push-latest',
-          'probes, push-tagged, push-latest',
-          'fts, push-tagged, push-latest',
-          'fts-cron/Dockerfile_cpp, push-tagged, push-latest',
-          'fts-cron/Dockerfile_java, push-tagged, push-latest, custom-tag=java',
-          'ssh, push-tagged, push-latest',
-          'xrootd, push-tagged, push-latest',
-          'xrootd-noauth, push-tagged, push-latest',
-          'fs, rucio-clients:release-#TAG, prepend-rucio, push-tagged, push-latest',
-          'webdav, push-latest'
+          'clients, prepend-rucio, prepend-release, push-tagged, force-latest',
+          'server, prepend-rucio, prepend-release, push-tagged, force-latest, push-alma9-latest',
+          'daemons, prepend-rucio, prepend-release, push-tagged, force-latest, push-alma9-latest',
+          'dev, prepend-rucio, prepend-release, push-tagged, force-latest, push-alma9-latest',
+          'ui, prepend-rucio, prepend-release, push-tagged, force-latest',
+          'webui, prepend-rucio, prepend-release, push-tagged, force-latest',
+          'init, prepend-rucio, prepend-release, push-tagged, force-latest',
+          'probes, push-tagged, force-latest',
+          'fts, push-tagged, force-latest',
+          'fts-cron/Dockerfile_cpp, push-tagged, force-latest',
+          'fts-cron/Dockerfile_java, push-tagged, force-latest, custom-tag=java',
+          'ssh, push-tagged, force-latest',
+          'xrootd, push-tagged, force-latest',
+          'xrootd-noauth, push-tagged, force-latest',
+          'fs, rucio-clients:release-#TAG, prepend-rucio, push-tagged, force-latest',
+          'webdav, force-latest'
         ]
       fail-fast: false
 
@@ -46,7 +44,13 @@ jobs:
       - name: Get the release tag from GitHub
         id: release_tag
         run: echo ::set-output name=version::${GITHUB_REF/refs\/tags\//}
-
+
+      - name: Get the latest tag
+        id: latest_tag
+        run: echo ::set-output name=version::$(git describe --tags `git rev-list --tags --max-count=1`)
+        # It is important for the current commit state in the master-arm64-actions branch, to correspond to the latest
+        # fetched tag version. Otherwise, the release version will not be correct.
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
@@ -86,7 +90,7 @@ jobs:
           # prepend-rucio: adds `rucio-` to image name
           # prepend-release: adds `release-` to image name
           # push-tagged: adds the $GIT_TAG to image name
-          # push-latest: if present, and the latest commit SHA on master branch 
+          # push-latest: if present, and the latest commit SHA on master (arm64) branch 
           #              matches the last commmit SHA of the current tag, adds release 
           #              version to image name.
           # force-latest: if present, the latest tag will be force pushed to Docker Hub
@@ -105,14 +109,33 @@ jobs:
           # | 1               	| 1           	| :release-$GIT_TAG 	|
 
           CONTEXT=$(echo "${{ matrix.context }}")
+          echo "CONTEXT=$CONTEXT"
+          
           CONTEXT_DIR=$(echo "${CONTEXT}" | cut -d, -f1 )
+          echo "CONTEXT_DIR=$CONTEXT_DIR"
+          
           PARENT=$(echo "${CONTEXT}" | cut -d, -f2 | xargs)
-          GIT_TAG=${{ steps.release_tag.outputs.version }}
-          BASE_TAG="${{ github.repository_owner}}/"
+          echo "PARENT=$PARENT"
+          
+          GIT_TAG=${{ steps.latest_tag.outputs.version }}
+          echo "GIT_TAG=$GIT_TAG"
+          
+          DOCKER_HUB_USER_OVERRIDE=dxen
+          
+          BASE_TAG="$DOCKER_HUB_USER_OVERRIDE/"
+          echo "BASE_TAG=$BASE_TAG"
+          
           PARENT_IMAGE_AVAILIBILITY_INTERVAL=60
+          echo "PARENT_IMAGE_AVAILIBILITY_INTERVAL=$PARENT_IMAGE_AVAILIBILITY_INTERVAL"
+          
           MAX_ATTEMPTS=10
+          echo "MAX_ATTEMPTS=$MAX_ATTEMPTS"
+          
           GIT_TAG_COMMIT_SHA=$(echo $(git rev-list -n 1 $GIT_TAG))
-          GIT_MASTER_LATEST_COMMIT_SHA=$(echo $(git ls-remote --heads origin | grep master  |  awk -F '\t' '{print $1}'))
+          echo "GIT_TAG_COMMIT_SHA=$GIT_TAG_COMMIT_SHA"
+          
+          GIT_MASTER_ARM64_LATEST_COMMIT_SHA=$(echo $(git ls-remote --heads origin | grep master-arm64-actions  |  awk -F '\t' '{print $1}'))
+          echo "GIT_MASTER_ARM64_LATEST_COMMIT_SHA=$GIT_MASTER_ARM64_LATEST_COMMIT_SHA"
 
           # if git tag begins with webui-{semantic version}, then treat context as a webui release
           if [[ $GIT_TAG =~ ^webui-([0-9]+.[0-9]+.[0-9]+)$ ]]; then
@@ -163,14 +186,14 @@ jobs:
             IMAGE_TAG+="$BASE_TAG:release-$GIT_TAG"
           fi
 
-          if [[ "${CONTEXT}" =~ .*"push-latest".* &&  $GIT_MASTER_LATEST_COMMIT_SHA == $GIT_TAG_COMMIT_SHA  ]]; then
-              echo "There's a match between the SHA of latest commit on Master ($GIT_MASTER_LATEST_COMMIT_SHA) and \
+          if [[ "${CONTEXT}" =~ .*"push-latest".* &&  $GIT_MASTER_ARM64_LATEST_COMMIT_SHA == $GIT_TAG_COMMIT_SHA  ]]; then
+              echo "There's a match between the SHA of latest commit on Master (arm64) ($GIT_MASTER_ARM64_LATEST_COMMIT_SHA) and \
                     latest commit on current tag $TAG ($GIT_TAG_COMMIT_SHA)"
               echo "An image with latest tag will be pushed to Docker Hub"
               LATEST_TAG="$BASE_TAG:latest"
           elif [[ "${CONTEXT}" =~ .*"push-latest".*  ]]; then
-              echo "Note: Image tagged latest will not be pushed as commit SHA's on master and current tag do not match!!"
-              echo "Latest commit on Master: $GIT_MASTER_LATEST_COMMIT_SHA"
+              echo "Note: Image tagged latest will not be pushed as commit SHA's on master (arm64) and current tag do not match!!"
+              echo "Latest commit on Master (arm64): $GIT_MASTER_ARM64_LATEST_COMMIT_SHA"
               echo "Latest commit on $GIT_TAG: $GIT_TAG_COMMIT_SHA"
           fi
 
@@ -223,11 +246,11 @@ jobs:
             PARENT=${PARENT/\#TAG/$GIT_TAG}
             fi
 
-            PARENT="${{ github.repository_owner }}/${PARENT}"
+            PARENT="${DOCKER_HUB_USER_OVERRIDE}/${PARENT}"
             for i in $(seq 1 1 $MAX_ATTEMPTS)
             do
                 echo "Attempt ${i}: Waiting for $PARENT to be available on Docker Hub"
-                docker pull $PARENT > /dev/null && echo "$PARENT found" && break || echo "Parent not found! Waiting $PARENT_IMAGE_AVAILIBILITY_INTERVAL seconds"
+                docker pull --platform linux/arm64 $PARENT > /dev/null && echo "$PARENT found" && break || echo "Parent not found! Waiting $PARENT_IMAGE_AVAILIBILITY_INTERVAL seconds"
                 sleep $PARENT_IMAGE_AVAILIBILITY_INTERVAL
 
                 if [[ $i == $MAX_ATTEMPTS ]]; then
@@ -257,4 +280,4 @@ jobs:
           push: true
           tags: '${{ steps.release.outputs.tags }}'
           build-args: TAG=${{ steps.release.outputs.git_tag }}
-
+          platforms: linux/arm64
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+.idea/
diff --git a/daemons/Dockerfile b/daemons/Dockerfile
@@ -5,19 +5,28 @@
 # You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
 
 FROM almalinux:9 as rpm_builder
-    RUN dnf upgrade -y
-    RUN dnf install -y rpm-build rpmdevtools yum-utils epel-release.noarch
-    RUN yum-config-manager --enable crb
+
+RUN dnf upgrade -y
+RUN dnf install -y rpm-build rpmdevtools yum-utils epel-release.noarch
+RUN yum-config-manager --enable crb
+
+# Conditional execution based on architecture
+RUN if [ "$(uname -m)" != "aarch64" ]; then  \
+    echo "Non-arm64 architecture detected, building OpenSSL from source and rebuilding davix"; \
     # BUILD and install openssl 3.1
-    ADD openssl.spec    /root/rpmbuild/SPECS/openssl.spec
-    RUN yum-builddep -y /root/rpmbuild/SPECS/openssl.spec
-    RUN spectool -g     /root/rpmbuild/SPECS/openssl.spec --directory /root/rpmbuild/SOURCES/
-    RUN rpmbuild -bb    /root/rpmbuild/SPECS/openssl.spec
-    RUN dnf install -y  /root/rpmbuild/RPMS/*/*
+    ADD openssl.spec /root/rpmbuild/SPECS/openssl.spec; \
+    yum-builddep -y /root/rpmbuild/SPECS/openssl.spec; \
+    spectool -g /root/rpmbuild/SPECS/openssl.spec --directory /root/rpmbuild/SOURCES/; \
+    rpmbuild -bb /root/rpmbuild/SPECS/openssl.spec; \
+    dnf install -y /root/rpmbuild/RPMS/*/*; \
     # REBUILD davix to use openssl 3.1
-    RUN yumdownloader --source davix
-    RUN yum-builddep -y --srpm davix*.src.rpm
-    RUN rpmbuild --rebuild davix-*.src.rpm
+    yumdownloader --source davix; \
+    yum-builddep -y --srpm davix*.src.rpm; \
+    rpmbuild --rebuild davix-*.src.rpm; \
+else \
+    echo "arm64 architecture detected, skipping custom OpenSSL build"; \
+    dnf install -y openssl-3.1.4 \
+fi
 
 FROM almalinux:9
 
@@ -52,23 +61,34 @@ RUN rpm -i https://download.oracle.com/otn_software/linux/instantclient/1912000/
     echo "/usr/lib/oracle/19/client64/lib" >/etc/ld.so.conf.d/oracle.conf; \
     ldconfig
 
+# Set default environment variable that can be overridden
+ENV USE_DAVIX_WITH_OPENSSL31=False
+
+# Install additional packages for arm64 architecture and set environment variables conditionally
+RUN if [ "$(uname -m)" = "aarch64" ]; then \
+        dnf install -y gcc python3-devel && \
+        dnf clean all && \
+        rm -rf /var/cache/dnf; \
+    else \
+        USE_DAVIX_WITH_OPENSSL31=True; \
+    fi
+
+# TODO: The following step is required for custom building cases (i.e. non arm64)
+#COPY --from=rpm_builder /root/rpmbuild/RPMS/x86_64/*.rpm /tmp/rpms/
+
 RUN python3 -m pip install --no-cache-dir --upgrade pip && \
-    python3 -m pip install --no-cache-dir --upgrade setuptools
-RUN python3 -m pip install --no-cache-dir --pre rucio[oracle,mysql,postgresql,globus]==$TAG
+    python3 -m pip install --no-cache-dir --upgrade setuptools && \
+    python3 -m pip install --no-cache-dir --pre rucio[oracle,mysql,postgresql,globus]==$TAG && \
+    python3 -m pip install --no-cache-dir j2cli
 
-RUN python3 -m pip install --no-cache-dir j2cli
 ADD rucio.config.default.cfg /tmp/
 ADD start-daemon.sh /
 
 RUN update-crypto-policies --set DEFAULT:SHA1
 
 RUN mkdir /var/log/rucio
 
-COPY --from=rpm_builder /root/rpmbuild/RPMS/x86_64/*.rpm /tmp/rpms/
-
 VOLUME /var/log/rucio
 VOLUME /opt/rucio/etc
 
-ENV USE_DAVIX_WITH_OPENSSL31=True
-
 ENTRYPOINT ["/start-daemon.sh"]
diff --git a/fs/Dockerfile b/fs/Dockerfile
@@ -1,5 +1,5 @@
 ARG TAG
-FROM rucio/rucio-clients:release-$TAG
+FROM dxen/rucio-clients:release-$TAG
 
 USER root
 RUN dnf install -y git cmake3 libcurl-devel fuse-devel tree

diff --git a/fts-cron/Dockerfile_cpp b/fts-cron/Dockerfile_cpp
@@ -7,13 +7,18 @@ RUN yum install -y epel-release.noarch http://linuxsoft.cern.ch/wlcg/centos7/x86
     yum clean all && \
     rm -rf /var/cache/yum
 
+# Configure yum to skip unavailable repositories
+RUN yum-config-manager --save --setopt=wlcg.skip_if_unavailable=true
+
 RUN yum update -y && \
     yum upgrade -y && \
     yum clean all && \
     rm -rf /var/cache/yum
 
-RUN yum -y install https://repo.ius.io/ius-release-el7.rpm &&\
-    yum install -y httpd python36u-pip python36-mod_wsgi libaio gcc python36-devel.x86_64 mod_ssl openssl-devel.x86_64 python36-m2crypto libnsl.x86_64 patch.x86_64 xrootd-client && \
+# Temporarily disable IUS repository
+RUN yum install -y httpd libaio gcc  \
+    python36 python36-pip python36-mod_wsgi python36-devel python36-setuptools python36-requests python36-m2crypto mod_ssl  \
+    openssl-devel libnsl patch xrootd-client && \
     yum clean all && \
     rm -rf /var/cache/yum
 
@@ -23,8 +28,7 @@ RUN chmod +x /usr/bin/kubectl
 
 # Install VOMS and FTS clients for delegating proxies
 RUN yum -y install ca-certificates.noarch lcg-CA ca_* fetch-crl voms-clients-cpp fts-rest-cli \
-    wlcg-iam-lsc-atlas wlcg-iam-vomses-atlas wlcg-voms-atlas wlcg-iam-lsc-cms wlcg-iam-vomses-cms wlcg-voms-cms \
-    python-pip python-setuptools python-requests && \
+    wlcg-iam-lsc-atlas wlcg-iam-vomses-atlas wlcg-voms-atlas wlcg-iam-lsc-cms wlcg-iam-vomses-cms wlcg-voms-cms && \
     yum clean all && \
     rm -rf /var/cache/yum
 

diff --git a/fts-cron/Dockerfile_java b/fts-cron/Dockerfile_java
@@ -7,13 +7,18 @@ RUN yum install -y epel-release.noarch http://linuxsoft.cern.ch/wlcg/centos7/x86
     yum clean all && \
     rm -rf /var/cache/yum
 
+# Configure yum to skip unavailable repositories
+RUN yum-config-manager --save --setopt=wlcg.skip_if_unavailable=true
+
 RUN yum update -y && \
     yum upgrade -y && \
     yum clean all && \
     rm -rf /var/cache/yum
 
-RUN yum -y install https://repo.ius.io/ius-release-el7.rpm &&\
-    yum install -y httpd python36u-pip python36-mod_wsgi libaio gcc python36-devel.x86_64 mod_ssl openssl-devel.x86_64 python36-m2crypto libnsl.x86_64 patch.x86_64 xrootd-client && \
+# Temporarily disable IUS repository
+RUN yum install -y httpd libaio gcc  \
+    python36 python36-pip python36-mod_wsgi python36-devel python36-setuptools python36-requests python36-m2crypto mod_ssl  \
+    openssl-devel libnsl patch xrootd-client && \
     yum clean all && \
     rm -rf /var/cache/yum
 
@@ -23,8 +28,7 @@ RUN chmod +x /usr/bin/kubectl
 
 # Install VOMS and FTS clients for delegating proxies
 RUN yum -y install ca-certificates.noarch lcg-CA ca_* fetch-crl voms-clients-java fts-rest-cli \
-    wlcg-iam-lsc-atlas wlcg-iam-vomses-atlas wlcg-voms-atlas wlcg-iam-lsc-cms wlcg-iam-vomses-cms wlcg-voms-cms \
-    python-pip python-setuptools python-requests && \
+    wlcg-iam-lsc-atlas wlcg-iam-vomses-atlas wlcg-voms-atlas wlcg-iam-lsc-cms wlcg-iam-vomses-cms wlcg-voms-cms && \
     yum clean all && \
     rm -rf /var/cache/yum