diff --git a/.github/workflows/docker-auto-build.yml b/.github/workflows/docker-auto-build.yml index 79c763e..534e0e0 100644 --- a/.github/workflows/docker-auto-build.yml +++ b/.github/workflows/docker-auto-build.yml @@ -1,8 +1,6 @@ name: docker-auto-build on: - release: - types: [created] workflow_dispatch: jobs: @@ -11,22 +9,22 @@ jobs: strategy: matrix: context: [ - 'clients, prepend-rucio, prepend-release, push-tagged, push-latest', - 'server, prepend-rucio, prepend-release, push-tagged, push-latest, push-alma9-latest', - 'daemons, prepend-rucio, prepend-release, push-tagged, push-latest, push-alma9-latest', - 'dev, prepend-rucio, prepend-release, push-tagged, push-latest, push-alma9-latest', - 'ui, prepend-rucio, prepend-release, push-tagged, push-latest', - 'webui, prepend-rucio, prepend-release, push-tagged, push-latest', - 'init, prepend-rucio, prepend-release, push-tagged, push-latest', - 'probes, push-tagged, push-latest', - 'fts, push-tagged, push-latest', - 'fts-cron/Dockerfile_cpp, push-tagged, push-latest', - 'fts-cron/Dockerfile_java, push-tagged, push-latest, custom-tag=java', - 'ssh, push-tagged, push-latest', - 'xrootd, push-tagged, push-latest', - 'xrootd-noauth, push-tagged, push-latest', - 'fs, rucio-clients:release-#TAG, prepend-rucio, push-tagged, push-latest', - 'webdav, push-latest' + 'clients, prepend-rucio, prepend-release, push-tagged, force-latest', + 'server, prepend-rucio, prepend-release, push-tagged, force-latest, push-alma9-latest', + 'daemons, prepend-rucio, prepend-release, push-tagged, force-latest, push-alma9-latest', + 'dev, prepend-rucio, prepend-release, push-tagged, force-latest, push-alma9-latest', + 'ui, prepend-rucio, prepend-release, push-tagged, force-latest', + 'webui, prepend-rucio, prepend-release, push-tagged, force-latest', + 'init, prepend-rucio, prepend-release, push-tagged, force-latest', + 'probes, push-tagged, force-latest', + 'fts, push-tagged, force-latest', + 'fts-cron/Dockerfile_cpp, push-tagged, force-latest', + 'fts-cron/Dockerfile_java, push-tagged, force-latest, custom-tag=java', + 'ssh, push-tagged, force-latest', + 'xrootd, push-tagged, force-latest', + 'xrootd-noauth, push-tagged, force-latest', + 'fs, rucio-clients:release-#TAG, prepend-rucio, push-tagged, force-latest', + 'webdav, force-latest' ] fail-fast: false @@ -46,7 +44,13 @@ jobs: - name: Get the release tag from GitHub id: release_tag run: echo ::set-output name=version::${GITHUB_REF/refs\/tags\//} - + + - name: Get the latest tag + id: latest_tag + run: echo ::set-output name=version::$(git describe --tags `git rev-list --tags --max-count=1`) + # It is important for the current commit state in the master-arm64-actions branch, to correspond to the latest + # fetched tag version. Otherwise, the release version will not be correct. + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -86,7 +90,7 @@ jobs: # prepend-rucio: adds `rucio-` to image name # prepend-release: adds `release-` to image name # push-tagged: adds the $GIT_TAG to image name - # push-latest: if present, and the latest commit SHA on master branch + # push-latest: if present, and the latest commit SHA on master (arm64) branch # matches the last commmit SHA of the current tag, adds release # version to image name. # force-latest: if present, the latest tag will be force pushed to Docker Hub @@ -105,14 +109,33 @@ jobs: # | 1 | 1 | :release-$GIT_TAG | CONTEXT=$(echo "${{ matrix.context }}") + echo "CONTEXT=$CONTEXT" + CONTEXT_DIR=$(echo "${CONTEXT}" | cut -d, -f1 ) + echo "CONTEXT_DIR=$CONTEXT_DIR" + PARENT=$(echo "${CONTEXT}" | cut -d, -f2 | xargs) - GIT_TAG=${{ steps.release_tag.outputs.version }} - BASE_TAG="${{ github.repository_owner}}/" + echo "PARENT=$PARENT" + + GIT_TAG=${{ steps.latest_tag.outputs.version }} + echo "GIT_TAG=$GIT_TAG" + + DOCKER_HUB_USER_OVERRIDE=dxen + + BASE_TAG="$DOCKER_HUB_USER_OVERRIDE/" + echo "BASE_TAG=$BASE_TAG" + PARENT_IMAGE_AVAILIBILITY_INTERVAL=60 + echo "PARENT_IMAGE_AVAILIBILITY_INTERVAL=$PARENT_IMAGE_AVAILIBILITY_INTERVAL" + MAX_ATTEMPTS=10 + echo "MAX_ATTEMPTS=$MAX_ATTEMPTS" + GIT_TAG_COMMIT_SHA=$(echo $(git rev-list -n 1 $GIT_TAG)) - GIT_MASTER_LATEST_COMMIT_SHA=$(echo $(git ls-remote --heads origin | grep master | awk -F '\t' '{print $1}')) + echo "GIT_TAG_COMMIT_SHA=$GIT_TAG_COMMIT_SHA" + + GIT_MASTER_ARM64_LATEST_COMMIT_SHA=$(echo $(git ls-remote --heads origin | grep master-arm64-actions | awk -F '\t' '{print $1}')) + echo "GIT_MASTER_ARM64_LATEST_COMMIT_SHA=$GIT_MASTER_ARM64_LATEST_COMMIT_SHA" # if git tag begins with webui-{semantic version}, then treat context as a webui release if [[ $GIT_TAG =~ ^webui-([0-9]+.[0-9]+.[0-9]+)$ ]]; then @@ -163,14 +186,14 @@ jobs: IMAGE_TAG+="$BASE_TAG:release-$GIT_TAG" fi - if [[ "${CONTEXT}" =~ .*"push-latest".* && $GIT_MASTER_LATEST_COMMIT_SHA == $GIT_TAG_COMMIT_SHA ]]; then - echo "There's a match between the SHA of latest commit on Master ($GIT_MASTER_LATEST_COMMIT_SHA) and \ + if [[ "${CONTEXT}" =~ .*"push-latest".* && $GIT_MASTER_ARM64_LATEST_COMMIT_SHA == $GIT_TAG_COMMIT_SHA ]]; then + echo "There's a match between the SHA of latest commit on Master (arm64) ($GIT_MASTER_ARM64_LATEST_COMMIT_SHA) and \ latest commit on current tag $TAG ($GIT_TAG_COMMIT_SHA)" echo "An image with latest tag will be pushed to Docker Hub" LATEST_TAG="$BASE_TAG:latest" elif [[ "${CONTEXT}" =~ .*"push-latest".* ]]; then - echo "Note: Image tagged latest will not be pushed as commit SHA's on master and current tag do not match!!" - echo "Latest commit on Master: $GIT_MASTER_LATEST_COMMIT_SHA" + echo "Note: Image tagged latest will not be pushed as commit SHA's on master (arm64) and current tag do not match!!" + echo "Latest commit on Master (arm64): $GIT_MASTER_ARM64_LATEST_COMMIT_SHA" echo "Latest commit on $GIT_TAG: $GIT_TAG_COMMIT_SHA" fi @@ -223,11 +246,11 @@ jobs: PARENT=${PARENT/\#TAG/$GIT_TAG} fi - PARENT="${{ github.repository_owner }}/${PARENT}" + PARENT="${DOCKER_HUB_USER_OVERRIDE}/${PARENT}" for i in $(seq 1 1 $MAX_ATTEMPTS) do echo "Attempt ${i}: Waiting for $PARENT to be available on Docker Hub" - docker pull $PARENT > /dev/null && echo "$PARENT found" && break || echo "Parent not found! Waiting $PARENT_IMAGE_AVAILIBILITY_INTERVAL seconds" + docker pull --platform linux/arm64 $PARENT > /dev/null && echo "$PARENT found" && break || echo "Parent not found! Waiting $PARENT_IMAGE_AVAILIBILITY_INTERVAL seconds" sleep $PARENT_IMAGE_AVAILIBILITY_INTERVAL if [[ $i == $MAX_ATTEMPTS ]]; then @@ -257,4 +280,4 @@ jobs: push: true tags: '${{ steps.release.outputs.tags }}' build-args: TAG=${{ steps.release.outputs.git_tag }} - \ No newline at end of file + platforms: linux/arm64 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9f11b75 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea/ diff --git a/daemons/Dockerfile b/daemons/Dockerfile index 4e33068..a4e216e 100644 --- a/daemons/Dockerfile +++ b/daemons/Dockerfile @@ -5,19 +5,28 @@ # You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 FROM almalinux:9 as rpm_builder - RUN dnf upgrade -y - RUN dnf install -y rpm-build rpmdevtools yum-utils epel-release.noarch - RUN yum-config-manager --enable crb + +RUN dnf upgrade -y +RUN dnf install -y rpm-build rpmdevtools yum-utils epel-release.noarch +RUN yum-config-manager --enable crb + +# Conditional execution based on architecture +RUN if [ "$(uname -m)" != "aarch64" ]; then \ + echo "Non-arm64 architecture detected, building OpenSSL from source and rebuilding davix"; \ # BUILD and install openssl 3.1 - ADD openssl.spec /root/rpmbuild/SPECS/openssl.spec - RUN yum-builddep -y /root/rpmbuild/SPECS/openssl.spec - RUN spectool -g /root/rpmbuild/SPECS/openssl.spec --directory /root/rpmbuild/SOURCES/ - RUN rpmbuild -bb /root/rpmbuild/SPECS/openssl.spec - RUN dnf install -y /root/rpmbuild/RPMS/*/* + ADD openssl.spec /root/rpmbuild/SPECS/openssl.spec; \ + yum-builddep -y /root/rpmbuild/SPECS/openssl.spec; \ + spectool -g /root/rpmbuild/SPECS/openssl.spec --directory /root/rpmbuild/SOURCES/; \ + rpmbuild -bb /root/rpmbuild/SPECS/openssl.spec; \ + dnf install -y /root/rpmbuild/RPMS/*/*; \ # REBUILD davix to use openssl 3.1 - RUN yumdownloader --source davix - RUN yum-builddep -y --srpm davix*.src.rpm - RUN rpmbuild --rebuild davix-*.src.rpm + yumdownloader --source davix; \ + yum-builddep -y --srpm davix*.src.rpm; \ + rpmbuild --rebuild davix-*.src.rpm; \ +else \ + echo "arm64 architecture detected, skipping custom OpenSSL build"; \ + dnf install -y openssl-3.1.4 \ +fi FROM almalinux:9 @@ -52,11 +61,26 @@ RUN rpm -i https://download.oracle.com/otn_software/linux/instantclient/1912000/ echo "/usr/lib/oracle/19/client64/lib" >/etc/ld.so.conf.d/oracle.conf; \ ldconfig +# Set default environment variable that can be overridden +ENV USE_DAVIX_WITH_OPENSSL31=False + +# Install additional packages for arm64 architecture and set environment variables conditionally +RUN if [ "$(uname -m)" = "aarch64" ]; then \ + dnf install -y gcc python3-devel && \ + dnf clean all && \ + rm -rf /var/cache/dnf; \ + else \ + USE_DAVIX_WITH_OPENSSL31=True; \ + fi + +# TODO: The following step is required for custom building cases (i.e. non arm64) +#COPY --from=rpm_builder /root/rpmbuild/RPMS/x86_64/*.rpm /tmp/rpms/ + RUN python3 -m pip install --no-cache-dir --upgrade pip && \ - python3 -m pip install --no-cache-dir --upgrade setuptools -RUN python3 -m pip install --no-cache-dir --pre rucio[oracle,mysql,postgresql,globus]==$TAG + python3 -m pip install --no-cache-dir --upgrade setuptools && \ + python3 -m pip install --no-cache-dir --pre rucio[oracle,mysql,postgresql,globus]==$TAG && \ + python3 -m pip install --no-cache-dir j2cli -RUN python3 -m pip install --no-cache-dir j2cli ADD rucio.config.default.cfg /tmp/ ADD start-daemon.sh / @@ -64,11 +88,7 @@ RUN update-crypto-policies --set DEFAULT:SHA1 RUN mkdir /var/log/rucio -COPY --from=rpm_builder /root/rpmbuild/RPMS/x86_64/*.rpm /tmp/rpms/ - VOLUME /var/log/rucio VOLUME /opt/rucio/etc -ENV USE_DAVIX_WITH_OPENSSL31=True - ENTRYPOINT ["/start-daemon.sh"] diff --git a/fs/Dockerfile b/fs/Dockerfile index 5af6b5e..3c48efa 100644 --- a/fs/Dockerfile +++ b/fs/Dockerfile @@ -1,5 +1,5 @@ ARG TAG -FROM rucio/rucio-clients:release-$TAG +FROM dxen/rucio-clients:release-$TAG USER root RUN dnf install -y git cmake3 libcurl-devel fuse-devel tree diff --git a/fts-cron/Dockerfile_cpp b/fts-cron/Dockerfile_cpp index 2c86778..a77b503 100644 --- a/fts-cron/Dockerfile_cpp +++ b/fts-cron/Dockerfile_cpp @@ -7,13 +7,18 @@ RUN yum install -y epel-release.noarch http://linuxsoft.cern.ch/wlcg/centos7/x86 yum clean all && \ rm -rf /var/cache/yum +# Configure yum to skip unavailable repositories +RUN yum-config-manager --save --setopt=wlcg.skip_if_unavailable=true + RUN yum update -y && \ yum upgrade -y && \ yum clean all && \ rm -rf /var/cache/yum -RUN yum -y install https://repo.ius.io/ius-release-el7.rpm &&\ - yum install -y httpd python36u-pip python36-mod_wsgi libaio gcc python36-devel.x86_64 mod_ssl openssl-devel.x86_64 python36-m2crypto libnsl.x86_64 patch.x86_64 xrootd-client && \ +# Temporarily disable IUS repository +RUN yum install -y httpd libaio gcc \ + python36 python36-pip python36-mod_wsgi python36-devel python36-setuptools python36-requests python36-m2crypto mod_ssl \ + openssl-devel libnsl patch xrootd-client && \ yum clean all && \ rm -rf /var/cache/yum @@ -23,8 +28,7 @@ RUN chmod +x /usr/bin/kubectl # Install VOMS and FTS clients for delegating proxies RUN yum -y install ca-certificates.noarch lcg-CA ca_* fetch-crl voms-clients-cpp fts-rest-cli \ - wlcg-iam-lsc-atlas wlcg-iam-vomses-atlas wlcg-voms-atlas wlcg-iam-lsc-cms wlcg-iam-vomses-cms wlcg-voms-cms \ - python-pip python-setuptools python-requests && \ + wlcg-iam-lsc-atlas wlcg-iam-vomses-atlas wlcg-voms-atlas wlcg-iam-lsc-cms wlcg-iam-vomses-cms wlcg-voms-cms && \ yum clean all && \ rm -rf /var/cache/yum diff --git a/fts-cron/Dockerfile_java b/fts-cron/Dockerfile_java index 0e0440f..85105c9 100644 --- a/fts-cron/Dockerfile_java +++ b/fts-cron/Dockerfile_java @@ -7,13 +7,18 @@ RUN yum install -y epel-release.noarch http://linuxsoft.cern.ch/wlcg/centos7/x86 yum clean all && \ rm -rf /var/cache/yum +# Configure yum to skip unavailable repositories +RUN yum-config-manager --save --setopt=wlcg.skip_if_unavailable=true + RUN yum update -y && \ yum upgrade -y && \ yum clean all && \ rm -rf /var/cache/yum -RUN yum -y install https://repo.ius.io/ius-release-el7.rpm &&\ - yum install -y httpd python36u-pip python36-mod_wsgi libaio gcc python36-devel.x86_64 mod_ssl openssl-devel.x86_64 python36-m2crypto libnsl.x86_64 patch.x86_64 xrootd-client && \ +# Temporarily disable IUS repository +RUN yum install -y httpd libaio gcc \ + python36 python36-pip python36-mod_wsgi python36-devel python36-setuptools python36-requests python36-m2crypto mod_ssl \ + openssl-devel libnsl patch xrootd-client && \ yum clean all && \ rm -rf /var/cache/yum @@ -23,8 +28,7 @@ RUN chmod +x /usr/bin/kubectl # Install VOMS and FTS clients for delegating proxies RUN yum -y install ca-certificates.noarch lcg-CA ca_* fetch-crl voms-clients-java fts-rest-cli \ - wlcg-iam-lsc-atlas wlcg-iam-vomses-atlas wlcg-voms-atlas wlcg-iam-lsc-cms wlcg-iam-vomses-cms wlcg-voms-cms \ - python-pip python-setuptools python-requests && \ + wlcg-iam-lsc-atlas wlcg-iam-vomses-atlas wlcg-voms-atlas wlcg-iam-lsc-cms wlcg-iam-vomses-cms wlcg-voms-cms && \ yum clean all && \ rm -rf /var/cache/yum diff --git a/fts/Dockerfile b/fts/Dockerfile index 513e03c..1be40a2 100644 --- a/fts/Dockerfile +++ b/fts/Dockerfile @@ -1,22 +1,170 @@ FROM centos:7 -# Install FTS -RUN yum install -y epel-release.noarch -ADD fts3-testing.repo /etc/yum.repos.d/fts3-testing.repo -RUN curl https://fts-repo.web.cern.ch/fts-repo/fts3-depend-el7.repo > /etc/yum.repos.d/fts3-depend-el7.repo -RUN curl https://dmc-repo.web.cern.ch/dmc-repo/dmc-el7.repo > /etc/yum.repos.d/dmc-el7.repo +# Define an ARG for bypassing the cache +ARG CACHE_TS=1 + RUN yum upgrade -y && \ - yum install -y centos-release-scl + yum install -y yum-utils epel-release.noarch centos-release-scl && \ + yum-config-manager --save --setopt=epel.skip_if_unavailable=false && \ + echo 'skip_missing_names_on_install=0' >> /etc/yum.conf + +ADD fts3-testing.repo /etc/yum.repos.d/fts3-testing.repo +RUN echo -e "[fts3-depend-el7]\nname=FTS3 Dependencies\nbaseurl=https://fts-repo.web.cern.ch/fts-repo/fts-depend/el7/x86_64\ngpgcheck=0\nenabled=1\nprotect=0" > /etc/yum.repos.d/fts3-depend-el7.repo +RUN echo -e "[dmc-el7]\nname=DMC Production Repository\nbaseurl=https://dmc-repo.web.cern.ch/dmc-repo/el7/x86_64\ngpgcheck=0\nenabled=1\nprotect=0" > /etc/yum.repos.d/dmc-el7.repo + +# ************************************************** +# Compile required libraries from the official repos + +# Install supporting packages +RUN yum install -y gcc python36-devel mysql-devel rpm-build rpmdevtools wget + +# Create RPM build environment +RUN rpmdev-setuptree + +###################### +# mysqlclient==1.4.6 +###################### +RUN wget https://files.pythonhosted.org/packages/d0/97/7326248ac8d5049968bf4ec708a5d3d4806e412a42e74160d7f266a3e03a/mysqlclient-1.4.6.tar.gz && \ + tar xzf mysqlclient-1.4.6.tar.gz +RUN cd mysqlclient-1.4.6 && \ + python3 setup.py bdist_rpm && \ + mv dist/mysqlclient-1.4.6-1.aarch64.rpm /tmp/ && \ + yum install -y /tmp/mysqlclient-1.4.6-1.aarch64.rpm && \ + rm -rf /mysqlclient-1.4.6 /tmp/mysqlclient-1.4.6-1.aarch64.rpm +#RUN python3 -c "import MySQLdb; print('Successfully imported MySQLdb!')" + +###################### +# SQLAlchemy==1.1.15 +###################### +RUN wget https://files.pythonhosted.org/packages/c2/f6/11fcc1ce19a7cb81b1c9377f4e27ce3813265611922e355905e57c44d164/SQLAlchemy-1.1.15.tar.gz && \ + tar xzf SQLAlchemy-1.1.15.tar.gz +RUN cd SQLAlchemy-1.1.15 && \ + python3 setup.py bdist_rpm && \ + mv dist/SQLAlchemy-1.1.15-1.aarch64.rpm /tmp/ && \ + yum install -y /tmp/SQLAlchemy-1.1.15-1.aarch64.rpm && \ + rm -rf /SQLAlchemy-1.1.15 /tmp/SQLAlchemy-1.1.15-1.aarch64.rpm +#RUN python3 -c "import sqlalchemy; print('Successfully imported sqlalchemy!')" + +###################### +# MarkupSafe==0.23 +###################### +RUN wget https://files.pythonhosted.org/packages/c0/41/bae1254e0396c0cc8cf1751cb7d9afc90a602353695af5952530482c963f/MarkupSafe-0.23.tar.gz -O /root/rpmbuild/SOURCES/MarkupSafe-0.23.tar.gz + +# We use custom SPECS to set the name required by other packages +COPY aarch64-files/MarkupSafe.spec /root/rpmbuild/SPECS/ +RUN rpmbuild -ba /root/rpmbuild/SPECS/MarkupSafe.spec && \ + mv /root/rpmbuild/RPMS/aarch64/MarkupSafe-0.23-1.aarch64.rpm /tmp/ && \ + yum install -y /tmp/MarkupSafe-0.23-1.aarch64.rpm +#RUN python3 -c "import markupsafe; print('Successfully imported markupsafe!')" + +###################### +# Jinja2==2.11.1 +###################### +RUN wget https://files.pythonhosted.org/packages/d8/03/e491f423379ea14bb3a02a5238507f7d446de639b623187bccc111fbecdf/Jinja2-2.11.1.tar.gz && \ + tar xzf Jinja2-2.11.1.tar.gz +RUN cd Jinja2-2.11.1 && \ + python3 setup.py bdist_rpm && \ + mv dist/Jinja2-2.11.1-1.noarch.rpm /tmp/ && \ + yum install -y /tmp/Jinja2-2.11.1-1.noarch.rpm && \ + rm -rf /Jinja2-2.11.1 /tmp/Jinja2-2.11.1-1.noarch.rpm +#RUN python3 -c "import jinja2; print('Successfully imported jinja2!')" + +###################### +# Werkzeug==1.0.1 +###################### +RUN wget https://files.pythonhosted.org/packages/10/27/a33329150147594eff0ea4c33c2036c0eadd933141055be0ff911f7f8d04/Werkzeug-1.0.1.tar.gz && \ + tar xzf Werkzeug-1.0.1.tar.gz +RUN cd Werkzeug-1.0.1 && \ + python3 setup.py bdist_rpm && \ + mv dist/Werkzeug-1.0.1-1.noarch.rpm /tmp/ && \ + yum install -y /tmp/Werkzeug-1.0.1-1.noarch.rpm && \ + rm -rf /Werkzeug-1.0.1 /tmp/Werkzeug-1.0.1-1.noarch.rpm +#RUN python3 -c "import werkzeug; print('Successfully imported werkzeug!')" + +###################### +# itsdangerous==1.1.0 +###################### +RUN wget https://files.pythonhosted.org/packages/68/1a/f27de07a8a304ad5fa817bbe383d1238ac4396da447fa11ed937039fa04b/itsdangerous-1.1.0.tar.gz && \ + tar xzf itsdangerous-1.1.0.tar.gz +RUN cd itsdangerous-1.1.0 && \ + python3 setup.py bdist_rpm && \ + mv dist/itsdangerous-1.1.0-1.noarch.rpm /tmp/ && \ + yum install -y /tmp/itsdangerous-1.1.0-1.noarch.rpm && \ + rm -rf /itsdangerous-1.1.0 /tmp/itsdangerous-1.1.0-1.noarch.rpm +#RUN python3 -c "import itsdangerous; print('Successfully imported itsdangerous!')" + +###################### +# click==6.7 +###################### +RUN wget https://files.pythonhosted.org/packages/95/d9/c3336b6b5711c3ab9d1d3a80f1a3e2afeb9d8c02a7166462f6cc96570897/click-6.7.tar.gz && \ + tar xzf click-6.7.tar.gz +RUN cd click-6.7 && \ + python3 setup.py bdist_rpm && \ + mv dist/click-6.7-1.noarch.rpm /tmp/ && \ + yum install -y /tmp/click-6.7-1.noarch.rpm && \ + rm -rf /click-6.7 /tmp/click-6.7-1.noarch.rpm +#RUN python3 -c "import click; print('Successfully imported click!')" + +###################### +# python36-flask==1.1.4 +###################### +RUN wget https://files.pythonhosted.org/packages/4d/5b/2d145f5fe718b2f15ebe69240538f06faa8bbb76488bf962091db1f7a26d/Flask-1.1.4.tar.gz -O /root/rpmbuild/SOURCES/Flask-1.1.4.tar.gz + +# We use custom SPECS to set the name required by other packages +COPY aarch64-files/flask.spec /root/rpmbuild/SPECS/ +RUN rpmbuild -ba /root/rpmbuild/SPECS/flask.spec && \ + mv /root/rpmbuild/RPMS/noarch/Flask-1.1.4-1.noarch.rpm /tmp/ && \ + yum install -y /tmp/Flask-1.1.4-1.noarch.rpm +#RUN python3 -c "import flask; print('Successfully imported flask!')" + +###################### +# M2Crypto==0.35.2 +###################### +RUN wget https://files.pythonhosted.org/packages/74/18/3beedd4ac48b52d1a4d12f2a8c5cf0ae342ce974859fba838cbbc1580249/M2Crypto-0.35.2.tar.gz -O /root/rpmbuild/SOURCES/M2Crypto-0.35.2.tar.gz + +# We use custom SPECS to set the name required by other packages +COPY aarch64-files/m2crypto.spec /root/rpmbuild/SPECS/ +RUN rpmbuild -ba /root/rpmbuild/SPECS/m2crypto.spec && \ + mv /root/rpmbuild/RPMS/aarch64/M2Crypto-0.35.2-1.aarch64.rpm /tmp/ && \ + yum install -y /tmp/M2Crypto-0.35.2-1.aarch64.rpm +#RUN python3 -c "import M2Crypto; print('Successfully imported M2Crypto!')" +# ************************************************** + + +# Download, install and clean the required core dependencies +RUN wget https://buildlogs.centos.org/c7-rh-python36.aarch64/rh-python36/20181031211245/2.0-1.el7.aarch64/rh-python36-runtime-2.0-1.el7.aarch64.rpm && \ + wget https://buildlogs.centos.org/c7-rh-python36.aarch64/rh-python36-python-setuptools/20181101145603/36.5.0-1.el7.aarch64/rh-python36-python-setuptools-36.5.0-1.el7.noarch.rpm && \ + wget https://cbs.centos.org/kojifiles/packages/httpd24-httpd/2.4.6/25.bs1.el7/aarch64/httpd24-httpd-tools-2.4.6-25.bs1.el7.aarch64.rpm && \ + wget https://cbs.centos.org/kojifiles/packages/httpd24/1.1/19.el7/aarch64/httpd24-runtime-1.1-19.el7.aarch64.rpm && \ + wget https://cbs.centos.org/kojifiles/packages/httpd24-httpd/2.4.6/25.bs1.el7/aarch64/httpd24-httpd-2.4.6-25.bs1.el7.aarch64.rpm && \ + wget https://cbs.centos.org/kojifiles/packages/rh-python36-mod_wsgi/4.5.18/1.bs1.el7/aarch64/rh-python36-mod_wsgi-4.5.18-1.bs1.el7.aarch64.rpm + +RUN yum install -y rh-python36-runtime-2.0-1.el7.aarch64.rpm && \ + yum install -y rh-python36-python-setuptools-36.5.0-1.el7.noarch.rpm && \ + yum install -y httpd24-httpd-tools-2.4.6-25.bs1.el7.aarch64.rpm && \ + yum install -y httpd24-runtime-1.1-19.el7.aarch64.rpm && \ + yum install -y httpd24-httpd-2.4.6-25.bs1.el7.aarch64.rpm && \ + yum install -y rh-python36-mod_wsgi-4.5.18-1.bs1.el7.aarch64.rpm + +RUN rm -f \ + rh-python36-runtime-2.0-1.el7.aarch64.rpm \ + rh-python36-python-setuptools-36.5.0-1.el7.noarch.rpm \ + httpd24-httpd-tools-2.4.6-25.bs1.el7.aarch64.rpm \ + httpd24-runtime-1.1-19.el7.aarch64.rpm \ + httpd24-httpd-2.4.6-25.bs1.el7.aarch64.rpm \ + rh-python36-mod_wsgi-4.5.18-1.bs1.el7.aarch64.rpm + +# Install required packages RUN yum install -y \ mysql \ multitail \ - gfal2-plugin* \ - fts-libs-3.13.0-r2405271344gitc6d02cc.el7.cern.x86_64 \ - fts-server-3.13.0-r2405271344gitc6d02cc.el7.cern.x86_64 \ - fts-rest-server-3.13.0-r2405271058git798429f.el7.cern.noarch \ - fts-monitoring \ - fts-mysql-3.13.0-r2405271344gitc6d02cc.el7.cern.x86_64 \ - fts-msg-3.13.0-r2405271344gitc6d02cc.el7.cern.x86_64 \ + gfal2-plugin* \ + fts-libs \ + fts-server \ + fts-rest-server-3.13.0-r2405271058git798429f.el7.cern.noarch \ + fts-monitoring \ + fts-mysql \ + fts-msg \ zeromq RUN yum clean all diff --git a/fts/aarch64-files/MarkupSafe.spec b/fts/aarch64-files/MarkupSafe.spec new file mode 100644 index 0000000..c6e976a --- /dev/null +++ b/fts/aarch64-files/MarkupSafe.spec @@ -0,0 +1,136 @@ +%define name MarkupSafe +%define version 0.23 +%define unmangled_version 0.23 +%define unmangled_version 0.23 +%define release 1 + +Summary: Implements a XML/HTML/XHTML Markup safe string for Python +Name: %{name} +Version: %{version} +Release: %{release} +Provides: python36-markupsafe = %{version}-%{release} +Provides: MarkupSafe = %{version}-%{release} +Source0: %{name}-%{unmangled_version}.tar.gz +License: BSD +Group: Development/Libraries +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot +Prefix: %{_prefix} +Vendor: Armin Ronacher +Url: http://github.com/mitsuhiko/markupsafe + +%description +MarkupSafe +========== + +Implements a unicode subclass that supports HTML strings: + +>>> from markupsafe import Markup, escape +>>> escape("") +Markup(u'<script>alert(document.cookie);</script>') +>>> tmpl = Markup("%s") +>>> tmpl % "Peter > Lustig" +Markup(u'Peter > Lustig') + +If you want to make an object unicode that is not yet unicode +but don't want to lose the taint information, you can use the +`soft_unicode` function. (On Python 3 you can also use `soft_str` which +is a different name for the same function). + +>>> from markupsafe import soft_unicode +>>> soft_unicode(42) +u'42' +>>> soft_unicode(Markup('foo')) +Markup(u'foo') + +HTML Representations +-------------------- + +Objects can customize their HTML markup equivalent by overriding +the `__html__` function: + +>>> class Foo(object): +... def __html__(self): +... return 'Nice' +... +>>> escape(Foo()) +Markup(u'Nice') +>>> Markup(Foo()) +Markup(u'Nice') + +Silent Escapes +-------------- + +Since MarkupSafe 0.10 there is now also a separate escape function +called `escape_silent` that returns an empty string for `None` for +consistency with other systems that return empty strings for `None` +when escaping (for instance Pylons' webhelpers). + +If you also want to use this for the escape method of the Markup +object, you can create your own subclass that does that:: + + from markupsafe import Markup, escape_silent as escape + + class SilentMarkup(Markup): + __slots__ = () + + @classmethod + def escape(cls, s): + return cls(escape(s)) + +New-Style String Formatting +--------------------------- + +Starting with MarkupSafe 0.21 new style string formats from Python 2.6 and +3.x are now fully supported. Previously the escape behavior of those +functions was spotty at best. The new implementations operates under the +following algorithm: + +1. if an object has an ``__html_format__`` method it is called as + replacement for ``__format__`` with the format specifier. It either + has to return a string or markup object. +2. if an object has an ``__html__`` method it is called. +3. otherwise the default format system of Python kicks in and the result + is HTML escaped. + +Here is how you can implement your own formatting:: + + class User(object): + + def __init__(self, id, username): + self.id = id + self.username = username + + def __html_format__(self, format_spec): + if format_spec == 'link': + return Markup('{1}').format( + self.id, + self.__html__(), + ) + elif format_spec: + raise ValueError('Invalid format spec') + return self.__html__() + + def __html__(self): + return Markup('{0}').format(self.username) + +And to format that user: + +>>> user = User(1, 'foo') +>>> Markup('

User: {0:link}').format(user) +Markup(u'

User: foo') + + +%prep +%setup -n %{name}-%{unmangled_version} -n %{name}-%{unmangled_version} + +%build +env CFLAGS="$RPM_OPT_FLAGS" python3 setup.py build + +%install +python3 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES + +%clean +rm -rf $RPM_BUILD_ROOT + +%files -f INSTALLED_FILES +%defattr(-,root,root) diff --git a/fts/aarch64-files/flask.spec b/fts/aarch64-files/flask.spec new file mode 100644 index 0000000..713bee3 --- /dev/null +++ b/fts/aarch64-files/flask.spec @@ -0,0 +1,46 @@ +%define name Flask +%define version 1.1.4 +%define unmangled_version 1.1.4 +%define unmangled_version 1.1.4 +%define release 1 + +Summary: A simple framework for building complex web applications. +Name: %{name} +Version: %{version} +Release: %{release} +Provides: python36-flask = %{version}-%{release} +Source0: %{name}-%{unmangled_version}.tar.gz +License: BSD-3-Clause +Group: Development/Libraries +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot +Prefix: %{_prefix} +BuildArch: noarch +Vendor: Pallets +Url: https://palletsprojects.com/p/flask/ + +%description +Flask is a lightweight `WSGI`_ web application framework. It is designed +to make getting started quick and easy, with the ability to scale up to +complex applications. It began as a simple wrapper around `Werkzeug`_ +and `Jinja`_ and has become one of the most popular Python web +application frameworks. + +Flask offers suggestions, but doesn't enforce any dependencies or +project layout. It is up to the developer to choose the tools and +libraries they want to use. There are many extensions provided by the +community that make adding new functionality easy. + +%prep +%setup -n %{name}-%{unmangled_version} -n %{name}-%{unmangled_version} + +%build +python3 setup.py build + +%install +python3 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES + +%clean +rm -rf $RPM_BUILD_ROOT + +%files -f INSTALLED_FILES +%defattr(-,root,root) diff --git a/fts/aarch64-files/m2crypto.spec b/fts/aarch64-files/m2crypto.spec new file mode 100644 index 0000000..5fa6658 --- /dev/null +++ b/fts/aarch64-files/m2crypto.spec @@ -0,0 +1,42 @@ +%define name M2Crypto +%define version 0.35.2 +%define unmangled_version 0.35.2 +%define unmangled_version 0.35.2 +%define release 1 + +Summary: M2Crypto: A Python crypto and SSL toolkit +Name: %{name} +Version: %{version} +Release: %{release} +Provides: python36-m2crypto = %{version}-%{release} +Source0: %{name}-%{unmangled_version}.tar.gz +License: MIT +Group: Development/Libraries +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot +Prefix: %{_prefix} +Vendor: Matej Cepl +Url: https://gitlab.com/m2crypto/m2crypto + +%description +M2Crypto is the most complete Python wrapper for OpenSSL featuring RSA, DSA, +DH, EC, HMACs, message digests, symmetric ciphers (including AES); SSL +functionality to implement clients and servers; HTTPS extensions to Python's +httplib, urllib, and xmlrpclib; unforgeable HMAC'ing AuthCookies for web +session management; FTP/TLS client and server; S/MIME; M2Crypto can also be +used to provide SSL for Twisted. Smartcards supported through the Engine +interface. + +%prep +%setup -n %{name}-%{unmangled_version} -n %{name}-%{unmangled_version} + +%build +env CFLAGS="$RPM_OPT_FLAGS" python3 setup.py build + +%install +python3 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES + +%clean +rm -rf $RPM_BUILD_ROOT + +%files -f INSTALLED_FILES +%defattr(-,root,root) diff --git a/probes/Dockerfile b/probes/Dockerfile index f6809c6..85a080d 100644 --- a/probes/Dockerfile +++ b/probes/Dockerfile @@ -13,7 +13,6 @@ WORKDIR /tmp ADD oic.rpm /tmp RUN dnf install -y epel-release.noarch && \ - dnf upgrade -y && \ dnf install -y \ git \ gcc \ @@ -22,15 +21,43 @@ RUN dnf install -y epel-release.noarch && \ openssl-devel \ procps-ng \ python-devel \ - python-pip && \ + python-pip \ + make \ + go \ + tar \ + gzip \ + rsync \ + wget && \ dnf clean all && \ rm -rf /var/cache/dnf -RUN rpm -i https://download.oracle.com/otn_software/linux/instantclient/1912000/oracle-instantclient19.12-basiclite-19.12.0.0.0-1.x86_64.rpm; \ +RUN rpm -i https://download.oracle.com/otn_software/linux/instantclient/1923000/oracle-instantclient19.23-basic-19.23.0.0.0-1.aarch64.rpm; \ echo "/usr/lib/oracle/19/client64/lib" >/etc/ld.so.conf.d/oracle.conf; \ ldconfig -RUN rpm -i https://github.com/dshearer/jobber/releases/download/v1.4.0/jobber-1.4.0-1.el7.x86_64.rpm +# Set Go environment variables +ENV GOPATH=/root/go +ENV GOBIN=$GOPATH/bin + +# Create necessary directories +RUN mkdir -p $GOBIN && mkdir -p $GOPATH/src/github.com/dshearer + +# Install dep (Go dependency management tool) +RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh + +# Download and extract the jobber tarball +RUN wget https://github.com/dshearer/jobber/archive/refs/tags/v1.4.0.tar.gz && \ + tar -xzf v1.4.0.tar.gz && \ + mv jobber-1.4.0 $GOPATH/src/github.com/dshearer/jobber && \ + cd $GOPATH/src/github.com/dshearer/jobber && \ + go mod init github.com/dshearer/jobber && \ + go mod tidy && \ + go mod vendor && \ + make && \ + make install + +# Clean up +RUN rm -rf /tmp/* RUN python3 -m pip install --no-cache-dir --upgrade pip RUN python3 -m pip install --no-cache-dir --upgrade setuptools diff --git a/server/Dockerfile b/server/Dockerfile index c918e7e..ac9424e 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -28,6 +28,13 @@ RUN rpm -i https://download.oracle.com/otn_software/linux/instantclient/1912000/ echo "/usr/lib/oracle/19/client64/lib" >/etc/ld.so.conf.d/oracle.conf; \ ldconfig +# Install additional packages for arm64 architecture +RUN if [ "$(uname -m)" = "aarch64" ]; then \ + dnf install -y gcc make python3-devel; \ + dnf clean all; \ + rm -rf /var/cache/dnf; \ +fi + RUN python3 -m pip install --no-cache-dir --upgrade pip && \ python3 -m pip install --no-cache-dir --upgrade setuptools RUN python3 -m pip install --no-cache-dir --pre rucio[oracle,mysql,postgresql]==$TAG