Allow most pipeline scripts to be run locally

dmlc · Nov 20, 2024 · 2c2d47d · 2c2d47d
1 parent 9c43544
commit 2c2d47d
Show file tree

Hide file tree

Showing 22 changed files with 158 additions and 150 deletions.
diff --git a/.github/workflows/jvm_tests.yml b/.github/workflows/jvm_tests.yml
@@ -69,6 +69,12 @@ jobs:
         env:
           CONTAINER_ID: xgb-ci.manylinux2014_${{ matrix.arch }}
       - run: bash ops/pipeline/build-jvm-manylinux2014.sh ${{ matrix.arch }}
+      - name: Upload libxgboost4j.so
+        run: |
+          libname=lib/libxgboost4j_linux_${{ matrix.arch }}_${{ github.sha }}.so
+          mv -v lib/libxgboost4j.so ${libname}
+          bash ops/pipeline/publish-artifact.sh ${libname} \
+            s3://xgboost-nightly-builds/${{ env.BRANCH_NAME }}/libxgboost4j/
 
   build-jvm-gpu:
     name: Build libxgboost4j.so with CUDA
@@ -102,15 +108,22 @@ jobs:
         include:
           - description: "MacOS (Apple Silicon)"
             script: ops/pipeline/build-jvm-macos-apple-silicon.sh
+            libname: libxgboost4j_m1_${{ github.sha }}.dylib
             runner: macos-14
           - description: "MacOS (Intel)"
             script: ops/pipeline/build-jvm-macos-intel.sh
+            libname: libxgboost4j_intel_${{ github.sha }}.dylib
             runner: macos-13
     steps:
       - uses: actions/checkout@v4.2.2
         with:
           submodules: "true"
       - run: bash ${{ matrix.script }}
+      - name: Upload libxgboost4j.dylib
+        run: |
+          mv -v lib/libxgboost4j.dylib ${{ matrix.libname }}
+          bash ops/pipeline/publish-artifact.sh ${{ matrix.libname }} \
+            s3://xgboost-nightly-builds/${{ env.BRANCH_NAME }}/libxgboost4j/
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_IAM_S3_UPLOADER }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_IAM_S3_UPLOADER }}
@@ -137,6 +150,10 @@ jobs:
           COMMAND: download
           KEY: build-jvm-gpu
       - run: bash ops/pipeline/build-jvm-doc.sh
+      - name: Upload JVM doc
+        run: |
+          bash ops/pipeline/publish-artifact.sh jvm-packages/${{ env.BRANCH_NAME }}.tar.bz2 \
+            s3://xgboost-docs/
 
   build-test-jvm-packages:
     name: Build and test JVM packages (Linux)
@@ -279,6 +296,6 @@ jobs:
           COMMAND: download
           KEY: ${{ matrix.artifact_from }}
       - name: Deploy JVM packages to S3
-        run: >-
+        run: |
           bash ops/pipeline/deploy-jvm-packages.sh ${{ matrix.variant }} \
             ${{ matrix.container_id }}
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -93,6 +93,10 @@ jobs:
         env:
           COMMAND: upload
           KEY: build-cpu-arm64
+      - name: Upload Python wheel
+        run: |
+          bash ops/pipeline/publish-artifact.sh python-package/dist/*.whl \
+            s3://xgboost-nightly-builds/${{ env.BRANCH_NAME }}/
 
   build-cuda:
     name: Build CUDA + manylinux_2_28_x86_64 wheel
@@ -122,6 +126,13 @@ jobs:
         env:
           COMMAND: upload
           KEY: build-cuda
+      - name: Upload Python wheel
+        run: |
+          for file in python-package/dist/*.whl python-package/dist/meta.json
+          do
+            bash ops/pipeline/publish-artifact.sh "${file}" \
+              s3://xgboost-nightly-builds/${{ env.BRANCH_NAME }}/
+          done
 
   build-cuda-with-rmm:
     name: Build CUDA with RMM
@@ -149,6 +160,10 @@ jobs:
         env:
           COMMAND: upload
           KEY: build-cuda-with-rmm
+      - name: Upload Python wheel
+        run: |
+          bash ops/pipeline/publish-artifact.sh python-package/dist/*.whl \
+            s3://xgboost-nightly-builds/experimental_build_with_rmm/
 
   build-manylinux2014:
     name: Build manylinux2014_${{ matrix.arch }} wheel
@@ -175,6 +190,13 @@ jobs:
         env:
           CONTAINER_ID: xgb-ci.manylinux2014_${{ matrix.arch }}
       - run: bash ops/pipeline/build-manylinux2014.sh ${{ matrix.arch }}
+      - name: Upload Python wheel
+        run: |
+          for wheel in python-package/dist/*.whl
+          do
+            bash ops/pipeline/publish-artifact.sh "${wheel}" \
+              s3://xgboost-nightly-builds/${{ env.BRANCH_NAME }}/
+          done
 
   build-gpu-rpkg:
     name: Build GPU-enabled R package
@@ -193,6 +215,11 @@ jobs:
         env:
           CONTAINER_ID: xgb-ci.gpu_build_r_rockylinux8
       - run: bash ops/pipeline/build-gpu-rpkg.sh
+      - name: Upload R tarball
+        run: |
+          bash ops/pipeline/publish-artifact.sh xgboost_r_gpu_linux_*.tar.gz \
+            s3://xgboost-nightly-builds/${{ env.BRANCH_NAME }}/
+
 
   test-cpp-gpu:
     name: >-

diff --git a/ops/pipeline/build-cpu-arm64.sh b/ops/pipeline/build-cpu-arm64.sh
@@ -2,12 +2,16 @@
 
 set -euox pipefail
 
+if [[ -z "${GITHUB_SHA:-}" ]]
+then
+  echo "Make sure to set environment variable GITHUB_SHA"
+  exit 1
+fi
+
 WHEEL_TAG=manylinux_2_28_aarch64
 
 echo "--- Build CPU code targeting ARM64"
 
-source ops/pipeline/enforce-ci.sh
-
 echo "--- Build libxgboost from the source"
 python3 ops/docker_run.py \
   --container-id xgb-ci.aarch64 \
@@ -46,10 +50,3 @@ python3 ops/docker_run.py \
   --container-id xgb-ci.aarch64 \
   -- bash -c \
   "unzip -l python-package/dist/*.whl | grep libgomp  || exit -1"
-
-echo "--- Upload Python wheel"
-if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
-then
-  aws s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/${BRANCH_NAME}/ \
-    --acl public-read --no-progress
-fi
diff --git a/ops/pipeline/build-cpu.sh b/ops/pipeline/build-cpu.sh
@@ -2,8 +2,6 @@
 
 set -euox pipefail
 
-source ops/pipeline/enforce-ci.sh
-
 echo "--- Build CPU code"
 
 # This step is not necessary, but here we include it, to ensure that

diff --git a/ops/pipeline/build-cuda-with-rmm.sh b/ops/pipeline/build-cuda-with-rmm.sh
@@ -2,9 +2,15 @@
 
 set -euox pipefail
 
-WHEEL_TAG=manylinux_2_28_x86_64
+if [[ -z "${GITHUB_SHA:-}" ]]
+then
+  echo "Make sure to set environment variable GITHUB_SHA"
+  exit 1
+fi
+
+source ops/pipeline/classify-git-branch.sh
 
-source ops/pipeline/enforce-ci.sh
+WHEEL_TAG=manylinux_2_28_x86_64
 
 echo "--- Build with CUDA with RMM"
 
@@ -43,7 +49,7 @@ python3 ops/script/rename_whl.py  \
 
 echo "--- Audit binary wheel to ensure it's compliant with ${WHEEL_TAG} standard"
 python3 ops/docker_run.py \
-  --container-id xgb-ci.$WHEEL_TAG \
+  --container-id xgb-ci.${WHEEL_TAG} \
   -- auditwheel repair \
   --plat ${WHEEL_TAG} python-package/dist/*.whl
 python3 ops/script/rename_whl.py  \
@@ -53,13 +59,6 @@ python3 ops/script/rename_whl.py  \
 mv -v wheelhouse/*.whl python-package/dist/
 # Make sure that libgomp.so is vendored in the wheel
 python3 ops/docker_run.py \
-  --container-id xgb-ci.$WHEEL_TAG \
+  --container-id xgb-ci.${WHEEL_TAG} \
   -- bash -c \
   "unzip -l python-package/dist/*.whl | grep libgomp  || exit -1"
-
-echo "--- Upload Python wheel"
-if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
-then
-  aws s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/experimental_build_with_rmm/ \
-    --acl public-read --no-progress
-fi
diff --git a/ops/pipeline/build-cuda.sh b/ops/pipeline/build-cuda.sh
@@ -2,9 +2,15 @@
 
 set -euox pipefail
 
+if [[ -z "${GITHUB_SHA:-}" ]]
+then
+  echo "Make sure to set environment variable GITHUB_SHA"
+  exit 1
+fi
+
 WHEEL_TAG=manylinux_2_28_x86_64
 
-source ops/pipeline/enforce-ci.sh
+source ops/pipeline/classify-git-branch.sh
 
 echo "--- Build with CUDA"
 
@@ -59,21 +65,20 @@ python3 ops/docker_run.py \
   --container-id xgb-ci.manylinux_2_28_x86_64 \
   -- bash -c "unzip -l python-package/dist/*.whl | grep libgomp  || exit -1"
 
+# Generate the meta info which includes xgboost version and the commit info
+python3 ops/docker_run.py \
+--container-id xgb-ci.gpu_build_rockylinux8 \
+-- python ops/script/format_wheel_meta.py \
+  --wheel-path python-package/dist/*.whl  \
+  --commit-hash ${GITHUB_SHA}  \
+  --platform-tag ${WHEEL_TAG}  \
+  --meta-path python-package/dist/
+
 echo "--- Upload Python wheel"
 if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
 then
   aws s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/${BRANCH_NAME}/ \
     --acl public-read --no-progress
-
-  # Generate the meta info which includes xgboost version and the commit info
-  python3 ops/docker_run.py \
-  --container-id xgb-ci.gpu_build_rockylinux8 \
-  -- python ops/script/format_wheel_meta.py \
-    --wheel-path python-package/dist/*.whl  \
-    --commit-hash ${GITHUB_SHA}  \
-    --platform-tag ${WHEEL_TAG}  \
-    --meta-path python-package/dist/
   aws s3 cp python-package/dist/meta.json s3://xgboost-nightly-builds/${BRANCH_NAME}/ \
     --acl public-read --no-progress
 fi
-echo "-- Stash C++ test executable (testxgboost)"
diff --git a/ops/pipeline/build-gpu-rpkg.sh b/ops/pipeline/build-gpu-rpkg.sh
@@ -2,17 +2,14 @@
 
 set -euox pipefail
 
-source ops/pipeline/enforce-ci.sh
+if [[ -z "${GITHUB_SHA:-}" ]]
+then
+  echo "Make sure to set environment variable GITHUB_SHA"
+  exit 1
+fi
 
 echo "--- Build XGBoost R package with CUDA"
 python3 ops/docker_run.py \
   --container-id xgb-ci.gpu_build_r_rockylinux8 \
   -- ops/pipeline/build-gpu-rpkg-impl.sh \
   ${GITHUB_SHA}
-
-if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
-then
-  echo "--- Upload R tarball"
-  aws s3 cp xgboost_r_gpu_linux_*.tar.gz s3://xgboost-nightly-builds/${BRANCH_NAME}/ \
-    --acl public-read --no-progress
-fi
diff --git a/ops/pipeline/build-jvm-doc.sh b/ops/pipeline/build-jvm-doc.sh
@@ -1,19 +1,24 @@
 #!/bin/bash
 ## Build docs for the JVM packages and package it in a tarball
-## Note: Note: this script assumes that the user has already built libxgboost4j.so
+## Note: this script assumes that the user has already built libxgboost4j.so
 ## and place it in the lib/ directory.
 
 set -euox pipefail
 
-source ops/pipeline/enforce-ci.sh
-
 echo "--- Build JVM packages doc"
+
+if [[ -z ${BRANCH_NAME:-} ]]
+then
+  echo "Make sure to define environment variable BRANCH_NAME."
+  exit 1
+fi
+
+if [[ ! -f lib/libxgboost4j.so ]]
+then
+  echo "Must place libxgboost4j.so in lib/ first"
+  exit 2
+fi
+
 python3 ops/docker_run.py \
   --container-id xgb-ci.jvm_gpu_build \
   -- ops/pipeline/build-jvm-doc-impl.sh ${BRANCH_NAME}
-if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
-then
-  echo "--- Upload JVM packages doc"
-  aws s3 cp jvm-packages/${BRANCH_NAME}.tar.bz2 \
-    s3://xgboost-docs/${BRANCH_NAME}.tar.bz2 --acl public-read --no-progress
-fi
diff --git a/ops/pipeline/build-jvm-gpu.sh b/ops/pipeline/build-jvm-gpu.sh
@@ -3,7 +3,7 @@
 
 set -euo pipefail
 
-source ops/pipeline/enforce-ci.sh
+source ops/pipeline/classify-git-branch.sh
 
 echo "--- Build libxgboost4j.so with CUDA"
 

diff --git a/ops/pipeline/build-jvm-macos-apple-silicon.sh b/ops/pipeline/build-jvm-macos-apple-silicon.sh
@@ -3,8 +3,6 @@
 
 set -euox pipefail
 
-source ops/pipeline/enforce-ci.sh
-
 # Display system info
 echo "--- Display system information"
 set -x
@@ -27,18 +25,3 @@ popd
 rm -rf build
 otool -L lib/libxgboost.dylib
 set +x
-
-echo "--- Upload libxgboost4j.dylib"
-set -x
-pushd lib
-libname=libxgboost4j_m1_${GITHUB_SHA}.dylib
-mv -v libxgboost4j.dylib ${libname}
-
-if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
-then
-  aws s3 cp ${libname} \
-    s3://xgboost-nightly-builds/${BRANCH_NAME}/libxgboost4j/ \
-    --acl public-read --no-progress
-fi
-popd
-set +x
diff --git a/ops/pipeline/build-jvm-macos-intel.sh b/ops/pipeline/build-jvm-macos-intel.sh
@@ -3,8 +3,6 @@
 
 set -euox pipefail
 
-source ops/pipeline/enforce-ci.sh
-
 # Display system info
 echo "--- Display system information"
 set -x
@@ -26,19 +24,3 @@ ninja -v
 popd
 rm -rf build
 otool -L lib/libxgboost.dylib
-set +x
-
-echo "--- Upload libxgboost4j.dylib"
-set -x
-pushd lib
-libname=libxgboost4j_intel_${GITHUB_SHA}.dylib
-mv -v libxgboost4j.dylib ${libname}
-
-if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
-then
-  aws s3 cp ${libname} \
-    s3://xgboost-nightly-builds/${BRANCH_NAME}/libxgboost4j/ \
-    --acl public-read --no-progress
-fi
-popd
-set +x
diff --git a/ops/pipeline/build-jvm-manylinux2014.sh b/ops/pipeline/build-jvm-manylinux2014.sh
@@ -3,9 +3,8 @@
 
 set -euox pipefail
 
-source ops/pipeline/enforce-ci.sh
-
-if [ $# -ne 1 ]; then
+if [[ $# -ne 1 ]]
+then
   echo "Usage: $0 {x86_64,aarch64}"
   exit 1
 fi
@@ -24,15 +23,3 @@ python3 ops/docker_run.py \
   "cd build && cmake .. -DJVM_BINDINGS=ON -DUSE_OPENMP=ON && make -j$(nproc)"
 ldd lib/libxgboost4j.so
 objdump -T lib/libxgboost4j.so | grep GLIBC_ | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -Vu
-
-echo "--- Upload libxgboost4j.so"
-pushd lib
-libname=libxgboost4j_linux_${arch}_${GITHUB_SHA}.so
-mv -v libxgboost4j.so ${libname}
-if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
-then
-  aws s3 cp ${libname} \
-    s3://xgboost-nightly-builds/${BRANCH_NAME}/libxgboost4j/ \
-    --acl public-read --no-progress
-fi
-popd