Next-generation CI/CD pipelines with RunsOn #13474
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: XGBoost CI | |
| on: [push, pull_request] | |
| permissions: | |
| contents: read # to fetch code (actions/checkout) | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| BRANCH_NAME: >- | |
| ${{ github.event.pull_request.number && 'PR-' }}${{ github.event.pull_request.number || github.ref_name }} | |
| USE_DOCKER_CACHE: 1 | |
| jobs: | |
| build-containers: | |
| name: Build CI containers (${{ matrix.container_id }}) | |
| runs-on: | |
| - runs-on | |
| - runner=${{ matrix.runner }} | |
| - run-id=${{ github.run_id }} | |
| - tag=main-build-containers-${{ matrix.container_id }} | |
| strategy: | |
| max-parallel: 2 | |
| matrix: | |
| container_id: | |
| - xgb-ci.gpu_build_rockylinux8 | |
| - xgb-ci.gpu_build_r_rockylinux8 | |
| - xgb-ci.gpu | |
| - xgb-ci.gpu_dev_ver | |
| - xgb-ci.cpu | |
| - xgb-ci.manylinux_2_28_x86_64 | |
| - xgb-ci.manylinux2014_x86_64 | |
| runner: [linux-amd64-cpu] | |
| include: | |
| - container_id: xgb-ci.manylinux2014_aarch64 | |
| runner: linux-arm64-cpu | |
| - container_id: xgb-ci.aarch64 | |
| runner: linux-arm64-cpu | |
| steps: | |
| # Restart Docker daemon so that it recognizes the ephemeral disks | |
| - run: sudo systemctl restart docker | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: "true" | |
| - name: Build ${{ matrix.container_id }} | |
| run: bash ops/docker_build.sh | |
| env: | |
| CONTAINER_ID: ${{ matrix.container_id }} | |
| build-cpu: | |
| name: Build CPU | |
| needs: build-containers | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - runner=linux-amd64-cpu | |
| steps: | |
| # Restart Docker daemon so that it recognizes the ephemeral disks | |
| - run: sudo systemctl restart docker | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: "true" | |
| - name: Fetch container from cache | |
| run: bash ops/docker_build.sh | |
| env: | |
| CONTAINER_ID: xgb-ci.cpu | |
| - run: bash ops/pipeline/build-cpu.sh | |
| - name: Stash CLI executable | |
| run: bash ops/pipeline/stash-artifacts.sh ./xgboost | |
| env: | |
| COMMAND: upload | |
| KEY: build-cpu | |
| build-cpu-arm64: | |
| name: Build CPU ARM64 + manylinux_2_28_aarch64 wheel | |
| needs: build-containers | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - runner=linux-arm64-cpu | |
| steps: | |
| # Restart Docker daemon so that it recognizes the ephemeral disks | |
| - run: sudo systemctl restart docker | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: "true" | |
| - name: Fetch container from cache | |
| run: bash ops/docker_build.sh | |
| env: | |
| CONTAINER_ID: xgb-ci.aarch64 | |
| - run: bash ops/pipeline/build-cpu-arm64.sh | |
| - name: Stash files | |
| run: bash ops/pipeline/stash-artifacts.sh ./xgboost python-package/dist/*.whl | |
| env: | |
| COMMAND: upload | |
| KEY: build-cpu-arm64 | |
| - name: Upload Python wheel | |
| run: | | |
| bash ops/pipeline/publish-artifact.sh python-package/dist/*.whl \ | |
| s3://xgboost-nightly-builds/${{ env.BRANCH_NAME }}/ | |
| build-cuda: | |
| name: Build CUDA + manylinux_2_28_x86_64 wheel | |
| needs: build-containers | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - runner=linux-amd64-cpu | |
| steps: | |
| # Restart Docker daemon so that it recognizes the ephemeral disks | |
| - run: sudo systemctl restart docker | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: "true" | |
| - name: Fetch container from cache | |
| run: bash ops/docker_build.sh | |
| env: | |
| CONTAINER_ID: xgb-ci.gpu_build_rockylinux8 | |
| - name: Fetch container from cache | |
| run: bash ops/docker_build.sh | |
| env: | |
| CONTAINER_ID: xgb-ci.manylinux_2_28_x86_64 | |
| - run: bash ops/pipeline/build-cuda.sh | |
| - name: Stash files | |
| run: | | |
| bash ops/pipeline/stash-artifacts.sh \ | |
| build/testxgboost ./xgboost python-package/dist/*.whl | |
| env: | |
| COMMAND: upload | |
| KEY: build-cuda | |
| - name: Upload Python wheel | |
| run: | | |
| for file in python-package/dist/*.whl python-package/dist/meta.json | |
| do | |
| bash ops/pipeline/publish-artifact.sh "${file}" \ | |
| s3://xgboost-nightly-builds/${{ env.BRANCH_NAME }}/ | |
| done | |
| build-cuda-with-rmm: | |
| name: Build CUDA with RMM | |
| needs: build-containers | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - runner=linux-amd64-cpu | |
| steps: | |
| # Restart Docker daemon so that it recognizes the ephemeral disks | |
| - run: sudo systemctl restart docker | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: "true" | |
| - name: Fetch container from cache | |
| run: bash ops/docker_build.sh | |
| env: | |
| CONTAINER_ID: xgb-ci.gpu_build_rockylinux8 | |
| - name: Fetch container from cache | |
| run: bash ops/docker_build.sh | |
| env: | |
| CONTAINER_ID: xgb-ci.manylinux_2_28_x86_64 | |
| - run: bash ops/pipeline/build-cuda-with-rmm.sh | |
| - name: Stash files | |
| run: bash ops/pipeline/stash-artifacts.sh build/testxgboost | |
| env: | |
| COMMAND: upload | |
| KEY: build-cuda-with-rmm | |
| - name: Upload Python wheel | |
| run: | | |
| bash ops/pipeline/publish-artifact.sh python-package/dist/*.whl \ | |
| s3://xgboost-nightly-builds/experimental_build_with_rmm/ | |
| build-manylinux2014: | |
| name: Build manylinux2014_${{ matrix.arch }} wheel | |
| needs: build-containers | |
| runs-on: | |
| - runs-on | |
| - runner=${{ matrix.runner }} | |
| - run-id=${{ github.run_id }} | |
| - tag=main-build-manylinux2014-${{ matrix.arch }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - arch: aarch64 | |
| runner: linux-arm64-cpu | |
| - arch: x86_64 | |
| runner: linux-amd64-cpu | |
| steps: | |
| # Restart Docker daemon so that it recognizes the ephemeral disks | |
| - run: sudo systemctl restart docker | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: "true" | |
| - name: Fetch container from cache | |
| run: bash ops/docker_build.sh | |
| env: | |
| CONTAINER_ID: xgb-ci.manylinux2014_${{ matrix.arch }} | |
| - run: bash ops/pipeline/build-manylinux2014.sh ${{ matrix.arch }} | |
| - name: Upload Python wheel | |
| run: | | |
| for wheel in python-package/dist/*.whl | |
| do | |
| bash ops/pipeline/publish-artifact.sh "${wheel}" \ | |
| s3://xgboost-nightly-builds/${{ env.BRANCH_NAME }}/ | |
| done | |
| build-gpu-rpkg: | |
| name: Build GPU-enabled R package | |
| needs: build-containers | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - runner=linux-amd64-cpu | |
| steps: | |
| # Restart Docker daemon so that it recognizes the ephemeral disks | |
| - run: sudo systemctl restart docker | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: "true" | |
| - name: Fetch container from cache | |
| run: bash ops/docker_build.sh | |
| env: | |
| CONTAINER_ID: xgb-ci.gpu_build_r_rockylinux8 | |
| - run: bash ops/pipeline/build-gpu-rpkg.sh | |
| - name: Upload R tarball | |
| run: | | |
| bash ops/pipeline/publish-artifact.sh xgboost_r_gpu_linux_*.tar.gz \ | |
| s3://xgboost-nightly-builds/${{ env.BRANCH_NAME }}/ | |
| test-cpp-gpu: | |
| name: >- | |
| Run Google Tests with GPUs | |
| (Suite ${{ matrix.suite }}, Runner ${{ matrix.runner }}) | |
| needs: [build-cuda, build-cuda-with-rmm] | |
| runs-on: | |
| - runs-on | |
| - runner=${{ matrix.runner }} | |
| - run-id=${{ github.run_id }} | |
| - tag=main-test-cpp-gpu-${{ matrix.suite }} | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 2 | |
| matrix: | |
| include: | |
| - suite: gpu | |
| runner: linux-amd64-gpu | |
| artifact_from: build-cuda | |
| - suite: gpu-rmm | |
| runner: linux-amd64-gpu | |
| artifact_from: build-cuda-with-rmm | |
| - suite: mgpu | |
| runner: linux-amd64-mgpu | |
| artifact_from: build-cuda | |
| steps: | |
| # Restart Docker daemon so that it recognizes the ephemeral disks | |
| - run: sudo systemctl restart docker | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: "true" | |
| - name: Fetch container from cache | |
| run: bash ops/docker_build.sh | |
| env: | |
| CONTAINER_ID: xgb-ci.gpu | |
| - name: Unstash gtest | |
| run: | | |
| bash ops/pipeline/stash-artifacts.sh build/testxgboost | |
| chmod +x build/testxgboost | |
| env: | |
| COMMAND: download | |
| KEY: ${{ matrix.artifact_from }} | |
| - run: bash ops/pipeline/test-cpp-gpu.sh ${{ matrix.suite }} | |
| test-python-wheel: | |
| name: Run Python tests (${{ matrix.description }}) | |
| needs: [build-cuda, build-cpu-arm64] | |
| runs-on: | |
| - runs-on | |
| - runner=${{ matrix.runner }} | |
| - run-id=${{ github.run_id }} | |
| - tag=main-test-python-wheel-${{ matrix.description }} | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 2 | |
| matrix: | |
| include: | |
| - description: single-gpu | |
| container: xgb-ci.gpu | |
| suite: gpu | |
| runner: linux-amd64-gpu | |
| artifact_from: build-cuda | |
| - description: single-gpu-nightly-deps | |
| container: xgb-ci.gpu_dev_ver | |
| suite: gpu | |
| runner: linux-amd64-gpu | |
| artifact_from: build-cuda | |
| - description: multiple-gpu | |
| container: xgb-ci.gpu | |
| suite: mgpu | |
| runner: linux-amd64-mgpu | |
| artifact_from: build-cuda | |
| - description: multiple-gpu-nightly-deps | |
| container: xgb-ci.gpu_dev_ver | |
| suite: mgpu | |
| runner: linux-amd64-mgpu | |
| artifact_from: build-cuda | |
| - description: cpu-amd64 | |
| container: xgb-ci.cpu | |
| suite: cpu | |
| runner: linux-amd64-cpu | |
| artifact_from: build-cuda | |
| - description: cpu-arm64 | |
| container: xgb-ci.aarch64 | |
| suite: cpu-arm64 | |
| runner: linux-arm64-cpu | |
| artifact_from: build-cpu-arm64 | |
| steps: | |
| # Restart Docker daemon so that it recognizes the ephemeral disks | |
| - run: sudo systemctl restart docker | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: "true" | |
| - name: Fetch container from cache | |
| run: bash ops/docker_build.sh | |
| env: | |
| CONTAINER_ID: ${{ matrix.container }} | |
| - name: Unstash Python wheel | |
| run: | | |
| bash ops/pipeline/stash-artifacts.sh python-package/dist/*.whl ./xgboost | |
| chmod +x ./xgboost | |
| env: | |
| COMMAND: download | |
| KEY: ${{ matrix.artifact_from }} | |
| - name: Run Python tests, ${{ matrix.description }} | |
| run: bash ops/pipeline/test-python-wheel.sh ${{ matrix.suite }} ${{ matrix.container }} |