remote-benchmarks.yml

# Copyright (c) 2023-2024 Deephaven Data Labs and Patent Pending

# Run benchmarks on a remote system for the release the project is using
# - This workflow is reusable with run_type set to release or nightly
# - Scripts ending in "-local" run on the github runner
# - Scripts ending in "-remote" run are executed on the remote test server through ssh

name: Benchmark Test on Docker Deephaven

on:
  workflow_call:
    inputs:
      run_type:
        required: true
        type: string
      docker_image:
       required: true
       type: string
      run_label:
        required: true
        type: string
      test_package:
        required: true
        type: string
      test_class_regex:
        required: true
        type: string
      test_iterations:
        required: true
        type: string
      scale_row_count:
        required: true
        type: string
      distribution:
        required: true
        type: string
      test_device_addr:
        required: true
        type: string
      config_options:
        required: true
        type: string
        
env:
  HOST: "${{ inputs.test_device_addr == '<default>' && secrets.BENCHMARK_HOST || inputs.test_device_addr }}"

concurrency:
  group: ${{ inputs.test_device_addr }}
  cancel-in-progress: ${{ inputs.run_type == 'nightly' }}

jobs:
  setup-benchmarks:
    runs-on: ubuntu-22.04
    outputs:
      matrix-iterations: ${{ steps.matrix-iterations.outputs.matrix-iterations }}
    env:
      SD: .github/scripts
      USER: ${{secrets.BENCHMARK_USER}}
      REPO: ${{github.repository}}
      BRANCH: ${{github.ref_name}}
      RUN_TYPE: ${{inputs.run_type}}
      DOCKER_IMG: ${{inputs.docker_image}}
      TEST_ITERS: ${{inputs.test_iterations}}
      CONFIG_OPTS: "${{inputs.config_options}}"

    steps:
    - uses: actions/checkout@v4
    - name: Set up JDK 21
      uses: actions/setup-java@v4
      with:
        java-version: '21'
        distribution: 'temurin'
        
    - name: Setup Local Scripts
      run: |
        sudo chmod +x ${SD}/*
        ${SD}/setup-ssh-local.sh ${HOST} "${{secrets.BENCHMARK_KEY}}"

    - name: Run Remote Test Server Setup
      run: |
        ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} setup-test-server-remote ${REPO} ${BRANCH} ${RUN_TYPE} "${DOCKER_IMG}"

    - name: Run Remote Server Distribution Build
      if: ${{ !contains(env.DOCKER_IMG, '@sha256:') && contains(env.DOCKER_IMG, ':') }}
      run: |
        ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-server-distribution-remote "${DOCKER_IMG}"
        
    - name: Run Remote Docker Image Build
      if: ${{ !contains(env.DOCKER_IMG, '@sha256:') && contains(env.DOCKER_IMG, ':') }}
      run: |
        ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-docker-image-remote

    - name: Start Remote Deephaven Server
      run: | 
        ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} manage-deephaven-remote start "${DOCKER_IMG}" "${CONFIG_OPTS}"

    - name: Run Remote Benchmark Artifact Build
      run: |
        ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} build-benchmark-artifact-remote
        
    - name: Set Iteration Matrix Output Variable
      id: matrix-iterations
      run: | 
        ${SD}/matrix_array.sh matrix-iterations ${RUN_TYPE} ${TEST_ITERS} >> "$GITHUB_OUTPUT"

  run-benchmarks:
    needs: setup-benchmarks
    runs-on: ubuntu-22.04
    strategy:
      max-parallel: 1
      matrix:
        tag: ${{ fromJson(needs.setup-benchmarks.outputs.matrix-iterations) }}
    env: 
      SD: .github/scripts
      USER: ${{secrets.BENCHMARK_USER}}
      RUN_TYPE: ${{inputs.run_type}}
      TEST_PKG: ${{inputs.test_package}}
      TEST_RGX: "${{inputs.test_class_regex}}"
      ROW_CNT: ${{inputs.scale_row_count}}
      DISTRIB: ${{inputs.distribution}}

    steps:
    - uses: actions/checkout@v4
    
    - name: Setup Local Scripts
      run: |
        sudo chmod +x ${SD}/*
        ${SD}/setup-ssh-local.sh ${HOST} "${{secrets.BENCHMARK_KEY}}"
    
    - name: Run Remote Benchmarks
      run: |
        ${SD}/run-ssh-local.sh ${HOST} ${USER} ${SD} run-benchmarks-remote ${RUN_TYPE} "${TEST_PKG}" "${TEST_RGX}" ${ROW_CNT} ${DISTRIB} ${{ matrix.tag }}

  report-benchmarks:
    needs: run-benchmarks
    runs-on: ubuntu-22.04
    env: 
      SD: .github/scripts
      USER: ${{secrets.BENCHMARK_USER}}
      RUN_TYPE: ${{inputs.run_type}}
      DOCKER_IMG: ${{inputs.docker_image}}
      RUN_LABEL: ${{inputs.run_label}}
      ACTOR: "${{ inputs.run_type == 'adhoc' && github.actor || github.repository_owner }}" 
    
    steps:
    - uses: actions/checkout@v4
    
    - name: Set up JDK 21
      uses: actions/setup-java@v4
      with:
        java-version: '21'
        distribution: 'temurin'
    
    - name: Setup Local Scripts
      run: |
        sudo chmod +x ${SD}/*
        ${SD}/setup-ssh-local.sh ${HOST} "${{secrets.BENCHMARK_KEY}}"
    
    - name: Fetch Benchmark Results and Prepare for Upload
      id: fetch-results
      run: |
        ${SD}/fetch-results-local.sh ${HOST} ${USER} ${SD} ${RUN_TYPE} "${ACTOR}" "${RUN_LABEL}" "${DOCKER_IMG}"
        cat fetch-results-local.out >> "$GITHUB_OUTPUT"

    - name: Authorize GCloud Credentials
      uses: google-github-actions/auth@v2
      with:
        credentials_json: ${{secrets.BENCHMARK_GCLOUD}}
        
    - name: Set up Cloud SDK
      uses: google-github-actions/setup-gcloud@v2

    - name: Upload Benchmark Results to GCloud
      uses: google-github-actions/upload-cloud-storage@v2
      with:
        path: ${{env.RUN_TYPE}}
        destination: deephaven-benchmark
        process_gcloudignore: false
        
    - name: Sync GCloud with Demo NFS
      if: ${{ env.RUN_TYPE != 'adhoc' }}
      run: |
        gcloud compute ssh --zone "us-central1-a" --project "deephaven-oss" dhc-demo-nfs-client --command="sudo gsutil -m rsync -d -r gs://deephaven-benchmark /nfs/deephaven-benchmark"
        gcloud compute ssh --zone "us-central1-a" --project "deephaven-oss" dhc-demo-nfs-client --command="sudo find /nfs/deephaven-benchmark -mindepth 1 -type d -empty -delete"
        
    - name: Publish Slack Notification Tables
      run: |
        ${SD}/run-publish-local.sh ${RUN_TYPE} "${{secrets.BENCHMARK_SLACK_CHANNEL}}" "${{secrets.BENCHMARK_SLACK_TOKEN}}" 

    - name: Archive Results
      uses: actions/upload-artifact@v4
      with:
        name: Benchmarks and Logs for Set ${{steps.fetch-results.outputs.SET_LABEL}}
        path: |
          ${{env.RUN_TYPE}}/**/*
          logs/**/*