Design Changes for Dir Struct, Tagged Iterations, Metrics (#317)

.github/resources/nightly-scale-benchmark.properties

@@ -18,7 +18,7 @@ kafka.consumer.addr=redpanda:29092
 default.completion.timeout=10 minutes
 
 # Default data distribution for column data (random, ascending, descending, runlength)
-default.data.distribution=random
+default.data.distribution=${baseDistrib}
 
 # Slows down record generation (Used for experiments not full test runs)
 generator.pause.per.row=0 millis
@@ -27,7 +27,7 @@ generator.pause.per.row=0 millis
 record.compression=LZ4
 
 # Row count to scale tests (Tests can override but typically do not)
-scale.row.count=10000000
+scale.row.count=${baseRowCount}
 
 # True: Use a timestamp for the parent directory of each test run
 # False: Overwrite previous test results for each test run

.github/resources/release-scale-benchmark.properties

@@ -18,7 +18,7 @@ kafka.consumer.addr=redpanda:29092
 default.completion.timeout=10 minutes
 
 # Default data distribution for column data (random, ascending, descending, runlength)
-default.data.distribution=random
+default.data.distribution=${baseDistrib}
 
 # Slows down record generation (Used for experiments not full test runs)
 generator.pause.per.row=0 millis
@@ -27,7 +27,7 @@ generator.pause.per.row=0 millis
 record.compression=LZO
 
 # Row count to scale tests (Tests can override but typically do not)
-scale.row.count=10000000
+scale.row.count=${baseRowCount}
 
 # True: Use a timestamp for the parent directory of each test run
 # False: Overwrite previous test results for each test run

.github/scripts/base62.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+
+# Convert the given Base 10 number to Base 62 characters
+# ex. base62.sh 1718738365297350992 
+# ex. ./base62.sh $(date +%s%03N) 
+
+DECNUM=$1
+BASE62=($(echo {0..9} {A..Z} {a..z}))
+
+for i in $(bc <<< "obase=62; ${DECNUM}"); do
+  echo -n ${BASE62[$(( 10#$i ))]}
+done && echo
+

.github/scripts/build-server-distribution-remote.sh

@@ -7,6 +7,11 @@ set -o pipefail
 # The supplied argument can be an image name or <owner>::<branch>
 # Ensure that the artifacts and Deephaven version are available in standard directories
 
+if [[ $# != 1 ]]; then
+  echo "$0: Missing docker image/branch argument"
+  exit 1
+fi
+
 HOST=`hostname`
 GIT_DIR=/root/git
 DEEPHAVEN_DIR=/root/deephaven
@@ -19,11 +24,6 @@ if [ ! -d "${DEEPHAVEN_DIR}" ]; then
   exit 1
 fi
 
-if [[ $# != 1 ]]; then
-  echo "$0: Missing docker image/branch argument"
-  exit 1
-fi
-
 title () { echo; echo $1; }
 
 OWNER=$(sed 's/'"${BRANCH_DELIM}"'.*//g' <<< "${DOCKER_IMG}")

.github/scripts/fetch-results-local.sh

@@ -7,34 +7,42 @@ set -o nounset
 # Fetches Benchmark results and logs from the remote test server and
 # compresses the runs before upload
 
+if [[ $# != 7 ]]; then
+  echo "$0: Missing host, user, run type, script dir, actor, docker img, or run label arguments"
+  exit 1
+fi
+
 HOST=$1
 USER=$2
-RUN_TYPE=$3
-ACTOR=${4:-}
-RUN_LABEL=${5:-}
+SCRIPT_DIR=$3
+RUN_TYPE=$4
+ACTOR=$5
+RUN_LABEL=${6:-$(echo -n "set-"; ${SCRIPT_DIR}/base62.sh $(date +%s%03N))}
+DOCKER_IMG=$7
 RUN_DIR=/root/run
 
-if [[ $# != 3 ]] && [[ $# != 5 ]]; then
-    echo "$0: Missing host, user, run type, actor, or run label arguments"
-    exit 1
+# Get the date for the Set Label, since Github Workflows don't have 'with: ${{github.date}}'
+if [ "${RUN_LABEL}" = "<date>" ]; then
+  RUN_LABEL=$(date '+%Y-%m-%d')
+fi
+
+# Get the version for the Set Label, since Github Workflows don't have 'with: ${{github.date}}'
+if [ "${RUN_LABEL}" = "<version>" ]; then
+  vers=${DOCKER_IMG}
+  major=$(printf '%02d\n' $(echo ${vers} | cut -d "." -f 1))
+  minor=$(printf '%03d\n' $(echo ${vers} | cut -d "." -f 2))
+  patch=$(printf '%02d\n' $(echo ${vers} | cut -d "." -f 3))
+  RUN_LABEL="${major}.${minor}.${patch}"
 fi
 
 # Pull results from the benchmark server
 scp -r ${USER}@${HOST}:${RUN_DIR}/results .
 scp -r ${USER}@${HOST}:${RUN_DIR}/logs .
 scp -r ${USER}@${HOST}:${RUN_DIR}/*.jar .
 
-# If the RUN_TYPE is adhoc, userfy the destination directory
-DEST_DIR=${RUN_TYPE}
-if [ "${RUN_TYPE}" = "adhoc" ]; then
-    if [ -z "${ACTOR}" ] || [ -z "${RUN_LABEL}" ]; then
-        echo "$0: Missing actor of run label argument for adhoc run type"
-        exit 1
-    fi
-    DEST_DIR=${RUN_TYPE}/${ACTOR}/${RUN_LABEL}
-    mkdir -p ${DEST_DIR}
-fi
-
+# Move the results into the destination directory
+DEST_DIR=${RUN_TYPE}/${ACTOR}/${RUN_LABEL}
+mkdir -p ${DEST_DIR}
 rm -rf ${DEST_DIR}
 mv results/ ${DEST_DIR}/
 
@@ -46,13 +54,14 @@ TMP_SVG_DIR=${DEST_DIR}/tmp-svg
 mkdir -p ${TMP_SVG_DIR}
 mv ${DEST_DIR}/*.svg ${TMP_SVG_DIR}
 mv ${TMP_SVG_DIR}/${RUN_TYPE}-benchmark-summary.svg ${DEST_DIR}/benchmark-summary.svg
+cp ${DEST_DIR}/benchmark-summary.svg ${DEST_DIR}/../
 rm -rf ${TMP_SVG_DIR}
 
 # Compress CSV and Test Logs
 for runId in `find ${DEST_DIR}/ -name "run-*"`
 do
-    (cd ${runId}; gzip *.csv)
-    (cd ${runId}/test-logs; tar -zcvf test-logs.tgz *; mv test-logs.tgz ../)
-    rm -rf ${runId}/test-logs/
+  (cd ${runId}; gzip *.csv)
+  (cd ${runId}/test-logs; tar -zcvf test-logs.tgz *; mv test-logs.tgz ../)
+  rm -rf ${runId}/test-logs/
 done
 

.github/scripts/manage-deephaven-remote.sh

@@ -7,6 +7,11 @@ set -o pipefail
 # The directives argument can be start or stop
 # The supplied image argument can be an image name or <owner>::<branch>
 
+if [[ $# != 2 ]]; then
+  echo "$0: Missing docker directive or image/branch argument"
+  exit 1
+fi
+
 HOST=`hostname`
 DEEPHAVEN_DIR=/root/deephaven
 DIRECTIVE=$1
@@ -18,11 +23,6 @@ if [ ! -d "${DEEPHAVEN_DIR}" ]; then
   exit 1
 fi
 
-if [[ $# != 2 ]]; then
-  echo "$0: Missing docker directive or image/branch argument"
-  exit 1
-fi
-
 title () { echo; echo $1; }
 
 title "- Setting up Remote Docker Image on ${HOST} -"

.github/scripts/run-benchmarks-remote.sh

@@ -18,7 +18,8 @@ TEST_PATTERN="$3"
 ROW_COUNT=$4
 DISTRIB=$5
 ITERATIONS=$6
-HOST=`hostname`
+TAG_ITERS=4
+HOST=$(hostname)
 RUN_DIR=/root/run
 DEEPHAVEN_DIR=/root/deephaven
 
@@ -43,11 +44,16 @@ title "-- Running Benchmarks --"
 cd ${RUN_DIR}
 cat ${RUN_TYPE}-scale-benchmark.properties | sed 's|${baseRowCount}|'"${ROW_COUNT}|g" | sed 's|${baseDistrib}|'"${DISTRIB}|g" > scale-benchmark.properties
 
-for i in `seq 1 ${ITERATIONS}`
-do
+for i in `seq 1 ${ITERATIONS}`; do
   java -Dbenchmark.profile=scale-benchmark.properties -jar deephaven-benchmark-*.jar -cp standard-tests.jar -p ${TEST_PACKAGE} -n "${TEST_PATTERN}"
 done
 
+if [ "${RUN_TYPE}" = "nightly" ] || [ "${RUN_TYPE}" = "release" ]; then
+  for i in `seq 1 ${TAG_ITERS}`; do
+    java -Dbenchmark.profile=scale-benchmark.properties -jar deephaven-benchmark-*.jar -cp standard-tests.jar -p ${TEST_PACKAGE} -t "Iterate"
+  done
+fi
+
 title "-- Getting Docker Logs --"
 mkdir -p ${RUN_DIR}/logs
 cd ${DEEPHAVEN_DIR};

.github/scripts/run-publish-local.sh

@@ -6,6 +6,7 @@ set -o nounset
 
 # Run queries that publish a secret slack channel. Queries operation exclusively
 # the deephaven-benchmark GCloud bucket
+
 if [[ $# != 3 ]]; then
   echo "$0: Missing run type or slack-channel or slack-uri arguments"
   exit 1

.github/scripts/run-ssh-local.sh

@@ -8,16 +8,16 @@ set -o nounset
 # local working directory. Also, this script wraps arguments provided for the
 # remote scripts in single-quotes to avoid syntax errors.
 
-HOST=$1
-USER=$2
-SCRIPT_DIR=$3
-SCRIPT_NAME=$4
-
 if [[ $# -lt 4 ]]; then
 	echo "$0: Missing host, user, script dir, or script name argument"
 	exit 1
 fi
 
+HOST=$1
+USER=$2
+SCRIPT_DIR=$3
+SCRIPT_NAME=$4
+
 args=()
 for i in ${@:5}; do
   args+=("'"$i"'")

.github/workflows/adhoc-remote-benchmarks.yml

@@ -16,9 +16,10 @@ on:
        default: 'edge'
        type: string
      run_label:
-       description: 'Run Label'
+       description: 'Set Label'
        required: true
        type: string
+       default: ''
      test_package:
        description: 'Benchmark Test Package'
        required: true

.github/workflows/compare-remote-benchmarks.yml

@@ -20,7 +20,7 @@ jobs:
     with:
       run_type: compare
       docker_image: ${{ inputs.docker_image }}
-      run_label: ""
+      run_label: "<version>"
       test_package: "io.deephaven.benchmark.tests.compare"
       test_class_regex: "^(Test.*|.+[.$]Test.*|.*Tests?)$"
       test_iterations: 1

.github/workflows/nightly-remote-benchmarks.yml

@@ -14,7 +14,7 @@ jobs:
     with:
       run_type: nightly
       docker_image: edge
-      run_label: ""
+      run_label: "<date>"
       test_package: "io.deephaven.benchmark.tests.standard"
       test_class_regex: "^(Test.*|.+[.$]Test.*|.*Tests?)$"
       test_iterations: 1

.github/workflows/release-remote-benchmarks.yml

@@ -10,7 +10,7 @@ on:
      docker_image:
        description: 'Docker Image Name'
        required: true
-       default: '0.32.0'
+       default: '0.35.0'
        type: string
 
 jobs:
@@ -19,7 +19,7 @@ jobs:
     with:
       run_type: release
       docker_image: ${{ inputs.docker_image }}
-      run_label: ""
+      run_label: "<version>"
       test_package: "io.deephaven.benchmark.tests.standard"
       test_class_regex: "^(Test.*|.+[.$]Test.*|.*Tests?)$"
       test_iterations: 1

.github/workflows/remote-benchmarks.yml

@@ -95,7 +95,7 @@ jobs:
         
     - name: Fetch Benchmark Results and Prepare for Upload
       run: |
-        ${SD}/fetch-results-local.sh ${HOST} ${USER} ${RUN_TYPE} "${ACTOR}" "${RUN_LABEL}"
+        ${SD}/fetch-results-local.sh ${HOST} ${USER} ${SD} ${RUN_TYPE} "${ACTOR}" "${RUN_LABEL}" "${DOCKER_IMG}"
 
     - name: Authorize GCloud Credentials
       uses: google-github-actions/auth@v1
@@ -104,13 +104,6 @@ jobs:
 
     - name: Set up Cloud SDK
       uses: google-github-actions/setup-gcloud@v1
-
-    - name: Backup Existing Benchmark Summary SVG
-      run: |
-        SUMMARY_PREFIX=gs://deephaven-benchmark/${RUN_TYPE}/benchmark-summary
-        if gsutil stat ${SUMMARY_PREFIX}.svg &>/dev/null; then
-          gsutil mv ${SUMMARY_PREFIX}.svg ${SUMMARY_PREFIX}.prev.svg &>/dev/null
-        fi
 
     - name: Upload Benchmark Results to GCloud
       uses: google-github-actions/upload-cloud-storage@v1

README.md

@@ -1,7 +1,7 @@
 # Deephaven Benchmark
 
 [Summary of Latest Successful Nightly Benchmarks](docs/NightlySummary.md)
-![Operation Rate Change Tracking By Release](https://storage.googleapis.com/deephaven-benchmark/nightly/benchmark-summary.svg?)
+![Operation Rate Change Tracking By Release](https://storage.googleapis.com/deephaven-benchmark/nightly/deephaven/benchmark-summary.svg?)
 ([See Other Deephaven Summaries Below](#other-deephaven-summaries))
 
 The Benchmark framework provides support for gathering performance measurements and statistics for operations on tabular data.  It uses the JUnit
@@ -75,4 +75,4 @@ used for nightly Deephaven benchmarks. Either way the submission of the result t
 ## Other Deephaven Summaries
 
 [Summary of Comparison Benchmarks](docs/ComparisonSummary.md)
-![Operation Rate Product Comparison](https://storage.googleapis.com/deephaven-benchmark/compare/benchmark-summary.svg?)
+![Operation Rate Product Comparison](https://storage.googleapis.com/deephaven-benchmark/compare/deephaven/benchmark-summary.svg?)

docs/ComparisonSummary.md

@@ -1,6 +1,6 @@
 # Comparison Benchmark Summary
 
-![Operation Rate Comparison](https://storage.googleapis.com/deephaven-benchmark/compare/benchmark-summary.svg?)
+![Operation Rate Comparison](https://storage.googleapis.com/deephaven-benchmark/compare/deephaven/benchmark-summary.svg?)
 
 ## Comparison Table Organization
 

docs/NightlySummary.md

@@ -1,6 +1,6 @@
 # Nightly Benchmark Summary
 
-![Operation Rate Change Tracking By Release](https://storage.googleapis.com/deephaven-benchmark/nightly/benchmark-summary.svg?)
+![Operation Rate Change Tracking By Release](https://storage.googleapis.com/deephaven-benchmark/nightly/deephaven/benchmark-summary.svg?)
 
 ## Summary Table Organization
 

src/it/java/io/deephaven/benchmark/tests/experimental/mergescale/ScaleTestRunner.java

@@ -36,6 +36,8 @@ void runTest(String testName, String tableName, long baseRowCount, long rowCount
         from deephaven import new_table, merge, garbage_collect, agg
         from deephaven.column import int_col, float_col
         from deephaven.parquet import read
+        
+        bench_api_metrics_init()
 
         ${table} = ${readTable}
         garbage_collect()
@@ -45,11 +47,9 @@ void runTest(String testName, String tableName, long baseRowCount, long rowCount
            agg.pct(0.90, ['Percentile3=int1M'])
         ]
         
-        bench_api_metrics_snapshot()
         begin_time = time.perf_counter_ns()
         result = ${table}.agg_by(aggs, by=['str250', 'str640'])
         end_time = time.perf_counter_ns()
-        bench_api_metrics_snapshot()
         standard_metrics = bench_api_metrics_collect()
         
         stats = new_table([
@@ -85,7 +85,7 @@ void restartDocker(Bench api) {
         var controller = new DeephavenDockerController(api.property("docker.compose.file", ""), api.property("deephaven.addr", ""));
         if (!controller.restartService())
             return;
-        var metrics = new Metrics(Timer.now(), "test-runner", "setup", "docker");
+        var metrics = new Metrics(Timer.now(), "test-runner", "setup.docker");
         metrics.set("restart", timer.duration().toMillis(), "standard");
         api.metrics().add(metrics);
     }