Automatic Python Versions to Compare SVG Template

.github/resources/compare-benchmark-docker-compose.yml

@@ -2,7 +2,7 @@ version: "3.4"
 
 services:
   deephaven:
-    image: ghcr.io/deephaven/server:0.27.1
+    image: ghcr.io/deephaven/server:0.28.0
     ports:
       - "${DEEPHAVEN_PORT:-10000}:10000"
     volumes:

.github/resources/release-benchmark-docker-compose.yml

@@ -2,7 +2,7 @@ version: "3.4"
 
 services:
   deephaven:
-    image: ghcr.io/deephaven/server:0.27.1
+    image: ghcr.io/deephaven/server:0.28.0
     ports:
       - "${DEEPHAVEN_PORT:-10000}:10000"
     volumes:

src/it/java/io/deephaven/benchmark/tests/compare/CompareTestRunner.java

@@ -28,7 +28,7 @@
 public class CompareTestRunner {
     final Object testInst;
     final Set<String> requiredPackages = new LinkedHashSet<>();
-    final Map<String,String> downloadFiles = new LinkedHashMap<>();
+    final Map<String, String> downloadFiles = new LinkedHashMap<>();
     private Bench api = null;
 
     public CompareTestRunner(Object testInst) {
@@ -43,8 +43,16 @@ public CompareTestRunner(Object testInst) {
     public Bench api() {
         return api;
     }
-
-    public void addDownloadFiles(String sourceUri, String destDir) {
+
+    /**
+     * Download and place the given file into the environment Deephaven is running in. If the destination directory is
+     * specified as a relative path, the download file will be placed relative to the root of the virtual environment
+     * this test runner is using for python scripts and pip installs.
+     * 
+     * @param sourceUri a URI (ex. https://repo1.maven.org/maven2/org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9-tests.jar)
+     * @param destDir a directory to place the downloaded file into
+     */
+    public void addDownloadFile(String sourceUri, String destDir) {
         downloadFiles.put(sourceUri, destDir);
     }
 
@@ -72,7 +80,8 @@ public void initPython(String... packages) {
         restartDocker(1);
         initialize(testInst);
         requiredPackages.addAll(Arrays.asList(packages));
-        requiredPackages.add("install-jdk");
+        if (Arrays.stream(packages).anyMatch(p -> p.startsWith("jdk")))
+            requiredPackages.add("install-jdk");
     }
 
     /**
@@ -120,32 +129,69 @@ public void test(String name, long expectedRowCount, String setup, String operat
      * these packages are installed, Deephaven will only be used as an agent to run command line python code
      */
     void installRequiredPackages() {
+        var pipPackagesMarker = "--- Bench Pip Installed Versions ---";
         var pipPackages = requiredPackages.stream().filter(p -> !isJdkPackage(p)).toList();
 
         var query = """
         text = '''PACKAGES='${pipPackages}'
         VENV_PATH=~/deephaven-benchmark-venv
+        PIP_INSTALL_VERSIONS=pip-install-versions.txt
         rm -rf ${VENV_PATH}/*
         python3 -m venv ${VENV_PATH}
         cd ${VENV_PATH}
         for PKG in ${PACKAGES}; do
             ./bin/pip install ${PKG}
+            ./bin/pip list | grep -E "^${PKG}\s+.*" >> ${PIP_INSTALL_VERSIONS}
         done
+        echo "${pipPackagesMarker}"
+        cat ${PIP_INSTALL_VERSIONS}
         '''
         save_file('setup-benchmark-workspace.sh', text)
-        run_script('bash', 'setup-benchmark-workspace.sh')
+        result = run_script('bash', 'setup-benchmark-workspace.sh')
+
+        pip_versions = new_table([
+            string_col("versions", [result]),
+        ])
         """;
         query = query.replace("${pipPackages}", String.join(" ", pipPackages));
-        api.query(query).execute();
+        query = query.replace("${pipPackagesMarker}", pipPackagesMarker);
+        api.query(query).fetchAfter("pip_versions", table -> {
+            boolean isPastMarker = false;
+            for (String line : table.getValue(0, "versions").toString().lines().toList()) {
+                line = line.trim();
+                if (!isPastMarker && line.equals(pipPackagesMarker)) {
+                    isPastMarker = true;
+                    continue;
+                }
+                if (!isPastMarker)
+                    continue;
+
+                String[] s = line.split("\\s+");
+                if (s.length > 1)
+                    api.platform().add("python-dh-agent", "pip." + s[0] + ".version", s[1]);
+            }
+        }).execute();
 
         requiredPackages.forEach(p -> installJavaPackage(p));
-        downloadFiles.forEach((s,d) -> placeDownloadFile(s, d));
+        downloadFiles.forEach((s, d) -> placeDownloadFile(s, d));
     }
-
+
+    /**
+     * Determine if the given package descriptor has the form of a java package descriptor.
+     * 
+     * @param javaDescr a package descriptor like 'jdk-11'
+     * @return true if the given descriptor describes a java package, otherwise false
+     */
     boolean isJdkPackage(String javaDescr) {
         return javaDescr.matches("jdk-[0-9]+");
     }
 
+    /**
+     * Download and install a java package according to the descriptor (@see isJdkPackage()) into the virtual
+     * environment where python and pip are installed.
+     * 
+     * @param javaDescr a description like 'jdk-11'
+     */
     void installJavaPackage(String javaDescr) {
         if (!isJdkPackage(javaDescr))
             return;
@@ -164,7 +210,14 @@ void installJavaPackage(String javaDescr) {
         query = query.replace("${version}", String.join(" ", version));
         api.query(query).execute();
     }
-
+
+    /**
+     * Download and place the given source URL to the given destination directory. (@see addDownloadFile()) This method
+     * uses python on the Deephaven server to download and place.
+     * 
+     * @param sourceUri the file to download
+     * @param destDir the directory to put the downloaded file in
+     */
     void placeDownloadFile(String sourceUri, String destDir) {
         var query = """
         text = '''
@@ -285,7 +338,7 @@ void initialize(Object testInst) {
         import subprocess, os, stat, time
         from pathlib import Path
         from deephaven import new_table, garbage_collect
-        from deephaven.column import long_col, double_col
+        from deephaven.column import long_col, double_col, string_col
 
         user_home = str(Path.home())
         benchmark_home = user_home + '/deephaven-benchmark-venv'

src/it/java/io/deephaven/benchmark/tests/compare/Setup.java

@@ -47,6 +47,6 @@ static public void addDownloadJar(CompareTestRunner r, String prod, String artif
         var destDir = "lib/python3.10/site-packages/pyflink/lib";
         var apacheUri = "https://repo1.maven.org/maven2/org/apache/";
         var uri = apacheUri + prod + '/' + artifact + '/' + version + '/' + artifact + '-' + version + ".jar";
-        r.addDownloadFiles(uri, destDir);
+        r.addDownloadFile(uri, destDir);
     }
 }

src/main/java/io/deephaven/benchmark/api/Bench.java

@@ -17,14 +17,18 @@
 
 /**
  * The root accessor class for the API. Use <code>Bench.create(this)</code> in a typical JUnit test to start things off
+ * <p/>
+ * Bench API methods are not thread-safe, nor are they intended to be. It makes no sense to run benchmark tests in
+ * parallel. If parallel tests are desired to shorten overall test time, use the standalone uber-jar and select separate
+ * sets of test packages to run on different systems simultaneously.
  */
 final public class Bench {
     static final public String rootOutputDir = "results";
     static final public String resultFileName = "benchmark-results.csv";
     static final public String metricsFileName = "benchmark-metrics.csv";
+    static final public String platformFileName = "benchmark-platform.csv";
     static final Profile profile = new Profile();
     static final public Path outputDir = initializeOutputDirectory();
-    static final Platform platform = new Platform(outputDir);
 
     static public Bench create(Object testInst) {
         Bench v = new Bench(testInst.getClass());
@@ -35,6 +39,7 @@ static public Bench create(Object testInst) {
     final Object testInst;
     final BenchResult result;
     final BenchMetrics metrics;
+    final BenchPlatform platform;
     final QueryLog queryLog;
     final List<Future<Metrics>> futures = new ArrayList<>();
     final List<Closeable> closeables = new ArrayList<>();
@@ -44,6 +49,7 @@ static public Bench create(Object testInst) {
         this.testInst = testInst;
         this.result = new BenchResult(outputDir);
         this.metrics = new BenchMetrics(outputDir);
+        this.platform = new BenchPlatform(outputDir);
         this.queryLog = new QueryLog(outputDir, testInst);
     }
 
@@ -156,14 +162,23 @@ public BenchResult result() {
     }
 
     /**
-     * Get the result for this Benchmark instance (e.g. test) used for collecting rates
+     * Get the metrics for this Benchmark instance (e.g. test) used for collecting metric values
      * 
-     * @return the result instance
+     * @return the metrics instance
      */
     public BenchMetrics metrics() {
         return metrics;
     }
 
+    /**
+     * Get the platform for this Benchmark instance (e.g. test) used for collecting platform properties
+     * 
+     * @return the platform instance
+     */
+    public BenchPlatform platform() {
+        return platform;
+    }
+
     /**
      * Has this Bench api instance been closed along with all connectors and files opened since creating the instance
      * 
@@ -221,7 +236,7 @@ static private Path initializeOutputDirectory() {
         Path dir = Paths.get(rootOutputDir);
         if (isTimestamped)
             dir = dir.resolve(Ids.runId());
-        Filer.deleteAll(dir);
+        Filer.delete(dir);
         try {
             return Files.createDirectories(dir);
         } catch (Exception ex) {

src/main/java/io/deephaven/benchmark/api/Platform.java → src/main/java/io/deephaven/benchmark/api/BenchPlatform.java

@@ -5,28 +5,34 @@
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import io.deephaven.benchmark.connect.ResultTable;
 import io.deephaven.benchmark.util.Filer;
+import io.deephaven.benchmark.util.Numbers;
 import io.deephaven.engine.exceptions.ArgumentException;
 
 /**
  * Collects various properties about the running client and server used during a benchmark run and stores them in the
- * benchmark results directory.
+ * benchmark results directory. Since properties are potentially collected from multiple tests, and there is a single
+ * platform property file for an entire test run, properties, once added, are not permitted to be overwritten.
  */
-class Platform {
-    static final String platformFileName = "benchmark-platform.csv";
-    static final String[] header = {"origin", "name", "value"};
+public class BenchPlatform {
+    static final Map<String, Property> properties = new LinkedHashMap<>();
+    static boolean hasBeenCommitted = false;
     final Path platformFile;
-    private boolean hasBeenCommitted = false;
+
 
     /**
      * Initialize platform detail collection with the default result file name.
      * 
      * @param parent the parent directory of the platform file
      */
-    Platform(Path parent) {
-        this(parent, platformFileName);
+    BenchPlatform(Path parent) {
+        this(parent, Bench.platformFileName);
     }
 
     /**
@@ -35,24 +41,31 @@ class Platform {
      * @param parent the parent directory of the platform file
      * @param platformFileName the name the file to store platform properties
      */
-    Platform(Path parent, String platformFileName) {
+    BenchPlatform(Path parent, String platformFileName) {
         this.platformFile = parent.resolve(platformFileName);
     }
 
+    public BenchPlatform add(String origin, String name, Object value) {
+        benchApiAddProperty(properties, origin, name, value);
+        return this;
+    }
+
     /**
-     * Ensure that collected plaform properties have been saved
+     * Ensure that collected platform properties have been saved
      */
     void commit() {
-        if (hasBeenCommitted)
-            return;
-        hasBeenCommitted = true;
-        try (BufferedWriter out = Files.newBufferedWriter(platformFile)) {
-            out.write(String.join(",", header));
-            out.newLine();
-            writeTestProps(out);
-            writeEngineProps(out);
-        } catch (Exception ex) {
-            throw new RuntimeException("Failed to write platform file: " + platformFile, ex);
+        if (!hasBeenCommitted) {
+            hasBeenCommitted = true;
+            Filer.delete(platformFile);
+            writeLine(new Property("origin", "name", "value", new AtomicBoolean(true)), platformFile);
+            addTestProps(properties);
+            addEngineProps(properties);
+        }
+        for (Property prop : properties.values()) {
+            if (!prop.isWritten().get()) {
+                writeLine(prop, platformFile);
+                prop.isWritten().set(true);
+            }
         }
     }
 
@@ -62,7 +75,7 @@ void commit() {
      * @param query the query used to get/make the property table
      * @return a cached result table containing properties
      */
-    ResultTable fetchResult(String query) {
+    protected ResultTable fetchResult(String query) {
         Bench api = new Bench(Bench.class);
         api.setName("# Write Platform Details"); // # means skip adding to results file
 
@@ -96,11 +109,12 @@ String getDeephavenVersion(Object dhInst, String pomResource) {
         return v.matches("[0-9]+\\.[0-9]+\\.[0-9]+") ? v : "Unknown";
     }
 
-    private void writeTestProps(BufferedWriter benchApiProps) throws Exception {
+    private void addTestProps(Map<String, Property> benchApiProps) {
         var dhInst = new ArgumentException();
         var benchApiOrigin = "test-runner";
         var deephavenVersion = getDeephavenVersion(dhInst, "/META-INF/maven/io.deephaven/deephaven-benchmark/pom.xml");
 
+        // Java Properties (These match the Python calls in addEngineProps
         benchApiAddProperty(benchApiProps, benchApiOrigin, "java.version", System.getProperty("java.version"));
         benchApiAddProperty(benchApiProps, benchApiOrigin, "java.vm.name", System.getProperty("java.vm.name"));
         benchApiAddProperty(benchApiProps, benchApiOrigin, "java.class.version",
@@ -113,7 +127,7 @@ private void writeTestProps(BufferedWriter benchApiProps) throws Exception {
         benchApiAddProperty(benchApiProps, benchApiOrigin, "deephaven.version", deephavenVersion);
     }
 
-    private void writeEngineProps(BufferedWriter out) throws Exception {
+    private void addEngineProps(Map<String, Property> benchApiProps) {
         var query = """
         import jpy
         from deephaven import new_table, input_table
@@ -155,13 +169,38 @@ def benchApiAddProperty(prop_table, origin, name, value):
             String origin = t.getValue(r, "origin").toString();
             String name = t.getValue(r, "name").toString();
             String value = t.getValue(r, "value").toString();
-            benchApiAddProperty(out, origin, name, value);
+            benchApiAddProperty(properties, origin, name, value);
         }
     }
 
-    private void benchApiAddProperty(BufferedWriter out, String type, String name, Object value) throws Exception {
-        out.write(String.join(",", type, name, value.toString()));
-        out.newLine();
+    private void benchApiAddProperty(Map<String, Property> properties, String origin, String name, Object value) {
+        var v = formatValue(name, value);
+        var prop = new Property(origin, name, v, new AtomicBoolean(false));
+        properties.putIfAbsent(prop.getName(), prop);
+    }
+
+    static void writeLine(Property prop, Path file) {
+        try (BufferedWriter out = Files.newBufferedWriter(file, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) {
+            out.write(String.join(",", prop.origin(), prop.name(), prop.value()));
+            out.newLine();
+        } catch (Exception ex) {
+            throw new RuntimeException("Failed to write result to file: " + file, ex);
+        }
+    }
+
+    private String formatValue(String name, Object value) {
+        switch (name) {
+            case "java.max.memory":
+                return Numbers.formatBytesToGigs(value);
+            default:
+                return value.toString();
+        }
+    }
+
+    record Property(String origin, String name, String value, AtomicBoolean isWritten) {
+        String getName() {
+            return origin + ">>" + name;
+        }
     }
 
 }