diff --git a/.github/resources/compare-benchmark-docker-compose.yml b/.github/resources/compare-benchmark-docker-compose.yml index 86c764c4..e595d8de 100644 --- a/.github/resources/compare-benchmark-docker-compose.yml +++ b/.github/resources/compare-benchmark-docker-compose.yml @@ -2,7 +2,7 @@ version: "3.4" services: deephaven: - image: ghcr.io/deephaven/server:0.27.1 + image: ghcr.io/deephaven/server:0.28.0 ports: - "${DEEPHAVEN_PORT:-10000}:10000" volumes: diff --git a/.github/resources/release-benchmark-docker-compose.yml b/.github/resources/release-benchmark-docker-compose.yml index 86c764c4..e595d8de 100644 --- a/.github/resources/release-benchmark-docker-compose.yml +++ b/.github/resources/release-benchmark-docker-compose.yml @@ -2,7 +2,7 @@ version: "3.4" services: deephaven: - image: ghcr.io/deephaven/server:0.27.1 + image: ghcr.io/deephaven/server:0.28.0 ports: - "${DEEPHAVEN_PORT:-10000}:10000" volumes: diff --git a/src/it/java/io/deephaven/benchmark/tests/compare/CompareTestRunner.java b/src/it/java/io/deephaven/benchmark/tests/compare/CompareTestRunner.java index 764e3182..cc08561b 100644 --- a/src/it/java/io/deephaven/benchmark/tests/compare/CompareTestRunner.java +++ b/src/it/java/io/deephaven/benchmark/tests/compare/CompareTestRunner.java @@ -28,7 +28,7 @@ public class CompareTestRunner { final Object testInst; final Set requiredPackages = new LinkedHashSet<>(); - final Map downloadFiles = new LinkedHashMap<>(); + final Map downloadFiles = new LinkedHashMap<>(); private Bench api = null; public CompareTestRunner(Object testInst) { @@ -43,8 +43,16 @@ public CompareTestRunner(Object testInst) { public Bench api() { return api; } - - public void addDownloadFiles(String sourceUri, String destDir) { + + /** + * Download and place the given file into the environment Deephaven is running in. If the destination directory is + * specified as a relative path, the download file will be placed relative to the root of the virtual environment + * this test runner is using for python scripts and pip installs. + * + * @param sourceUri a URI (ex. https://repo1.maven.org/maven2/org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9-tests.jar) + * @param destDir a directory to place the downloaded file into + */ + public void addDownloadFile(String sourceUri, String destDir) { downloadFiles.put(sourceUri, destDir); } @@ -72,7 +80,8 @@ public void initPython(String... packages) { restartDocker(1); initialize(testInst); requiredPackages.addAll(Arrays.asList(packages)); - requiredPackages.add("install-jdk"); + if (Arrays.stream(packages).anyMatch(p -> p.startsWith("jdk"))) + requiredPackages.add("install-jdk"); } /** @@ -120,32 +129,69 @@ public void test(String name, long expectedRowCount, String setup, String operat * these packages are installed, Deephaven will only be used as an agent to run command line python code */ void installRequiredPackages() { + var pipPackagesMarker = "--- Bench Pip Installed Versions ---"; var pipPackages = requiredPackages.stream().filter(p -> !isJdkPackage(p)).toList(); var query = """ text = '''PACKAGES='${pipPackages}' VENV_PATH=~/deephaven-benchmark-venv + PIP_INSTALL_VERSIONS=pip-install-versions.txt rm -rf ${VENV_PATH}/* python3 -m venv ${VENV_PATH} cd ${VENV_PATH} for PKG in ${PACKAGES}; do ./bin/pip install ${PKG} + ./bin/pip list | grep -E "^${PKG}\s+.*" >> ${PIP_INSTALL_VERSIONS} done + echo "${pipPackagesMarker}" + cat ${PIP_INSTALL_VERSIONS} ''' save_file('setup-benchmark-workspace.sh', text) - run_script('bash', 'setup-benchmark-workspace.sh') + result = run_script('bash', 'setup-benchmark-workspace.sh') + + pip_versions = new_table([ + string_col("versions", [result]), + ]) """; query = query.replace("${pipPackages}", String.join(" ", pipPackages)); - api.query(query).execute(); + query = query.replace("${pipPackagesMarker}", pipPackagesMarker); + api.query(query).fetchAfter("pip_versions", table -> { + boolean isPastMarker = false; + for (String line : table.getValue(0, "versions").toString().lines().toList()) { + line = line.trim(); + if (!isPastMarker && line.equals(pipPackagesMarker)) { + isPastMarker = true; + continue; + } + if (!isPastMarker) + continue; + + String[] s = line.split("\\s+"); + if (s.length > 1) + api.platform().add("python-dh-agent", "pip." + s[0] + ".version", s[1]); + } + }).execute(); requiredPackages.forEach(p -> installJavaPackage(p)); - downloadFiles.forEach((s,d) -> placeDownloadFile(s, d)); + downloadFiles.forEach((s, d) -> placeDownloadFile(s, d)); } - + + /** + * Determine if the given package descriptor has the form of a java package descriptor. + * + * @param javaDescr a package descriptor like 'jdk-11' + * @return true if the given descriptor describes a java package, otherwise false + */ boolean isJdkPackage(String javaDescr) { return javaDescr.matches("jdk-[0-9]+"); } + /** + * Download and install a java package according to the descriptor (@see isJdkPackage()) into the virtual + * environment where python and pip are installed. + * + * @param javaDescr a description like 'jdk-11' + */ void installJavaPackage(String javaDescr) { if (!isJdkPackage(javaDescr)) return; @@ -164,7 +210,14 @@ void installJavaPackage(String javaDescr) { query = query.replace("${version}", String.join(" ", version)); api.query(query).execute(); } - + + /** + * Download and place the given source URL to the given destination directory. (@see addDownloadFile()) This method + * uses python on the Deephaven server to download and place. + * + * @param sourceUri the file to download + * @param destDir the directory to put the downloaded file in + */ void placeDownloadFile(String sourceUri, String destDir) { var query = """ text = ''' @@ -285,7 +338,7 @@ void initialize(Object testInst) { import subprocess, os, stat, time from pathlib import Path from deephaven import new_table, garbage_collect - from deephaven.column import long_col, double_col + from deephaven.column import long_col, double_col, string_col user_home = str(Path.home()) benchmark_home = user_home + '/deephaven-benchmark-venv' diff --git a/src/it/java/io/deephaven/benchmark/tests/compare/Setup.java b/src/it/java/io/deephaven/benchmark/tests/compare/Setup.java index fc5483bb..2ddf8292 100644 --- a/src/it/java/io/deephaven/benchmark/tests/compare/Setup.java +++ b/src/it/java/io/deephaven/benchmark/tests/compare/Setup.java @@ -47,6 +47,6 @@ static public void addDownloadJar(CompareTestRunner r, String prod, String artif var destDir = "lib/python3.10/site-packages/pyflink/lib"; var apacheUri = "https://repo1.maven.org/maven2/org/apache/"; var uri = apacheUri + prod + '/' + artifact + '/' + version + '/' + artifact + '-' + version + ".jar"; - r.addDownloadFiles(uri, destDir); + r.addDownloadFile(uri, destDir); } } diff --git a/src/main/java/io/deephaven/benchmark/api/Bench.java b/src/main/java/io/deephaven/benchmark/api/Bench.java index cf113c91..3548316a 100644 --- a/src/main/java/io/deephaven/benchmark/api/Bench.java +++ b/src/main/java/io/deephaven/benchmark/api/Bench.java @@ -17,14 +17,18 @@ /** * The root accessor class for the API. Use Bench.create(this) in a typical JUnit test to start things off + *

+ * Bench API methods are not thread-safe, nor are they intended to be. It makes no sense to run benchmark tests in + * parallel. If parallel tests are desired to shorten overall test time, use the standalone uber-jar and select separate + * sets of test packages to run on different systems simultaneously. */ final public class Bench { static final public String rootOutputDir = "results"; static final public String resultFileName = "benchmark-results.csv"; static final public String metricsFileName = "benchmark-metrics.csv"; + static final public String platformFileName = "benchmark-platform.csv"; static final Profile profile = new Profile(); static final public Path outputDir = initializeOutputDirectory(); - static final Platform platform = new Platform(outputDir); static public Bench create(Object testInst) { Bench v = new Bench(testInst.getClass()); @@ -35,6 +39,7 @@ static public Bench create(Object testInst) { final Object testInst; final BenchResult result; final BenchMetrics metrics; + final BenchPlatform platform; final QueryLog queryLog; final List> futures = new ArrayList<>(); final List closeables = new ArrayList<>(); @@ -44,6 +49,7 @@ static public Bench create(Object testInst) { this.testInst = testInst; this.result = new BenchResult(outputDir); this.metrics = new BenchMetrics(outputDir); + this.platform = new BenchPlatform(outputDir); this.queryLog = new QueryLog(outputDir, testInst); } @@ -156,14 +162,23 @@ public BenchResult result() { } /** - * Get the result for this Benchmark instance (e.g. test) used for collecting rates + * Get the metrics for this Benchmark instance (e.g. test) used for collecting metric values * - * @return the result instance + * @return the metrics instance */ public BenchMetrics metrics() { return metrics; } + /** + * Get the platform for this Benchmark instance (e.g. test) used for collecting platform properties + * + * @return the platform instance + */ + public BenchPlatform platform() { + return platform; + } + /** * Has this Bench api instance been closed along with all connectors and files opened since creating the instance * @@ -221,7 +236,7 @@ static private Path initializeOutputDirectory() { Path dir = Paths.get(rootOutputDir); if (isTimestamped) dir = dir.resolve(Ids.runId()); - Filer.deleteAll(dir); + Filer.delete(dir); try { return Files.createDirectories(dir); } catch (Exception ex) { diff --git a/src/main/java/io/deephaven/benchmark/api/Platform.java b/src/main/java/io/deephaven/benchmark/api/BenchPlatform.java similarity index 68% rename from src/main/java/io/deephaven/benchmark/api/Platform.java rename to src/main/java/io/deephaven/benchmark/api/BenchPlatform.java index 13f670a9..6379aac0 100644 --- a/src/main/java/io/deephaven/benchmark/api/Platform.java +++ b/src/main/java/io/deephaven/benchmark/api/BenchPlatform.java @@ -5,28 +5,34 @@ import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import io.deephaven.benchmark.connect.ResultTable; import io.deephaven.benchmark.util.Filer; +import io.deephaven.benchmark.util.Numbers; import io.deephaven.engine.exceptions.ArgumentException; /** * Collects various properties about the running client and server used during a benchmark run and stores them in the - * benchmark results directory. + * benchmark results directory. Since properties are potentially collected from multiple tests, and there is a single + * platform property file for an entire test run, properties, once added, are not permitted to be overwritten. */ -class Platform { - static final String platformFileName = "benchmark-platform.csv"; - static final String[] header = {"origin", "name", "value"}; +public class BenchPlatform { + static final Map properties = new LinkedHashMap<>(); + static boolean hasBeenCommitted = false; final Path platformFile; - private boolean hasBeenCommitted = false; + /** * Initialize platform detail collection with the default result file name. * * @param parent the parent directory of the platform file */ - Platform(Path parent) { - this(parent, platformFileName); + BenchPlatform(Path parent) { + this(parent, Bench.platformFileName); } /** @@ -35,24 +41,31 @@ class Platform { * @param parent the parent directory of the platform file * @param platformFileName the name the file to store platform properties */ - Platform(Path parent, String platformFileName) { + BenchPlatform(Path parent, String platformFileName) { this.platformFile = parent.resolve(platformFileName); } + public BenchPlatform add(String origin, String name, Object value) { + benchApiAddProperty(properties, origin, name, value); + return this; + } + /** - * Ensure that collected plaform properties have been saved + * Ensure that collected platform properties have been saved */ void commit() { - if (hasBeenCommitted) - return; - hasBeenCommitted = true; - try (BufferedWriter out = Files.newBufferedWriter(platformFile)) { - out.write(String.join(",", header)); - out.newLine(); - writeTestProps(out); - writeEngineProps(out); - } catch (Exception ex) { - throw new RuntimeException("Failed to write platform file: " + platformFile, ex); + if (!hasBeenCommitted) { + hasBeenCommitted = true; + Filer.delete(platformFile); + writeLine(new Property("origin", "name", "value", new AtomicBoolean(true)), platformFile); + addTestProps(properties); + addEngineProps(properties); + } + for (Property prop : properties.values()) { + if (!prop.isWritten().get()) { + writeLine(prop, platformFile); + prop.isWritten().set(true); + } } } @@ -62,7 +75,7 @@ void commit() { * @param query the query used to get/make the property table * @return a cached result table containing properties */ - ResultTable fetchResult(String query) { + protected ResultTable fetchResult(String query) { Bench api = new Bench(Bench.class); api.setName("# Write Platform Details"); // # means skip adding to results file @@ -96,11 +109,12 @@ String getDeephavenVersion(Object dhInst, String pomResource) { return v.matches("[0-9]+\\.[0-9]+\\.[0-9]+") ? v : "Unknown"; } - private void writeTestProps(BufferedWriter benchApiProps) throws Exception { + private void addTestProps(Map benchApiProps) { var dhInst = new ArgumentException(); var benchApiOrigin = "test-runner"; var deephavenVersion = getDeephavenVersion(dhInst, "/META-INF/maven/io.deephaven/deephaven-benchmark/pom.xml"); + // Java Properties (These match the Python calls in addEngineProps benchApiAddProperty(benchApiProps, benchApiOrigin, "java.version", System.getProperty("java.version")); benchApiAddProperty(benchApiProps, benchApiOrigin, "java.vm.name", System.getProperty("java.vm.name")); benchApiAddProperty(benchApiProps, benchApiOrigin, "java.class.version", @@ -113,7 +127,7 @@ private void writeTestProps(BufferedWriter benchApiProps) throws Exception { benchApiAddProperty(benchApiProps, benchApiOrigin, "deephaven.version", deephavenVersion); } - private void writeEngineProps(BufferedWriter out) throws Exception { + private void addEngineProps(Map benchApiProps) { var query = """ import jpy from deephaven import new_table, input_table @@ -155,13 +169,38 @@ def benchApiAddProperty(prop_table, origin, name, value): String origin = t.getValue(r, "origin").toString(); String name = t.getValue(r, "name").toString(); String value = t.getValue(r, "value").toString(); - benchApiAddProperty(out, origin, name, value); + benchApiAddProperty(properties, origin, name, value); } } - private void benchApiAddProperty(BufferedWriter out, String type, String name, Object value) throws Exception { - out.write(String.join(",", type, name, value.toString())); - out.newLine(); + private void benchApiAddProperty(Map properties, String origin, String name, Object value) { + var v = formatValue(name, value); + var prop = new Property(origin, name, v, new AtomicBoolean(false)); + properties.putIfAbsent(prop.getName(), prop); + } + + static void writeLine(Property prop, Path file) { + try (BufferedWriter out = Files.newBufferedWriter(file, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) { + out.write(String.join(",", prop.origin(), prop.name(), prop.value())); + out.newLine(); + } catch (Exception ex) { + throw new RuntimeException("Failed to write result to file: " + file, ex); + } + } + + private String formatValue(String name, Object value) { + switch (name) { + case "java.max.memory": + return Numbers.formatBytesToGigs(value); + default: + return value.toString(); + } + } + + record Property(String origin, String name, String value, AtomicBoolean isWritten) { + String getName() { + return origin + ">>" + name; + } } } diff --git a/src/main/java/io/deephaven/benchmark/run/BenchmarkMain.java b/src/main/java/io/deephaven/benchmark/run/BenchmarkMain.java index f30943f0..95ba4732 100644 --- a/src/main/java/io/deephaven/benchmark/run/BenchmarkMain.java +++ b/src/main/java/io/deephaven/benchmark/run/BenchmarkMain.java @@ -33,18 +33,20 @@ static int main1(String[] args) { setSystemProperties(); int exitCode = ConsoleLauncher.execute(System.out, System.err, args).getExitCode(); if (exitCode == 0) { - Path outputDir = Paths.get(Bench.rootOutputDir); - URL csv = new ResultSummary(outputDir).summarize(); - toSummarySvg(csv, "standard", outputDir, "nightly"); - toSummarySvg(csv, "standard", outputDir, "release"); - toSummarySvg(csv, "compare", outputDir, "compare"); + Path d = Paths.get(Bench.rootOutputDir); + URL platformCsv = new ResultSummary(d, "platform-summary-results.csv", Bench.platformFileName).summarize(); + URL benchmarkCsv = new ResultSummary(d, "benchmark-summary-results.csv", Bench.resultFileName).summarize(); + toSummarySvg(platformCsv, benchmarkCsv, "standard", d, "nightly"); + toSummarySvg(platformCsv, benchmarkCsv, "standard", d, "release"); + toSummarySvg(platformCsv, benchmarkCsv, "compare", d, "compare"); } return exitCode; } - static void toSummarySvg(URL csv, String tmpPrefix, Path outputDir, String outputPrefix) { + static void toSummarySvg(URL platformCsv, URL benchCsv, String tmpPrefix, Path outputDir, String outputPrefix) { URL svgTemplate = BenchmarkMain.class.getResource("profile/" + tmpPrefix + "-benchmark-summary.template.svg"); - new SvgSummary(csv, svgTemplate, outputDir.resolve(outputPrefix + "-benchmark-summary.svg")).summarize(); + new SvgSummary(platformCsv, benchCsv, svgTemplate, outputDir.resolve(outputPrefix + "-benchmark-summary.svg")) + .summarize(); } // Set system properties for running from the command line diff --git a/src/main/java/io/deephaven/benchmark/run/ResultSummary.java b/src/main/java/io/deephaven/benchmark/run/ResultSummary.java index 9ac62171..03cac669 100644 --- a/src/main/java/io/deephaven/benchmark/run/ResultSummary.java +++ b/src/main/java/io/deephaven/benchmark/run/ResultSummary.java @@ -10,17 +10,18 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; -import io.deephaven.benchmark.api.Bench; import io.deephaven.benchmark.util.Ids; class ResultSummary { static final String headerPrefix = "run-id"; final Path rootDir; final Path summaryFile; + final String runResultFileName; - ResultSummary(Path rootDir) { + ResultSummary(Path rootDir, String summaryFileName, String runResultFileName) { this.rootDir = rootDir; - this.summaryFile = getSummaryFile(rootDir, "benchmark-summary-results.csv"); + this.summaryFile = getSummaryFile(rootDir, summaryFileName); + this.runResultFileName = runResultFileName; } URL summarize() { @@ -68,7 +69,7 @@ List getResultFiles(Path rootDir) { return; if (!Files.isDirectory(d)) return; - Path resultFile = d.resolve(Bench.resultFileName); + Path resultFile = d.resolve(runResultFileName); if (Files.exists(resultFile)) resultFiles.add(resultFile); }); diff --git a/src/main/java/io/deephaven/benchmark/run/SvgSummary.java b/src/main/java/io/deephaven/benchmark/run/SvgSummary.java index 328f5cc1..4ec4b43e 100644 --- a/src/main/java/io/deephaven/benchmark/run/SvgSummary.java +++ b/src/main/java/io/deephaven/benchmark/run/SvgSummary.java @@ -24,7 +24,8 @@ */ class SvgSummary { final String varRegex = "\\$\\{([^}]+)\\}"; - final Map benchmarks; + final Map benchmarks; + final Map platformProps; final String svgTemplate; final Path outputDir; final Path svgFile; @@ -36,8 +37,9 @@ class SvgSummary { * @param svgTemplate the template containing references to the benchmark data * @param svgFile the svg file path to produce */ - SvgSummary(URL benchmarkCsv, URL svgTemplate, Path svgFile) { - this.benchmarks = readBenchmarks(benchmarkCsv); + SvgSummary(URL platformCsv, URL benchmarkCsv, URL svgTemplate, Path svgFile) { + this.benchmarks = readSummaryCsv(benchmarkCsv, "benchmark_name"); + this.platformProps = readSummaryCsv(platformCsv, "origin", "name"); this.svgTemplate = Filer.getURLText(svgTemplate); this.outputDir = svgFile.getParent(); this.svgFile = svgFile; @@ -57,27 +59,31 @@ void summarize() { var split = match.group(1).split("=>"); if (split.length < 2) return "$0"; - var benchName = split[0].trim(); + var lookupName = split[0].trim(); var columnName = split[1].trim(); - var benchmark = benchmarks.get(benchName); - if (benchmark == null) - return "$0"; - return Numbers.formatNumber(benchmark.getValue(columnName)); + var platformProp = platformProps.get(lookupName); + var benchmark = benchmarks.get(lookupName); + if (platformProp != null) + return platformProp.getValue(columnName); + if (benchmark != null) + return Numbers.formatNumber(benchmark.getValue(columnName)); + return "$0"; + }); Filer.putFileText(svgFile, out); } - private Map readBenchmarks(URL csv) { + private Map readSummaryCsv(URL csv, String... nameColumns) { var header = new HashMap(); - var benchmarks = new HashMap(); + var benchmarks = new HashMap(); var csvLines = Filer.getURLText(csv).lines().toList(); for (int i = 0, n = csvLines.size(); i < n; i++) { String[] values = csvLines.get(i).split(","); if (i == 0) { IntStream.range(0, values.length).forEach(pos -> header.put(values[pos], pos)); } else { - var next = new Benchmark(header, values); + var next = new Row(header, values, nameColumns); var existing = benchmarks.get(next.getName()); if (next.isNewerThan(existing)) benchmarks.put(next.getName(), next); @@ -89,30 +95,38 @@ private Map readBenchmarks(URL csv) { private String replacePlatformVars(String str) { DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd"); str = str.replace("${run_date}", dtf.format(LocalDateTime.now())); - str = str.replace("${dh_threads}", "16"); - str = str.replace("${dh_heap}", "24G".toLowerCase()); str = str.replace("${os_name}", "Ubuntu 22.04.1 LTS".toLowerCase().replace(" ", "-")); return str.replace("${benchmark_count}", "" + benchmarks.size()); } - record Benchmark(Map header, String[] values) { + record PlatformProp(Map header, String[] values) { String getValue(String colName) { Integer index = header.get(colName); if (index == null) - throw new RuntimeException("Undefined benchmark column name: " + colName); + throw new RuntimeException("Undefined platform column name: " + colName); return values[index].trim(); } + } - String getName() { - return getValue("benchmark_name"); + record Row(Map header, String[] values, String... lookupColumns) { + String getValue(String colName) { + Integer index = header.get(colName); + if (index == null) + throw new RuntimeException("Undefined benchmark column name: " + colName); + return values[index].trim(); } - long getTimestamp() { - return Numbers.parseNumber(getValue("timestamp")).longValue(); + String getName() { + var name = ""; + for (int i = 0, n = lookupColumns.length; i < n; i++) { + var v = getValue(lookupColumns[i]); + name += (i == 0) ? v : (">>" + v); + } + return name; } - boolean isNewerThan(Benchmark other) { - return (other == null) ? true : (getTimestamp() > other.getTimestamp()); + boolean isNewerThan(Row other) { + return (other == null) ? true : (getValue("run-id").compareTo(other.getValue("run-id")) > 0); } } diff --git a/src/main/java/io/deephaven/benchmark/util/Filer.java b/src/main/java/io/deephaven/benchmark/util/Filer.java index 3af252c8..f14a85a9 100644 --- a/src/main/java/io/deephaven/benchmark/util/Filer.java +++ b/src/main/java/io/deephaven/benchmark/util/Filer.java @@ -13,21 +13,24 @@ * Utility class for manipulating files and directories. */ public class Filer { - /** - * Delete the given directory recursively + * Delete the given file or directory. Directories are deleted recursively * - * @param dir the directory to delete + * @param path the directory to delete */ - static public void deleteAll(Path dir) { + static public void delete(Path path) { try { - if (!Files.exists(dir)) + if (!Files.exists(path)) return; - Files.walk(dir) - .sorted(Comparator.reverseOrder()) - .map(Path::toFile).forEach(File::delete); + if (!Files.isDirectory(path)) { + Files.deleteIfExists(path); + } else { + Files.walk(path) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile).forEach(File::delete); + } } catch (Exception ex) { - throw new RuntimeException("Failed to delete directory: " + dir); + throw new RuntimeException("Failed to delete path: " + path); } } diff --git a/src/main/java/io/deephaven/benchmark/util/Numbers.java b/src/main/java/io/deephaven/benchmark/util/Numbers.java index 9b81116f..dd2baa71 100644 --- a/src/main/java/io/deephaven/benchmark/util/Numbers.java +++ b/src/main/java/io/deephaven/benchmark/util/Numbers.java @@ -52,4 +52,17 @@ static public String formatNumber(Object val) { } } + /** + * Format a given byte count into Gigabytes. ex. 1g, 200g + * + * @param bytes the byte count to convert + * @return a string of the form 1g + */ + static public String formatBytesToGigs(Object val) { + if (val == null) + return null; + long bytes = parseNumber(val).longValue(); + return "" + (bytes / 1024 / 1024 / 1024) + "g"; + } + } diff --git a/src/main/resources/io/deephaven/benchmark/run/profile/compare-benchmark-summary.template.svg b/src/main/resources/io/deephaven/benchmark/run/profile/compare-benchmark-summary.template.svg index 2a7afd00..bb5bc1c3 100644 --- a/src/main/resources/io/deephaven/benchmark/run/profile/compare-benchmark-summary.template.svg +++ b/src/main/resources/io/deephaven/benchmark/run/profile/compare-benchmark-summary.template.svg @@ -109,7 +109,11 @@ - + + + + @@ -119,7 +123,7 @@ - +
BenchmarkDeephaven (rows/sec)PyArrow (rows/sec)Pandas (rows/sec)
BenchmarkDeephaven ${deephaven-engine>>deephaven.version=>value} (rows/sec)PyArrow ${python-dh-agent>>pip.pyarrow.version=>value} (rows/sec) + Pandas ${python-dh-agent>>pip.pandas.version=>value} (rows/sec)
Filters${Deephaven Filter=>op_rate}${PyArrow Filter=>op_rate}${Pandas Filter=>op_rate}
Sort${Deephaven Sort=>op_rate}${PyArrow Sort=>op_rate}${Pandas Sort=>op_rate}
* cpu-threads=${dh_threads} heap=${dh_heap} os=${os_name} full-benchmark-set=${benchmark_count}
* cpu-threads=${deephaven-engine>>available.processors=>value} heap=${deephaven-engine>>java.max.memory=>value} os=${os_name} full-benchmark-set=${benchmark_count}
diff --git a/src/test/java/io/deephaven/benchmark/api/PlatformTest.java b/src/test/java/io/deephaven/benchmark/api/BenchPlatformTest.java similarity index 80% rename from src/test/java/io/deephaven/benchmark/api/PlatformTest.java rename to src/test/java/io/deephaven/benchmark/api/BenchPlatformTest.java index a19a5a5a..053c542e 100644 --- a/src/test/java/io/deephaven/benchmark/api/PlatformTest.java +++ b/src/test/java/io/deephaven/benchmark/api/BenchPlatformTest.java @@ -9,10 +9,10 @@ import io.deephaven.benchmark.connect.CachedResultTable; import io.deephaven.benchmark.connect.ResultTable; -public class PlatformTest { +public class BenchPlatformTest { @Test - public void ensureCommit() throws Exception { + public void commit() throws Exception { Path outParent = Paths.get(getClass().getResource("test-profile.properties").toURI()).getParent(); var platform = new LocalPlatform(outParent, "platform-test.out"); platform.commit(); @@ -22,10 +22,17 @@ public void ensureCommit() throws Exception { assertEquals("origin,name,value", lines.get(0), "Wrong header"); assertTrue(lines.get(1).matches("test-runner,java.version,[0-9.]+"), "Wrong values: " + lines.get(1)); assertTrue(lines.get(3).matches("test-runner,java.class.version,[0-9.]+"), "Wrong values: " + lines.get(3)); - assertTrue(lines.get(7).matches("test-runner,java.max.memory,[0-9]+"), "Wrong values: " + lines.get(7)); + assertTrue(lines.get(7).matches("test-runner,java.max.memory,[0-9]+g"), "Wrong values: " + lines.get(7)); assertEquals("deephaven-engine,java.version,17.0.5", lines.get(9), "Wrong values"); assertEquals("deephaven-engine,java.class.version,61.0", lines.get(11), "Wrong values"); - assertEquals("deephaven-engine,java.max.memory,25769803776", lines.get(15), "Wrong values"); + assertEquals("deephaven-engine,java.max.memory,24g", lines.get(15), "Wrong values"); + + platform.add("deephaven-engine", "no.repeat.no.overwrite", "100"); + platform.add("deephaven-engine", "no.repeat.no.overwrite", "200"); + platform.commit(); + + lines = Files.readAllLines(outParent.resolve("platform-test.out")); + assertEquals("deephaven-engine,no.repeat.no.overwrite,100", lines.get(16), "Wrong values"); } @Test @@ -37,7 +44,7 @@ public void getDeephavenVersionFromPom() throws Exception { } - static class LocalPlatform extends Platform { + static class LocalPlatform extends BenchPlatform { LocalPlatform(Path dir, String fileName) { super(dir, fileName); } diff --git a/src/test/java/io/deephaven/benchmark/run/ResultSummaryTest.java b/src/test/java/io/deephaven/benchmark/run/ResultSummaryTest.java index 308796d5..87a0f819 100644 --- a/src/test/java/io/deephaven/benchmark/run/ResultSummaryTest.java +++ b/src/test/java/io/deephaven/benchmark/run/ResultSummaryTest.java @@ -5,6 +5,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import org.junit.jupiter.api.Test; +import io.deephaven.benchmark.api.Bench; import io.deephaven.benchmark.util.Filer; public class ResultSummaryTest { @@ -12,7 +13,7 @@ public class ResultSummaryTest { @Test public void summarize() throws Exception { Path rootDir = Paths.get(getClass().getResource("findme.txt").toURI()).resolveSibling("test-data"); - var summary = new ResultSummary(rootDir); + var summary = new ResultSummary(rootDir, "test-summarize-result.csv", Bench.resultFileName); summary.summarize(); assertEquals(""" run-id,name,timestamp,duration,test-rate diff --git a/src/test/java/io/deephaven/benchmark/run/SvgSummaryTest.java b/src/test/java/io/deephaven/benchmark/run/SvgSummaryTest.java index d8289be6..ebbd9af3 100644 --- a/src/test/java/io/deephaven/benchmark/run/SvgSummaryTest.java +++ b/src/test/java/io/deephaven/benchmark/run/SvgSummaryTest.java @@ -12,9 +12,10 @@ public class SvgSummaryTest { @Test public void summarize() throws Exception { var template = getClass().getResource("test-summary.template.svg"); - var csv = getClass().getResource("test-benchmark-results.csv"); + var benchmarkCsv = getClass().getResource("test-benchmark-results.csv"); + var platformCsv = getClass().getResource("test-platform-results.csv"); var svg = Paths.get(template.toURI()).resolveSibling("benchmark-summary.svg"); - var summary = new SvgSummary(csv, template, svg); + var summary = new SvgSummary(platformCsv, benchmarkCsv, template, svg); Files.deleteIfExists(summary.svgFile); summary.summarize(); assertEquals(""" diff --git a/src/test/java/io/deephaven/benchmark/util/FilerTest.java b/src/test/java/io/deephaven/benchmark/util/FilerTest.java index bd8f9c63..773cd46a 100644 --- a/src/test/java/io/deephaven/benchmark/util/FilerTest.java +++ b/src/test/java/io/deephaven/benchmark/util/FilerTest.java @@ -1,12 +1,32 @@ /* Copyright (c) 2022-2023 Deephaven Data Labs and Patent Pending */ package io.deephaven.benchmark.util; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.*; import java.net.URL; import java.nio.file.*; import org.junit.Test; public class FilerTest { + @Test + public void delete() throws Exception { + Path p = Paths.get(getClass().getResource("filertest.txt").toURI()).resolveSibling("deleteme.txt"); + Filer.putFileText(p, "This should be deleted soon"); + assertTrue(Files.exists(p), "Must exist in order to be deleted"); + + Filer.delete(p); + assertFalse(Files.exists(p), "Failed to delete file"); + + Path dir = p.resolveSibling("test-dir"); + Path dir2 = dir.resolve("test-sub-dir"); + Path file1 = dir2.resolve("deleteme.txt"); + Files.createDirectories(dir2); + Filer.putFileText(file1, "This should be deleted soon"); + assertTrue(Files.exists(file1), "Must exist in order to be deleted"); + + Filer.delete(dir); + assertFalse(Files.exists(dir), "Failed to delete dir"); + assertFalse(Files.exists(file1), "Failed to delete file"); + } @Test public void getFileText() throws Exception { diff --git a/src/test/java/io/deephaven/benchmark/util/NumbersTest.java b/src/test/java/io/deephaven/benchmark/util/NumbersTest.java index d3eec56d..ccb9b0bc 100644 --- a/src/test/java/io/deephaven/benchmark/util/NumbersTest.java +++ b/src/test/java/io/deephaven/benchmark/util/NumbersTest.java @@ -26,4 +26,10 @@ public void formatNumber() { assertEquals("0", Numbers.formatNumber("0"), "Should be 0"); } + @Test + public void formatBytesToGigs() { + assertNull(Numbers.formatBytesToGigs(null), "Should be null"); + assertEquals("24g", Numbers.formatBytesToGigs(25769803776L)); + } + } diff --git a/src/test/resources/io/deephaven/benchmark/run/test-benchmark-results.csv b/src/test/resources/io/deephaven/benchmark/run/test-benchmark-results.csv index a42c54d8..0b14fdc1 100644 --- a/src/test/resources/io/deephaven/benchmark/run/test-benchmark-results.csv +++ b/src/test/resources/io/deephaven/benchmark/run/test-benchmark-results.csv @@ -1,6 +1,6 @@ -benchmark_name,origin,timestamp,test_duration,op_duration,op_rate,row_count -AvgBy- 2 Groups 160K Unique Combos Int -Static,deephaven-engine,1688376932899,21.3360,12.0680,14915478,180000000 -AvgBy- 2 Groups 160K Unique Combos Int -Inc,deephaven-engine,1688376954235,21.2960,12.4870,9609994,120000000 -MedianBy- 2 Group 160K Unique Combos Float -Static,deephaven-engine,1688375192283,24.9090,16.6020,2409348,40000000 -MedianBy- 2 Group 160K Unique Combos Float -Inc,deephaven-engine,1688375217193,26.0260,17.9630,2226799,40000000 -NoOp- 20 Double Cols JSON Append,deephaven-engine,1688374272565,26.0520,11.8600,210792,2500000 \ No newline at end of file +run_id,benchmark_name,origin,timestamp,test_duration,op_duration,op_rate,row_count +1,AvgBy- 2 Groups 160K Unique Combos Int -Static,deephaven-engine,1688376932899,21.3360,12.0680,14915478,180000000 +2,AvgBy- 2 Groups 160K Unique Combos Int -Inc,deephaven-engine,1688376954235,21.2960,12.4870,9609994,120000000 +3,MedianBy- 2 Group 160K Unique Combos Float -Static,deephaven-engine,1688375192283,24.9090,16.6020,2409348,40000000 +4,MedianBy- 2 Group 160K Unique Combos Float -Inc,deephaven-engine,1688375217193,26.0260,17.9630,2226799,40000000 +5,NoOp- 20 Double Cols JSON Append,deephaven-engine,1688374272565,26.0520,11.8600,210792,2500000 \ No newline at end of file diff --git a/src/test/resources/io/deephaven/benchmark/run/test-platform-results.csv b/src/test/resources/io/deephaven/benchmark/run/test-platform-results.csv new file mode 100644 index 00000000..7bd3aaed --- /dev/null +++ b/src/test/resources/io/deephaven/benchmark/run/test-platform-results.csv @@ -0,0 +1,18 @@ +run_id,origin,name,value +1,test-runner,java.version,17.0.8.1 +1,test-runner,java.vm.name,OpenJDK 64-Bit Server VM +1,test-runner,java.class.version,61.0 +1,test-runner,os.name,Linux +1,test-runner,os.version,5.17.0-1035-oem +1,test-runner,available.processors,24 +1,test-runner,java.max.memory,8g +1,test-runner,deephaven.version,0.28.0 +1,deephaven-engine,python.version,3.10.12 +1,deephaven-engine,java.version,17.0.8.1 +1,deephaven-engine,java.vm.name,OpenJDK 64-Bit Server VM +1,deephaven-engine,java.class.version,61.0 +1,deephaven-engine,os.name,Linux +1,deephaven-engine,os.version,5.17.0-1035-oem +1,deephaven-engine,available.processors,16 +1,deephaven-engine,java.max.memory,24g +1,deephaven-engine,deephaven.version,0.28.0 \ No newline at end of file diff --git a/src/test/resources/io/deephaven/benchmark/run/test-summary.template.svg b/src/test/resources/io/deephaven/benchmark/run/test-summary.template.svg index a78c5112..bdd9745c 100644 --- a/src/test/resources/io/deephaven/benchmark/run/test-summary.template.svg +++ b/src/test/resources/io/deephaven/benchmark/run/test-summary.template.svg @@ -18,7 +18,7 @@ Avg By Row1${AvgBy- 2 Groups 160K Unique Combos Int -Static=>op_rate}${AvgBy- 2 Groups 160K Unique Combos Int -Inc=>op_rate} Median By Row2${MedianBy- 2 Group 160K Unique Combos Float -Static=>op_rate}${MedianBy- 2 Group 160K Unique Combos Float -Inc=>op_rate} - * threads=${dh_threads} heap=${dh_heap} os=${os_name} benchmark-count=${benchmark_count} + * threads=${deephaven-engine>>available.processors=>value} heap=${deephaven-engine>>java.max.memory=>value} os=${os_name} benchmark-count=${benchmark_count}