S3 benchmarks and Service Selection (#339)

deephaven · Aug 30, 2024 · 2a1d7aa · 2a1d7aa
1 parent ddc8bb6
commit 2a1d7aa
Show file tree

Hide file tree

Showing 14 changed files with 474 additions and 62 deletions.
diff --git a/.github/resources/adhoc-benchmark-docker-compose.yml b/.github/resources/adhoc-benchmark-docker-compose.yml
@@ -30,3 +30,39 @@ services:
     - 8082:8082
     - 9092:9092
     - 29092:29092
+
+  minio-server:
+    image: minio/minio:RELEASE.2024-07-04T14-25-45Z
+    command: server /minio --console-address ":9001"
+    hostname: minio
+    environment:
+      MINIO_DOMAIN: minio
+    networks:
+      default:
+        aliases:
+          - data.minio
+    expose:
+    - "9000"
+    - "9001"
+    ports:
+    - 9000:9000
+    - 9001:9001
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+    volumes:
+    - ./minio:/minio
+
+  minio-bucket:
+    image: minio/mc
+    depends_on:
+      - minio-server
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc alias set endpoint http://minio:9000 minioadmin minioadmin;
+      /usr/bin/mc mb endpoint/data;
+      /usr/bin/mc anonymous set public endpoint/data;
+      exit 0;
+      "
diff --git a/.github/resources/nightly-benchmark-docker-compose.yml b/.github/resources/nightly-benchmark-docker-compose.yml
@@ -30,3 +30,39 @@ services:
     - 8082:8082
     - 9092:9092
     - 29092:29092
+
+  minio-server:
+    image: minio/minio:RELEASE.2024-07-04T14-25-45Z
+    command: server /minio --console-address ":9001"
+    hostname: minio
+    environment:
+      MINIO_DOMAIN: minio
+    networks:
+      default:
+        aliases:
+          - data.minio
+    expose:
+    - "9000"
+    - "9001"
+    ports:
+    - 9000:9000
+    - 9001:9001
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+    volumes:
+    - ./minio:/minio
+
+  minio-bucket:
+    image: minio/mc
+    depends_on:
+      - minio-server
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc alias set endpoint http://minio:9000 minioadmin minioadmin;
+      /usr/bin/mc mb endpoint/data;
+      /usr/bin/mc anonymous set public endpoint/data;
+      exit 0;
+      "
diff --git a/.github/resources/release-benchmark-docker-compose.yml b/.github/resources/release-benchmark-docker-compose.yml
@@ -30,3 +30,39 @@ services:
     - 8082:8082
     - 9092:9092
     - 29092:29092
+
+  minio-server:
+    image: minio/minio:RELEASE.2024-07-04T14-25-45Z
+    command: server /minio --console-address ":9001"
+    hostname: minio
+    environment:
+      MINIO_DOMAIN: minio
+    networks:
+      default:
+        aliases:
+          - data.minio
+    expose:
+    - "9000"
+    - "9001"
+    ports:
+    - 9000:9000
+    - 9001:9001
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+    volumes:
+    - ./minio:/minio
+
+  minio-bucket:
+    image: minio/mc
+    depends_on:
+      - minio-server
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc alias set endpoint http://minio:9000 minioadmin minioadmin;
+      /usr/bin/mc mb endpoint/data;
+      /usr/bin/mc anonymous set public endpoint/data;
+      exit 0;
+      "
diff --git a/src/it/java/io/deephaven/benchmark/tests/compare/CompareTestRunner.java b/src/it/java/io/deephaven/benchmark/tests/compare/CompareTestRunner.java
@@ -9,7 +9,9 @@
 import java.util.concurrent.atomic.AtomicReference;
 import io.deephaven.benchmark.api.Bench;
 import io.deephaven.benchmark.controller.DeephavenDockerController;
+import io.deephaven.benchmark.metric.Metrics;
 import io.deephaven.benchmark.util.Filer;
+import io.deephaven.benchmark.util.Timer;
 
 /**
  * A wrapper for the Bench api that allows running tests for the purpose of comparing Deephaven to other products that
@@ -55,11 +57,11 @@ public void addDownloadFile(String sourceUri, String destDir) {
      * @param columnNames the columns in both tables to included
      */
     public void initDeephaven(int rowCountFactor, String leftTable, String rightTable, String... columnNames) {
-        restartDocker();
+        restartServices();
         generateTable(rowCountFactor, leftTable, columnNames);
         if (rightTable != null)
             generateTable(rowCountFactor, rightTable, columnNames);
-        restartDocker();
+        restartServices();
         initialize(testInst);
     }
 
@@ -303,6 +305,7 @@ Result runTest(String name, String query, String setup, String operation, String
         if (api == null)
             throw new RuntimeException("Initialize with initDeephaven() or initPython()s before running the test");
         api.setName(name);
+        stopUnusedServices();
         query = query.replace("${setupQueries}", setup);
         query = query.replace("${operation}", operation);
         query = query.replace("${mainSizeGetter}", mainSizeGetter);
@@ -352,23 +355,22 @@ def run_script(runner, script_name):
         api.query(query).execute();
     }
 
-    void restartDocker() {
-        var api = Bench.create("# Docker Restart");
+    void restartServices() {
+        var api = Bench.create("# Services Restart");
         try {
-            api.setName("# Docker Restart");
-            var controller = new DeephavenDockerController(api.property("docker.compose.file", ""),
+            api.setName("# Services Restart");
+            var c = new DeephavenDockerController(api.property("docker.compose.file", ""),
                     api.property("deephaven.addr", ""));
-            if (!controller.restartService())
-                return;
+            c.restartService();
         } finally {
             api.close();
         }
     }
 
     void restartDocker(int heapGigs) {
-        var api = Bench.create("# Docker Restart");
+        var api = Bench.create("# Services Restart");
         try {
-            api.setName("# Docker Restart " + heapGigs + "G");
+            api.setName("# Services Restart " + heapGigs + "G");
             String dockerComposeFile = api.property("docker.compose.file", "");
             String deephavenHostPort = api.property("deephaven.addr", "");
             if (dockerComposeFile.isBlank() || deephavenHostPort.isBlank())
@@ -381,6 +383,16 @@ void restartDocker(int heapGigs) {
         }
     }
 
+    void stopUnusedServices() {
+        var timer = api.timer();
+        var c = new DeephavenDockerController(api.property("docker.compose.file", ""), api.property("deephaven.addr", ""));
+        if (!c.stopService(Set.of("deephaven")))
+            return;
+        var metrics = new Metrics(Timer.now(), "test-runner", "setup.services");
+        metrics.set("stop", timer.duration().toMillis(), "standard");
+        api.metrics().add(metrics);
+    }
+
     // Replace heap (e.g. -Xmx64g) in docker-compose.yml with new heap value
     String makeHeapAdjustedDockerCompose(String dockerComposeFile, int heapGigs) {
         Path sourceComposeFile = Paths.get(dockerComposeFile);

diff --git a/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java b/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java
@@ -25,6 +25,7 @@ final public class StandardTestRunner {
     final List<String> supportTables = new ArrayList<>();
     final List<String> setupQueries = new ArrayList<>();
     final List<String> preOpQueries = new ArrayList<>();
+    final Set<String> requiredServices = new TreeSet<>(List.of("deephaven"));
     private String mainTable = "source";
     private Bench api;
     private Controller controller;
@@ -89,6 +90,11 @@ public void groupedTable(String name, String... groups) {
         mainTable = name;
         generateTable(name, null, groups);
     }
+
+    public void setServices(String... services) {
+        requiredServices.clear();
+        requiredServices.addAll(Arrays.asList(services));
+    }
 
     /**
      * Add a query to be run directly after the main table is loaded. It is not measured. This query can transform the
@@ -289,6 +295,8 @@ Result runTest(String name, String query, String operation, String read, String.
         if (api.isClosed())
             initialize(testInst);
         api.setName(name);
+        stopUnusedServices(requiredServices);
+
         query = query.replace("${readTable}", read);
         query = query.replace("${mainTable}", mainTable);
         query = query.replace("${loadSupportTables}", loadSupportTables());
@@ -318,7 +326,7 @@ Result runTest(String name, String query, String operation, String read, String.
             api.result().test("deephaven-engine", result.get().elapsedTime(), result.get().loadedRowCount());
             return result.get();
         } finally {
-            addDockerLog(api);
+            addServiceLog(api);
             api.close();
         }
     }
@@ -353,36 +361,45 @@ void initialize(Object testInst) {
         this.api = Bench.create(testInst);
         this.controller = new DeephavenDockerController(api.property("docker.compose.file", ""),
                 api.property("deephaven.addr", ""));
-        restartDocker();
+        restartServices();
         api.query(query).execute();
     }
 
-    void addDockerLog(Bench api) {
+    void addServiceLog(Bench api) {
         var timer = api.timer();
         var logText = controller.getLog();
         if (logText.isBlank())
             return;
         api.log().add("deephaven-engine", logText);
-        var metrics = new Metrics(Timer.now(), "test-runner", "teardown.docker");
+        var metrics = new Metrics(Timer.now(), "test-runner", "teardown.services");
         metrics.set("log", timer.duration().toMillis(), "standard");
         api.metrics().add(metrics);
     }
 
-    void restartDocker() {
+    void restartServices() {
         var timer = api.timer();
         if (!controller.restartService())
             return;
-        var metrics = new Metrics(Timer.now(), "test-runner", "setup.docker");
+        var metrics = new Metrics(Timer.now(), "test-runner", "setup.services");
         metrics.set("restart", timer.duration().toMillis(), "standard");
         api.metrics().add(metrics);
     }
+
+    void stopUnusedServices(Set<String> keepServices) {
+        var timer = api.timer();
+        if (!controller.stopService(keepServices))
+            return;
+        var metrics = new Metrics(Timer.now(), "test-runner", "setup.services");
+        metrics.set("stop", timer.duration().toMillis(), "standard");
+        api.metrics().add(metrics);
+    }
 
     void generateTable(String name, String distribution, String[] groups) {
         var isNew = generateNamedTable(name, distribution, groups);
         if (isNew) {
             if (!api.isClosed()) {
                 api.setName("# Data Table Generation " + name);
-                addDockerLog(api);
+                addServiceLog(api);
                 api.close();
             }
             initialize(testInst);