test: Increased timeout for S3 unit tests (deephaven#6335)

Should help fix some of the sporadic TimeoutExceptions that we get innightly jobs with the S3 unit tests.
devinrsmith · Nov 5, 2024 · b41039a · b41039a
1 parent c19a4a3
commit b41039a
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 7 deletions.
diff --git a/...ions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/S3WarehouseSqliteCatalogBase.java b/...ions/iceberg/s3/src/test/java/io/deephaven/iceberg/util/S3WarehouseSqliteCatalogBase.java
@@ -18,6 +18,8 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import static io.deephaven.extensions.s3.testlib.S3Helper.TIMEOUT_SECONDS;
+
 public abstract class S3WarehouseSqliteCatalogBase extends SqliteCatalogBase {
 
     public abstract S3Instructions s3Instructions();
@@ -31,7 +33,8 @@ protected IcebergCatalogAdapter catalogAdapter(TestInfo testInfo, Path rootDir,
         final String catalogName = methodName + "-catalog";
         final String bucket = methodName.toLowerCase(Locale.US) + "-bucket";
         try (final S3AsyncClient client = s3AsyncClient()) {
-            client.createBucket(CreateBucketRequest.builder().bucket(bucket).build()).get(5, TimeUnit.SECONDS);
+            client.createBucket(CreateBucketRequest.builder().bucket(bucket).build())
+                    .get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
         }
         properties.put(CatalogProperties.WAREHOUSE_LOCATION, "s3://" + bucket + "/warehouse");
         properties.put(CatalogProperties.FILE_IO_IMPL, S3FileIO.class.getName());

diff --git a/extensions/s3/src/test/java/io/deephaven/extensions/s3/testlib/S3Helper.java b/extensions/s3/src/test/java/io/deephaven/extensions/s3/testlib/S3Helper.java
@@ -26,6 +26,12 @@
 import java.util.stream.Collectors;
 
 public final class S3Helper {
+
+    /**
+     * Timeout in seconds for S3 operations for testing.
+     */
+    public static final int TIMEOUT_SECONDS = 30;
+
     public static void uploadDirectory(
             S3AsyncClient s3AsyncClient,
             Path dir,
@@ -59,7 +65,8 @@ private static void uploadDirectory(
     public static void deleteAllKeys(S3AsyncClient s3AsyncClient, String bucket)
             throws ExecutionException, InterruptedException, TimeoutException {
         ListObjectsV2Response response = s3AsyncClient
-                .listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).build()).get(5, TimeUnit.SECONDS);
+                .listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).build())
+                .get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
         final List<CompletableFuture<?>> futures = new ArrayList<>();
         while (true) {
             final List<ObjectIdentifier> deletes = response.contents()
@@ -80,12 +87,13 @@ public static void deleteAllKeys(S3AsyncClient s3AsyncClient, String bucket)
             }
             response = s3AsyncClient.listObjectsV2(
                     ListObjectsV2Request.builder().bucket(bucket).continuationToken(nextContinuationToken).build())
-                    .get(5, TimeUnit.SECONDS);
+                    .get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
         }
         if (futures.isEmpty()) {
             return;
         }
-        CompletableFuture.allOf(futures.stream().toArray(CompletableFuture[]::new)).get(5, TimeUnit.SECONDS);
+        CompletableFuture.allOf(futures.stream().toArray(CompletableFuture[]::new))
+                .get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
     }
 
     private static ObjectIdentifier objectId(String o) {

diff --git a/...sions/s3/src/test/java/io/deephaven/extensions/s3/testlib/S3SeekableChannelTestSetup.java b/...sions/s3/src/test/java/io/deephaven/extensions/s3/testlib/S3SeekableChannelTestSetup.java
@@ -25,6 +25,8 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import static io.deephaven.extensions.s3.testlib.S3Helper.TIMEOUT_SECONDS;
+
 public abstract class S3SeekableChannelTestSetup {
 
     protected ExecutorService executor;
@@ -39,19 +41,21 @@ protected final void doSetUp() throws ExecutionException, InterruptedException,
         executor = Executors.newCachedThreadPool();
         bucket = UUID.randomUUID().toString();
         asyncClient = s3AsyncClient();
-        asyncClient.createBucket(CreateBucketRequest.builder().bucket(bucket).build()).get(5, TimeUnit.SECONDS);
+        asyncClient.createBucket(CreateBucketRequest.builder().bucket(bucket).build())
+                .get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
     }
 
     protected final void doTearDown() throws ExecutionException, InterruptedException, TimeoutException {
         S3Helper.deleteAllKeys(asyncClient, bucket);
-        asyncClient.deleteBucket(DeleteBucketRequest.builder().bucket(bucket).build()).get(5, TimeUnit.SECONDS);
+        asyncClient.deleteBucket(DeleteBucketRequest.builder().bucket(bucket).build())
+                .get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
         asyncClient.close();
         executor.shutdownNow();
     }
 
     protected void uploadDirectory(final Path directory, final String prefix)
             throws ExecutionException, InterruptedException, TimeoutException {
-        S3Helper.uploadDirectory(asyncClient, directory, bucket, prefix, Duration.ofSeconds(5));
+        S3Helper.uploadDirectory(asyncClient, directory, bucket, prefix, Duration.ofSeconds(TIMEOUT_SECONDS));
     }
 
     protected final URI uri(String key) {