Fully enable proxy

uploader/Dockerfile

@@ -16,4 +16,4 @@ COPY --from=build target/uploader-*.jar /dataupload.jar
 # Time zone
 ENV TZ="US/Eastern"
 
-ENTRYPOINT ["java", "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005", "-Xmx8192m", "-jar", "/dataupload.jar"]
+ENTRYPOINT java $DEBUG_VARS $PROXY_VARS -Xmx8192m -jar /dataupload.jar

uploader/docker-compose.yml

@@ -2,7 +2,7 @@
 version: '3'
 services:
   uploader:
-    image: lukesikinabch/gic-uploader:1.2.0
+    image: lukesikinabch/gic-uploader:1.2.1
     container_name: uploader
     ports:
       - "5005:5005"

uploader/env-proto

@@ -20,3 +20,6 @@ INSTITUTIONS=bch-dev
 HOME_INSTITUTION_NAME=bch-dev
 HOME_INSTITUTION_DISPLAY=BCH
 HOME_INSTITUTION_LONG_DISPLAY=Boston Children's Hospital
+
+DEBUG_VARS=
+PROXY_VARS=

uploader/pom.xml

@@ -57,6 +57,11 @@
 			<artifactId>kms</artifactId>
 			<version>${aws.version}</version>
 		</dependency>
+		<dependency>
+			<groupId>software.amazon.awssdk</groupId>
+			<artifactId>apache-client</artifactId>
+			<version>${aws.version}</version>
+		</dependency>
 		<dependency>
 			<groupId>software.amazon.encryption.s3</groupId>
 			<artifactId>amazon-s3-encryption-client-java</artifactId>

uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSConfiguration.java

@@ -9,7 +9,9 @@
 import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.util.StringUtils;
 import software.amazon.awssdk.auth.credentials.*;
+import software.amazon.awssdk.http.SdkHttpClient;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.sts.StsClient;
 import software.amazon.awssdk.services.sts.StsClientBuilder;
@@ -18,7 +20,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 @ConditionalOnProperty(name = "production", havingValue = "true")
@@ -53,16 +54,27 @@ public class AWSConfiguration {
     @Autowired
     private ConfigurableApplicationContext context;
 
+    @Autowired(required = false)
+    private SdkHttpClient sdkHttpClient;
+
+    @Value("${http.proxyUser:}")
+    private String proxyUser;
+
     @Bean
     @ConditionalOnProperty(name = "production", havingValue = "true")
     public StsClient stsClients(
         @Autowired AwsCredentials credentials,
         @Autowired StsClientBuilder stsClientBuilder
     ) {
-        return stsClientBuilder
+        StsClientBuilder builder = stsClientBuilder
             .region(Region.US_EAST_1)
-            .credentialsProvider(StaticCredentialsProvider.create(credentials))
-            .build();
+            .credentialsProvider(StaticCredentialsProvider.create(credentials));
+
+        if (StringUtils.hasLength(proxyUser)) {
+            builder.httpClient(sdkHttpClient);
+        }
+
+        return builder.build();
     }
 
     @Bean

uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/SelfRefreshingS3Client.java

@@ -11,8 +11,10 @@
 import org.springframework.stereotype.Service;
 import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
 import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.http.SdkHttpClient;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
 import software.amazon.awssdk.services.sts.StsClient;
 import software.amazon.awssdk.services.sts.model.AssumeRoleRequest;
 import software.amazon.awssdk.services.sts.model.AssumeRoleResponse;
@@ -54,6 +56,9 @@ public class SelfRefreshingS3Client {
     @Autowired
     StatusService statusService;
 
+    @Autowired(required = false)
+    private SdkHttpClient sdkHttpClient;
+
     @PostConstruct
     private void refreshClient() {
         locks = roleARNs.keySet().stream()
@@ -95,12 +100,14 @@ void refreshClient(String siteName) {
             .expirationTime(credentials.expiration())
             .build();
         StaticCredentialsProvider provider = StaticCredentialsProvider.create(sessionCredentials);
-        S3Client client = S3Client.builder()
+        S3ClientBuilder builder = S3Client.builder()
             .credentialsProvider(provider)
-            .region(Region.US_EAST_1)
-            .build();
+            .region(Region.US_EAST_1);
+        if (sdkHttpClient != null) {
+            builder.httpClient(sdkHttpClient);
+        }
         LOG.info("Created S3 client");
-        clients.put(siteName, client);
+        clients.put(siteName, builder.build());
         // now that client is refreshed, unlock for reading
         LOG.info("Unlocking s3 client. Session refreshed");
         locks.get(siteName).writeLock().unlock();

uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClient.java

@@ -5,17 +5,20 @@
 import edu.harvard.dbmi.avillach.dataupload.hpds.hpdsartifactsdonotchange.Query;
 import edu.harvard.dbmi.avillach.domain.GeneralQueryRequest;
 import edu.harvard.dbmi.avillach.domain.QueryRequest;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.entity.StringEntity;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.net.URI;
-import java.net.http.HttpClient;
-import java.net.http.HttpRequest;
-import java.net.http.HttpRequest.BodyPublishers;
-import java.net.http.HttpResponse;
+import java.util.Optional;
 
 @Service
 public class HPDSClient {
@@ -27,6 +30,9 @@ public class HPDSClient {
     @Autowired
     private HttpClient client;
 
+    @Autowired
+    private HttpClientContext context;
+
     public boolean writePhenotypicData(Query query) {
         return writeData(query, "phenotypic");
     }
@@ -44,13 +50,11 @@ public boolean initializeQuery(Query query) {
             return false;
         }
 
-        HttpRequest request = HttpRequest.newBuilder()
-            .uri(URI.create(HPDS_URI + "query/sync"))
-            .POST(BodyPublishers.ofString(body))
-            .setHeader("Content-Type", "application/json")
-            .build();
+        Optional<HttpPost> maybePost = createPost(HPDS_URI + "query/sync", body);
 
-        return sendAndVerifyRequest(request);
+        return maybePost
+            .map(this::sendAndVerifyRequest)
+            .orElse(false);
     }
 
     private boolean writeData(Query query, String mode) {
@@ -59,20 +63,31 @@ private boolean writeData(Query query, String mode) {
             return false;
         }
 
-        HttpRequest request = HttpRequest.newBuilder()
-            .uri(URI.create(HPDS_URI + "write/" + mode))
-            .POST(BodyPublishers.ofString(body))
-            .setHeader("Content-Type", "application/json")
-            .build();
+        Optional<HttpPost> maybePost = createPost(HPDS_URI + "write/" + mode, body);
+
+        return maybePost
+            .map(this::sendAndVerifyRequest)
+            .orElse(false);
+    }
+
+    private Optional<HttpPost> createPost(String uri, String body) {
+        HttpPost request = new HttpPost(URI.create(uri));
+        try {
+            request.setEntity(new StringEntity(body));
+        } catch (UnsupportedEncodingException e) {
+            LOG.error("Error making request body", e);
+            return Optional.empty();
+        }
+        request.setHeader("Content-Type", "application/json");
 
-        return sendAndVerifyRequest(request);
+        return Optional.of(request);
     }
 
-    private boolean sendAndVerifyRequest(HttpRequest request) {
+    private boolean sendAndVerifyRequest(HttpPost request) {
         try {
-            HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
-            return response.statusCode() == 200;
-        } catch (IOException | InterruptedException e) {
+            HttpResponse response = client.execute(request, context);
+            return response.getStatusLine().getStatusCode() == 200;
+        } catch (IOException e) {
             LOG.error("Error making request", e);
             return false;
         }

uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HttpClientConfig.java

@@ -1,15 +1,75 @@
 package edu.harvard.dbmi.avillach.dataupload.hpds;
 
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
-
-import java.net.http.HttpClient;
+import org.springframework.util.StringUtils;
+import software.amazon.awssdk.http.SdkHttpClient;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.http.apache.ProxyConfiguration;
 
 @Configuration
 public class HttpClientConfig {
+    private static final Logger LOG = LoggerFactory.getLogger(HttpClientConfig.class);
+
+    @Value("${http.proxyUser:}")
+    private String proxyUser;
+
+    @Value("${http.proxyPassword:}")
+    private String proxyPassword;
 
     @Bean
     public HttpClient getHttpClient() {
-        return HttpClient.newHttpClient();
+        if (!StringUtils.hasLength(proxyUser)) {
+            return HttpClients.createDefault();
+        }
+        LOG.info("Found proxy user {}, will configure proxy", proxyUser);
+        PoolingHttpClientConnectionManager manager = new PoolingHttpClientConnectionManager();
+        manager.setMaxTotal(100);
+        return HttpClients
+            .custom()
+            .setConnectionManager(new PoolingHttpClientConnectionManager())
+            .useSystemProperties()
+            .build();
+    }
+
+
+    @Bean
+    public SdkHttpClient getSdkClient() {
+        if (!StringUtils.hasLength(proxyUser)) {
+            return null;
+        }
+        LOG.info("Found proxy user {}, will configure sdk proxy", proxyUser);
+        ProxyConfiguration proxy = ProxyConfiguration.builder()
+            .useSystemPropertyValues(true)
+            .username(proxyUser)
+            .password(proxyPassword)
+            .build();
+        return ApacheHttpClient.builder()
+            .proxyConfiguration(proxy)
+            .build();
+    }
+
+    @Bean
+    public HttpClientContext getClientConfig() {
+        if (StringUtils.hasLength(proxyUser) && StringUtils.hasLength(proxyPassword)) {
+            HttpClientContext httpClientContext = HttpClientContext.create();
+            CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+            credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(proxyUser, proxyPassword));
+            httpClientContext.setCredentialsProvider(credentialsProvider);
+
+            return httpClientContext;
+        }
+        return HttpClientContext.create();
     }
 }

uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadService.java

@@ -40,6 +40,9 @@ public class DataUploadService {
     @Value("${aws.s3.access_key_id}")
     private String keyId;
 
+    @Value("${institution.name}")
+    private String home;
+
     @Autowired
     private SelfRefreshingS3Client s3;
 
@@ -52,9 +55,6 @@ public class DataUploadService {
     @Autowired
     private Path sharingRoot;
 
-    @Value("${institution.name}")
-    private String home;
-
     @Autowired
     private Map<String, SiteAWSInfo> roleARNs;
 

uploader/src/main/resources/application.properties

@@ -20,4 +20,4 @@ file_sharing_root=${FILE_SHARING_ROOT}
 institution.name=${HOME_INSTITUTION_NAME}
 institution.short-display=${HOME_INSTITUTION_DISPLAY}
 institution.long-display=${HOME_INSTITUTION_LONG_DISPLAY}
-server.port=${PORT:80}
+server.port=${PORT:80}