Add ability to start and stop a database docker container for use whi… (

#36)

pom.xml

@@ -147,6 +147,33 @@
             <scope>compile</scope>
         </dependency>
 
+        <!-- docker java -->
+        <dependency>
+            <groupId>com.github.docker-java</groupId>
+            <artifactId>docker-java</artifactId>
+            <version>3.3.4</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.github.docker-java</groupId>
+                    <artifactId>docker-java-transport-jersey</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.github.docker-java</groupId>
+                    <artifactId>docker-java-transport-netty</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.github.docker-java</groupId>
+            <artifactId>docker-java-api</artifactId>
+            <version>3.3.4</version>
+        </dependency>
+        <dependency>
+            <groupId>com.github.docker-java</groupId>
+            <artifactId>docker-java-transport-httpclient5</artifactId>
+            <version>3.3.4</version>
+        </dependency>
+
         <!-- test dependencies -->
         <dependency>
             <groupId>org.hamcrest</groupId>

src/main/java/org/pih/petl/DockerConnector.java

@@ -0,0 +1,84 @@
+/**
+ * This Source Code Form is subject to the terms of the Mozilla Public License,
+ * v. 2.0. If a copy of the MPL was not distributed with this file, You can
+ * obtain one at http://mozilla.org/MPL/2.0/. OpenMRS is also distributed under
+ * the terms of the Healthcare Disclaimer located at http://openmrs.org/license.
+ *
+ * Copyright (C) OpenMRS Inc. OpenMRS is a registered trademark and the OpenMRS
+ * graphic logo is a trademark of OpenMRS Inc.
+ */
+package org.pih.petl;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.model.Container;
+import com.github.dockerjava.core.DefaultDockerClientConfig;
+import com.github.dockerjava.core.DockerClientConfig;
+import com.github.dockerjava.core.DockerClientImpl;
+import com.github.dockerjava.httpclient5.ApacheDockerHttpClient;
+import com.github.dockerjava.transport.DockerHttpClient;
+import org.apache.commons.io.IOUtils;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Utility methods useful for manipulating SQL statements
+ */
+public class DockerConnector implements Closeable {
+
+    private final DockerClientConfig dockerClientConfig;
+    private final DockerHttpClient dockerHttpClient;
+    private final DockerClient dockerClient;
+
+    private DockerConnector() {
+        dockerClientConfig = DefaultDockerClientConfig.createDefaultConfigBuilder().build();
+        dockerHttpClient = new ApacheDockerHttpClient.Builder()
+                .dockerHost(dockerClientConfig.getDockerHost())
+                .sslConfig(dockerClientConfig.getSSLConfig())
+                .maxConnections(100)
+                .connectionTimeout(Duration.ofSeconds(30))
+                .responseTimeout(Duration.ofSeconds(45))
+                .build();
+        dockerClient = DockerClientImpl.getInstance(dockerClientConfig, dockerHttpClient);
+    }
+
+    public static DockerConnector open() {
+        return new DockerConnector();
+    }
+
+    public void close() {
+        IOUtils.closeQuietly(dockerClient);
+    }
+
+    public List<Container> getContainers() {
+        return dockerClient.listContainersCmd().withShowAll(true).exec();
+    }
+
+    public Container getContainer(String containerName) {
+        for (Container container : getContainers()) {
+            List<String> names = Arrays.asList(container.getNames());
+            if (names.contains(containerName) || names.contains("/" + containerName)) {
+                return container;
+            }
+        }
+        return null;
+    }
+
+    public boolean containerExists(String containerName) {
+        return getContainer(containerName) != null;
+    }
+
+    public boolean isContainerRunning(Container container) {
+        return "running".equalsIgnoreCase(container.getState());
+    }
+
+    public void startContainer(Container container) {
+        dockerClient.startContainerCmd(container.getId()).exec();
+    }
+
+    public void stopContainer(Container container) {
+        dockerClient.stopContainerCmd(container.getId()).exec();
+    }
+}

src/main/java/org/pih/petl/job/RunMultipleJob.java

@@ -1,12 +1,16 @@
 package org.pih.petl.job;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import com.github.dockerjava.api.model.Container;
+import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.pih.petl.DockerConnector;
 import org.pih.petl.api.EtlService;
 import org.pih.petl.api.JobExecution;
 import org.pih.petl.api.JobExecutionTask;
 import org.pih.petl.api.JobExecutor;
+import org.pih.petl.job.config.DataSource;
 import org.pih.petl.job.config.JobConfig;
 import org.pih.petl.job.config.JobConfigReader;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -33,7 +37,9 @@ public class RunMultipleJob implements PetlJob {
     public void execute(final JobExecution jobExecution) throws Exception {
         JobConfigReader configReader = new JobConfigReader(etlService.getApplicationConfig(), jobExecution.getJobConfig());
         JobExecutor jobExecutor = new JobExecutor(etlService, 1);
+        List<String> containersStarted = new ArrayList<>();
         try {
+            containersStarted = startContainersIfNecessary(configReader);
             List<JsonNode> jobTemplates = configReader.getList("jobs");
             log.debug("Executing " + jobTemplates.size() + " jobs");
             List<JobExecutionTask> tasks = new ArrayList<>();
@@ -46,7 +52,60 @@ public void execute(final JobExecution jobExecution) throws Exception {
             jobExecutor.executeInSeries(tasks);
         }
         finally {
+            stopContainers(containersStarted);
             jobExecutor.shutdown();
         }
     }
+
+    /**
+     * starts any containers referenced in datasources that are not already started
+     * @return a list of container names that were newly started
+     */
+    private List<String> startContainersIfNecessary(JobConfigReader configReader) {
+        List<String> ret = new ArrayList<>();
+        for (DataSource dataSource : configReader.getDataSources("datasources")) {
+            String containerName = dataSource.getContainerName();
+            if (StringUtils.isNotBlank(containerName)) {
+                log.info("Checking if container '" + containerName + "' is started");
+                try (DockerConnector docker = DockerConnector.open()) {
+                    Container container = docker.getContainer(containerName);
+                    if (container != null) {
+                        if (docker.isContainerRunning(container)) {
+                            log.info("Container '" + containerName + "' is already running");
+                        }
+                        else {
+                            log.info("Container '" + containerName + "' is not already running, starting it");
+                            docker.startContainer(container);
+                            ret.add(containerName);
+                            log.info("Container started");
+                        }
+                    }
+                    else {
+                        log.warn("No container named " + containerName + " found, skipping");
+                    }
+                }
+            }
+        }
+        return ret;
+    }
+
+    private void stopContainers(List<String> containersToStop) {
+        if (containersToStop != null) {
+            for (String containerName : containersToStop) {
+                log.info("Stopping previously started container " + containerName);
+                try (DockerConnector docker = DockerConnector.open()) {
+                    Container container = docker.getContainer(containerName);
+                    if (container != null) {
+                        if (docker.isContainerRunning(container)) {
+                            docker.stopContainer(container);
+                            log.info("Container '" + containerName + "' stopped");
+                        }
+                        else {
+                            log.info("Container '" + containerName + "' is not running");
+                        }
+                    }
+                }
+            }
+        }
+    }
 }

src/main/java/org/pih/petl/job/config/DataSource.java

@@ -34,6 +34,7 @@ public class DataSource {
     private String url; // Alternative to the above piecemeal settings
     private String user;
     private String password;
+    private String containerName; // If this database is in a docker container, can configure the container name here
 
     //***** CONSTRUCTORS *****
 
@@ -267,4 +268,12 @@ public String getPassword() {
     public void setPassword(String password) {
         this.password = password;
     }
+
+    public String getContainerName() {
+        return containerName;
+    }
+
+    public void setContainerName(String containerName) {
+        this.containerName = containerName;
+    }
 }

src/main/java/org/pih/petl/job/config/JobConfigReader.java

@@ -104,6 +104,14 @@ public DataSource getDataSource(String... keys) {
         return appConfig.getEtlDataSource(path);
     }
 
+    public List<DataSource> getDataSources(String... keys) {
+        List<DataSource> ret = new ArrayList<>();
+        for (String path : getStringList(keys)) {
+            ret.add(appConfig.getEtlDataSource(path));
+        }
+        return ret;
+    }
+
     /**
      * Convenience to get the configuration of a given setting as a String
      */

src/test/java/org/pih/petl/DockerConnectorTest.java

@@ -0,0 +1,18 @@
+package org.pih.petl;
+
+import com.github.dockerjava.api.model.Container;
+
+import java.util.Arrays;
+
+public class DockerConnectorTest {
+
+    public void testIsRunning() {
+        try (DockerConnector docker = DockerConnector.open()) {
+            for (Container container : docker.getContainers()) {
+                System.out.println("Container: " + Arrays.asList(container.getNames()));
+                System.out.println("Is running: " + docker.isContainerRunning(container));
+            }
+        }
+    }
+
+}