diff --git a/pom.xml b/pom.xml index 47fa82c..3ffeaf3 100644 --- a/pom.xml +++ b/pom.xml @@ -147,6 +147,33 @@ compile + + + com.github.docker-java + docker-java + 3.3.4 + + + com.github.docker-java + docker-java-transport-jersey + + + com.github.docker-java + docker-java-transport-netty + + + + + com.github.docker-java + docker-java-api + 3.3.4 + + + com.github.docker-java + docker-java-transport-httpclient5 + 3.3.4 + + org.hamcrest diff --git a/src/main/java/org/pih/petl/DockerConnector.java b/src/main/java/org/pih/petl/DockerConnector.java new file mode 100644 index 0000000..82044ea --- /dev/null +++ b/src/main/java/org/pih/petl/DockerConnector.java @@ -0,0 +1,84 @@ +/** + * This Source Code Form is subject to the terms of the Mozilla Public License, + * v. 2.0. If a copy of the MPL was not distributed with this file, You can + * obtain one at http://mozilla.org/MPL/2.0/. OpenMRS is also distributed under + * the terms of the Healthcare Disclaimer located at http://openmrs.org/license. + * + * Copyright (C) OpenMRS Inc. OpenMRS is a registered trademark and the OpenMRS + * graphic logo is a trademark of OpenMRS Inc. + */ +package org.pih.petl; + +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.model.Container; +import com.github.dockerjava.core.DefaultDockerClientConfig; +import com.github.dockerjava.core.DockerClientConfig; +import com.github.dockerjava.core.DockerClientImpl; +import com.github.dockerjava.httpclient5.ApacheDockerHttpClient; +import com.github.dockerjava.transport.DockerHttpClient; +import org.apache.commons.io.IOUtils; + +import java.io.Closeable; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; + +/** + * Utility methods useful for manipulating SQL statements + */ +public class DockerConnector implements Closeable { + + private final DockerClientConfig dockerClientConfig; + private final DockerHttpClient dockerHttpClient; + private final DockerClient dockerClient; + + private DockerConnector() { + dockerClientConfig = DefaultDockerClientConfig.createDefaultConfigBuilder().build(); + dockerHttpClient = new ApacheDockerHttpClient.Builder() + .dockerHost(dockerClientConfig.getDockerHost()) + .sslConfig(dockerClientConfig.getSSLConfig()) + .maxConnections(100) + .connectionTimeout(Duration.ofSeconds(30)) + .responseTimeout(Duration.ofSeconds(45)) + .build(); + dockerClient = DockerClientImpl.getInstance(dockerClientConfig, dockerHttpClient); + } + + public static DockerConnector open() { + return new DockerConnector(); + } + + public void close() { + IOUtils.closeQuietly(dockerClient); + } + + public List getContainers() { + return dockerClient.listContainersCmd().withShowAll(true).exec(); + } + + public Container getContainer(String containerName) { + for (Container container : getContainers()) { + List names = Arrays.asList(container.getNames()); + if (names.contains(containerName) || names.contains("/" + containerName)) { + return container; + } + } + return null; + } + + public boolean containerExists(String containerName) { + return getContainer(containerName) != null; + } + + public boolean isContainerRunning(Container container) { + return "running".equalsIgnoreCase(container.getState()); + } + + public void startContainer(Container container) { + dockerClient.startContainerCmd(container.getId()).exec(); + } + + public void stopContainer(Container container) { + dockerClient.stopContainerCmd(container.getId()).exec(); + } +} diff --git a/src/main/java/org/pih/petl/job/RunMultipleJob.java b/src/main/java/org/pih/petl/job/RunMultipleJob.java index 88d203e..d2e1e2a 100644 --- a/src/main/java/org/pih/petl/job/RunMultipleJob.java +++ b/src/main/java/org/pih/petl/job/RunMultipleJob.java @@ -1,12 +1,16 @@ package org.pih.petl.job; import com.fasterxml.jackson.databind.JsonNode; +import com.github.dockerjava.api.model.Container; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.pih.petl.DockerConnector; import org.pih.petl.api.EtlService; import org.pih.petl.api.JobExecution; import org.pih.petl.api.JobExecutionTask; import org.pih.petl.api.JobExecutor; +import org.pih.petl.job.config.DataSource; import org.pih.petl.job.config.JobConfig; import org.pih.petl.job.config.JobConfigReader; import org.springframework.beans.factory.annotation.Autowired; @@ -33,7 +37,9 @@ public class RunMultipleJob implements PetlJob { public void execute(final JobExecution jobExecution) throws Exception { JobConfigReader configReader = new JobConfigReader(etlService.getApplicationConfig(), jobExecution.getJobConfig()); JobExecutor jobExecutor = new JobExecutor(etlService, 1); + List containersStarted = new ArrayList<>(); try { + containersStarted = startContainersIfNecessary(configReader); List jobTemplates = configReader.getList("jobs"); log.debug("Executing " + jobTemplates.size() + " jobs"); List tasks = new ArrayList<>(); @@ -46,7 +52,60 @@ public void execute(final JobExecution jobExecution) throws Exception { jobExecutor.executeInSeries(tasks); } finally { + stopContainers(containersStarted); jobExecutor.shutdown(); } } + + /** + * starts any containers referenced in datasources that are not already started + * @return a list of container names that were newly started + */ + private List startContainersIfNecessary(JobConfigReader configReader) { + List ret = new ArrayList<>(); + for (DataSource dataSource : configReader.getDataSources("datasources")) { + String containerName = dataSource.getContainerName(); + if (StringUtils.isNotBlank(containerName)) { + log.info("Checking if container '" + containerName + "' is started"); + try (DockerConnector docker = DockerConnector.open()) { + Container container = docker.getContainer(containerName); + if (container != null) { + if (docker.isContainerRunning(container)) { + log.info("Container '" + containerName + "' is already running"); + } + else { + log.info("Container '" + containerName + "' is not already running, starting it"); + docker.startContainer(container); + ret.add(containerName); + log.info("Container started"); + } + } + else { + log.warn("No container named " + containerName + " found, skipping"); + } + } + } + } + return ret; + } + + private void stopContainers(List containersToStop) { + if (containersToStop != null) { + for (String containerName : containersToStop) { + log.info("Stopping previously started container " + containerName); + try (DockerConnector docker = DockerConnector.open()) { + Container container = docker.getContainer(containerName); + if (container != null) { + if (docker.isContainerRunning(container)) { + docker.stopContainer(container); + log.info("Container '" + containerName + "' stopped"); + } + else { + log.info("Container '" + containerName + "' is not running"); + } + } + } + } + } + } } diff --git a/src/main/java/org/pih/petl/job/config/DataSource.java b/src/main/java/org/pih/petl/job/config/DataSource.java index 99296a0..4aad9b4 100644 --- a/src/main/java/org/pih/petl/job/config/DataSource.java +++ b/src/main/java/org/pih/petl/job/config/DataSource.java @@ -34,6 +34,7 @@ public class DataSource { private String url; // Alternative to the above piecemeal settings private String user; private String password; + private String containerName; // If this database is in a docker container, can configure the container name here //***** CONSTRUCTORS ***** @@ -267,4 +268,12 @@ public String getPassword() { public void setPassword(String password) { this.password = password; } + + public String getContainerName() { + return containerName; + } + + public void setContainerName(String containerName) { + this.containerName = containerName; + } } diff --git a/src/main/java/org/pih/petl/job/config/JobConfigReader.java b/src/main/java/org/pih/petl/job/config/JobConfigReader.java index e36f58f..3ab6fd4 100644 --- a/src/main/java/org/pih/petl/job/config/JobConfigReader.java +++ b/src/main/java/org/pih/petl/job/config/JobConfigReader.java @@ -104,6 +104,14 @@ public DataSource getDataSource(String... keys) { return appConfig.getEtlDataSource(path); } + public List getDataSources(String... keys) { + List ret = new ArrayList<>(); + for (String path : getStringList(keys)) { + ret.add(appConfig.getEtlDataSource(path)); + } + return ret; + } + /** * Convenience to get the configuration of a given setting as a String */ diff --git a/src/test/java/org/pih/petl/DockerConnectorTest.java b/src/test/java/org/pih/petl/DockerConnectorTest.java new file mode 100644 index 0000000..dcd1f90 --- /dev/null +++ b/src/test/java/org/pih/petl/DockerConnectorTest.java @@ -0,0 +1,18 @@ +package org.pih.petl; + +import com.github.dockerjava.api.model.Container; + +import java.util.Arrays; + +public class DockerConnectorTest { + + public void testIsRunning() { + try (DockerConnector docker = DockerConnector.open()) { + for (Container container : docker.getContainers()) { + System.out.println("Container: " + Arrays.asList(container.getNames())); + System.out.println("Is running: " + docker.isContainerRunning(container)); + } + } + } + +}