From 6d9b5bd3a3d947ded754f3595ca6665d61336a61 Mon Sep 17 00:00:00 2001 From: Andrew Mains Date: Mon, 29 Aug 2022 16:14:48 -0400 Subject: [PATCH] etcd_docker 5: Incorporate docker based etcd approach into docker integration tests. PR 5 for https://github.com/m3db/m3/issues/4144 This PR makes the docker integration tests use containerized etcd. Previously, these relied on M3DB running an embbeded etcd server. There's no inherent need for this, and it opens us up to dependency issues as described in the linked github issue. Note: there are a handful that require multiple servers; these are currently "skipped" (commented). I intend to bring those back at a later date.. commit-id:e67a5172 --- .../aggregator/docker-compose.yml | 22 ++++++-- .../aggregator/m3aggregator.yml | 2 +- .../aggregator/m3coordinator.yml | 2 +- .../aggregator/test.sh | 2 + .../aggregator_legacy/docker-compose.yml | 18 +++++-- .../aggregator_legacy/m3aggregator.yml | 2 +- .../aggregator_legacy/m3coordinator.yml | 2 +- .../aggregator_legacy/test.sh | 3 ++ .../carbon/docker-compose.yml | 18 +++++-- .../carbon/m3coordinator.yml | 2 +- .../docker-integration-tests/carbon/test.sh | 5 +- .../cold_writes_simple/docker-compose.yml | 18 +++++-- .../cold_writes_simple/m3coordinator.yml | 2 +- .../cold_writes_simple/test.sh | 5 +- scripts/docker-integration-tests/common.sh | 2 + .../docker-compose.yml | 18 +++++-- .../m3coordinator.yml | 2 +- .../coordinator_config_rules/test.sh | 3 +- .../coordinator_noop/docker-compose.yml | 42 +++++---------- .../coordinator_noop/m3coordinator.yml | 2 +- .../coordinator_noop/test.sh | 3 +- .../docker-compose.yml | 42 +++++---------- .../docker-compose-etcd.yml | 15 ++++++ .../m3coordinator.Dockerfile | 2 +- .../m3dbnode.Dockerfile | 4 +- .../multi_cluster_write/docker-compose.yml | 24 ++++++++- .../m3coordinator-cluster-a.yml | 2 +- .../m3coordinator-cluster-b.yml | 2 +- .../m3dbnode-cluster-a.yml | 6 +-- .../m3dbnode-cluster-b.yml | 2 +- .../docker-compose.yml | 52 ++++++++----------- .../m3aggregator.yml | 2 +- .../m3coordinator-admin.yml | 2 +- .../m3coordinator.yml | 2 +- .../prom_remote_write_backend/test.sh | 2 +- .../prometheus/docker-compose.yml | 20 +++++-- .../prometheus/m3coordinator.yml | 2 +- .../prometheus_replication/docker-compose.yml | 30 +++++++++-- .../m3coordinator01.yml | 2 +- .../m3coordinator02.yml | 2 +- .../prometheus_replication/test.sh | 10 ++-- .../query_fanout/docker-compose.yml | 24 ++++++++- .../repair/docker-compose.yml | 20 +++++-- .../repair/m3coordinator.yml | 2 +- .../repair/m3dbnode.yml | 7 +-- .../docker-integration-tests/repair/test.sh | 1 + .../replication/docker-compose.yml | 24 ++++++++- .../replication/m3coordinator-cluster-a.yml | 2 +- .../replication/m3coordinator-cluster-b.yml | 2 +- .../replication/m3dbnode-cluster-a.yml | 4 +- .../replication/m3dbnode-cluster-b.yml | 4 +- scripts/docker-integration-tests/run.sh | 20 +++++-- scripts/docker-integration-tests/setup.sh | 8 +-- .../simple_v2_batch_apis/docker-compose.yml | 18 +++++-- .../simple_v2_batch_apis/m3coordinator.yml | 2 +- .../config/m3dbnode-local-docker-etcd.yml | 11 ++++ .../m3coordinator-local-docker-etcd.yml | 16 ++++++ 57 files changed, 384 insertions(+), 181 deletions(-) create mode 100644 scripts/docker-integration-tests/docker-compose-etcd.yml create mode 100644 src/dbnode/config/m3dbnode-local-docker-etcd.yml create mode 100644 src/query/config/m3coordinator-local-docker-etcd.yml diff --git a/scripts/docker-integration-tests/aggregator/docker-compose.yml b/scripts/docker-integration-tests/aggregator/docker-compose.yml index c93b41ee25..355e1895c3 100644 --- a/scripts/docker-integration-tests/aggregator/docker-compose.yml +++ b/scripts/docker-integration-tests/aggregator/docker-compose.yml @@ -1,17 +1,27 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" - "7201" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" - "0.0.0.0:7201:7201" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd m3coordinator01: expose: - "7202" @@ -26,6 +36,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd m3aggregator01: expose: - "6001" @@ -38,6 +50,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3aggregator02: networks: - backend @@ -46,5 +60,7 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/aggregator/m3aggregator.yml b/scripts/docker-integration-tests/aggregator/m3aggregator.yml index c2f26330bd..1aa1c7c091 100644 --- a/scripts/docker-integration-tests/aggregator/m3aggregator.yml +++ b/scripts/docker-integration-tests/aggregator/m3aggregator.yml @@ -38,7 +38,7 @@ kvClient: autoSyncInterval: 10m dialTimeout: 1m endpoints: - - dbnode01:2379 + - etcd:2379 runtimeOptions: kvConfig: diff --git a/scripts/docker-integration-tests/aggregator/m3coordinator.yml b/scripts/docker-integration-tests/aggregator/m3coordinator.yml index dfc757b8c4..a1e44edda4 100644 --- a/scripts/docker-integration-tests/aggregator/m3coordinator.yml +++ b/scripts/docker-integration-tests/aggregator/m3coordinator.yml @@ -24,7 +24,7 @@ clusters: autoSyncInterval: 10m dialTimeout: 1m endpoints: - - dbnode01:2379 + - etcd:2379 downsample: rules: diff --git a/scripts/docker-integration-tests/aggregator/test.sh b/scripts/docker-integration-tests/aggregator/test.sh index a65b912c8a..a5b44122c8 100755 --- a/scripts/docker-integration-tests/aggregator/test.sh +++ b/scripts/docker-integration-tests/aggregator/test.sh @@ -14,6 +14,8 @@ echo "Pull containers required for test" docker pull $PROMREMOTECLI_IMAGE docker pull $JQ_IMAGE +docker-compose -f ${COMPOSE_FILE} up -d etcd + echo "Run m3dbnode" docker-compose -f ${COMPOSE_FILE} up -d dbnode01 diff --git a/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml b/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml index c93b41ee25..ca6606f627 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml @@ -1,17 +1,23 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" - "7201" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" - "0.0.0.0:7201:7201" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd m3coordinator01: expose: - "7202" @@ -26,6 +32,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd m3aggregator01: expose: - "6001" @@ -38,6 +46,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3aggregator02: networks: - backend @@ -46,5 +56,7 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml index 569ea9a9c2..e10c7cbe96 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml @@ -57,7 +57,7 @@ kvClient: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 runtimeOptions: kvConfig: diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml index 281662faca..d15fd1a78c 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml @@ -22,7 +22,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 downsample: remoteAggregator: diff --git a/scripts/docker-integration-tests/aggregator_legacy/test.sh b/scripts/docker-integration-tests/aggregator_legacy/test.sh index e101eae186..7f5050005b 100755 --- a/scripts/docker-integration-tests/aggregator_legacy/test.sh +++ b/scripts/docker-integration-tests/aggregator_legacy/test.sh @@ -7,6 +7,9 @@ REVISION=$(git rev-parse HEAD) COMPOSE_FILE="$M3_PATH"/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml export REVISION +echo "Run etcd" +docker-compose -f ${COMPOSE_FILE} up -d etcd + echo "Run m3dbnode" docker-compose -f ${COMPOSE_FILE} up -d dbnode01 diff --git a/scripts/docker-integration-tests/carbon/docker-compose.yml b/scripts/docker-integration-tests/carbon/docker-compose.yml index 53a28f0b88..e6b97c2e0f 100644 --- a/scripts/docker-integration-tests/carbon/docker-compose.yml +++ b/scripts/docker-integration-tests/carbon/docker-compose.yml @@ -1,15 +1,25 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,5 +34,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/carbon/m3coordinator.yml b/scripts/docker-integration-tests/carbon/m3coordinator.yml index 7f33d8701c..b069fbf7b6 100644 --- a/scripts/docker-integration-tests/carbon/m3coordinator.yml +++ b/scripts/docker-integration-tests/carbon/m3coordinator.yml @@ -9,7 +9,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 carbon: findResultsIncludeBothExpandableAndLeaf: true diff --git a/scripts/docker-integration-tests/carbon/test.sh b/scripts/docker-integration-tests/carbon/test.sh index f1c499a71e..6c811cb3b8 100755 --- a/scripts/docker-integration-tests/carbon/test.sh +++ b/scripts/docker-integration-tests/carbon/test.sh @@ -10,8 +10,7 @@ EXPECTED_PATH=$SCRIPT_PATH/expected export REVISION echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d coordinator01 +docker-compose -f ${COMPOSE_FILE} up -d # Think of this as a defer func() in golang METRIC_EMIT_PID="-1" @@ -152,7 +151,7 @@ ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "wait_carbon_values_accum # Now test the max datapoints behavior using max of four datapoints (4x 5s resolution = 20s) end=$(date +%s) -start=$(($end-20)) +start=$(($end-20)) # 1. no max datapoints set, should not adjust number of datapoints coming back ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'stat.already-aggregated.foo' 42 $start $end" # 2. max datapoints with LTTB, should be an existing value (i.e. 42) diff --git a/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml b/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml index 53a28f0b88..e6b97c2e0f 100644 --- a/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml +++ b/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml @@ -1,15 +1,25 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,5 +34,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml index cc33cf4021..dc075a01fe 100644 --- a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml +++ b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml @@ -13,4 +13,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 diff --git a/scripts/docker-integration-tests/cold_writes_simple/test.sh b/scripts/docker-integration-tests/cold_writes_simple/test.sh index 999ef1b20d..235c870e2b 100755 --- a/scripts/docker-integration-tests/cold_writes_simple/test.sh +++ b/scripts/docker-integration-tests/cold_writes_simple/test.sh @@ -8,9 +8,8 @@ SCRIPT_PATH="$M3_PATH"/scripts/docker-integration-tests/cold_writes_simple COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml export REVISION -echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01 +echo "Run etcd, m3dbnode and m3coordinator containers" +docker-compose -f "${COMPOSE_FILE}" up -d --renew-anon-volumes # Think of this as a defer func() in golang function defer { diff --git a/scripts/docker-integration-tests/common.sh b/scripts/docker-integration-tests/common.sh index 918ab3b76a..ee8651584f 100644 --- a/scripts/docker-integration-tests/common.sh +++ b/scripts/docker-integration-tests/common.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + # Retries a command a configurable number of times with backoff. # # The retry count is given by ATTEMPTS (default 3), the initial backoff diff --git a/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml b/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml index 53a28f0b88..e6b97c2e0f 100644 --- a/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml +++ b/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml @@ -1,15 +1,25 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,5 +34,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml index 952c9a7ddf..3ca941aa8d 100644 --- a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml +++ b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml @@ -9,7 +9,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 downsample: rules: diff --git a/scripts/docker-integration-tests/coordinator_config_rules/test.sh b/scripts/docker-integration-tests/coordinator_config_rules/test.sh index a1590983c8..48dce08870 100755 --- a/scripts/docker-integration-tests/coordinator_config_rules/test.sh +++ b/scripts/docker-integration-tests/coordinator_config_rules/test.sh @@ -16,8 +16,7 @@ docker pull $PROMREMOTECLI_IMAGE docker pull $JQ_IMAGE echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d coordinator01 +docker-compose -f ${COMPOSE_FILE} up -d # Think of this as a defer func() in golang function defer { diff --git a/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml b/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml index ee8207bd26..f75413892f 100644 --- a/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml +++ b/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml @@ -1,5 +1,15 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend coordinator01: expose: - "7201" @@ -10,33 +20,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml" - etcd01: - expose: - - "2379-2380" - ports: - - "0.0.0.0:2379-2380:2379-2380" - networks: - - backend - image: quay.io/coreos/etcd:v3.4.3 - command: - - "etcd" - - "--name" - - "etcd01" - - "--listen-peer-urls" - - "http://0.0.0.0:2380" - - "--listen-client-urls" - - "http://0.0.0.0:2379" - - "--advertise-client-urls" - - "http://etcd01:2379" - - "--initial-cluster-token" - - "etcd-cluster-1" - - "--initial-advertise-peer-urls" - - "http://etcd01:2380" - - "--initial-cluster" - - "etcd01=http://etcd01:2380" - - "--initial-cluster-state" - - "new" - - "--data-dir" - - "/var/lib/etcd" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml index c3c08c0104..6a261ecf26 100644 --- a/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml +++ b/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml @@ -23,7 +23,7 @@ clusterManagement: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 tagOptions: idScheme: quoted diff --git a/scripts/docker-integration-tests/coordinator_noop/test.sh b/scripts/docker-integration-tests/coordinator_noop/test.sh index 738ea863d0..0f91ec91da 100755 --- a/scripts/docker-integration-tests/coordinator_noop/test.sh +++ b/scripts/docker-integration-tests/coordinator_noop/test.sh @@ -9,8 +9,7 @@ COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml export REVISION echo "Run coordinator with no etcd" -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01 -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes etcd01 +docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes function defer { docker-compose -f ${COMPOSE_FILE} down || echo "unable to shutdown containers" # CI fails to stop all containers sometimes diff --git a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml index e155eb5334..323289a1a6 100644 --- a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml +++ b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml @@ -1,5 +1,15 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" @@ -14,33 +24,7 @@ services: - M3DB_HOST_ID=dbnode01 volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" - etcd01: - expose: - - "2379-2380" - ports: - - "0.0.0.0:2379-2380:2379-2380" - networks: - - backend - image: quay.io/coreos/etcd:v3.4.3 - command: - - "etcd" - - "--name" - - "etcd01" - - "--listen-peer-urls" - - "http://0.0.0.0:2380" - - "--listen-client-urls" - - "http://0.0.0.0:2379" - - "--advertise-client-urls" - - "http://etcd01:2379" - - "--initial-cluster-token" - - "etcd-cluster-1" - - "--initial-advertise-peer-urls" - - "http://etcd01:2380" - - "--initial-cluster" - - "etcd01=http://etcd01:2380" - - "--initial-cluster-state" - - "new" - - "--data-dir" - - "/var/lib/etcd" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/docker-compose-etcd.yml b/scripts/docker-integration-tests/docker-compose-etcd.yml new file mode 100644 index 0000000000..90af19b550 --- /dev/null +++ b/scripts/docker-integration-tests/docker-compose-etcd.yml @@ -0,0 +1,15 @@ +version: "3.5" +services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend + +networks: + backend: diff --git a/scripts/docker-integration-tests/m3coordinator.Dockerfile b/scripts/docker-integration-tests/m3coordinator.Dockerfile index 0319613977..489030dbea 100644 --- a/scripts/docker-integration-tests/m3coordinator.Dockerfile +++ b/scripts/docker-integration-tests/m3coordinator.Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer="The M3DB Authors " RUN mkdir -p /bin RUN mkdir -p /etc/m3coordinator ADD ./m3coordinator /bin/ -ADD ./m3coordinator-local-etcd.yml /etc/m3coordinator/m3coordinator.yml +ADD ./m3coordinator-local-docker-etcd.yml /etc/m3coordinator/m3coordinator.yml EXPOSE 7201/tcp 7203/tcp diff --git a/scripts/docker-integration-tests/m3dbnode.Dockerfile b/scripts/docker-integration-tests/m3dbnode.Dockerfile index a352ad4bf5..6d430fe803 100644 --- a/scripts/docker-integration-tests/m3dbnode.Dockerfile +++ b/scripts/docker-integration-tests/m3dbnode.Dockerfile @@ -4,9 +4,9 @@ LABEL maintainer="The M3DB Authors " RUN mkdir -p /bin RUN mkdir -p /etc/m3dbnode ADD ./m3dbnode /bin/ -ADD ./m3dbnode-local-etcd.yml /etc/m3dbnode/m3dbnode.yml +ADD ./m3dbnode-local-docker-etcd.yml /etc/m3dbnode/m3dbnode.yml -EXPOSE 2379/tcp 2380/tcp 7201/tcp 7203/tcp 9000-9004/tcp +EXPOSE 7201/tcp 7203/tcp 9000-9004/tcp ENV PANIC_ON_INVARIANT_VIOLATED=true diff --git a/scripts/docker-integration-tests/multi_cluster_write/docker-compose.yml b/scripts/docker-integration-tests/multi_cluster_write/docker-compose.yml index b5b52d922d..5d9e35cfcb 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/docker-compose.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/docker-compose.yml @@ -1,5 +1,15 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend cluster_a_dbnode01: expose: - "9000-9004" @@ -14,6 +24,8 @@ services: - M3DB_HOST_ID=cluster_a_m3db_local_1 volumes: - "./m3dbnode-cluster-a.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd cluster_a_dbnode02: expose: - "9000-9004" @@ -28,6 +40,8 @@ services: - M3DB_HOST_ID=cluster_a_m3db_local_2 volumes: - "./m3dbnode-cluster-a.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd cluster_a_coordinator01: expose: - "7201" @@ -42,6 +56,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-cluster-a.yml/:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd cluster_b_dbnode01: expose: - "9000-9004" @@ -56,6 +72,8 @@ services: - M3DB_HOST_ID=cluster_b_m3db_local_1 volumes: - "./m3dbnode-cluster-b.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd cluster_b_dbnode02: expose: - "9000-9004" @@ -70,6 +88,8 @@ services: - M3DB_HOST_ID=cluster_b_m3db_local_2 volumes: - "./m3dbnode-cluster-b.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd cluster_b_coordinator01: expose: - "7201" @@ -84,5 +104,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-cluster-b.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml index e393f2ef22..8fb9bb136a 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml @@ -13,4 +13,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - cluster_a_dbnode01:2379 + - cluster_a_etcd:2379 diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml index 7933a84078..facf8fa580 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml @@ -13,4 +13,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - cluster_b_dbnode01:2379 + - cluster_b_etcd:2379 diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml index 13aac06336..052e908401 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml @@ -14,7 +14,7 @@ db: etcdClusters: - zone: embedded endpoints: - - cluster_a_dbnode01:2379 + - cluster_a_etcd:2379 - service: env: default_env zone: embedded @@ -23,7 +23,7 @@ db: etcdClusters: - zone: embedded endpoints: - - cluster_b_dbnode01:2379 + - cluster_b_etcd:2379 async: true discovery: @@ -36,7 +36,7 @@ db: etcdClusters: - zone: embedded endpoints: - - cluster_a_dbnode01:2379 + - cluster_a_etcd:2379 seedNodes: initialCluster: - hostID: cluster_a_m3db_local_1 diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml index 9e59d22898..d37ae5171a 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml @@ -13,7 +13,7 @@ db: etcdClusters: - zone: embedded endpoints: - - cluster_b_dbnode01:2379 + - cluster_b_etcd:2379 seedNodes: initialCluster: - hostID: cluster_b_m3db_local_1 diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml b/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml index 6c38ae39de..7bf3fc841f 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml @@ -1,5 +1,15 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend m3aggregator01: expose: - "6001" @@ -12,6 +22,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3aggregator02: expose: - "6002" @@ -24,6 +36,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3coordinator01: expose: - "7202" @@ -34,6 +48,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd coordinatoradmin: expose: - "7201" @@ -44,6 +60,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-admin.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd prometheusraw: expose: - "9090" @@ -60,6 +78,8 @@ services: - "--web.console.libraries=/usr/share/prometheus/console_libraries" - "--web.console.templates=/usr/share/prometheus/consoles" - "--enable-feature=remote-write-receiver" + depends_on: + - etcd prometheusagg: expose: - "9091" @@ -76,33 +96,7 @@ services: - "--web.console.libraries=/usr/share/prometheus/console_libraries" - "--web.console.templates=/usr/share/prometheus/consoles" - "--enable-feature=remote-write-receiver" - etcd01: - expose: - - "2379-2380" - ports: - - "0.0.0.0:2379-2380:2379-2380" - networks: - - backend - image: quay.io/coreos/etcd:v3.4.3 - command: - - "etcd" - - "--name" - - "etcd01" - - "--listen-peer-urls" - - "http://0.0.0.0:2380" - - "--listen-client-urls" - - "http://0.0.0.0:2379" - - "--advertise-client-urls" - - "http://etcd01:2379" - - "--initial-cluster-token" - - "etcd-cluster-1" - - "--initial-advertise-peer-urls" - - "http://etcd01:2380" - - "--initial-cluster" - - "etcd01=http://etcd01:2380" - - "--initial-cluster-state" - - "new" - - "--data-dir" - - "/var/lib/etcd" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml index 1d77b0a035..37ac3f2aa4 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml @@ -40,7 +40,7 @@ kvClient: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 runtimeOptions: kvConfig: diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml index c3c08c0104..6a261ecf26 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml @@ -23,7 +23,7 @@ clusterManagement: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 tagOptions: idScheme: quoted diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml index d6c54c8430..935bc8ed5e 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml @@ -36,7 +36,7 @@ clusterManagement: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 tagOptions: idScheme: quoted diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/test.sh b/scripts/docker-integration-tests/prom_remote_write_backend/test.sh index 208bb88565..7ba4c0435e 100755 --- a/scripts/docker-integration-tests/prom_remote_write_backend/test.sh +++ b/scripts/docker-integration-tests/prom_remote_write_backend/test.sh @@ -20,7 +20,7 @@ docker pull $PROMREMOTECLI_IMAGE trap 'cleanup ${COMPOSE_FILE} ${TEST_SUCCESS}' EXIT echo "Run ETCD" -docker-compose -f "${COMPOSE_FILE}" up -d etcd01 +docker-compose -f "${COMPOSE_FILE}" up -d etcd echo "Run Coordinator in Admin mode" docker-compose -f "${COMPOSE_FILE}" up -d coordinatoradmin diff --git a/scripts/docker-integration-tests/prometheus/docker-compose.yml b/scripts/docker-integration-tests/prometheus/docker-compose.yml index 03b839832b..a859edea16 100644 --- a/scripts/docker-integration-tests/prometheus/docker-compose.yml +++ b/scripts/docker-integration-tests/prometheus/docker-compose.yml @@ -1,17 +1,27 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,6 +34,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd prometheus01: expose: - "9090" @@ -34,5 +46,7 @@ services: image: prom/prometheus:latest volumes: - "./prometheus.yml:/etc/prometheus/prometheus.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/prometheus/m3coordinator.yml b/scripts/docker-integration-tests/prometheus/m3coordinator.yml index 79f849a151..7ebe2792b3 100644 --- a/scripts/docker-integration-tests/prometheus/m3coordinator.yml +++ b/scripts/docker-integration-tests/prometheus/m3coordinator.yml @@ -13,7 +13,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 query: restrictTags: diff --git a/scripts/docker-integration-tests/prometheus_replication/docker-compose.yml b/scripts/docker-integration-tests/prometheus_replication/docker-compose.yml index 333ee79238..3619670d13 100644 --- a/scripts/docker-integration-tests/prometheus_replication/docker-compose.yml +++ b/scripts/docker-integration-tests/prometheus_replication/docker-compose.yml @@ -1,15 +1,25 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -22,16 +32,21 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator01.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + # The coordinator talks to both DB clusters to facilitate replication, and thus depends on both of them. + - etcd + - dbnode01 + - dbnode02 dbnode02: expose: - "19000-19004" - - "12379-12380" ports: - "0.0.0.0:19000-19004:9000-9004" - - "0.0.0.0:12379-12380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator02: expose: - "17201" @@ -44,5 +59,10 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator02.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + # The coordinator talks to both DB clusters to facilitate replication, and thus depends on both of them. + - etcd + - dbnode01 + - dbnode02 networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml b/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml index 426653f6cc..4b323bdd1e 100644 --- a/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml +++ b/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml @@ -14,4 +14,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 diff --git a/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml b/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml index d7a81d02b4..122077927b 100644 --- a/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml +++ b/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml @@ -9,4 +9,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode02:2379 + - etcd:2379 diff --git a/scripts/docker-integration-tests/prometheus_replication/test.sh b/scripts/docker-integration-tests/prometheus_replication/test.sh index 3d3c72407f..e1b8b6d619 100755 --- a/scripts/docker-integration-tests/prometheus_replication/test.sh +++ b/scripts/docker-integration-tests/prometheus_replication/test.sh @@ -14,14 +14,12 @@ echo "Pull containers required for test" docker pull $PROMREMOTECLI_IMAGE docker pull $JQ_IMAGE -echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d dbnode02 -docker-compose -f ${COMPOSE_FILE} up -d coordinator01 -docker-compose -f ${COMPOSE_FILE} up -d coordinator02 +echo "Run etcd, m3dbnode and m3coordinator containers" +docker-compose -f ${COMPOSE_FILE} up -d function defer { - docker-compose -f ${COMPOSE_FILE} down || echo "unable to shutdown containers" # CI fails to stop all containers sometimes + : +# docker-compose -f ${COMPOSE_FILE} down || echo "unable to shutdown containers" # CI fails to stop all containers sometimes } trap defer EXIT diff --git a/scripts/docker-integration-tests/query_fanout/docker-compose.yml b/scripts/docker-integration-tests/query_fanout/docker-compose.yml index c5cba80180..7a9b82e9fb 100644 --- a/scripts/docker-integration-tests/query_fanout/docker-compose.yml +++ b/scripts/docker-integration-tests/query_fanout/docker-compose.yml @@ -1,5 +1,15 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode-cluster-a: expose: - "9000-9004" @@ -10,6 +20,8 @@ services: networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator-cluster-a: expose: - "7201" @@ -24,6 +36,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-cluster-a.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd dbnode-cluster-b: expose: - "9000-9004" @@ -34,6 +48,8 @@ services: networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator-cluster-b: expose: - "7201" @@ -48,6 +64,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-cluster-b.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd dbnode-cluster-c: expose: - "9000-9004" @@ -58,6 +76,8 @@ services: networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator-cluster-c: expose: - "7201" @@ -72,5 +92,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-cluster-c.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/repair/docker-compose.yml b/scripts/docker-integration-tests/repair/docker-compose.yml index b91284adcf..58ec4d27fb 100644 --- a/scripts/docker-integration-tests/repair/docker-compose.yml +++ b/scripts/docker-integration-tests/repair/docker-compose.yml @@ -1,9 +1,18 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9012:9002" - "0.0.0.0:9013:9003" @@ -14,10 +23,11 @@ services: - M3DB_HOST_ID=m3db_local_1 volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd dbnode02: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9022:9002" - "0.0.0.0:9023:9003" @@ -28,6 +38,8 @@ services: - M3DB_HOST_ID=m3db_local_2 volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -42,5 +54,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/repair/m3coordinator.yml b/scripts/docker-integration-tests/repair/m3coordinator.yml index cc33cf4021..dc075a01fe 100644 --- a/scripts/docker-integration-tests/repair/m3coordinator.yml +++ b/scripts/docker-integration-tests/repair/m3coordinator.yml @@ -13,4 +13,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 diff --git a/scripts/docker-integration-tests/repair/m3dbnode.yml b/scripts/docker-integration-tests/repair/m3dbnode.yml index 29dcc22d4b..62f3e88cf8 100644 --- a/scripts/docker-integration-tests/repair/m3dbnode.yml +++ b/scripts/docker-integration-tests/repair/m3dbnode.yml @@ -13,12 +13,7 @@ db: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 - seedNodes: - initialCluster: - - hostID: m3db_local_1 - endpoint: http://dbnode01:2380 - + - etcd:2379 # Enable repairs. repair: enabled: true diff --git a/scripts/docker-integration-tests/repair/test.sh b/scripts/docker-integration-tests/repair/test.sh index ee32d632a9..eff434dc7f 100755 --- a/scripts/docker-integration-tests/repair/test.sh +++ b/scripts/docker-integration-tests/repair/test.sh @@ -9,6 +9,7 @@ COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml export REVISION echo "Run m3dbnode and m3coordinator containers" +docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes etcd docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode01 docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode02 docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01 diff --git a/scripts/docker-integration-tests/replication/docker-compose.yml b/scripts/docker-integration-tests/replication/docker-compose.yml index b5b52d922d..5d9e35cfcb 100644 --- a/scripts/docker-integration-tests/replication/docker-compose.yml +++ b/scripts/docker-integration-tests/replication/docker-compose.yml @@ -1,5 +1,15 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend cluster_a_dbnode01: expose: - "9000-9004" @@ -14,6 +24,8 @@ services: - M3DB_HOST_ID=cluster_a_m3db_local_1 volumes: - "./m3dbnode-cluster-a.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd cluster_a_dbnode02: expose: - "9000-9004" @@ -28,6 +40,8 @@ services: - M3DB_HOST_ID=cluster_a_m3db_local_2 volumes: - "./m3dbnode-cluster-a.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd cluster_a_coordinator01: expose: - "7201" @@ -42,6 +56,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-cluster-a.yml/:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd cluster_b_dbnode01: expose: - "9000-9004" @@ -56,6 +72,8 @@ services: - M3DB_HOST_ID=cluster_b_m3db_local_1 volumes: - "./m3dbnode-cluster-b.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd cluster_b_dbnode02: expose: - "9000-9004" @@ -70,6 +88,8 @@ services: - M3DB_HOST_ID=cluster_b_m3db_local_2 volumes: - "./m3dbnode-cluster-b.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd cluster_b_coordinator01: expose: - "7201" @@ -84,5 +104,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-cluster-b.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml index e393f2ef22..8fb9bb136a 100644 --- a/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml @@ -13,4 +13,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - cluster_a_dbnode01:2379 + - cluster_a_etcd:2379 diff --git a/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml index 7933a84078..facf8fa580 100644 --- a/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml @@ -13,4 +13,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - cluster_b_dbnode01:2379 + - cluster_b_etcd:2379 diff --git a/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml index 52e75125c5..7039d9ca27 100644 --- a/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml @@ -13,7 +13,7 @@ db: etcdClusters: - zone: embedded endpoints: - - cluster_a_dbnode01:2379 + - cluster_a_etcd:2379 seedNodes: initialCluster: - hostID: cluster_a_m3db_local_1 @@ -40,4 +40,4 @@ db: etcdClusters: - zone: embedded endpoints: - - cluster_b_dbnode01:2379 + - cluster_b_etcd:2379 diff --git a/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml index 2d04a09719..b6aab43943 100644 --- a/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml @@ -13,7 +13,7 @@ db: etcdClusters: - zone: embedded endpoints: - - cluster_b_dbnode01:2379 + - cluster_b_etcd:2379 seedNodes: initialCluster: - hostID: cluster_b_m3db_local_1 @@ -42,4 +42,4 @@ db: etcdClusters: - zone: embedded endpoints: - - cluster_a_dbnode01:2379 + - cluster_a_etcd:2379 diff --git a/scripts/docker-integration-tests/run.sh b/scripts/docker-integration-tests/run.sh index bead15b207..a687e5bce9 100755 --- a/scripts/docker-integration-tests/run.sh +++ b/scripts/docker-integration-tests/run.sh @@ -4,14 +4,24 @@ set -ex TESTS=( scripts/docker-integration-tests/cold_writes_simple/test.sh - scripts/docker-integration-tests/prometheus_replication/test.sh + # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. + # scripts/docker-integration-tests/prometheus_replication/test.sh + scripts/docker-integration-tests/carbon/test.sh scripts/docker-integration-tests/aggregator/test.sh scripts/docker-integration-tests/aggregator_legacy/test.sh - scripts/docker-integration-tests/query_fanout/test.sh - scripts/docker-integration-tests/repair/test.sh - scripts/docker-integration-tests/replication/test.sh - scripts/docker-integration-tests/multi_cluster_write/test.sh + + # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. + # scripts/docker-integration-tests/query_fanout/test.sh + + scripts/docker-integration-tests/repair/test.sh + + # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. + # scripts/docker-integration-tests/replication/test.sh + + # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. + # scripts/docker-integration-tests/multi_cluster_write/test.sh + scripts/docker-integration-tests/coordinator_config_rules/test.sh scripts/docker-integration-tests/coordinator_noop/test.sh scripts/docker-integration-tests/prom_remote_write_backend/test.sh diff --git a/scripts/docker-integration-tests/setup.sh b/scripts/docker-integration-tests/setup.sh index 76659a49a2..d90ecb4967 100755 --- a/scripts/docker-integration-tests/setup.sh +++ b/scripts/docker-integration-tests/setup.sh @@ -15,8 +15,8 @@ mkdir -p ./bin # by keeping all the required files in ./bin, it makes the build context # for docker much smaller -cp ./src/query/config/m3coordinator-local-etcd.yml ./bin -cp ./src/dbnode/config/m3dbnode-local-etcd.yml ./bin +cp ./src/query/config/m3coordinator-local-docker-etcd.yml ./bin +cp ./src/dbnode/config/m3dbnode-local-docker-etcd.yml ./bin cp ./src/aggregator/config/m3aggregator.yml ./bin # build images @@ -26,7 +26,9 @@ function build_image { local svc=$1 echo "creating image for $svc" make ${svc}-linux-amd64 - docker build -t "${svc}_integration:${REVISION}" -f ./scripts/docker-integration-tests/${svc}.Dockerfile ./bin + docker build \ + --no-cache \ + -t "${svc}_integration:${REVISION}" -f ./scripts/docker-integration-tests/${svc}.Dockerfile ./bin } if [[ "$SERVICE" != "" ]]; then diff --git a/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml b/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml index bce43d559e..d2c7844c8c 100644 --- a/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml +++ b/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml @@ -1,15 +1,25 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -22,5 +32,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml index 7d5cb12598..945b26e3b5 100644 --- a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml +++ b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml @@ -13,5 +13,5 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 useV2BatchAPIs: true diff --git a/src/dbnode/config/m3dbnode-local-docker-etcd.yml b/src/dbnode/config/m3dbnode-local-docker-etcd.yml new file mode 100644 index 0000000000..83650bc894 --- /dev/null +++ b/src/dbnode/config/m3dbnode-local-docker-etcd.yml @@ -0,0 +1,11 @@ +coordinator: {} +"db": + discovery: + "config": + "service": + "etcdClusters": + - "endpoints": ["http://etcd:2379"] + "zone": "embedded" + "service": "m3db" + "zone": "embedded" + "env": "default_env" \ No newline at end of file diff --git a/src/query/config/m3coordinator-local-docker-etcd.yml b/src/query/config/m3coordinator-local-docker-etcd.yml new file mode 100644 index 0000000000..c25dff706c --- /dev/null +++ b/src/query/config/m3coordinator-local-docker-etcd.yml @@ -0,0 +1,16 @@ +clusters: + - namespaces: + - namespace: default + type: unaggregated + retention: 48h + client: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - http://etcd:2379