diff --git a/build.sh b/build.sh index a1ed4841..a8343e4d 100644 --- a/build.sh +++ b/build.sh @@ -1,58 +1,107 @@ #!/bin/bash -echo "===============================================================================" -echo "Building Oasis..." -echo "===============================================================================" -mvn clean install -DskipTests - -echo "===============================================================================" -echo "Build the base java image" -echo "===============================================================================" -docker build -t oasis/base-java -f ./buildscripts/docker/base-java.dockerfile . +do_log() { + echo "===============================================================================" + echo "$1" + echo "===============================================================================" +} + +proj_build_status() { + case $1 in + 'events-api') echo 'true';; + 'stats-api') echo 'true';; + 'engine') echo 'true';; + 'feeder') echo 'true';; + *) echo 'true' + esac +} +skip_build_project=false +do_clean=true +skip_tests=true +build_base_java_image=true +compose_hide_logs="kafka,zookeeper" + +# ///////////////////// BUILD PROJECT //////////////////////////////////// + +if [ "$skip_build_project" != "true" ]; then + MVN_ARGS="" + if [ "$do_clean" == "true" ]; then + MVN_ARGS+="clean" + fi + MVN_ARGS+=" install " + if [ "$skip_tests" == "true" ]; then + MVN_ARGS+=" -DskipTests " + fi + + do_log "⚙️ Building Oasis..." + mvn $MVN_ARGS +else + echo "🔸 Skipped: Building maven project" +fi + +# ///////////////////// BUILD DOCKER IMAGES //////////////////////////////////// + +do_log "🚢 Creating Containers..." + +if [ "$build_base_java_image" == "true" ]; then + do_log "🛠️ Build the base java image" + docker build -t oasis/base-java -f ./buildscripts/docker/base-java.dockerfile . +else + echo "🔸 Skipped: Building base java image" +fi cp externals/kafka-stream/target/libs/* buildscripts/modules cp externals/kafka-stream/target/oasis-ext-kafkastream.jar buildscripts/modules -echo "===============================================================================" -echo "Building Events API Docker Image..." -echo "===============================================================================" -cd services/events-api -docker build -t oasis/events-api . - -cd ../.. - -echo "===============================================================================" -echo "Building Admin/Stats API Docker Image..." -echo "===============================================================================" -cd services/stats-api -docker build -t oasis/stats-api . - -cd ../.. - -echo "===============================================================================" -echo "Building Feeder Docker Image..." -echo "===============================================================================" -cd services/feeder -docker build -t oasis/feeder . - -cd ../.. - -echo "===============================================================================" -echo "Building Engine Docker Image..." -echo "===============================================================================" -cd engine -docker build -t oasis/engine . - -cd .. +if [ "$(proj_build_status events-api)" == "true" ]; then + do_log "🛠️ Building Events API Docker Image..." + cd services/events-api || exit 1 + docker build -t oasis/events-api . + cd ../.. +else + echo "🔸 Skipped: Building events-api" +fi + +if [ "$(proj_build_status stats-api)" == "true" ]; then + do_log "🛠 Building Admin/Stats API Docker Image..." + cd services/stats-api || exit 1 + docker build -t oasis/stats-api . + cd ../.. +else + echo "🔸 Skipped: Building stats-api" +fi + +if [ "$(proj_build_status feeder)" == "true" ]; then + do_log "🛠️ Building Feeder Docker Image..." + cd services/feeder || exit 1 + docker build -t oasis/feeder . + cd ../.. +else + echo "🔸 Skipped: Building feeder" +fi + +if [ "$(proj_build_status engine)" == "true" ]; then + do_log "🛠️ Building Engine Docker Image..." + cd engine || exit 1 + docker build -t oasis/engine . + cd .. +else + echo "🔸 Skipped: Building engine" +fi mkdir -p .tmpdata/enginedb mkdir -p .tmpdata/cache -echo "===============================================================================" -echo "Starting Oasis..." -echo "===============================================================================" -docker compose up --no-attach kafka --no-attach zookeeper --no-attach stats-api +do_log "📣 Starting Oasis..." + +IFS=',' read -ra LOGS <<< "$compose_hide_logs" +args_logs="" +for log in "${LOGS[@]}"; do + args_logs+="--no-attach $log " +done + +docker compose up $args_logs diff --git a/buildscripts/engine.yml b/buildscripts/engine.yml index 9d19e973..8a9b7a56 100644 --- a/buildscripts/engine.yml +++ b/buildscripts/engine.yml @@ -5,7 +5,7 @@ oasis: eventstream: impl: "io.github.oasis.ext.kafkastream.KafkaStreamFactory" configs: - brokerUrls: "kafka:9092" + brokerUrls: "localhost:9092" maxConsumerThreadPoolSize: 4 @@ -18,11 +18,11 @@ oasis: engineEventConsumer: props: - redis: - host: enginedb - port: 6379 + enginedb: + configs: + url: "redis://localhost:6379" - pool: - max: 16 - maxIdle: 8 - minIdle: 4 \ No newline at end of file + pool: + max: 16 + maxIdle: 8 + minIdle: 4 \ No newline at end of file diff --git a/buildscripts/events-api.yml b/buildscripts/events-api.yml index 7804bc4b..ab46f234 100644 --- a/buildscripts/events-api.yml +++ b/buildscripts/events-api.yml @@ -13,7 +13,7 @@ http: oasis: adminApi: - baseUrl: "http://stats-api:8010/api" + baseUrl: "http://localhost:8010/api" eventSourceGet: "/admin/event-source" playerGet: "/players" @@ -23,10 +23,10 @@ oasis: secretKey: "eventapi" - dispatcher: + eventstream: impl: "oasis:io.github.oasis.ext.kafkastream.KafkaStreamFactory" configs: - brokerUrls: "kafka:9092" + brokerUrls: "localhost:9092" maxConsumerThreadPoolSize: 4 @@ -58,7 +58,7 @@ oasis: cache: impl: "oasis:io.github.oasis.services.events.db.RedisVerticle" configs: - connectionString: "redis://apicache:6381" + connectionString: "redis://localhost:6379" maxPoolSize: 16 maxWaitingHandlers: 16 diff --git a/buildscripts/feeder.yml b/buildscripts/feeder.yml index 36a0e298..371758f2 100644 --- a/buildscripts/feeder.yml +++ b/buildscripts/feeder.yml @@ -1,6 +1,6 @@ oasis: adminApi: - baseUrl: "http://stats-api:8010/api" + baseUrl: "http://localhost:8010/api" eventSourceGet: "/admin/event-sources/" playerGet: "/players/" @@ -21,7 +21,7 @@ oasis: eventstream: impl: "io.github.oasis.ext.kafkastream.KafkaStreamFactory" configs: - brokerUrls: "kafka:9092" + brokerUrls: "localhost:9092" feedStreamConsumer: groupId: feed-consumer-group diff --git a/buildscripts/stats-api.yml b/buildscripts/stats-api.yml index 0e4696de..b41ffa6f 100644 --- a/buildscripts/stats-api.yml +++ b/buildscripts/stats-api.yml @@ -12,8 +12,7 @@ oasis: defaultApiKey: "root:root,eventapi:eventapi,feeder:feeder:7" cache: - host: "apicache" - port: 6379 + url: "redis://localhost:6381" connectionRetries: 5 connectionRetryDelay: 5000 @@ -24,8 +23,7 @@ oasis: minIdle: 4 enginedb: - host: "enginedb" - port: 6379 + url: "redis://localhost:6379" connectionRetries: 5 connectionRetryDelay: 5000 @@ -35,24 +33,24 @@ oasis: maxIdle: 16 minIdle: 4 - dispatcher: + eventstream: impl: "io.github.oasis.ext.kafkastream.KafkaStreamFactory" configs: - brokerUrls: "kafka:9092" + brokerUrls: "localhost:9092" maxConsumerThreadPoolSize: 4 gameEventsConsumer: - # groupId: "" + # groupId: "" - # If not specified, this will take engine id - # instanceId: "" + # If not specified, this will take engine id + # instanceId: "" - # These props will directly feed to Kafka property configs - # Check kafka consumer documentation for supported props. - # props: - #"max.poll.records": 10, - #"session.timeout.ms": 86400000 + # These props will directly feed to Kafka property configs + # Check kafka consumer documentation for supported props. + # props: + #"max.poll.records": 10, + #"session.timeout.ms": 86400000 broadcastConsumer: # groupId: "" diff --git a/core/src/main/java/io/github/oasis/core/configs/OasisConfigs.java b/core/src/main/java/io/github/oasis/core/configs/OasisConfigs.java index e41ab4fe..4e4bed83 100644 --- a/core/src/main/java/io/github/oasis/core/configs/OasisConfigs.java +++ b/core/src/main/java/io/github/oasis/core/configs/OasisConfigs.java @@ -41,7 +41,7 @@ */ public class OasisConfigs implements Serializable { - private static final String DEFAULT_ENV_PREFIX = "OASIS_"; + private static final String DEFAULT_ENV_PREFIX = "O_"; public static final String GAME_SUPERVISOR_COUNT = "oasis.supervisors.game"; public static final String RULE_SUPERVISOR_COUNT = "oasis.supervisors.rule"; @@ -62,14 +62,22 @@ public class OasisConfigs implements Serializable { private DocumentContext ctx; - private OasisConfigs(Map configValues, String envVarPrefix, Supplier> envVarSupplier) { - this.envVarPrefix = Objects.toString(envVarPrefix, DEFAULT_ENV_PREFIX); + private OasisConfigs(Map configValues, String theEnvVarPrefix, Supplier> envVarSupplier) { + this.envVarPrefix = deriveEnvVarPrefix(theEnvVarPrefix); this.configValues = configValues; this.loadEnvVariables(Objects.requireNonNullElse(envVarSupplier, defaultEnvVarSupplier)); this.createConfigContext(configValues); } + private String deriveEnvVarPrefix(String theEnvVarPrefix) { + if (theEnvVarPrefix != null) { + return theEnvVarPrefix; + } + + return System.getProperty("oasis.config.env.prefix", DEFAULT_ENV_PREFIX); + } + private void createConfigContext(Map configValues) { var jsonObject = new JsonObject(configValues); ctx = JsonPath.parse(jsonObject.toJson(), jsonPathConf); @@ -92,7 +100,7 @@ private Optional readFromEnvVar(String prop) { } public Map getAll() { - return Collections.unmodifiableMap(configValues); + return Collections.unmodifiableMap(resolveWithEnvVars(configValues, null)); } public boolean hasProperty(String property) { @@ -150,6 +158,7 @@ public Map getObject(String property) { return resolveWithEnvVars(value, property); } + @SuppressWarnings("unchecked") private Map resolveWithEnvVars(Map configs, String baseProperty) { var result = new HashMap(); @@ -166,6 +175,8 @@ private Map resolveWithEnvVars(Map configs, Stri result.put(k, getFloat(concat(baseProperty, k), (Float) v)); } else if (v instanceof Double) { result.put(k, getDouble(concat(baseProperty, k), (Double) v)); + } else if (v instanceof Map) { + result.put(k, resolveWithEnvVars((Map) v, concat(baseProperty, k))); } else { result.put(k, v); } @@ -182,6 +193,9 @@ private String concat(String base, String property) { if (base == null) { return "$." + property; } else { + if (base.startsWith("$.")) { + return base.substring(2) + "." + property; + } return base + "." + property; } } diff --git a/core/src/test/java/io/github/oasis/core/configs/OasisConfigTest.java b/core/src/test/java/io/github/oasis/core/configs/OasisConfigTest.java index d62c5454..1c6df224 100644 --- a/core/src/test/java/io/github/oasis/core/configs/OasisConfigTest.java +++ b/core/src/test/java/io/github/oasis/core/configs/OasisConfigTest.java @@ -26,8 +26,8 @@ void testConfigsWithoutEnvVariables() { void testConfigsWithEnvVariablesDefaultPrefix() { ClassLoader clsLoader = Thread.currentThread().getContextClassLoader(); var envVars = new HashMap(); - envVars.put("OASIS_OASIS_ENGINE_ID", "overridden-value"); - envVars.put("OASIS_NX_IN_FILE", "value-2"); + envVars.put("O_OASIS_ENGINE_ID", "overridden-value"); + envVars.put("O_NX_IN_FILE", "value-2"); var oc = new OasisConfigs.Builder().withCustomEnvOverrides(envVars) .buildFromYamlResource("configs/sample.yaml", clsLoader); @@ -49,10 +49,10 @@ void testConfigsWithEnvVariablesDefaultPrefix() { void testGetResolvedObject() { ClassLoader clsLoader = Thread.currentThread().getContextClassLoader(); var envVars = new HashMap(); - envVars.put("OASIS_OASIS_ENGINE_ID", "overridden-value"); - envVars.put("OASIS_OASIS_ENGINE_AUTORESTART", "false"); - envVars.put("OASIS_OASIS_ENGINE_INTERVAL", "100"); - envVars.put("OASIS_OASIS_ENGINE_PI", "4.124"); + envVars.put("O_OASIS_ENGINE_ID", "overridden-value"); + envVars.put("O_OASIS_ENGINE_AUTORESTART", "false"); + envVars.put("O_OASIS_ENGINE_INTERVAL", "100"); + envVars.put("O_OASIS_ENGINE_PI", "4.124"); var oc = new OasisConfigs.Builder().withCustomEnvOverrides(envVars) .buildFromYamlResource("configs/sample.yaml", clsLoader); @@ -63,6 +63,24 @@ void testGetResolvedObject() { assertEquals("overridden-value", engineConf.get("id")); } + @Test + void testGetAllResolved() { + ClassLoader clsLoader = Thread.currentThread().getContextClassLoader(); + var envVars = new HashMap(); + envVars.put("O_OASIS_ENGINE_ID", "overridden-value"); + envVars.put("O_OASIS_ENGINE_AUTORESTART", "false"); + envVars.put("O_OASIS_ENGINE_INTERVAL", "100"); + envVars.put("O_OASIS_ENGINE_PI", "4.124"); + var oc = new OasisConfigs.Builder().withCustomEnvOverrides(envVars) + .buildFromYamlResource("configs/sample.yaml", clsLoader); + + var engineConf = new OasisConfigs.Builder().buildWithConfigs(oc.getAll()); + assertFalse(engineConf.getBoolean("oasis.engine.autoRestart", true)); + assertEquals(4.124, engineConf.getDouble("oasis.engine.pi", 0.0)); + assertEquals(100, engineConf.getInt("oasis.engine.interval", 0)); + assertEquals("overridden-value", engineConf.get("oasis.engine.id", null)); + } + @Test void testCustomEnvPrefix() { ClassLoader clsLoader = Thread.currentThread().getContextClassLoader(); diff --git a/docker-compose.yml b/docker-compose.yml index af305238..a52a9ea5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,6 +51,9 @@ services: - "./buildscripts/events-api.yml:/etc/oasis/events-api.yml" environment: - "WAIT_HOSTS=apicache:6381,kafka:9092" + - "O_OASIS_CACHE_CONFIGS_CONNECTIONSTRING=redis://apicache:6381" + - "O_OASIS_EVENTSTREAM_CONFIGS_BROKERURLS=kafka:9092" + - "O_OASIS_ADMINAPI_BASEURL=http://stats-api:8010/api" depends_on: - apicache - kafka @@ -65,6 +68,9 @@ services: - "./buildscripts/modules:/etc/oasis/modules" environment: - "WAIT_HOSTS=apicache:6381,kafka:9092" + - "O_OASIS_ENGINEDB_URL=redis://enginedb:6379" + - "O_OASIS_CACHE_URL=redis://apicache:6381" + - "O_OASIS_EVENTSTREAM_CONFIGS_BROKERURLS=kafka:9092" depends_on: - apicache - kafka @@ -73,6 +79,9 @@ services: image: oasis/engine environment: - "WAIT_HOSTS=enginedb:6379,kafka:9092" + - "O_OASIS_ENGINE_ID=engine_in_docker_compose" + - "O_OASIS_ENGINEDB_CONFIGS_URL=redis://enginedb:6379" + - "O_OASIS_EVENTSTREAM_CONFIGS_BROKERURLS=kafka:9092" volumes: - "./buildscripts/engine.yml:/etc/oasis/engine.yml" - "./buildscripts/modules:/etc/oasis/modules" @@ -84,6 +93,8 @@ services: image: oasis/feeder environment: - "WAIT_HOSTS=kafka:9092" + - "O_OASIS_EVENTSTREAM_CONFIGS_BROKERURLS=kafka:9092" + - "O_OASIS_ADMINAPI_BASEURL=http://stats-api:8010/api" volumes: - "./buildscripts/feeder.yml:/etc/oasis/feeder.yml" - "./buildscripts/modules:/etc/oasis/modules" diff --git a/engine/src/main/java/io/github/oasis/engine/OasisEngineRunner.java b/engine/src/main/java/io/github/oasis/engine/OasisEngineRunner.java index 2bfa10d2..e6a186e2 100644 --- a/engine/src/main/java/io/github/oasis/engine/OasisEngineRunner.java +++ b/engine/src/main/java/io/github/oasis/engine/OasisEngineRunner.java @@ -52,7 +52,7 @@ public static void main(String[] args) throws OasisException { discoverElements(builder); - Db dbPool = RedisDb.create(configs, "oasis.redis"); + Db dbPool = RedisDb.create(configs, "oasis.enginedb.configs"); dbPool.init(); builder.withDb(dbPool); diff --git a/engine/src/main/resources/engine-defaults.yml b/engine/src/main/resources/engine-defaults.yml index 22373eea..9d53fa76 100644 --- a/engine/src/main/resources/engine-defaults.yml +++ b/engine/src/main/resources/engine-defaults.yml @@ -25,11 +25,11 @@ oasis: trustAll: true protocol: "TLSv1.2" - redis: - host: localhost - port: 6379 + enginedb: + configs: + url: redis://localhost:6379 - pool: - max: 16 - maxIdle: 8 - minIdle: 4 \ No newline at end of file + pool: + max: 16 + maxIdle: 8 + minIdle: 4 \ No newline at end of file diff --git a/externals/kafka-stream/src/main/java/io/github/oasis/ext/kafkastream/KafkaEngineManagerSubscription.java b/externals/kafka-stream/src/main/java/io/github/oasis/ext/kafkastream/KafkaEngineManagerSubscription.java index 801b965d..25a93417 100644 --- a/externals/kafka-stream/src/main/java/io/github/oasis/ext/kafkastream/KafkaEngineManagerSubscription.java +++ b/externals/kafka-stream/src/main/java/io/github/oasis/ext/kafkastream/KafkaEngineManagerSubscription.java @@ -61,7 +61,7 @@ class KafkaEngineManagerSubscription implements EngineManagerSubscription { @Override public void init(OasisConfigs oasisConfigs) { - Map configs = oasisConfigs.getObject("oasis.dispatcher.configs"); + Map configs = oasisConfigs.getObject("oasis.eventstream.configs"); try { KafkaConfigs kafkaConfigs = KafkaUtils.parseFrom(configs); diff --git a/services/events-api/src/main/java/io/github/oasis/services/events/Constants.java b/services/events-api/src/main/java/io/github/oasis/services/events/Constants.java index 600ca591..b311c9b0 100644 --- a/services/events-api/src/main/java/io/github/oasis/services/events/Constants.java +++ b/services/events-api/src/main/java/io/github/oasis/services/events/Constants.java @@ -9,9 +9,9 @@ public final class Constants { static final String SYS_CONFIG_FILE = "oasis.config.file"; - static final String KEY_DISPATCHER = "dispatcher"; - static final String KEY_DISPATCHER_IMPL = "impl"; - static final String KEY_DISPATCHER_CONFIGS = "configs"; + static final String KEY_EVENTSTREAM = "eventstream"; + static final String KEY_EVENTSTREAM_IMPL = "impl"; + static final String KEY_EVENTSTREAM_CONFIGS = "configs"; public static final int DEFAULT_TTL_EVENT_SOURCE = 900; diff --git a/services/events-api/src/main/java/io/github/oasis/services/events/EventsApi.java b/services/events-api/src/main/java/io/github/oasis/services/events/EventsApi.java index c52e745e..5a7df96c 100644 --- a/services/events-api/src/main/java/io/github/oasis/services/events/EventsApi.java +++ b/services/events-api/src/main/java/io/github/oasis/services/events/EventsApi.java @@ -31,9 +31,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static io.github.oasis.services.events.Constants.KEY_DISPATCHER; -import static io.github.oasis.services.events.Constants.KEY_DISPATCHER_CONFIGS; -import static io.github.oasis.services.events.Constants.KEY_DISPATCHER_IMPL; +import static io.github.oasis.services.events.Constants.KEY_EVENTSTREAM; +import static io.github.oasis.services.events.Constants.KEY_EVENTSTREAM_CONFIGS; +import static io.github.oasis.services.events.Constants.KEY_EVENTSTREAM_IMPL; /** * @author Isuru Weerarathna @@ -57,10 +57,10 @@ public void start(Promise promise) { return vertx.deployVerticle(ClientVerticle.class, clientOptions); }) .compose(id -> { - JsonObject dispatcherConf = oasisConfigs.getJsonObject(KEY_DISPATCHER); + JsonObject dispatcherConf = oasisConfigs.getJsonObject(KEY_EVENTSTREAM); DeploymentOptions dispatcherConfigs = new DeploymentOptions() - .setConfig(dispatcherConf.getJsonObject(KEY_DISPATCHER_CONFIGS)); - return vertx.deployVerticle(dispatcherConf.getString(KEY_DISPATCHER_IMPL), dispatcherConfigs); + .setConfig(dispatcherConf.getJsonObject(KEY_EVENTSTREAM_CONFIGS)); + return vertx.deployVerticle(dispatcherConf.getString(KEY_EVENTSTREAM_IMPL), dispatcherConfigs); }) .compose(id -> { DeploymentOptions deploymentOptions = createHttpDeploymentOptions(httpConfigs); diff --git a/services/events-api/src/main/java/io/github/oasis/services/events/Main.java b/services/events-api/src/main/java/io/github/oasis/services/events/Main.java index 42c00ade..4c6891b3 100644 --- a/services/events-api/src/main/java/io/github/oasis/services/events/Main.java +++ b/services/events-api/src/main/java/io/github/oasis/services/events/Main.java @@ -19,6 +19,7 @@ package io.github.oasis.services.events; +import io.github.oasis.core.configs.OasisConfigs; import io.github.oasis.services.events.dispatcher.DispatcherFactory; import io.vertx.config.ConfigRetriever; import io.vertx.config.ConfigRetrieverOptions; @@ -42,31 +43,27 @@ public class Main { private static final Logger LOG = LoggerFactory.getLogger(Main.class); public static void main(String[] args) { - ConfigStoreOptions envConfigs = new ConfigStoreOptions().setType("env"); - ConfigStoreOptions jvmConfigs = new ConfigStoreOptions().setType("sys"); - ConfigStoreOptions fileConfigs = new ConfigStoreOptions() - .setType("file") - .setFormat("yaml") - .setConfig(new JsonObject().put("path", resolveConfigFileLocation())); + var configs = resolveConfigs(); + var jsonObject = new JsonObject(configs.getAll()); + var vertxOptions = new ConfigStoreOptions() + .setType("json") + .setConfig(jsonObject); - ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() - .addStore(jvmConfigs) - .addStore(envConfigs) - .addStore(fileConfigs); + ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions().addStore(vertxOptions); deploy(retrieverOptions); } - private static String resolveConfigFileLocation() { + private static OasisConfigs resolveConfigs() { String confPath = System.getenv(ENV_CONFIG_FILE); if (Objects.nonNull(confPath) && !confPath.isEmpty()) { - return confPath; + return new OasisConfigs.Builder().buildFromYamlFile(confPath); } confPath = System.getProperty(SYS_CONFIG_FILE); if (Objects.nonNull(confPath) && !confPath.isEmpty()) { - return confPath; + return new OasisConfigs.Builder().buildFromYamlFile(confPath); } - return "application.conf"; + return new OasisConfigs.Builder().buildFromYamlResource("defaults.yml"); } public static void deploy(ConfigRetrieverOptions configOptions) { diff --git a/services/events-api/src/main/java/io/github/oasis/services/events/client/AdminApiClient.java b/services/events-api/src/main/java/io/github/oasis/services/events/client/AdminApiClient.java index d8d3b8cc..9c92fc23 100644 --- a/services/events-api/src/main/java/io/github/oasis/services/events/client/AdminApiClient.java +++ b/services/events-api/src/main/java/io/github/oasis/services/events/client/AdminApiClient.java @@ -76,6 +76,7 @@ public AdminApiClient(WebClient webClient, JsonObject configs) { JsonObject adminApiConf = configs.getJsonObject("adminApi", new JsonObject()); String baseUrl = adminApiConf.getString("baseUrl"); + LOG.info("Admin api base url: {}", baseUrl); getPlayerInfoUrl = baseUrl + adminApiConf.getString("playerGet"); getEventSourceInfoUrl = baseUrl + adminApiConf.getString("eventSourceGet"); diff --git a/services/events-api/src/main/resources/application.conf b/services/events-api/src/main/resources/application.conf deleted file mode 100644 index 6b2023f7..00000000 --- a/services/events-api/src/main/resources/application.conf +++ /dev/null @@ -1,71 +0,0 @@ -http = { - port = 8050 - ssl = false -} - -oasis = { - - adminApi = { - baseUrl: "http://localhost:8081/api" - - eventSourceGet: "/admin/event-source" - playerGet: "/players" - - # authentication details of admin api - apiKey: "eventapi" - secretKey: "eventapi" - } - - dispatcher = { - impl = "oasis:io.github.oasis.ext.kafkastream.KafkaStreamFactory" - configs = { - brokerUrls: "localhost:9092" - - maxConsumerThreadPoolSize: 4 - - gameEventsConsumer = { - # groupId = "" - - # If not specified, this will take engine id - # instanceId = "" - - # These props will directly feed to Kafka property configs - # Check kafka consumer documentation for supported props. - props = { - #"max.poll.records": 10, - #"session.timeout.ms": 86400000 - } - } - - broadcastConsumer = { - # groupId = "" - - # These props will directly feed to Kafka property configs - # Check kafka consumer documentation for supported props. - props = { - } - } - - dispatcherConfigs = { - props = {} - } - - engineEventConsumer = { - props = {} - } - - } - } - - cache = { - impl = "oasis:io.github.oasis.services.events.db.RedisVerticle" - configs = { - connectionString: "redis://localhost:6379" - maxPoolSize: 4 - maxWaitingHandlers: 16 - - connectionRetries: 5 - connectionRetryDelay: 5000 - } - } -} \ No newline at end of file diff --git a/services/events-api/src/main/resources/defaults.yml b/services/events-api/src/main/resources/defaults.yml new file mode 100644 index 00000000..31470e8b --- /dev/null +++ b/services/events-api/src/main/resources/defaults.yml @@ -0,0 +1,70 @@ +http: + port: 8050 + ssl: false + + # Set this to true, if you want to disable event integrity check against valid event sources registered + # through admin api. By default, this will be set to false and it means, event integrity will be enforced. + # Setting this to true, will still want to register a event source in admin api for authentication purpose. + # Beware: Disabling integrity check should be done only if you can guarantee that this event api will + # never be exposed as a public api. Otherwise, external users may send events which causes the + # bad impact to the integrity of game play. + skipEventIntegrityCheck: false + +oasis: + + adminApi: + baseUrl: "http://localhost:8010/api" + + eventSourceGet: "/admin/event-source" + playerGet: "/players" + + # authentication details of admin api + apiKey: "eventapi" + secretKey: "eventapi" + + + eventstream: + impl: "oasis:io.github.oasis.ext.kafkastream.KafkaStreamFactory" + configs: + brokerUrls: "localhost:9092" + + maxConsumerThreadPoolSize: 4 + + gameEventsConsumer: + # groupId = "" + + # If not specified, this will take engine id + # instanceId = "" + + # These props will directly feed to Kafka property configs + # Check kafka consumer documentation for supported props. + props: + #"max.poll.records": 10, + #"session.timeout.ms": 86400000 + + broadcastConsumer: + # groupId = "" + + # These props will directly feed to Kafka property configs + # Check kafka consumer documentation for supported props. + props: + + dispatcherConfigs: + props: + + engineEventConsumer: + props: + + cache: + impl: "oasis:io.github.oasis.services.events.db.RedisVerticle" + configs: + connectionString: "redis://localhost:6379" + maxPoolSize: 16 + maxWaitingHandlers: 16 + + # expiration seconds for once cached event source data. + # Better to have a ttl based on your domain. + # Longer TTLs will suffer from event source updates in admin api. + # Shorter TTLs will suffer from unnecessary invocations to admin api. + # If this value is not set, no expiration will set for cached entries. + eventSourcesTTL: 900 diff --git a/services/events-api/src/test/java/io/github/oasis/services/events/AbstractTest.java b/services/events-api/src/test/java/io/github/oasis/services/events/AbstractTest.java index 3fa3ff23..fe2695e4 100644 --- a/services/events-api/src/test/java/io/github/oasis/services/events/AbstractTest.java +++ b/services/events-api/src/test/java/io/github/oasis/services/events/AbstractTest.java @@ -76,7 +76,7 @@ void beforeEach(Vertx vertx, VertxTestContext testContext, WireMockRuntimeInfo r JsonObject dispatcherConf = new JsonObject().put("impl", "test:any").put("configs", new JsonObject()); JsonObject testConfigs = new JsonObject() .put("http", new JsonObject().put("instances", 1).put("port", TEST_PORT)) - .put("oasis", new JsonObject().put("dispatcher", dispatcherConf) + .put("oasis", new JsonObject().put("eventstream", dispatcherConf) .put("cache", cacheConfigs).put("adminApi", adminConfigs)); modifyConfigs(testConfigs); dispatcherService = Mockito.spy(new TestDispatcherService()); diff --git a/services/events-api/src/test/java/io/github/oasis/services/events/dispatcher/ExternalDispatcherTest.java b/services/events-api/src/test/java/io/github/oasis/services/events/dispatcher/ExternalDispatcherTest.java index 283eecbb..be246c68 100644 --- a/services/events-api/src/test/java/io/github/oasis/services/events/dispatcher/ExternalDispatcherTest.java +++ b/services/events-api/src/test/java/io/github/oasis/services/events/dispatcher/ExternalDispatcherTest.java @@ -60,7 +60,7 @@ void testLoadSyncDispatcher(Vertx vertx, VertxTestContext testContext) { JsonObject dispatcherConf = new JsonObject().put("impl", "oasis:" + impl).put("configs", new JsonObject()); JsonObject testConfigs = new JsonObject() .put("http", new JsonObject().put("instances", 1).put("port", TEST_PORT)) - .put("oasis", new JsonObject().put("dispatcher", dispatcherConf).put("cache", getDefaultRedisCacheConfigs())); + .put("oasis", new JsonObject().put("eventstream", dispatcherConf).put("cache", getDefaultRedisCacheConfigs())); DeploymentOptions options = new DeploymentOptions().setConfig(testConfigs); vertx.registerVerticleFactory(new DispatcherFactory()); vertx.deployVerticle(new EventsApi(), options, testContext.succeedingThenComplete()); @@ -75,7 +75,7 @@ void testLoadASyncDispatcher(Vertx vertx, VertxTestContext testContext) { JsonObject dispatcherConf = new JsonObject().put("impl", "oasis:" + impl).put("configs", new JsonObject()); JsonObject testConfigs = new JsonObject() .put("http", new JsonObject().put("instances", 1).put("port", TEST_PORT)) - .put("oasis", new JsonObject().put("dispatcher", dispatcherConf).put("cache", getDefaultRedisCacheConfigs())); + .put("oasis", new JsonObject().put("eventstream", dispatcherConf).put("cache", getDefaultRedisCacheConfigs())); DeploymentOptions options = new DeploymentOptions().setConfig(testConfigs); vertx.registerVerticleFactory(new DispatcherFactory()); vertx.deployVerticle(new EventsApi(), options, testContext.succeedingThenComplete()); @@ -90,7 +90,7 @@ void testLoadVerticleDispatcher(Vertx vertx, VertxTestContext testContext) { JsonObject dispatcherConf = new JsonObject().put("impl", "oasis:" + impl).put("configs", new JsonObject()); JsonObject testConfigs = new JsonObject() .put("http", new JsonObject().put("instances", 1).put("port", TEST_PORT)) - .put("oasis", new JsonObject().put("dispatcher", dispatcherConf).put("cache", getDefaultRedisCacheConfigs())); + .put("oasis", new JsonObject().put("eventstream", dispatcherConf).put("cache", getDefaultRedisCacheConfigs())); DeploymentOptions options = new DeploymentOptions().setConfig(testConfigs); vertx.registerVerticleFactory(new DispatcherFactory()); vertx.deployVerticle(new EventsApi(), options, testContext.succeedingThenComplete()); @@ -105,7 +105,7 @@ void testLoadVerticleDispatcherNoConstructor(Vertx vertx, VertxTestContext testC JsonObject dispatcherConf = new JsonObject().put("impl", "oasis:" + impl).put("configs", new JsonObject()); JsonObject testConfigs = new JsonObject() .put("http", new JsonObject().put("instances", 1).put("port", TEST_PORT)) - .put("oasis", new JsonObject().put("dispatcher", dispatcherConf)); + .put("oasis", new JsonObject().put("eventstream", dispatcherConf)); DeploymentOptions options = new DeploymentOptions().setConfig(testConfigs); vertx.registerVerticleFactory(new DispatcherFactory()); vertx.deployVerticle(new EventsApi(), options, testContext.failing(e -> {})); @@ -120,7 +120,7 @@ void testUnknownDispatcher(Vertx vertx, VertxTestContext testContext) { JsonObject dispatcherConf = new JsonObject().put("impl", "oasis:" + impl).put("configs", new JsonObject()); JsonObject testConfigs = new JsonObject() .put("http", new JsonObject().put("instances", 1).put("port", TEST_PORT)) - .put("oasis", new JsonObject().put("dispatcher", dispatcherConf)); + .put("oasis", new JsonObject().put("eventstream", dispatcherConf)); DeploymentOptions options = new DeploymentOptions().setConfig(testConfigs); vertx.registerVerticleFactory(new DispatcherFactory()); vertx.deployVerticle(new EventsApi(), options, testContext.failing(e -> {})); diff --git a/services/events-api/src/test/resources/test-defaults.yml b/services/events-api/src/test/resources/test-defaults.yml new file mode 100644 index 00000000..dcc5a79c --- /dev/null +++ b/services/events-api/src/test/resources/test-defaults.yml @@ -0,0 +1,70 @@ +http: + port: 8050 + ssl: false + + # Set this to true, if you want to disable event integrity check against valid event sources registered + # through admin api. By default, this will be set to false and it means, event integrity will be enforced. + # Setting this to true, will still want to register a event source in admin api for authentication purpose. + # Beware: Disabling integrity check should be done only if you can guarantee that this event api will + # never be exposed as a public api. Otherwise, external users may send events which causes the + # bad impact to the integrity of game play. + skipEventIntegrityCheck: false + +oasis: + + adminApi: + baseUrl: "http://localhost:18050/api" + + eventSourceGet: "/admin/event-source" + playerGet: "/players" + + # authentication details of admin api + apiKey: "eventapi" + secretKey: "eventapi" + + + eventstream: + impl: "oasis:io.github.oasis.services.events.dispatcher.RabbitMQVerticle" + configs: + brokerUrls: "localhost:9092" + + maxConsumerThreadPoolSize: 4 + + gameEventsConsumer: + # groupId = "" + + # If not specified, this will take engine id + # instanceId = "" + + # These props will directly feed to Kafka property configs + # Check kafka consumer documentation for supported props. + props: + #"max.poll.records": 10, + #"session.timeout.ms": 86400000 + + broadcastConsumer: + # groupId = "" + + # These props will directly feed to Kafka property configs + # Check kafka consumer documentation for supported props. + props: + + dispatcherConfigs: + props: + + engineEventConsumer: + props: + + cache: + impl: "oasis:io.github.oasis.services.events.db.RedisVerticle" + configs: + connectionString: "redis://localhost:6379" + maxPoolSize: 16 + maxWaitingHandlers: 16 + + # expiration seconds for once cached event source data. + # Better to have a ttl based on your domain. + # Longer TTLs will suffer from event source updates in admin api. + # Shorter TTLs will suffer from unnecessary invocations to admin api. + # If this value is not set, no expiration will set for cached entries. + eventSourcesTTL: 900 diff --git a/services/feeder/pom.xml b/services/feeder/pom.xml index 9de2e4d6..37e03423 100644 --- a/services/feeder/pom.xml +++ b/services/feeder/pom.xml @@ -18,6 +18,11 @@ caffeine 3.1.1 + + ch.qos.logback + logback-classic + ${logback.version} + org.junit.jupiter diff --git a/services/feeder/src/main/java/io/github/oasis/services/feeds/Feeder.java b/services/feeder/src/main/java/io/github/oasis/services/feeds/Feeder.java index c79d10fb..bd7e30bf 100644 --- a/services/feeder/src/main/java/io/github/oasis/services/feeds/Feeder.java +++ b/services/feeder/src/main/java/io/github/oasis/services/feeds/Feeder.java @@ -215,7 +215,6 @@ private FeedDeliverable loadFeedDeliveryImplementation(OasisConfigs configs) { private static OasisConfigs loadConfigs() { var oasisConfigFilePath = resolveConfigFileLocation(); - System.out.println(oasisConfigFilePath); if (Texts.isEmpty(oasisConfigFilePath)) { LOG.error("Loading default configurations bundled with artifacts!"); throw new IllegalStateException("Cannot load Oasis feeder configurations! Config location is unspecified!"); diff --git a/services/feeder/src/main/java/io/github/oasis/services/feeds/services/ApiDataService.java b/services/feeder/src/main/java/io/github/oasis/services/feeds/services/ApiDataService.java index 489cde48..1e4fb800 100644 --- a/services/feeder/src/main/java/io/github/oasis/services/feeds/services/ApiDataService.java +++ b/services/feeder/src/main/java/io/github/oasis/services/feeds/services/ApiDataService.java @@ -29,6 +29,8 @@ import io.github.oasis.core.model.EventSource; import io.github.oasis.core.model.PlayerObject; import io.github.oasis.core.model.TeamObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.net.URI; @@ -43,6 +45,8 @@ public class ApiDataService implements DataService { + private static final Logger LOG = LoggerFactory.getLogger(ApiDataService.class); + private final ObjectMapper mapper = new ObjectMapper(); private HttpClient client; @@ -71,8 +75,10 @@ public void init(OasisConfigs configs) { this.urlTeamGet = configs.get("oasis.adminApi.teamGet", EMPTY); this.requestTimeout = configs.getInt("oasis.adminApi.requestTimeout", DEF_REQUEST_TIMEOUT); - int connectTimeout = configs.getInt("oasis.adminApi.connectTimeout", DEF_CONNECT_TIMEOUT); + + LOG.info("Creating admin api client via: {}", adminApiBaseUrl); + this.client = HttpClient.newBuilder() .version(HttpClient.Version.HTTP_1_1) .followRedirects(HttpClient.Redirect.NORMAL) diff --git a/services/stats-api/src/main/java/io/github/oasis/core/services/api/configs/StreamConfigs.java b/services/stats-api/src/main/java/io/github/oasis/core/services/api/configs/StreamConfigs.java index 7e5ef007..a0c4abfb 100644 --- a/services/stats-api/src/main/java/io/github/oasis/core/services/api/configs/StreamConfigs.java +++ b/services/stats-api/src/main/java/io/github/oasis/core/services/api/configs/StreamConfigs.java @@ -45,12 +45,12 @@ public class StreamConfigs { @Bean public EventStreamFactory createStreamFactory(OasisConfigs oasisConfigs) { - String dispatcherImpl = oasisConfigs.get("oasis.dispatcher.impl", null); + String dispatcherImpl = oasisConfigs.get("oasis.eventstream.impl", null); if (StringUtils.isBlank(dispatcherImpl)) { - throw new IllegalStateException("Mandatory dispatcher implementation has not specified!"); + throw new IllegalStateException("Mandatory eventstream implementation has not specified!"); } - LOG.info("Initializing dispatcher implementation {}...", dispatcherImpl); + LOG.info("Initializing eventstream implementation {}...", dispatcherImpl); if (StringUtils.startsWith(dispatcherImpl, "classpath:")) { String dispatcherClz = StringUtils.substringAfter(dispatcherImpl, "classpath:"); @@ -62,14 +62,14 @@ public EventStreamFactory createStreamFactory(OasisConfigs oasisConfigs) { @Bean public EventDispatcher createStreamDispatcher(OasisConfigs oasisConfigs, EventStreamFactory eventStreamFactory) throws Exception { - String dispatcherImpl = oasisConfigs.get("oasis.dispatcher.impl", null); + String dispatcherImpl = oasisConfigs.get("oasis.eventstream.impl", null); - LOG.info("Dispatcher loaded from {}", dispatcherImpl); + LOG.info("Eventstream loaded from {}", dispatcherImpl); EventDispatcher dispatcher = eventStreamFactory.getDispatcher(); - Map config = oasisConfigs.getObject("oasis.dispatcher.configs"); + Map config = oasisConfigs.getObject("oasis.eventstream.configs"); EventDispatcher.DispatcherContext context = () -> config; dispatcher.init(context); - LOG.info("Dispatcher {} successfully loaded!", dispatcherImpl); + LOG.info("Eventstream {} successfully loaded!", dispatcherImpl); return dispatcher; } @@ -87,18 +87,18 @@ private EventStreamFactory loadFromClasspathReflection(String dispatcherImpl) { return (EventStreamFactory) referredClz.getDeclaredConstructor().newInstance(); } catch (ReflectiveOperationException e) { - throw new IllegalStateException("Dispatcher implementation cannot be found in classpath! " + dispatcherImpl); + throw new IllegalStateException("Eventstream implementation cannot be found in classpath! " + dispatcherImpl); } } private EventStreamFactory loadFromServiceLoader(String dispatcherImpl) { return ServiceLoader.load(EventStreamFactory.class) .stream() - .peek(eventStreamFactoryProvider -> LOG.info("Found dispatcher implementation: {}", eventStreamFactoryProvider.type().getName())) + .peek(eventStreamFactoryProvider -> LOG.info("Found eventstream implementation: {}", eventStreamFactoryProvider.type().getName())) .filter(eventStreamFactoryProvider -> dispatcherImpl.equals(eventStreamFactoryProvider.type().getName())) .map(ServiceLoader.Provider::get) .findFirst() - .orElseThrow(() -> new IllegalStateException("Unknown dispatcher implementation provided! " + dispatcherImpl)); + .orElseThrow(() -> new IllegalStateException("Unknown eventstream implementation provided! " + dispatcherImpl)); } } diff --git a/services/stats-api/src/main/resources/api.yml b/services/stats-api/src/main/resources/api.yml index e5a5378d..28dbb623 100644 --- a/services/stats-api/src/main/resources/api.yml +++ b/services/stats-api/src/main/resources/api.yml @@ -40,7 +40,7 @@ oasis: maxIdle: 16 minIdle: 4 - dispatcher: + eventstream: impl: "io.github.oasis.ext.kafkastream.KafkaStreamFactory" configs: brokerUrls: "localhost:29092" diff --git a/services/stats-api/src/main/resources/application.conf b/services/stats-api/src/main/resources/application.conf deleted file mode 100644 index 9ae38f15..00000000 --- a/services/stats-api/src/main/resources/application.conf +++ /dev/null @@ -1,89 +0,0 @@ -http = { - port = 8050 - ssl = false -} - -oasis = { - db = { - engine: redis - admin: jdbc - } - - jdbc = { - url: "jdbc:h2:mem:test", - driver: "org.h2.Driver", - user: "sa", - password: "sa", - } - - defaultApiKeys = "root:root,eventapi:eventapi" - - cache { - host: "localhost" - port: 6379 - - connectionRetries: 5 - connectionRetryDelay: 5000 - - pool = { - max: 16 - maxIdle: 16 - minIdle: 4 - } - } - - enginedb { - host: "localhost" - port: 6379 - - connectionRetries: 5 - connectionRetryDelay: 5000 - - pool = { - max: 16 - maxIdle: 16 - minIdle: 4 - } - } - - dispatcher = { - impl = "io.github.oasis.ext.kafkastream.KafkaStreamFactory" - configs = { - brokerUrls: "localhost:29092" - - maxConsumerThreadPoolSize: 4 - - gameEventsConsumer = { - # groupId = "" - - # If not specified, this will take engine id - # instanceId = "" - - # These props will directly feed to Kafka property configs - # Check kafka consumer documentation for supported props. - props = { - #"max.poll.records": 10, - #"session.timeout.ms": 86400000 - } - } - - broadcastConsumer = { - # groupId = "" - - # These props will directly feed to Kafka property configs - # Check kafka consumer documentation for supported props. - props = { - } - } - - dispatcherConfigs = { - props = {} - } - - engineEventConsumer = { - props = {} - } - - } - } -} \ No newline at end of file diff --git a/services/stats-api/src/main/resources/logback.xml b/services/stats-api/src/main/resources/logback.xml new file mode 100644 index 00000000..6c7b2b9f --- /dev/null +++ b/services/stats-api/src/main/resources/logback.xml @@ -0,0 +1,57 @@ + + + + + + + INFO + + + [%date{ISO8601}] [%-5level] [%-38logger{10}] - %msg%n + + + + myapp.log + false + + myapp_%d{yyyy-MM-dd}.log + + + [%date{ISO8601}] [%-5level] [%-30logger{10}] - %msg%n + + + + + 8192 + true + + + + + + + + + + + + + + \ No newline at end of file diff --git a/services/stats-api/src/test/java/io/github/oasis/core/services/api/beans/RedisRepositoryTest.java b/services/stats-api/src/test/java/io/github/oasis/core/services/api/beans/RedisRepositoryTest.java index e0326161..7f81f6f2 100644 --- a/services/stats-api/src/test/java/io/github/oasis/core/services/api/beans/RedisRepositoryTest.java +++ b/services/stats-api/src/test/java/io/github/oasis/core/services/api/beans/RedisRepositoryTest.java @@ -75,7 +75,7 @@ class RedisRepositoryTest { @BeforeAll public static void beforeAll() { var configs = new OasisConfigs.Builder() - .withCustomEnvOverrides(Map.of("OASIS_OASIS_CACHE_URL", redisContainer.getRedisURI())) + .withCustomEnvOverrides(Map.of("O_OASIS_CACHE_URL", redisContainer.getRedisURI())) .buildFromYamlResource("test-configs.yml", Thread.currentThread().getContextClassLoader()); RedisDb redisDb = RedisDb.create(configs, "oasis.cache"); redisDb.init(); diff --git a/services/stats-api/src/test/java/io/github/oasis/core/services/api/beans/TestEngineConfigs.java b/services/stats-api/src/test/java/io/github/oasis/core/services/api/beans/TestEngineConfigs.java index be7d6a65..7a97a7b3 100644 --- a/services/stats-api/src/test/java/io/github/oasis/core/services/api/beans/TestEngineConfigs.java +++ b/services/stats-api/src/test/java/io/github/oasis/core/services/api/beans/TestEngineConfigs.java @@ -26,8 +26,8 @@ public class TestEngineConfigs { @Bean public OasisConfigs createOasisConfigs(RedisContainer redisContainer) { var overriddenMap = Map.of( - "OASIS_OASIS_CACHE_URL", redisContainer.getRedisURI(), - "OASIS_OASIS_ENGINEDB_URL", redisContainer.getRedisURI()); + "O_OASIS_CACHE_URL", redisContainer.getRedisURI(), + "O_OASIS_ENGINEDB_URL", redisContainer.getRedisURI()); return new OasisConfigs.Builder() .withCustomEnvOverrides(overriddenMap) .buildFromYamlResource("test-configs.yml"); diff --git a/services/stats-api/src/test/resources/logback.xml b/services/stats-api/src/test/resources/logback.xml index f1408cc5..76d8f689 100644 --- a/services/stats-api/src/test/resources/logback.xml +++ b/services/stats-api/src/test/resources/logback.xml @@ -48,6 +48,7 @@ + diff --git a/services/stats-api/src/test/resources/test-configs.yml b/services/stats-api/src/test/resources/test-configs.yml index c14172e5..b17ec0f1 100644 --- a/services/stats-api/src/test/resources/test-configs.yml +++ b/services/stats-api/src/test/resources/test-configs.yml @@ -16,7 +16,7 @@ oasis: defaultApiKeys: "root:root,admin:admin:7,guest:guest:0" cache: - url: redis://localhost:6390 + url: redis://localhost:6379 connectionRetries: 5 connectionRetryDelay: 5000 @@ -37,7 +37,7 @@ oasis: maxIdle: 16 minIdle: 4 - dispatcher: + eventstream: impl: "classpath:io.github.oasis.core.services.api.beans.TestDispatcherFactory" configs: brokerUrls: "localhost:29092"