From 1f2e28780db05b659de25ac9fb01d8774a724b28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Poniedzia=C5=82ek?= Date: Tue, 9 May 2023 15:11:41 +0200 Subject: [PATCH] Config parsing improvements (close #1252) --- .../transformer/stream/common/CliConfig.scala | 17 ++- .../transformer/stream/common/Config.scala | 12 +- .../stream/common/ConfigUtils.scala | 23 ++-- .../rdbloader/common/config/ConfigUtils.scala | 114 ++++++++---------- .../common/config/TransformerCliConfig.scala | 41 ++----- .../rdbloader/common/config/args.scala | 59 +++++++++ .../loader/databricks/ConfigSpec.scala | 8 +- .../snowplow/rdbloader/config/CliConfig.scala | 48 +++----- .../snowplow/rdbloader/config/Config.scala | 18 ++- .../src/test/resources/app-config.hocon | 22 ++++ .../test/resources/invalid-config.yml.base64 | 1 - .../resources/invalid-redshift.json.base64 | 1 - .../resources/invalid-resolver.json.base64 | 1 - .../loader/src/test/resources/resolver.hocon | 21 ++++ .../src/test/resources/resolver.json.base64 | 1 - .../test/resources/simplelogger.properties | 4 - .../test/resources/valid-config.yml.base64 | 1 - .../test/resources/valid-redshift.json.base64 | 1 - .../snowplow/rdbloader/ConfigSpec.scala | 24 +--- .../snowplow/rdbloader/SpecHelpers.scala | 28 +++-- .../rdbloader/config/CliConfigSpec.scala | 101 +++++++++++----- .../snowplow/loader/redshift/ConfigSpec.scala | 6 +- .../loader/snowflake/ConfigSpec.scala | 8 +- .../transformer/batch/CliConfig.scala | 44 ++++--- .../rdbloader/transformer/batch/Config.scala | 12 +- .../transformer/batch/ConfigSpec.scala | 30 +++-- .../stream/kinesis/ConfigSpec.scala | 4 +- .../stream/pubsub/ConfigSpec.scala | 4 +- project/Dependencies.scala | 8 +- 29 files changed, 366 insertions(+), 296 deletions(-) create mode 100644 modules/common/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/common/config/args.scala create mode 100644 modules/loader/src/test/resources/app-config.hocon delete mode 100644 modules/loader/src/test/resources/invalid-config.yml.base64 delete mode 100644 modules/loader/src/test/resources/invalid-redshift.json.base64 delete mode 100644 modules/loader/src/test/resources/invalid-resolver.json.base64 create mode 100644 modules/loader/src/test/resources/resolver.hocon delete mode 100644 modules/loader/src/test/resources/resolver.json.base64 delete mode 100644 modules/loader/src/test/resources/simplelogger.properties delete mode 100644 modules/loader/src/test/resources/valid-config.yml.base64 delete mode 100644 modules/loader/src/test/resources/valid-redshift.json.base64 diff --git a/modules/common-transformer-stream/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/CliConfig.scala b/modules/common-transformer-stream/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/CliConfig.scala index 41b3f40d3..3cae2875f 100644 --- a/modules/common-transformer-stream/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/CliConfig.scala +++ b/modules/common-transformer-stream/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/CliConfig.scala @@ -15,17 +15,16 @@ package com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common import cats.data.EitherT import cats.effect.Sync - -import com.snowplowanalytics.snowplow.rdbloader.common.config.TransformerCliConfig +import cats.implicits._ +import com.snowplowanalytics.snowplow.rdbloader.common.config.{ConfigUtils, TransformerCliConfig} object CliConfig { - implicit def configParsable[F[_]: Sync]: TransformerCliConfig.Parsable[F, Config] = - new TransformerCliConfig.Parsable[F, Config] { - def fromString(conf: String): EitherT[F, String, Config] = - Config.fromString(conf) - } - def loadConfigFrom[F[_]: Sync](name: String, description: String)(args: Seq[String]): EitherT[F, String, CliConfig] = - TransformerCliConfig.loadConfigFrom[F, Config](name, description, args) + for { + raw <- EitherT.fromEither[F](TransformerCliConfig.command(name, description).parse(args).leftMap(_.show)) + appConfig <- Config.parse(raw.config) + resolverConfig <- ConfigUtils.parseJsonF[F](raw.igluConfig) + duplicatesStorageConfig <- raw.duplicateStorageConfig.traverse(d => ConfigUtils.parseJsonF(d)) + } yield TransformerCliConfig(resolverConfig, duplicatesStorageConfig, appConfig) } diff --git a/modules/common-transformer-stream/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/Config.scala b/modules/common-transformer-stream/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/Config.scala index e4191c51a..b34d77aa3 100644 --- a/modules/common-transformer-stream/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/Config.scala +++ b/modules/common-transformer-stream/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/Config.scala @@ -13,16 +13,14 @@ package com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common import java.net.URI - import cats.implicits._ import cats.data.EitherT import cats.effect.Sync - +import com.snowplowanalytics.snowplow.rdbloader.common.config.args.HoconOrPath import io.circe._ import io.circe.generic.semiauto._ import scala.concurrent.duration.{Duration, FiniteDuration} - import com.snowplowanalytics.snowplow.rdbloader.common.telemetry.Telemetry import com.snowplowanalytics.snowplow.rdbloader.common.config.{ConfigUtils, TransformerConfig} import com.snowplowanalytics.snowplow.rdbloader.common.config.implicits._ @@ -45,13 +43,13 @@ final case class Config( object Config { - def fromString[F[_]: Sync](conf: String): EitherT[F, String, Config] = - fromString(conf, impureDecoders) + def parse[F[_]: Sync](config: HoconOrPath): EitherT[F, String, Config] = + parse(config, impureDecoders) - def fromString[F[_]: Sync](conf: String, decoders: Decoders): EitherT[F, String, Config] = { + def parse[F[_]: Sync](config: HoconOrPath, decoders: Decoders): EitherT[F, String, Config] = { import decoders._ for { - config <- ConfigUtils.fromStringF[F, Config](conf) + config <- ConfigUtils.parseAppConfigF[F, Config](config) _ <- EitherT.fromEither[F](TransformerConfig.formatsCheck(config.formats)) } yield config } diff --git a/modules/common-transformer-stream/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/ConfigUtils.scala b/modules/common-transformer-stream/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/ConfigUtils.scala index 6abeae3db..6ba43ccdb 100644 --- a/modules/common-transformer-stream/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/ConfigUtils.scala +++ b/modules/common-transformer-stream/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/common/ConfigUtils.scala @@ -12,29 +12,28 @@ */ package com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common -import java.nio.file.{Files, Paths} import cats.effect.IO -import io.circe.Decoder -import com.snowplowanalytics.snowplow.rdbloader.common.config.Region +import cats.effect.unsafe.implicits.global import com.snowplowanalytics.snowplow.rdbloader.common.RegionSpec +import com.snowplowanalytics.snowplow.rdbloader.common.config.args.HoconOrPath +import com.snowplowanalytics.snowplow.rdbloader.common.config.Region +import io.circe.Decoder -import cats.effect.unsafe.implicits.global +import java.nio.file.{Path, Paths} object ConfigUtils { - def getConfig[A](confPath: String, parse: String => Either[String, A]): Either[String, A] = - parse(readResource(confPath)) + def getConfigFromResource[A](resourcePath: String, parse: HoconOrPath => Either[String, A]): Either[String, A] = + parse(Right(pathOf(resourcePath))) - def readResource(resourcePath: String): String = { - val configExamplePath = Paths.get(getClass.getResource(resourcePath).toURI) - Files.readString(configExamplePath) - } + def pathOf(resource: String): Path = + Paths.get(getClass.getResource(resource).toURI) def testDecoders: Config.Decoders = new Config.Decoders { implicit def regionDecoder: Decoder[Region] = RegionSpec.testRegionConfigDecoder } - def testParseStreamConfig(conf: String): Either[String, Config] = - Config.fromString[IO](conf, testDecoders).value.unsafeRunSync() + def testParseStreamConfig(config: HoconOrPath): Either[String, Config] = + Config.parse[IO](config, testDecoders).value.unsafeRunSync() } diff --git a/modules/common/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/common/config/ConfigUtils.scala b/modules/common/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/common/config/ConfigUtils.scala index 374b3eb9c..da4e1ba72 100644 --- a/modules/common/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/common/config/ConfigUtils.scala +++ b/modules/common/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/common/config/ConfigUtils.scala @@ -14,84 +14,72 @@ */ package com.snowplowanalytics.snowplow.rdbloader.common.config -import java.nio.charset.StandardCharsets.UTF_8 -import java.util.Base64 - -import cats.Id +import cats.data.EitherT import cats.effect.Sync -import cats.data.{EitherT, ValidatedNel} import cats.syntax.either._ import cats.syntax.show._ - -import io.circe.parser.parse +import com.snowplowanalytics.snowplow.rdbloader.common.config.args._ +import com.typesafe.config.{Config => TypesafeConfig, ConfigFactory} import io.circe._ +import io.circe.config.syntax.CirceConfigOps -import pureconfig.module.circe._ -import pureconfig._ -import pureconfig.error._ - -import com.snowplowanalytics.iglu.client.Resolver +import java.nio.file.{Files, Path} +import scala.jdk.CollectionConverters._ object ConfigUtils { - private val Base64Decoder = Base64.getDecoder - def fromStringF[F[_]: Sync, A](conf: String)(implicit d: Decoder[A]): EitherT[F, String, A] = - EitherT(Sync[F].delay(fromString(conf))) + def parseAppConfigF[F[_]: Sync, A: Decoder](in: HoconOrPath): EitherT[F, String, A] = + EitherT(Sync[F].delay(parseAppConfig(in))) - def fromString[A](conf: String)(implicit d: Decoder[A]): Either[String, A] = - Either - .catchNonFatal { - val source = ConfigSource.string(conf) - namespaced( - ConfigSource.default(namespaced(source.withFallback(namespaced(ConfigSource.default)))) - ) - } - .leftMap(error => ConfigReaderFailures(CannotParse(s"Not valid HOCON. ${error.getMessage}", None))) - .flatMap { config => - config - .load[Json] - .flatMap { json => - json.as[A].leftMap(failure => ConfigReaderFailures(CannotParse(failure.show, None))) - } - } - .leftMap(_.prettyPrint()) + def parseJsonF[F[_]: Sync](in: HoconOrPath): EitherT[F, String, Json] = + EitherT(Sync[F].delay(parseJson(in))) + + def parseAppConfig[A: Decoder](in: HoconOrPath): Either[String, A] = + parseHoconOrPath(in, appConfigFallbacks) + + def parseJson(in: HoconOrPath): Either[String, Json] = + parseHoconOrPath[Json](in, identity) - def base64decode(str: String): Either[String, String] = + def hoconFromString(str: String): Either[String, DecodedHocon] = Either - .catchOnly[IllegalArgumentException](Base64Decoder.decode(str)) - .map(arr => new String(arr, UTF_8)) + .catchNonFatal(DecodedHocon(ConfigFactory.parseString(str))) .leftMap(_.getMessage) - object Base64Json { - def decode(str: String): ValidatedNel[String, Json] = - base64decode(str) - .flatMap(str => parse(str).leftMap(_.show)) - .toValidatedNel + private def parseHoconOrPath[A: Decoder]( + config: HoconOrPath, + fallbacks: TypesafeConfig => TypesafeConfig + ): Either[String, A] = + config match { + case Left(hocon) => + resolve[A](hocon, fallbacks) + case Right(path) => + readTextFrom(path) + .flatMap(hoconFromString) + .flatMap(hocon => resolve[A](hocon, fallbacks)) + } + + private def resolve[A: Decoder](hocon: DecodedHocon, fallbacks: TypesafeConfig => TypesafeConfig): Either[String, A] = { + val either = for { + resolved <- Either.catchNonFatal(hocon.value.resolve()).leftMap(_.getMessage) + merged <- Either.catchNonFatal(fallbacks(resolved)).leftMap(_.getMessage) + parsed <- merged.as[A].leftMap(_.show) + } yield parsed + either.leftMap(e => s"Cannot resolve config: $e") } - /** - * Optionally give precedence to configs wrapped in a "snowplow" block. To help avoid polluting - * config namespace - */ - private def namespaced(configObjSource: ConfigObjectSource): ConfigObjectSource = - ConfigObjectSource { - for { - configObj <- configObjSource.value() - conf = configObj.toConfig - } yield - if (conf.hasPath(Namespace)) - conf.getConfig(Namespace).withFallback(conf.withoutPath(Namespace)) - else - conf - } + private def readTextFrom(path: Path): Either[String, String] = + Either + .catchNonFatal(Files.readAllLines(path).asScala.mkString("\n")) + .leftMap(e => s"Error reading ${path.toAbsolutePath} file from filesystem: ${e.getMessage}") - def validateResolverJson(resolverJson: Json): ValidatedNel[String, Json] = - Resolver - .parse[Id](resolverJson) - .leftMap(_.show) - .map(_ => resolverJson) - .toValidatedNel + private def appConfigFallbacks(config: TypesafeConfig): TypesafeConfig = + namespaced(ConfigFactory.load(namespaced(config.withFallback(namespaced(ConfigFactory.load()))))) - // Used as an option prefix when reading system properties. - val Namespace = "snowplow" + private def namespaced(config: TypesafeConfig): TypesafeConfig = { + val namespace = "snowplow" + if (config.hasPath(namespace)) + config.getConfig(namespace).withFallback(config.withoutPath(namespace)) + else + config + } } diff --git a/modules/common/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/common/config/TransformerCliConfig.scala b/modules/common/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/common/config/TransformerCliConfig.scala index ad10bc45b..d90ab02b2 100644 --- a/modules/common/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/common/config/TransformerCliConfig.scala +++ b/modules/common/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/common/config/TransformerCliConfig.scala @@ -13,14 +13,10 @@ */ package com.snowplowanalytics.snowplow.rdbloader.common.config -import cats.Monad import cats.implicits._ -import cats.data.EitherT - import io.circe.Json - import com.monovore.decline.{Command, Opts} - +import com.snowplowanalytics.snowplow.rdbloader.common.config.args.HoconOrPath import com.snowplowanalytics.snowplow.rdbloader.generated.BuildInfo case class TransformerCliConfig[C]( @@ -30,29 +26,21 @@ case class TransformerCliConfig[C]( ) object TransformerCliConfig { - trait Parsable[F[_], C] { - def fromString(conf: String): EitherT[F, String, C] - } - case class RawConfig( - igluConfig: Json, - duplicateStorageConfig: Option[Json], - config: String + igluConfig: HoconOrPath, + duplicateStorageConfig: Option[HoconOrPath], + config: HoconOrPath ) val igluConfigOpt = Opts - .option[String]("iglu-config", "Base64-encoded Iglu Client JSON config", metavar = "") - .mapValidated(ConfigUtils.Base64Json.decode) - .mapValidated(ConfigUtils.validateResolverJson) + .option[HoconOrPath]("iglu-config", "Base64-encoded Iglu Client HOCON config", metavar = "resolver.hocon") val duplicatesOpt = Opts - .option[String]("duplicate-storage-config", "Base64-encoded Events Manifest JSON config", metavar = "") - .mapValidated(ConfigUtils.Base64Json.decode) + .option[HoconOrPath]("duplicate-storage-config", "Base64-encoded Events Manifest JSON config", metavar = "") .orNone - def configOpt = Opts - .option[String]("config", "base64-encoded config HOCON", "c", "config.hocon") - .mapValidated(x => ConfigUtils.base64decode(x).toValidatedNel) + val configOpt = Opts + .option[HoconOrPath]("config", "base64-encoded config HOCON", "c", "config.hocon") def rawConfigOpt: Opts[RawConfig] = (igluConfigOpt, duplicatesOpt, configOpt).mapN { (iglu, dupeStorage, target) => @@ -61,17 +49,4 @@ object TransformerCliConfig { def command(name: String, description: String): Command[RawConfig] = Command(s"$name-${BuildInfo.version}", description)(rawConfigOpt) - - def loadConfigFrom[F[_], C]( - name: String, - description: String, - args: Seq[String] - )(implicit M: Monad[F], - P: Parsable[F, C] - ): EitherT[F, String, TransformerCliConfig[C]] = - for { - raw <- EitherT.fromEither[F](command(name, description).parse(args).leftMap(_.show)) - conf <- P.fromString(raw.config) - } yield TransformerCliConfig(raw.igluConfig, raw.duplicateStorageConfig, conf) - } diff --git a/modules/common/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/common/config/args.scala b/modules/common/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/common/config/args.scala new file mode 100644 index 000000000..352d57b57 --- /dev/null +++ b/modules/common/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/common/config/args.scala @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2012-2023 Snowplow Analytics Ltd. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at + * http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and + * limitations there under. + */ +package com.snowplowanalytics.snowplow.rdbloader.common.config +import cats.data.{NonEmptyList, ValidatedNel} +import cats.implicits._ +import com.monovore.decline.Argument +import com.typesafe.config.{Config => TypesafeConfig} + +import java.nio.charset.StandardCharsets +import java.nio.file.Path +import java.util.Base64 + +object args { + + final case class DecodedHocon(value: TypesafeConfig) extends AnyVal + + type HoconOrPath = Either[DecodedHocon, Path] + + implicit val hoconOrPathArg: Argument[HoconOrPath] = + new Argument[HoconOrPath] { + def read(string: String): ValidatedNel[String, HoconOrPath] = { + val hocon = Argument[DecodedHocon].read(string).map(_.asLeft) + val path = Argument[Path].read(string).map(_.asRight) + val error = show"Value $string cannot be parsed as Base64 hocon neither as FS path" + hocon.orElse(path).leftMap(_ => NonEmptyList.one(error)) + } + + def defaultMetavar: String = "input" + } + + implicit val decodedHoconArg: Argument[DecodedHocon] = + new Argument[DecodedHocon] { + def read(string: String): ValidatedNel[String, DecodedHocon] = + tryToDecodeString(string) + .leftMap(_.getMessage) + .flatMap(ConfigUtils.hoconFromString) + .toValidatedNel + + def defaultMetavar: String = "base64" + } + + private def tryToDecodeString(string: String): Either[IllegalArgumentException, String] = + Either + .catchOnly[IllegalArgumentException](Base64.getDecoder.decode(string)) + .map(bytes => new String(bytes, StandardCharsets.UTF_8)) + +} diff --git a/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/ConfigSpec.scala b/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/ConfigSpec.scala index e32b7659b..0848cf95e 100644 --- a/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/ConfigSpec.scala +++ b/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/ConfigSpec.scala @@ -28,7 +28,7 @@ class ConfigSpec extends Specification { "fromString" should { "be able to parse extended AWS Databricks Loader config" in { - val result = getConfig("/loader/aws/databricks.config.reference.hocon", Config.fromString[IO]) + val result = getConfigFromResource("/loader/aws/databricks.config.reference.hocon", Config.parseAppConfig[IO]) val monitoring = exampleMonitoring.copy( snowplow = exampleMonitoring.snowplow.map(_.copy(appId = "databricks-loader")) ) @@ -50,7 +50,7 @@ class ConfigSpec extends Specification { } "be able to parse extended GCP Databricks Loader config" in { - val result = getConfig("/loader/gcp/databricks.config.reference.hocon", Config.fromString[IO]) + val result = getConfigFromResource("/loader/gcp/databricks.config.reference.hocon", Config.parseAppConfig[IO]) val monitoring = exampleMonitoring.copy( snowplow = exampleMonitoring.snowplow.map(_.copy(appId = "databricks-loader")), folders = exampleMonitoring.folders.map( @@ -78,7 +78,7 @@ class ConfigSpec extends Specification { } "be able to parse minimal AWS Snowflake Loader config" in { - val result = getConfig("/loader/aws/databricks.config.minimal.hocon", testParseConfig) + val result = getConfigFromResource("/loader/aws/databricks.config.minimal.hocon", testParseConfig) val storage = ConfigSpec.exampleStorage.copy( catalog = None, password = StorageTarget.PasswordConfig.PlainText("Supersecret1") @@ -105,7 +105,7 @@ class ConfigSpec extends Specification { } "be able to parse minimal GCP Snowflake Loader config" in { - val result = getConfig("/loader/gcp/databricks.config.minimal.hocon", testParseConfig) + val result = getConfigFromResource("/loader/gcp/databricks.config.minimal.hocon", testParseConfig) val storage = ConfigSpec.exampleStorage.copy( catalog = None, password = StorageTarget.PasswordConfig.PlainText("Supersecret1") diff --git a/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/config/CliConfig.scala b/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/config/CliConfig.scala index 1006dc84e..878f9a69a 100644 --- a/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/config/CliConfig.scala +++ b/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/config/CliConfig.scala @@ -15,10 +15,9 @@ package com.snowplowanalytics.snowplow.rdbloader.config import cats.effect.Sync import cats.data._ import cats.implicits._ - import io.circe.Json - import com.monovore.decline.{Command, Opts} +import com.snowplowanalytics.snowplow.rdbloader.common.config.args.HoconOrPath // This project import com.snowplowanalytics.snowplow.rdbloader.generated.BuildInfo @@ -42,41 +41,30 @@ case class CliConfig( object CliConfig { - case class RawCliConfig( - config: String, - dryRun: Boolean, - resolverConfig: Json + private final case class RawCliConfig( + appConfig: HoconOrPath, + resolverConfig: HoconOrPath, + dryRun: Boolean ) - val config = Opts - .option[String]("config", "base64-encoded HOCON configuration", "c", "config.hocon") - .mapValidated(x => ConfigUtils.base64decode(x).toValidatedNel) - val igluConfig = Opts - .option[String]("iglu-config", "base64-encoded string with Iglu resolver configuration JSON", "r", "resolver.json") - .mapValidated(ConfigUtils.Base64Json.decode) - .mapValidated(ConfigUtils.validateResolverJson) - val dryRun = Opts.flag("dry-run", "do not perform loading, just print SQL statements").orFalse + private val appConfig = Opts + .option[HoconOrPath]("config", "base64-encoded HOCON configuration", "c", "config.hocon") + + private val resolverConfig = Opts + .option[HoconOrPath]("iglu-config", "base64-encoded HOCON Iglu resolver configuration", "r", "resolver.hocon") + + private val dryRun = Opts.flag("dry-run", "do not perform loading, just print SQL statements").orFalse - val cliConfig = (config, dryRun, igluConfig).mapN { case (cfg, dry, iglu) => - RawCliConfig(cfg, dry, iglu) + private val cliConfig = (appConfig, resolverConfig, dryRun).mapN { case (cfg, iglu, dryRun) => + RawCliConfig(cfg, iglu, dryRun) } - val parser = Command[RawCliConfig](BuildInfo.name, BuildInfo.version)(cliConfig) + private val parser = Command[RawCliConfig](BuildInfo.name, BuildInfo.version)(cliConfig) - /** - * Parse raw CLI arguments into validated and transformed application config This is - * side-effecting function, it'll print to stdout all errors - * - * @param argv - * list of command-line arguments - * @return - * none if not all required arguments were passed or unknown arguments provided, some config - * error if arguments could not be transformed into application config some application config - * if everything was validated correctly - */ def parse[F[_]: Sync](argv: Seq[String]): EitherT[F, String, CliConfig] = for { raw <- EitherT.fromEither[F](parser.parse(argv).leftMap(_.show)) - conf <- Config.fromString[F](raw.config) - } yield CliConfig(conf, raw.dryRun, raw.resolverConfig) + appConfig <- Config.parseAppConfig[F](raw.appConfig) + resolverJson <- ConfigUtils.parseJsonF[F](raw.resolverConfig) + } yield CliConfig(appConfig, raw.dryRun, resolverJson) } diff --git a/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/config/Config.scala b/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/config/Config.scala index 280300f64..7478e4e1d 100644 --- a/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/config/Config.scala +++ b/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/config/Config.scala @@ -13,24 +13,19 @@ package com.snowplowanalytics.snowplow.rdbloader.config import java.net.URI - import scala.concurrent.duration.{Duration, FiniteDuration} - import cats.effect.Sync import cats.data.EitherT import cats.syntax.either._ import cats.syntax.option._ - import io.circe._ import io.circe.generic.semiauto._ - import org.http4s.{ParseFailure, Uri} - import cron4s.CronExpr import cron4s.circe._ - import com.snowplowanalytics.snowplow.rdbloader.common.telemetry.Telemetry import com.snowplowanalytics.snowplow.rdbloader.common.cloud.BlobStorage +import com.snowplowanalytics.snowplow.rdbloader.common.config.args.HoconOrPath import com.snowplowanalytics.snowplow.rdbloader.common.config.{ConfigUtils, Region} import com.snowplowanalytics.snowplow.rdbloader.config.Config._ @@ -58,12 +53,15 @@ object Config { val MetricsDefaultPrefix = "snowplow.rdbloader" - def fromString[F[_]: Sync](s: String): EitherT[F, String, Config[StorageTarget]] = - fromString(s, implicits().configDecoder) + def parseAppConfig[F[_]: Sync](config: HoconOrPath): EitherT[F, String, Config[StorageTarget]] = + parseAppConfig(config, implicits().configDecoder) - def fromString[F[_]: Sync](s: String, configDecoder: Decoder[Config[StorageTarget]]): EitherT[F, String, Config[StorageTarget]] = { + def parseAppConfig[F[_]: Sync]( + config: HoconOrPath, + configDecoder: Decoder[Config[StorageTarget]] + ): EitherT[F, String, Config[StorageTarget]] = { implicit val implConfigDecoder: Decoder[Config[StorageTarget]] = configDecoder - ConfigUtils.fromStringF[F, Config[StorageTarget]](s) + ConfigUtils.parseAppConfigF[F, Config[StorageTarget]](config) } final case class Schedule( diff --git a/modules/loader/src/test/resources/app-config.hocon b/modules/loader/src/test/resources/app-config.hocon new file mode 100644 index 000000000..488636465 --- /dev/null +++ b/modules/loader/src/test/resources/app-config.hocon @@ -0,0 +1,22 @@ +{ + "region": "us-east-1" + "messageQueue": "test-queue" + "storage" : { + "type": "redshift" + "port": 5432 + "jdbc": { "ssl": true } + "host": "redshift.amazonaws.com" + "maxError": 10, + "database": "snowplow" + "roleArn": "arn:aws:iam::123456789876:role/RedshiftLoadRole" + "schema": "atomic" + "username": "admin" + "password": ${sub.a} + "loadAuthMethod": { + "type": "NoCreds" + } + }, + "sub": { + "a": "Supersecret password from substitution!" + } +} \ No newline at end of file diff --git a/modules/loader/src/test/resources/invalid-config.yml.base64 b/modules/loader/src/test/resources/invalid-config.yml.base64 deleted file mode 100644 index 778ab3e9f..000000000 --- a/modules/loader/src/test/resources/invalid-config.yml.base64 +++ /dev/null @@ -1 +0,0 @@ -YXdzOg0KICAjIENyZWRlbnRpYWxzIGNhbiBiZSBoYXJkY29kZWQgb3Igc2V0IGluIGVudmlyb25tZW50IHZhcmlhYmxlcw0KICBhY2Nlc3Nfa2V5X2lkOiBBQUFBQUFBQUFBQUFBQUFBQUFBQQ0KICBzZWNyZXRfYWNjZXNzX2tleTogQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQg0KICBzMzoNCiAgICByZWdpb246IHVzLWVhc3QtMQ0KICAgIGJ1Y2tldHM6DQogICAgICBhc3NldHM6IHMzOi8vc25vd3Bsb3ctYWNtZS1zdG9yYWdlDQogICAgICBqc29ucGF0aF9hc3NldHM6IG5vdC1hLWJ1Y2tldCAjIEVSUk9SDQogICAgICBsb2c6IHMzOi8vc25vd3Bsb3ctYWNtZS1zdG9yYWdlL2xvZ3MNCiAgICAgIHJhdzoNCiAgICAgICAgaW46DQogICAgICAgICAgLSBzMzovL3Nub3dwbG93LWFjbWUtc3RvcmFnZS9pbnB1dA0KICAgICAgICBwcm9jZXNzaW5nOiBzMzovL3Nub3dwbG93LWFjbWUtc3RvcmFnZS9wcm9jZXNzaW5nDQogICAgICAgIGFyY2hpdmU6IHMzOi8vc25vd3Bsb3ctYWNtZS1zdG9yYWdlL2FyY2hpdmUNCiAgICAgIGVucmljaGVkOg0KICAgICAgICBnb29kOiBzMzovL3Nub3dwbG93LWFjbWUtc3RvcmFnZS9lbnJpY2hlZC9nb29kDQogICAgICAgIGJhZDogczM6Ly9zbm93cGxvdy1hY21lLXN0b3JhZ2UvZW5yaWNoZWQvYmFkDQogICAgICAgIGVycm9yczoNCiAgICAgICAgYXJjaGl2ZTogczM6Ly9zbm93cGxvdy1hY21lLXN0b3JhZ2UvZW5yaWNoZWQtYXJjaGl2ZQ0KICAgICAgc2hyZWRkZWQ6DQogICAgICAgIGdvb2Q6IHMzOi8vc25vd3Bsb3ctYWNtZS1zdG9yYWdlL3NocmVkZGVkL2dvb2QvDQogICAgICAgIGJhZDogczM6Ly9zbm93cGxvdy1hY21lLXN0b3JhZ2Uvc2hyZWRkZWQvYmFkLw0KICAgICAgICBlcnJvcnM6DQogICAgICAgIGFyY2hpdmU6IHMzOi8vc25vd3Bsb3ctYWNtZS1zdG9yYWdlL3NocmVkZGVkLWFyY2hpdmUvDQogIGVtcjoNCiAgICBhbWlfdmVyc2lvbjogNC41LjANCiAgICByZWdpb246IHVzLWVhc3QtMQ0KICAgIGpvYmZsb3dfcm9sZTogRU1SX0VDMl9EZWZhdWx0Um9sZSAjIENyZWF0ZWQgdXNpbmcgJCBhd3MgZW1yIGNyZWF0ZS1kZWZhdWx0LXJvbGVzDQogICAgc2VydmljZV9yb2xlOiBFTVJfRGVmYXVsdFJvbGUgICAgICMgQ3JlYXRlZCB1c2luZyAkIGF3cyBlbXIgY3JlYXRlLWRlZmF1bHQtcm9sZXMNCiAgICBwbGFjZW1lbnQ6ICAgICAgIyBTZXQgdGhpcyBpZiBub3QgcnVubmluZyBpbiBWUEMuIExlYXZlIGJsYW5rIG90aGVyd2lzZQ0KICAgIGVjMl9zdWJuZXRfaWQ6ICAjIFNldCB0aGlzIGlmIHJ1bm5pbmcgaW4gVlBDLiBMZWF2ZSBibGFuayBvdGhlcndpc2UNCiAgICBlYzJfa2V5X25hbWU6IGFudG9uLWVuZ2luZWVyaW5nDQogICAgYm9vdHN0cmFwOiBbXSAgICAgICAgICAgIyBTZXQgdGhpcyB0byBzcGVjaWZ5IGN1c3RvbSBib29zdHJhcCBhY3Rpb25zLiBMZWF2ZSBlbXB0eSBvdGhlcndpc2UNCiAgICBzb2Z0d2FyZToNCiAgICAgIGhiYXNlOiAgICAgICAgICAgICAgICAjIE9wdGlvbmFsLiBUbyBsYXVuY2ggb24gY2x1c3RlciwgcHJvdmlkZSB2ZXJzaW9uLCAiMC45Mi4wIiwga2VlcCBxdW90ZXMuIExlYXZlIGVtcHR5IG90aGVyd2lzZS4NCiAgICAgIGxpbmd1YWw6ICAgICAgICAgICAgICAjIE9wdGlvbmFsLiBUbyBsYXVuY2ggb24gY2x1c3RlciwgcHJvdmlkZSB2ZXJzaW9uLCAiMS4xIiwga2VlcCBxdW90ZXMuIExlYXZlIGVtcHR5IG90aGVyd2lzZS4NCiAgICAjIEFkanVzdCB5b3VyIEhhZG9vcCBjbHVzdGVyIGJlbG93DQogICAgam9iZmxvdzoNCiAgICAgIGpvYl9uYW1lOiBTbm93cGxvdyBFVEwNCiAgICAgIG1hc3Rlcl9pbnN0YW5jZV90eXBlOiBtMS5tZWRpdW0NCiAgICAgIGNvcmVfaW5zdGFuY2VfY291bnQ6IDINCiAgICAgIGNvcmVfaW5zdGFuY2VfdHlwZTogbTEubWVkaXVtDQogICAgICB0YXNrX2luc3RhbmNlX2NvdW50OiAwICMgSW5jcmVhc2UgdG8gdXNlIHNwb3QgaW5zdGFuY2VzDQogICAgICB0YXNrX2luc3RhbmNlX3R5cGU6IG0xLm1lZGl1bQ0KICAgICAgdGFza19pbnN0YW5jZV9iaWQ6IDAuMDE1ICMgSW4gVVNELiBBZGp1c3QgYmlkLCBvciBsZWF2ZSBibGFuayBmb3Igbm9uLXNwb3QtcHJpY2VkIChpLmUuIG9uLWRlbWFuZCkgdGFzayBpbnN0YW5jZXMNCiAgICBib290c3RyYXBfZmFpbHVyZV90cmllczogMyAjIE51bWJlciBvZiB0aW1lcyB0byBhdHRlbXB0IHRoZSBqb2IgaW4gdGhlIGV2ZW50IG9mIGJvb3RzdHJhcCBmYWlsdXJlcw0KICAgIGFkZGl0aW9uYWxfaW5mbzogICAgICAgICMgT3B0aW9uYWwgSlNPTiBzdHJpbmcgZm9yIHNlbGVjdGluZyBhZGRpdGlvbmFsIGZlYXR1cmVzDQpjb2xsZWN0b3JzOg0KICBmb3JtYXQ6IGNsai10b21jYXQgIyBGb3IgZXhhbXBsZTogJ2Nsai10b21jYXQnIGZvciB0aGUgQ2xvanVyZSBDb2xsZWN0b3IsICd0aHJpZnQnIGZvciBUaHJpZnQgcmVjb3JkcywgJ3Rzdi9jb20uYW1hem9uLmF3cy5jbG91ZGZyb250L3dkX2FjY2Vzc19sb2cnIGZvciBDbG91ZGZyb250IGFjY2VzcyBsb2dzIG9yICduZGpzb24vdXJiYW5haXJzaGlwLmNvbm5lY3QvdjEnIGZvciBVcmJhbkFpcnNoaXAgQ29ubmVjdCBldmVudHMNCmVucmljaDoNCiAgdmVyc2lvbnM6DQogICAgc3BhcmtfZW5yaWNoOiAxLjkuMCAjIFZlcnNpb24gb2YgdGhlIEhhZG9vcCBFbnJpY2htZW50IHByb2Nlc3MNCiAgY29udGludWVfb25fdW5leHBlY3RlZF9lcnJvcjogZmFsc2UgIyBTZXQgdG8gJ3RydWUnIChhbmQgc2V0IDpvdXRfZXJyb3JzOiBhYm92ZSkgaWYgeW91IGRvbid0IHdhbnQgYW55IGV4Y2VwdGlvbnMgdGhyb3duIGZyb20gRVRMDQogIG91dHB1dF9jb21wcmVzc2lvbjogTk9ORSAjIENvbXByZXNzaW9uIG9ubHkgc3VwcG9ydGVkIHdpdGggUmVkc2hpZnQsIHNldCB0byBOT05FIGlmIHlvdSBoYXZlIFBvc3RncmVzIHRhcmdldHMuIEFsbG93ZWQgZm9ybWF0czogTk9ORSwgR1pJUA0Kc3RvcmFnZToNCiAgdmVyc2lvbnM6DQogICAgcmRiX3NocmVkZGVyOiAwLjExLjAtcmM0ICMgVmVyc2lvbiBvZiB0aGUgSGFkb29wIFNocmVkZGluZyBwcm9jZXNzDQogICAgaGFkb29wX2VsYXN0aWNzZWFyY2g6IDAuMS4wICMgVmVyc2lvbiBvZiB0aGUgSGFkb29wIHRvIEVsYXN0aWNzZWFyY2ggY29weWluZyBwcm9jZXNzDQogIGRvd25sb2FkOg0KICAgIGZvbGRlcjogIyBQb3N0Z3Jlcy1vbmx5IGNvbmZpZyBvcHRpb24uIFdoZXJlIHRvIHN0b3JlIHRoZSBkb3dubG9hZGVkIGZpbGVzLiBMZWF2ZSBibGFuayBmb3IgUmVkc2hpZnQNCm1vbml0b3Jpbmc6DQogIHRhZ3M6IHt9ICMgTmFtZS12YWx1ZSBwYWlycyBkZXNjcmliaW5nIHRoaXMgam9iDQogIGxvZ2dpbmc6DQogICAgbGV2ZWw6IERFQlVHICMgWW91IGNhbiBvcHRpb25hbGx5IHN3aXRjaCB0byBJTkZPIGZvciBwcm9kdWN0aW9uDQogIHNub3dwbG93Og0KICAgIG1ldGhvZDogZ2V0DQogICAgYXBwX2lkOiBiYXRjaC1waXBlbGluZQ0KICAgIGNvbGxlY3Rvcjogc25wbG93LmFjbWUuY29tDQo= \ No newline at end of file diff --git a/modules/loader/src/test/resources/invalid-redshift.json.base64 b/modules/loader/src/test/resources/invalid-redshift.json.base64 deleted file mode 100644 index 8c00429b8..000000000 --- a/modules/loader/src/test/resources/invalid-redshift.json.base64 +++ /dev/null @@ -1 +0,0 @@ -ewogICAgInNjaGVtYSI6ICJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy5zdG9yYWdlL3JlZHNoaWZ0X2NvbmZpZy9qc29uc2NoZW1hLzEtMC0wIiwKICAgICJkYXRhIjogewogICAgICAgICJuYW1lIjogIkFXUyBSZWRzaGlmdCBlbnJpY2hlZCBldmVudHMgc3RvcmFnZSIsCiAgICAgICAgImhvc3QiOiAiYWJjZGUudXMtZWFzdC0xLnJlZHNoaWZ0LmFtYXpvbmF3cy5jb20iLAogICAgICAgICJkYXRhYmFzZSI6ICJzbm93cGxvdyIsCiAgICAgICAgInBvcnQiOiA1NDM5LAogICAgICAgICJzc2xNb2RlIjogIkRJU0FCTEUiLAogICAgICAgICJ1c2VybmFtZSI6ICJhZG1pbiIsCiAgICAgICAgInBhc3N3b3JkIjogInN1cGVyc2VjcmV0MSIsCiAgICAgICAgInNjaGVtYSI6ICJhdG9taWMiLAogICAgICAgICJtYXhFcnJvciI6ICIxIiwKICAgICAgICAiY29tcFJvd3MiOiAyMDAwMCwKICAgICAgICAicHVycG9zZSI6ICJFTlJJQ0hFRF9FVkVOVFMiCiAgICB9Cn0K \ No newline at end of file diff --git a/modules/loader/src/test/resources/invalid-resolver.json.base64 b/modules/loader/src/test/resources/invalid-resolver.json.base64 deleted file mode 100644 index d5023b1fc..000000000 --- a/modules/loader/src/test/resources/invalid-resolver.json.base64 +++ /dev/null @@ -1 +0,0 @@ -ewogICJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3MuaWdsdS9yZXNvbHZlci1jb25maWcvanNvbnNjaGVtYS8xLTAtMSIsCiAgImRhdGEiOiB7CiAgICAiY2FjaGVTaXplIjogNTAwLAogICAgInJlcG9zaXRvcmllcyI6IFsKICAgICAgewogICAgICAgICJuYW1lIjogIklnbHUgQ2VudHJhbCIsCiAgICAgICAgInByaW9yaXR5IjogMCwKICAgICAgICAidmVuZG9yUHJlZml4ZXMiOiBbICJjb20uc25vd3Bsb3dhbmFseXRpY3MiIF0sCiAgICAgICAgImNvbm5lY3Rpb24iOiB7CiAgICAgICAgICAiaHR0cCI6IHsKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJFbWJlZGRlZCBUZXN0IiwKICAgICAgICAicHJpb3JpdHkiOiAwLAogICAgICAgICJ2ZW5kb3JQcmVmaXhlcyI6IFsgImNvbS5zbm93cGxvd2FuYWx5dGljcyIgXSwKICAgICAgICAiY29ubmVjdGlvbiI6IHsKICAgICAgICAgICJlbWJlZGRlZCI6IHsKICAgICAgICAgICAgInBhdGgiOiAiL2VtYmVkIgogICAgICAgICAgfQogICAgICAgIH0KICAgICAgfQogICAgXQogIH0KfQ== \ No newline at end of file diff --git a/modules/loader/src/test/resources/resolver.hocon b/modules/loader/src/test/resources/resolver.hocon new file mode 100644 index 000000000..c7d2084f6 --- /dev/null +++ b/modules/loader/src/test/resources/resolver.hocon @@ -0,0 +1,21 @@ +{ + "schema": "iglu:com.snowplowanalytics.iglu/resolver-config/jsonschema/1-0-1", + "data": { + "cacheSize": 500, + "repositories": [ + { + "name": ${sub.a}, + "priority": 0, + "vendorPrefixes": [ "com.snowplowanalytics" ], + "connection": { + "http": { + "uri": "https://raw.githubusercontent.com/snowplow/iglu-central/feature/redshift-401" + } + } + } + ] + }, + "sub": { + "a": "Resolved name from substitution!" + } +} \ No newline at end of file diff --git a/modules/loader/src/test/resources/resolver.json.base64 b/modules/loader/src/test/resources/resolver.json.base64 deleted file mode 100644 index 848fe6970..000000000 --- a/modules/loader/src/test/resources/resolver.json.base64 +++ /dev/null @@ -1 +0,0 @@ -ewogICJzY2hlbWEiOiAiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3MuaWdsdS9yZXNvbHZlci1jb25maWcvanNvbnNjaGVtYS8xLTAtMSIsCiAgImRhdGEiOiB7CiAgICAiY2FjaGVTaXplIjogNTAwLAogICAgInJlcG9zaXRvcmllcyI6IFsKICAgICAgewogICAgICAgICJuYW1lIjogIklnbHUgQ2VudHJhbCIsCiAgICAgICAgInByaW9yaXR5IjogMCwKICAgICAgICAidmVuZG9yUHJlZml4ZXMiOiBbICJjb20uc25vd3Bsb3dhbmFseXRpY3MiIF0sCiAgICAgICAgImNvbm5lY3Rpb24iOiB7CiAgICAgICAgICAiaHR0cCI6IHsKICAgICAgICAgICAgInVyaSI6ICJodHRwczovL3Jhdy5naXRodWJ1c2VyY29udGVudC5jb20vc25vd3Bsb3cvaWdsdS1jZW50cmFsL2ZlYXR1cmUvcmVkc2hpZnQtNDAxIgogICAgICAgICAgfQogICAgICAgIH0KICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogIkVtYmVkZGVkIFRlc3QiLAogICAgICAgICJwcmlvcml0eSI6IDAsCiAgICAgICAgInZlbmRvclByZWZpeGVzIjogWyAiY29tLnNub3dwbG93YW5hbHl0aWNzIiBdLAogICAgICAgICJjb25uZWN0aW9uIjogewogICAgICAgICAgImVtYmVkZGVkIjogewogICAgICAgICAgICAicGF0aCI6ICIvZW1iZWQiCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICB9CiAgICBdCiAgfQp9Cg== diff --git a/modules/loader/src/test/resources/simplelogger.properties b/modules/loader/src/test/resources/simplelogger.properties deleted file mode 100644 index 9bb63dcb5..000000000 --- a/modules/loader/src/test/resources/simplelogger.properties +++ /dev/null @@ -1,4 +0,0 @@ -org.slf4j.simpleLogger.showLogName=false -org.slf4j.simpleLogger.showDateTime=true -org.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS -org.slf4j.simpleLogger.showThreadName=false diff --git a/modules/loader/src/test/resources/valid-config.yml.base64 b/modules/loader/src/test/resources/valid-config.yml.base64 deleted file mode 100644 index c5e79bd7c..000000000 --- a/modules/loader/src/test/resources/valid-config.yml.base64 +++ /dev/null @@ -1 +0,0 @@ -YXdzOg0KICBzMzoNCiAgICByZWdpb246IHVzLWVhc3QtMQ0KICAgIGJ1Y2tldHM6DQogICAgICBhc3NldHM6IHMzOi8vc25vd3Bsb3ctYWNtZS1zdG9yYWdlDQogICAgICBqc29ucGF0aF9hc3NldHM6ICMgSWYgeW91IGhhdmUgZGVmaW5lZCB5b3VyIG93biBKU09OIFNjaGVtYXMsIGFkZCB0aGUgczM6Ly8gcGF0aCB0byB5b3VyIG93biBKU09OIFBhdGggZmlsZXMgaW4geW91ciBvd24gYnVja2V0IGhlcmUNCiAgICAgIGxvZzogczM6Ly9zbm93cGxvdy1hY21lLXN0b3JhZ2UvbG9ncw0KICAgICAgcmF3Og0KICAgICAgICBpbjoNCiAgICAgICAgICAtIHMzOi8vc25vd3Bsb3ctYWNtZS1zdG9yYWdlL2lucHV0DQogICAgICAgIHByb2Nlc3Npbmc6IHMzOi8vc25vd3Bsb3ctYWNtZS1zdG9yYWdlL3Byb2Nlc3NpbmcNCiAgICAgICAgYXJjaGl2ZTogczM6Ly9zbm93cGxvdy1hY21lLXN0b3JhZ2UvYXJjaGl2ZQ0KICAgICAgZW5yaWNoZWQ6DQogICAgICAgIGdvb2Q6IHMzOi8vc25vd3Bsb3ctYWNtZS1zdG9yYWdlL2VucmljaGVkL2dvb2QNCiAgICAgICAgYmFkOiBzMzovL3Nub3dwbG93LWFjbWUtc3RvcmFnZS9lbnJpY2hlZC9iYWQNCiAgICAgICAgZXJyb3JzOg0KICAgICAgICBhcmNoaXZlOiBzMzovL3Nub3dwbG93LWFjbWUtc3RvcmFnZS9lbnJpY2hlZC1hcmNoaXZlDQogICAgICBzaHJlZGRlZDoNCiAgICAgICAgZ29vZDogczM6Ly9zbm93cGxvdy1hY21lLXN0b3JhZ2Uvc2hyZWRkZWQvZ29vZC8NCiAgICAgICAgYmFkOiBzMzovL3Nub3dwbG93LWFjbWUtc3RvcmFnZS9zaHJlZGRlZC9iYWQvDQogICAgICAgIGVycm9yczoNCiAgICAgICAgYXJjaGl2ZTogczM6Ly9zbm93cGxvdy1hY21lLXN0b3JhZ2Uvc2hyZWRkZWQtYXJjaGl2ZS8NCiAgZW1yOg0KICAgIGFtaV92ZXJzaW9uOiA0LjUuMA0KICAgIHJlZ2lvbjogdXMtZWFzdC0xDQogICAgam9iZmxvd19yb2xlOiBFTVJfRUMyX0RlZmF1bHRSb2xlICMgQ3JlYXRlZCB1c2luZyAkIGF3cyBlbXIgY3JlYXRlLWRlZmF1bHQtcm9sZXMNCiAgICBzZXJ2aWNlX3JvbGU6IEVNUl9EZWZhdWx0Um9sZSAgICAgIyBDcmVhdGVkIHVzaW5nICQgYXdzIGVtciBjcmVhdGUtZGVmYXVsdC1yb2xlcw0KICAgIHBsYWNlbWVudDogICAgICAjIFNldCB0aGlzIGlmIG5vdCBydW5uaW5nIGluIFZQQy4gTGVhdmUgYmxhbmsgb3RoZXJ3aXNlDQogICAgZWMyX3N1Ym5ldF9pZDogICMgU2V0IHRoaXMgaWYgcnVubmluZyBpbiBWUEMuIExlYXZlIGJsYW5rIG90aGVyd2lzZQ0KICAgIGVjMl9rZXlfbmFtZTogYW50b24tZW5naW5lZXJpbmcNCiAgICBib290c3RyYXA6IFtdICAgICAgICAgICAjIFNldCB0aGlzIHRvIHNwZWNpZnkgY3VzdG9tIGJvb3N0cmFwIGFjdGlvbnMuIExlYXZlIGVtcHR5IG90aGVyd2lzZQ0KICAgIHNvZnR3YXJlOg0KICAgICAgaGJhc2U6ICAgICAgICAgICAgICAgICMgT3B0aW9uYWwuIFRvIGxhdW5jaCBvbiBjbHVzdGVyLCBwcm92aWRlIHZlcnNpb24sICIwLjkyLjAiLCBrZWVwIHF1b3Rlcy4gTGVhdmUgZW1wdHkgb3RoZXJ3aXNlLg0KICAgICAgbGluZ3VhbDogICAgICAgICAgICAgICMgT3B0aW9uYWwuIFRvIGxhdW5jaCBvbiBjbHVzdGVyLCBwcm92aWRlIHZlcnNpb24sICIxLjEiLCBrZWVwIHF1b3Rlcy4gTGVhdmUgZW1wdHkgb3RoZXJ3aXNlLg0KICAgICMgQWRqdXN0IHlvdXIgSGFkb29wIGNsdXN0ZXIgYmVsb3cNCiAgICBqb2JmbG93Og0KICAgICAgam9iX25hbWU6IFNub3dwbG93IEVUTA0KICAgICAgbWFzdGVyX2luc3RhbmNlX3R5cGU6IG0xLm1lZGl1bQ0KICAgICAgY29yZV9pbnN0YW5jZV9jb3VudDogMg0KICAgICAgY29yZV9pbnN0YW5jZV90eXBlOiBtMS5tZWRpdW0NCiAgICAgIHRhc2tfaW5zdGFuY2VfY291bnQ6IDAgIyBJbmNyZWFzZSB0byB1c2Ugc3BvdCBpbnN0YW5jZXMNCiAgICAgIHRhc2tfaW5zdGFuY2VfdHlwZTogbTEubWVkaXVtDQogICAgICB0YXNrX2luc3RhbmNlX2JpZDogMC4wMTUgIyBJbiBVU0QuIEFkanVzdCBiaWQsIG9yIGxlYXZlIGJsYW5rIGZvciBub24tc3BvdC1wcmljZWQgKGkuZS4gb24tZGVtYW5kKSB0YXNrIGluc3RhbmNlcw0KICAgIGJvb3RzdHJhcF9mYWlsdXJlX3RyaWVzOiAzICMgTnVtYmVyIG9mIHRpbWVzIHRvIGF0dGVtcHQgdGhlIGpvYiBpbiB0aGUgZXZlbnQgb2YgYm9vdHN0cmFwIGZhaWx1cmVzDQogICAgYWRkaXRpb25hbF9pbmZvOiAgICAgICAgIyBPcHRpb25hbCBKU09OIHN0cmluZyBmb3Igc2VsZWN0aW5nIGFkZGl0aW9uYWwgZmVhdHVyZXMNCmNvbGxlY3RvcnM6DQogIGZvcm1hdDogY2xqLXRvbWNhdCAjIEZvciBleGFtcGxlOiAnY2xqLXRvbWNhdCcgZm9yIHRoZSBDbG9qdXJlIENvbGxlY3RvciwgJ3RocmlmdCcgZm9yIFRocmlmdCByZWNvcmRzLCAndHN2L2NvbS5hbWF6b24uYXdzLmNsb3VkZnJvbnQvd2RfYWNjZXNzX2xvZycgZm9yIENsb3VkZnJvbnQgYWNjZXNzIGxvZ3Mgb3IgJ25kanNvbi91cmJhbmFpcnNoaXAuY29ubmVjdC92MScgZm9yIFVyYmFuQWlyc2hpcCBDb25uZWN0IGV2ZW50cw0KZW5yaWNoOg0KICB2ZXJzaW9uczoNCiAgICBzcGFya19lbnJpY2g6IDEuOS4wICMgVmVyc2lvbiBvZiB0aGUgSGFkb29wIEVucmljaG1lbnQgcHJvY2Vzcw0KICBjb250aW51ZV9vbl91bmV4cGVjdGVkX2Vycm9yOiBmYWxzZSAjIFNldCB0byAndHJ1ZScgKGFuZCBzZXQgOm91dF9lcnJvcnM6IGFib3ZlKSBpZiB5b3UgZG9uJ3Qgd2FudCBhbnkgZXhjZXB0aW9ucyB0aHJvd24gZnJvbSBFVEwNCiAgb3V0cHV0X2NvbXByZXNzaW9uOiBOT05FICMgQ29tcHJlc3Npb24gb25seSBzdXBwb3J0ZWQgd2l0aCBSZWRzaGlmdCwgc2V0IHRvIE5PTkUgaWYgeW91IGhhdmUgUG9zdGdyZXMgdGFyZ2V0cy4gQWxsb3dlZCBmb3JtYXRzOiBOT05FLCBHWklQDQpzdG9yYWdlOg0KICB2ZXJzaW9uczoNCiAgICByZGJfc2hyZWRkZXI6IDAuMTIuMC1yYzQgIyBWZXJzaW9uIG9mIHRoZSBIYWRvb3AgU2hyZWRkaW5nIHByb2Nlc3MNCiAgICBoYWRvb3BfZWxhc3RpY3NlYXJjaDogMC4xLjAgIyBWZXJzaW9uIG9mIHRoZSBIYWRvb3AgdG8gRWxhc3RpY3NlYXJjaCBjb3B5aW5nIHByb2Nlc3MNCiAgZG93bmxvYWQ6DQogICAgZm9sZGVyOiAjIFBvc3RncmVzLW9ubHkgY29uZmlnIG9wdGlvbi4gV2hlcmUgdG8gc3RvcmUgdGhlIGRvd25sb2FkZWQgZmlsZXMuIExlYXZlIGJsYW5rIGZvciBSZWRzaGlmdA0KbW9uaXRvcmluZzoNCiAgdGFnczoge30gIyBOYW1lLXZhbHVlIHBhaXJzIGRlc2NyaWJpbmcgdGhpcyBqb2INCiAgbG9nZ2luZzoNCiAgICBsZXZlbDogREVCVUcgIyBZb3UgY2FuIG9wdGlvbmFsbHkgc3dpdGNoIHRvIElORk8gZm9yIHByb2R1Y3Rpb24NCiAgc25vd3Bsb3c6DQogICAgbWV0aG9kOiBnZXQNCiAgICBhcHBfaWQ6IGJhdGNoLXBpcGVsaW5lDQogICAgY29sbGVjdG9yOiBzbnBsb3cuYWNtZS5jb20NCg== diff --git a/modules/loader/src/test/resources/valid-redshift.json.base64 b/modules/loader/src/test/resources/valid-redshift.json.base64 deleted file mode 100644 index fca322956..000000000 --- a/modules/loader/src/test/resources/valid-redshift.json.base64 +++ /dev/null @@ -1 +0,0 @@ -ewoJInNjaGVtYSI6ICJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy5zdG9yYWdlL3JlZHNoaWZ0X2NvbmZpZy9qc29uc2NoZW1hLzUtMC0wIiwKCSJkYXRhIjogewoJCSJuYW1lIjogIkFXUyBSZWRzaGlmdCBlbnJpY2hlZCBldmVudHMgc3RvcmFnZSIsCgkJImlkIjogImUxN2MwZGVkLWVlZTctNDg0NS1hN2U2LThmZGM4OGQ1OTlkMCIsCgkJImhvc3QiOiAiYW5na29yLXdhdC1maW5hbC5jY3h2ZHB6MDF4bnIudXMtZWFzdC0xLnJlZHNoaWZ0LmFtYXpvbmF3cy5jb20iLAoJCSJkYXRhYmFzZSI6ICJzbm93cGxvdyIsCgkJInBvcnQiOiA1NDM5LAoJCSJqZGJjIjogewoJCQkic3NsIjogZmFsc2UKCQl9LAoJCSJ1c2VybmFtZSI6ICJhZG1pbiIsCgkJInBhc3N3b3JkIjogIlN1cGVyc2VjcmV0MSIsCgkJInNjaGVtYSI6ICJhdG9taWMiLAoJCSJyb2xlQXJuIjogImFybjphd3M6aWFtOjoxMjM0NTY3ODk4NzY6cm9sZS9SZWRzaGlmdExvYWRSb2xlIiwKCQkic3NoVHVubmVsIjogbnVsbCwKCQkibWF4RXJyb3IiOiAxLAoJCSJjb21wUm93cyI6IDIwMDAwLAogICAgICAgICAgICAgICAgImJsYWNrbGlzdFRhYnVsYXIiOiBudWxsLAogICAgICAgICAgICAgICAgIm1lc3NhZ2VRdWV1ZSI6ICJtZXNzYWdlLXF1ZXVlIiwKCQkicHVycG9zZSI6ICJFTlJJQ0hFRF9FVkVOVFMiCgl9Cn0= \ No newline at end of file diff --git a/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/ConfigSpec.scala b/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/ConfigSpec.scala index 51b857908..1f32c7325 100644 --- a/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/ConfigSpec.scala +++ b/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/ConfigSpec.scala @@ -13,27 +13,20 @@ package com.snowplowanalytics.snowplow.rdbloader import java.net.URI -import java.nio.file.{Files, Paths} - import scala.concurrent.duration._ - import cats.data.EitherT - import cats.effect.IO - import org.http4s.implicits._ - import com.snowplowanalytics.snowplow.rdbloader.common.telemetry.Telemetry import com.snowplowanalytics.snowplow.rdbloader.common.config.Region import com.snowplowanalytics.snowplow.rdbloader.common.RegionSpec import com.snowplowanalytics.snowplow.rdbloader.common.cloud.BlobStorage import com.snowplowanalytics.snowplow.rdbloader.config.{Config, StorageTarget} - import cron4s.Cron - import org.specs2.mutable.Specification - import cats.effect.unsafe.implicits.global +import com.snowplowanalytics.snowplow.rdbloader.SpecHelpers.fullPathOf +import com.snowplowanalytics.snowplow.rdbloader.common.config.args.HoconOrPath class ConfigSpec extends Specification { import ConfigSpec._ @@ -224,14 +217,9 @@ object ConfigSpec { exampleTelemetry ) - def getConfig[A](confPath: String, parse: String => EitherT[IO, String, A]): Either[String, A] = - parse(readResource(confPath)).value.unsafeRunSync() - - def readResource(resourcePath: String): String = { - val configExamplePath = Paths.get(getClass.getResource(resourcePath).toURI) - Files.readString(configExamplePath) - } + def getConfigFromResource[A](resourcePath: String, parse: HoconOrPath => EitherT[IO, String, A]): Either[String, A] = + parse(Right(fullPathOf(resourcePath))).value.unsafeRunSync() - def testParseConfig(conf: String): EitherT[IO, String, Config[StorageTarget]] = - Config.fromString[IO](conf, Config.implicits(RegionSpec.testRegionConfigDecoder).configDecoder) + def testParseConfig(config: HoconOrPath): EitherT[IO, String, Config[StorageTarget]] = + Config.parseAppConfig[IO](config, Config.implicits(RegionSpec.testRegionConfigDecoder).configDecoder) } diff --git a/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/SpecHelpers.scala b/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/SpecHelpers.scala index 2ced3be79..d4488eeda 100644 --- a/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/SpecHelpers.scala +++ b/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/SpecHelpers.scala @@ -13,23 +13,16 @@ package com.snowplowanalytics.snowplow.rdbloader import com.snowplowanalytics.snowplow.rdbloader.common.cloud.BlobStorage -import scala.io.Source.fromInputStream - import doobie.util.fragment.Fragment import doobie.util.update.Update0 - -import io.circe.jawn.parse import com.snowplowanalytics.snowplow.rdbloader.config.{Config, StorageTarget} -import com.snowplowanalytics.snowplow.rdbloader.config.CliConfig -object SpecHelpers { +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path, Paths} +import java.util.Base64 - val resolverConfig = fromInputStream(getClass.getResourceAsStream("/resolver.json.base64")).getLines.mkString("\n") - val invalidResolverConfig = fromInputStream(getClass.getResourceAsStream("/invalid-resolver.json.base64")).getLines.mkString("\n") - val resolverJson = - parse(new String(java.util.Base64.getDecoder.decode(resolverConfig))).getOrElse(throw new RuntimeException("Invalid resolver.json")) +object SpecHelpers { - val disableSsl = StorageTarget.RedshiftJdbc.empty.copy(ssl = Some(true)) val validConfig: Config[StorageTarget.Redshift] = Config( ConfigSpec.exampleRedshift, ConfigSpec.exampleCloud, @@ -44,7 +37,18 @@ object SpecHelpers { ConfigSpec.exampleFeatureFlags, ConfigSpec.exampleTelemetry ) - val validCliConfig: CliConfig = CliConfig(validConfig, false, resolverJson) + + def asB64(resourcePath: String): String = + encode(readResource(resourcePath)) + + def readResource(resourcePath: String): String = + Files.readString(fullPathOf(resourcePath)) + + def encode(string: String): String = + new String(Base64.getEncoder.encode(string.getBytes(StandardCharsets.UTF_8))) + + def fullPathOf(resource: String): Path = + Paths.get(getClass.getResource(resource).toURI) /** * Pretty prints a Scala value similar to its source represention. Particularly useful for case diff --git a/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/config/CliConfigSpec.scala b/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/config/CliConfigSpec.scala index 73318396a..c62e41ccb 100644 --- a/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/config/CliConfigSpec.scala +++ b/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/config/CliConfigSpec.scala @@ -12,47 +12,90 @@ */ package com.snowplowanalytics.snowplow.rdbloader.config -import java.util.Base64 - import cats.effect.IO +import io.circe.literal.JsonStringContext // specs2 +import cats.effect.unsafe.implicits.global +import com.snowplowanalytics.snowplow.rdbloader.SpecHelpers._ import org.specs2.mutable.Specification -import com.snowplowanalytics.snowplow.rdbloader.SpecHelpers._ -import com.snowplowanalytics.snowplow.rdbloader.ConfigSpec +class CliConfigSpec extends Specification { -import cats.effect.unsafe.implicits.global + private val expectedResolver = + json""" + { + "schema": "iglu:com.snowplowanalytics.iglu/resolver-config/jsonschema/1-0-1", + "data": { + "cacheSize": 500, + "repositories": [ + { + "name": "Resolved name from substitution!", + "priority": 0, + "vendorPrefixes": [ "com.snowplowanalytics" ], + "connection": { + "http": { + "uri": "https://raw.githubusercontent.com/snowplow/iglu-central/feature/redshift-401" + } + } + } + ] + }, + "sub": { + "a": "Resolved name from substitution!" + } + } + """ -class CliConfigSpec extends Specification { - val configB64 = new String( - Base64.getEncoder.encode( - ConfigSpec.readResource("/loader/aws/redshift.config.reference.hocon").getBytes - ) - ) - - "parse" should { - "parse valid configuration" in { - val cli = Array("--config", configB64, "--iglu-config", resolverConfig) - - val expected = CliConfig(validConfig, false, resolverJson) - val result = CliConfig.parse[IO](cli).value.unsafeRunSync() - result must beRight(expected) - } + val appConfigHocon = "/app-config.hocon" + val resolverHocon = "/resolver.hocon" - "parse CLI options with dry-run" in { - val cli = Array("--config", configB64, "--iglu-config", resolverConfig, "--dry-run") + "Cli config parse" should { + "return valid config for" >> { + "app config - base64, resolverConfig - base64" in { + assertValid( + appConfig = asB64(appConfigHocon), + resolverConfig = asB64(resolverHocon) + ) + } + "app config - full path, resolverConfig - full path" in { + assertValid( + appConfig = fullPathOf(appConfigHocon).toString, + resolverConfig = fullPathOf(resolverHocon).toString + ) + } + "app config - base64, resolverConfig - full path" in { + assertValid( + appConfig = asB64(appConfigHocon), + resolverConfig = fullPathOf(resolverHocon).toString + ) + } + "app config - full path, resolverConfig - full path" in { + assertValid( + appConfig = fullPathOf(appConfigHocon).toString, + resolverConfig = asB64(resolverHocon) + ) + } + "enabled dry run option" in { + val cli = Array("--config", asB64(appConfigHocon), "--iglu-config", asB64(resolverHocon), "--dry-run") - val expected = CliConfig(validConfig, true, resolverJson) - val result = CliConfig.parse[IO](cli).value.unsafeRunSync() - result must beRight(expected) + val result = CliConfig.parse[IO](cli).value.unsafeRunSync() + result must beRight.like { case CliConfig(_, dryRun, _) => + dryRun must beTrue + } + } } + } - "give error with invalid resolver" in { - val cli = Array("--config", configB64, "--iglu-config", invalidResolverConfig, "--dry-run") + def assertValid(appConfig: String, resolverConfig: String) = { + val cli = Array("--config", appConfig, "--iglu-config", resolverConfig) + val result = CliConfig.parse[IO](cli).value.unsafeRunSync() - val result = CliConfig.parse[IO](cli).value.unsafeRunSync() - result.isLeft must beTrue + result must beRight.like { case CliConfig(config, _, resolverConfig) => + config.storage.password.getUnencrypted must beEqualTo("Supersecret password from substitution!") + resolverConfig must beEqualTo(expectedResolver) } + } + } diff --git a/modules/redshift-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/redshift/ConfigSpec.scala b/modules/redshift-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/redshift/ConfigSpec.scala index b457dc7cf..eff8b0121 100644 --- a/modules/redshift-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/redshift/ConfigSpec.scala +++ b/modules/redshift-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/redshift/ConfigSpec.scala @@ -26,7 +26,7 @@ class ConfigSpec extends Specification { "fromString" should { "be able to parse extended Redshift config" in { - val result = getConfig("/loader/aws/redshift.config.reference.hocon", Config.fromString[IO]) + val result = getConfigFromResource("/loader/aws/redshift.config.reference.hocon", Config.parseAppConfig[IO]) val expected = Config( exampleRedshift, exampleCloud, @@ -45,7 +45,7 @@ class ConfigSpec extends Specification { } "be able to parse minimal config" in { - val result = getConfig("/loader/aws/redshift.config.minimal.hocon", testParseConfig) + val result = getConfigFromResource("/loader/aws/redshift.config.minimal.hocon", testParseConfig) val expected = Config( exampleRedshift, Config.Cloud.AWS(RegionSpec.DefaultTestRegion, exampleMessageQueue.copy(region = Some(RegionSpec.DefaultTestRegion))), @@ -64,7 +64,7 @@ class ConfigSpec extends Specification { } "give error when unknown region given" in { - val result = getConfig("/test.config1.hocon", Config.fromString[IO]) + val result = getConfigFromResource("/test.config1.hocon", Config.parseAppConfig[IO]) result must beLeft.like { case err => err must contain("unknown-region-1") } diff --git a/modules/snowflake-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/snowflake/ConfigSpec.scala b/modules/snowflake-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/snowflake/ConfigSpec.scala index a0cefaf51..77eadde8d 100644 --- a/modules/snowflake-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/snowflake/ConfigSpec.scala +++ b/modules/snowflake-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/snowflake/ConfigSpec.scala @@ -33,7 +33,7 @@ class ConfigSpec extends Specification { .copy(jdbcHost = Some("acme.eu-central-1.snowflake.com")) .copy(folderMonitoringStage = Some(StorageTarget.Snowflake.Stage("snowplow_folders_stage", None))) .copy(transformedStage = Some(StorageTarget.Snowflake.Stage("snowplow_stage", None))) - val result = getConfig("/loader/aws/snowflake.config.reference.hocon", Config.fromString[IO]) + val result = getConfigFromResource("/loader/aws/snowflake.config.reference.hocon", Config.parseAppConfig[IO]) val expected = Config( storage, exampleCloud, @@ -57,7 +57,7 @@ class ConfigSpec extends Specification { .copy(jdbcHost = Some("acme.eu-central-1.snowflake.com")) .copy(folderMonitoringStage = Some(StorageTarget.Snowflake.Stage("snowplow_folders_stage", None))) .copy(transformedStage = Some(StorageTarget.Snowflake.Stage("snowplow_stage", None))) - val result = getConfig("/loader/gcp/snowflake.config.reference.hocon", Config.fromString[IO]) + val result = getConfigFromResource("/loader/gcp/snowflake.config.reference.hocon", Config.parseAppConfig[IO]) val gcpCloud = Config.Cloud.GCP( Config.Cloud.GCP.Pubsub( subscription = "projects/project-id/subscriptions/subscription-id", @@ -93,7 +93,7 @@ class ConfigSpec extends Specification { } "be able to parse minimal AWS Snowflake Loader config" in { - val result = getConfig("/loader/aws/snowflake.config.minimal.hocon", testParseConfig) + val result = getConfigFromResource("/loader/aws/snowflake.config.minimal.hocon", testParseConfig) val expected = Config( exampleSnowflake .copy( @@ -115,7 +115,7 @@ class ConfigSpec extends Specification { } "be able to parse minimal GCP Snowflake Loader config" in { - val result = getConfig("/loader/gcp/snowflake.config.minimal.hocon", testParseConfig) + val result = getConfigFromResource("/loader/gcp/snowflake.config.minimal.hocon", testParseConfig) val gcpCloud = Config.Cloud.GCP( Config.Cloud.GCP.Pubsub( subscription = "projects/project-id/subscriptions/subscription-id", diff --git a/modules/transformer-batch/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/CliConfig.scala b/modules/transformer-batch/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/CliConfig.scala index 42bc919c7..ebde24740 100644 --- a/modules/transformer-batch/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/CliConfig.scala +++ b/modules/transformer-batch/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/CliConfig.scala @@ -13,29 +13,33 @@ */ package com.snowplowanalytics.snowplow.rdbloader.transformer.batch -import cats.data.EitherT -import cats.Id - -import com.snowplowanalytics.snowplow.rdbloader.common.config.TransformerCliConfig +import cats.implicits.{toBifunctorOps, toShow} +import com.snowplowanalytics.snowplow.rdbloader.common.config.{ConfigUtils, TransformerCliConfig} +import io.circe.Json object CliConfig { - implicit val configParsable: TransformerCliConfig.Parsable[Id, Config] = - new TransformerCliConfig.Parsable[Id, Config] { - def fromString(conf: String): EitherT[Id, String, Config] = - EitherT[Id, String, Config](Config.fromString(conf)) + def loadConfigFrom(name: String, description: String)(args: Seq[String]): Either[String, CliConfig] = + for { + raw <- TransformerCliConfig.command(name, description).parse(args).leftMap(_.show) + appConfig <- Config.parse(raw.config) + resolverConfig <- ConfigUtils.parseJson(raw.igluConfig) + duplicatesStorageConfig <- parseDuplicationConfig(raw) + cliConfig = TransformerCliConfig(resolverConfig, duplicatesStorageConfig, appConfig) + verified <- verifyDuplicationConfig(cliConfig) + } yield verified + + private def parseDuplicationConfig(raw: TransformerCliConfig.RawConfig): Either[String, Option[Json]] = + raw.duplicateStorageConfig match { + case Some(defined) => ConfigUtils.parseJson(defined).map(Some(_)) + case None => Right(None) } - def loadConfigFrom(name: String, description: String)(args: Seq[String]): Either[String, CliConfig] = - TransformerCliConfig - .loadConfigFrom[Id, Config](name, description, args) - .value - .flatMap { cli => - if (cli.duplicateStorageConfig.isDefined && !cli.config.deduplication.natural) - Left("Natural deduplication needs to be enabled when cross batch deduplication is enabled") - else if (cli.config.deduplication.synthetic != Config.Deduplication.Synthetic.None && !cli.config.deduplication.natural) - Left("Natural deduplication needs to be enabled when synthetic deduplication is enabled") - else - Right(cli) - } + private def verifyDuplicationConfig(cli: CliConfig): Either[String, CliConfig] = + if (cli.duplicateStorageConfig.isDefined && !cli.config.deduplication.natural) + Left("Natural deduplication needs to be enabled when cross batch deduplication is enabled") + else if (cli.config.deduplication.synthetic != Config.Deduplication.Synthetic.None && !cli.config.deduplication.natural) + Left("Natural deduplication needs to be enabled when synthetic deduplication is enabled") + else + Right(cli) } diff --git a/modules/transformer-batch/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/Config.scala b/modules/transformer-batch/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/Config.scala index e53433abf..995d9a632 100644 --- a/modules/transformer-batch/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/Config.scala +++ b/modules/transformer-batch/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/Config.scala @@ -14,15 +14,13 @@ package com.snowplowanalytics.snowplow.rdbloader.transformer.batch import java.net.URI import java.time.Instant - import cats.implicits._ - import io.circe._ import io.circe.generic.semiauto._ import scala.concurrent.duration.FiniteDuration - import com.snowplowanalytics.snowplow.rdbloader.common.Common +import com.snowplowanalytics.snowplow.rdbloader.common.config.args.HoconOrPath import com.snowplowanalytics.snowplow.rdbloader.common.config.{ConfigUtils, TransformerConfig} import com.snowplowanalytics.snowplow.rdbloader.common.config.TransformerConfig.Compression import com.snowplowanalytics.snowplow.rdbloader.common.config.Region @@ -41,13 +39,13 @@ final case class Config( ) object Config { - def fromString(conf: String): Either[String, Config] = - fromString(conf, impureDecoders) + def parse(config: HoconOrPath): Either[String, Config] = + parse(config, impureDecoders) - def fromString(conf: String, decoders: Decoders): Either[String, Config] = { + def parse(config: HoconOrPath, decoders: Decoders): Either[String, Config] = { import decoders._ for { - config <- ConfigUtils.fromString[Config](conf) + config <- ConfigUtils.parseAppConfig[Config](config) _ <- TransformerConfig.formatsCheck(config.formats) } yield config } diff --git a/modules/transformer-batch/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/ConfigSpec.scala b/modules/transformer-batch/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/ConfigSpec.scala index 9fd997a0b..8218fa5ff 100644 --- a/modules/transformer-batch/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/ConfigSpec.scala +++ b/modules/transformer-batch/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/batch/ConfigSpec.scala @@ -13,16 +13,15 @@ package com.snowplowanalytics.snowplow.rdbloader.transformer.batch import java.net.URI -import java.nio.file.{Files, Paths} +import java.nio.file.{Path, Paths} import java.time.Instant - import io.circe.Decoder import scala.concurrent.duration._ -import scala.collection.JavaConverters._ import com.snowplowanalytics.iglu.core.SchemaCriterion +import com.snowplowanalytics.snowplow.rdbloader.common.config.args.HoconOrPath import com.snowplowanalytics.snowplow.rdbloader.common.config.TransformerConfig.Validations -import com.snowplowanalytics.snowplow.rdbloader.common.config.{Region, TransformerConfig} +import com.snowplowanalytics.snowplow.rdbloader.common.config.{ConfigUtils, Region, TransformerConfig} import com.snowplowanalytics.snowplow.rdbloader.common.{LoaderMessage, RegionSpec} import org.specs2.mutable.Specification @@ -31,7 +30,7 @@ class ConfigSpec extends Specification { "config fromString" should { "be able to parse extended batch transformer config" in { - val result = getConfig("/transformer/aws/transformer.batch.config.reference.hocon", Config.fromString) + val result = getConfigFromResource("/transformer/aws/transformer.batch.config.reference.hocon", Config.parse) val expected = Config( exampleBatchInput, exampleOutput, @@ -47,7 +46,7 @@ class ConfigSpec extends Specification { } "be able to parse minimal batch transformer config" in { - val result = getConfig("/transformer/aws/transformer.batch.config.minimal.hocon", testParseBatchConfig) + val result = getConfigFromResource("/transformer/aws/transformer.batch.config.minimal.hocon", testParseBatchConfig) val expected = Config( exampleBatchInput, exampleDefaultOutput, @@ -63,7 +62,7 @@ class ConfigSpec extends Specification { } "give error when unknown region given" in { - val result = getConfig("/test.config1.hocon", Config.fromString) + val result = getConfigFromResource("/test.config1.hocon", Config.parse) result must beLeft(contain("unknown-region-1")) } @@ -94,7 +93,8 @@ class ConfigSpec extends Specification { val expected = "Following schema criterions overlap in different groups (TSV, JSON, skip): " + "iglu:com.acme/overlap/jsonschema/1-0-0, iglu:com.acme/overlap/jsonschema/1-*-*. " + "Make sure every schema can have only one format" - val result = Config.fromString(input) + val hocon = ConfigUtils.hoconFromString(input).right.get + val result = Config.parse(Left(hocon)) result must beLeft(expected) } } @@ -161,19 +161,17 @@ object TransformerConfigSpec { val exampleValidations = Validations(Some(Instant.parse("2021-11-18T11:00:00.00Z"))) val emptyValidations = Validations(None) - def getConfig[A](confPath: String, parse: String => Either[String, A]): Either[String, A] = - parse(readResource(confPath)) + def getConfigFromResource[A](resourcePath: String, parse: HoconOrPath => Either[String, A]): Either[String, A] = + parse(Right(pathOf(resourcePath))) - def readResource(resourcePath: String): String = { - val configExamplePath = Paths.get(getClass.getResource(resourcePath).toURI) - Files.readAllLines(configExamplePath).asScala.mkString("\n") - } + def pathOf(resource: String): Path = + Paths.get(getClass.getResource(resource).toURI) def testDecoders: Config.Decoders = new Config.Decoders { implicit def regionDecoder: Decoder[Region] = RegionSpec.testRegionConfigDecoder } - def testParseBatchConfig(conf: String): Either[String, Config] = - Config.fromString(conf, testDecoders) + def testParseBatchConfig(config: HoconOrPath): Either[String, Config] = + Config.parse(config, testDecoders) } diff --git a/modules/transformer-kinesis/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kinesis/ConfigSpec.scala b/modules/transformer-kinesis/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kinesis/ConfigSpec.scala index 88d3d13ef..1fa3e718d 100644 --- a/modules/transformer-kinesis/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kinesis/ConfigSpec.scala +++ b/modules/transformer-kinesis/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kinesis/ConfigSpec.scala @@ -40,7 +40,7 @@ class ConfigSpec extends Specification { "config fromString" should { "be able to parse extended transformer-kinesis config" in { val result = - getConfig("/transformer/aws/transformer.kinesis.config.reference.hocon", c => Config.fromString[IO](c).value.unsafeRunSync()) + getConfigFromResource("/transformer/aws/transformer.kinesis.config.reference.hocon", c => Config.parse[IO](c).value.unsafeRunSync()) val expected = Config( exampleStreamInput, exampleWindowPeriod, @@ -56,7 +56,7 @@ class ConfigSpec extends Specification { } "be able to parse minimal transformer-kinesis config" in { - val result = getConfig("/transformer/aws/transformer.kinesis.config.minimal.hocon", testParseStreamConfig) + val result = getConfigFromResource("/transformer/aws/transformer.kinesis.config.minimal.hocon", testParseStreamConfig) val expected = Config( exampleDefaultStreamInput, exampleWindowPeriod, diff --git a/modules/transformer-pubsub/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/pubsub/ConfigSpec.scala b/modules/transformer-pubsub/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/pubsub/ConfigSpec.scala index d783336e7..c86308298 100644 --- a/modules/transformer-pubsub/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/pubsub/ConfigSpec.scala +++ b/modules/transformer-pubsub/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/pubsub/ConfigSpec.scala @@ -35,7 +35,7 @@ class ConfigSpec extends Specification { "config fromString" should { "be able to parse extended transformer-pubsub config" in { val result = - getConfig("/transformer/gcp/transformer.pubsub.config.reference.hocon", c => Config.fromString[IO](c).value.unsafeRunSync()) + getConfigFromResource("/transformer/gcp/transformer.pubsub.config.reference.hocon", c => Config.parse[IO](c).value.unsafeRunSync()) val expected = Config( exampleStreamInput, exampleWindowPeriod, @@ -51,7 +51,7 @@ class ConfigSpec extends Specification { } "be able to parse minimal transformer-pubsub config" in { - val result = getConfig("/transformer/gcp/transformer.pubsub.config.minimal.hocon", testParseStreamConfig) + val result = getConfigFromResource("/transformer/gcp/transformer.pubsub.config.minimal.hocon", testParseStreamConfig) val expected = Config( exampleStreamInput, exampleWindowPeriod, diff --git a/project/Dependencies.scala b/project/Dependencies.scala index d3591fad5..b4b017815 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -21,9 +21,9 @@ object Dependencies { val igluCore = "1.1.1" val badrows = "2.2.0" val analyticsSdk = "3.1.0" - val pureconfig = "0.17.2" val cron4sCirce = "0.6.1" val circe = "0.14.1" + val circeConfig = "0.10.0" val cats = "2.2.0" val catsEffect = "3.3.14" val manifest = "0.3.0" @@ -93,10 +93,9 @@ object Dependencies { val igluCoreCirce = "com.snowplowanalytics" %% "iglu-core-circe" % V.igluCore val cats = "org.typelevel" %% "cats" % V.cats val circeCore = "io.circe" %% "circe-core" % V.circe + val circeConfig = "io.circe" %% "circe-config" % V.circeConfig val circeGeneric = "io.circe" %% "circe-generic" % V.circe val circeGenericExtra = "io.circe" %% "circe-generic-extras" % V.circe - val pureconfig = "com.github.pureconfig" %% "pureconfig" % V.pureconfig - val pureconfigCirce = "com.github.pureconfig" %% "pureconfig-circe" % V.pureconfig val cron4sCirce = ("com.github.alonsodomin.cron4s" %% "cron4s-circe" % V.cron4sCirce) .exclude("io.circe", "circe-core_2.12") // cron4s-circe lacks circe 0.13 support val fs2 = "co.fs2" %% "fs2-core" % V.fs2 @@ -213,11 +212,10 @@ object Dependencies { badrows, igluClient, catsEffectKernel, + circeConfig, circeGeneric, circeGenericExtra, circeLiteral, - pureconfig, - pureconfigCirce, cron4sCirce, schemaDdl, http4sCore,