Skip to content

Commit

Permalink
feat: added global and site-level variables (#445)
Browse files Browse the repository at this point in the history
<!--

Describe in detail the changes you are proposing, and the rationale.

-->

<!--

Link all GitHub issues fixed by this PR, and add references to prior
related PRs.

-->

Fixes #322

### NEW FEATURES | UPGRADE NOTES | ENHANCEMENTS | BUG FIXES |
EXPERIMENTS

<!--

Write a short description of your changes. Examples:

- Adds option to set global and site-level variables that will be merged
into components by key

--> 

-
  • Loading branch information
demeyerthom authored Sep 13, 2024
2 parents 8d7ccd8 + 38a4bc2 commit f27f228
Show file tree
Hide file tree
Showing 17 changed files with 314 additions and 48 deletions.
4 changes: 4 additions & 0 deletions .changes/unreleased/Added-20240910-154131.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
kind: Added
body: Added option to set global and site-level configurations. These will be merged
into the component variables during generation
time: 2024-09-10T15:41:31.371358062+02:00
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ require (
github.com/mitchellh/mapstructure v1.5.0
github.com/olekukonko/tablewriter v0.0.5
github.com/rs/zerolog v1.33.0
github.com/santhosh-tekuri/jsonschema/v6 v6.0.1
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
github.com/spf13/afero v1.11.0
github.com/spf13/cobra v1.8.1
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucVPgCo=
github.com/dominikbraun/graph v0.23.0/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc=
github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
Expand Down Expand Up @@ -510,6 +512,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke
github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 h1:PKK9DyHxif4LZo+uQSgXNqs0jj5+xZwwfKHgph2lxBw=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.1/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
Expand Down
15 changes: 15 additions & 0 deletions internal/cmd/generate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -154,3 +154,18 @@ func (s *GenerateTestSuite) TestGenerateWithAlias() {
assert.NoError(s.T(), err)
assert.NoError(s.T(), CompareDirectories(path.Join(workdir, "deployments"), path.Join(workdir, "expected")))
}

func (s *GenerateTestSuite) TestGenerateWithVariables() {
pwd, _ := os.Getwd()
workdir := path.Join(pwd, "testdata/cases/generate/with-variables")

cmd := RootCmd
cmd.SetArgs([]string{
"generate",
"--output-path", path.Join(workdir, "deployments"),
"--file", path.Join(workdir, "main.yaml"),
})
err := cmd.Execute()
assert.NoError(s.T(), err)
assert.NoError(s.T(), CompareDirectories(path.Join(workdir, "deployments"), path.Join(workdir, "expected")))
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# This file is auto-generated by MACH composer
# site: test-1
terraform {
backend "local" {
path = "./states/test-1.tfstate"
}
required_providers {
aws = {
version = "~> 3.74.1"
}
}
}

# File sources
# Resources
# Configuring AWS
provider "aws" {
region = "eu-west-1"
}

locals {
tags = {
Site = "test-1"
Environment = "test"
}
}

# Component: component
module "component" {
source = "{{ .PWD }}/testdata/modules/application"
variables = {
component_variable = "component_variable_value"
global_variable = "global_variable_value"
site_variable = "site_variable_value"
}
secrets = {
component_secret = "component_secret_value"
global_secret = "global_secret_value"
site_secret = "site_secret_value"
}
component_version = "test"
environment = "test"
site = "test-1"
tags = local.tags
providers = {
aws = aws,
}
}

output "component" {
description = "The module outputs for component"
sensitive = true
value = module.component
}
49 changes: 49 additions & 0 deletions internal/cmd/testdata/cases/generate/with-variables/main.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
mach_composer:
version: 1
plugins:
aws:
source: mach-composer/aws
version: 0.1.0

global:
cloud: "aws"
environment: test
terraform_config:
remote_state:
plugin: local
path: ./states
variables:
global_variable: "global_variable_value"
site_variable: "overridden_site_variable_value"
component_variable: "overridden_component_variable_value"
secrets:
global_secret: "global_secret_value"
site_secret: "overridden_site_secret_value"
component_secret: "overridden_component_secret_value"

sites:
- identifier: test-1
variables:
site_variable: "site_variable_value"
component_variable: "overridden_component_variable_value"
secrets:
site_secret: "site_secret_value"
component_secret: "overridden_component_secret_value"
aws:
account_id: "12345"
region: eu-west-1
components:
- name: component
variables:
component_variable: "component_variable_value"
secrets:
component_secret: "component_secret_value"

components:
- name: component
source: ./testdata/modules/application
version: "test"
branch: main
integrations:
- aws

4 changes: 4 additions & 0 deletions internal/config/global.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package config
import (
"fmt"
"github.com/mach-composer/mach-composer-cli/internal/cli"
"github.com/mach-composer/mach-composer-cli/internal/config/variable"
"github.com/mach-composer/mach-composer-cli/internal/utils"
"gopkg.in/yaml.v3"
)
Expand All @@ -12,6 +13,9 @@ type GlobalConfig struct {
Cloud string `yaml:"cloud"`
TerraformStateProvider string `yaml:"-"`
TerraformConfig *TerraformConfig `yaml:"terraform_config"`

Variables variable.VariablesMap `yaml:"variables"`
Secrets variable.VariablesMap `yaml:"secrets"`
}

type TerraformConfig struct {
Expand Down
16 changes: 8 additions & 8 deletions internal/config/parse_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,12 @@ func TestOpenBasic(t *testing.T) {
Name: "your-component",
Deployment: &Deployment{Type: DeploymentSite},
Variables: variable.VariablesMap{
"FOO_VAR": variable.MustCreateNewScalarVariable(t, "my-value"),
"BAR_VAR": variable.MustCreateNewScalarVariable(t, "${var.foo}"),
"MULTIPLE_VARS": variable.MustCreateNewScalarVariable(t, "${var.foo.bar} ${var.bar.foo}"),
"FOO_VAR": variable.MustCreateNewScalarVariable("my-value"),
"BAR_VAR": variable.MustCreateNewScalarVariable("${var.foo}"),
"MULTIPLE_VARS": variable.MustCreateNewScalarVariable("${var.foo.bar} ${var.bar.foo}"),
},
Secrets: variable.VariablesMap{
"MY_SECRET": variable.MustCreateNewScalarVariable(t, "secretvalue"),
"MY_SECRET": variable.MustCreateNewScalarVariable("secretvalue"),
},
Definition: &component,
},
Expand Down Expand Up @@ -145,12 +145,12 @@ func TestOpenComplex(t *testing.T) {
Name: "your-component",
Deployment: &Deployment{Type: DeploymentSite},
Variables: variable.VariablesMap{
"FOO_VAR": variable.MustCreateNewScalarVariable(t, "my-value"),
"BAR_VAR": variable.MustCreateNewScalarVariable(t, "${var.foo}"),
"MULTIPLE_VARS": variable.MustCreateNewScalarVariable(t, "${var.foo.bar} ${var.bar.foo}"),
"FOO_VAR": variable.MustCreateNewScalarVariable("my-value"),
"BAR_VAR": variable.MustCreateNewScalarVariable("${var.foo}"),
"MULTIPLE_VARS": variable.MustCreateNewScalarVariable("${var.foo.bar} ${var.bar.foo}"),
},
Secrets: variable.VariablesMap{
"MY_SECRET": variable.MustCreateNewScalarVariable(t, "secretvalue"),
"MY_SECRET": variable.MustCreateNewScalarVariable("secretvalue"),
},
Definition: &component,
},
Expand Down
62 changes: 60 additions & 2 deletions internal/config/schemas/schema-1.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,12 @@ definitions:
$ref: "#/definitions/TerraformConfig"
cloud:
type: string
variables:
$ref: "#/definitions/MachComposerVariables"
description: Global variables. These will be merged with the site specific variables, where the site variables will take precedence
secrets:
$ref: "#/definitions/MachComposerSecrets"
description: Global secrets. These will be merged with the site specific secrets, where the site secrets will take precedence

TerraformConfig:
type: object
Expand Down Expand Up @@ -137,6 +143,12 @@ definitions:
- $ref: "#/definitions/SiteEndpointConfig"
deployment:
$ref: "#/definitions/MachComposerDeployment"
variables:
$ref: "#/definitions/MachComposerVariables"
description: Site specific variables. These will be merged with the component variables, where the component variables will take precedence
secrets:
$ref: "#/definitions/MachComposerSecrets"
description: Site specific secrets. These will be merged with the component secrets, where the component secrets will take precedence
components:
type: array
items:
Expand All @@ -163,9 +175,9 @@ definitions:
name:
type: string
variables:
type: object
$ref: "#/definitions/MachComposerVariables"
secrets:
type: object
$ref: "#/definitions/MachComposerSecrets"
store_variables:
description: Commercetools store specific variables
deprecationMessage: The `store_variables` configuration is deprecated
Expand Down Expand Up @@ -227,6 +239,52 @@ definitions:
"[a-zA-Z0-9]+":
type: string

MachComposerVariables:
description: |
Variables are used to configure the components. They are passed to the terraform module as a terraform
variable with the name `variables` containing an object with the same fields:
```yaml
components:
- name: my-component
variables:
my_string_field: "my string value"
my_number_field: 42
```
```
variable "variables" {
type = object({
my_string_field = string
my_number_field = number
})
}
```
type: object

MachComposerSecrets:
description: |
Secrets are used to configure the components. They are passed to the terraform module as a terraform
variable with the name `secrets` containing an object with the same fields:
```yaml
components:
- name: my-component
secrets:
my_string_field: "my string value"
my_number_field: 42
```
```
variable "secrets" {
type = object({
my_string_field = string
my_number_field = number
})
}
```
type: object

MachComposerDeployment:
type: object
description: |
Expand Down
4 changes: 4 additions & 0 deletions internal/config/site.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package config

import (
"fmt"
"github.com/mach-composer/mach-composer-cli/internal/config/variable"
"github.com/rs/zerolog/log"

"github.com/elliotchance/pie/v2"
Expand Down Expand Up @@ -29,6 +30,9 @@ type SiteConfig struct {
Deployment *Deployment `yaml:"deployment"`
RawEndpoints map[string]any `yaml:"endpoints"`

Variables variable.VariablesMap `yaml:"variables"`
Secrets variable.VariablesMap `yaml:"secrets"`

Components SiteComponentConfigs `yaml:"components"`
}

Expand Down
14 changes: 14 additions & 0 deletions internal/config/variable/base.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,20 @@ const (

type VariablesMap map[string]Variable

// MergeVariablesMaps merges multiple VariablesMap into a single VariablesMap. The last map will override the previous
// ones. Only the keys on the first level are merged.
func MergeVariablesMaps(maps ...VariablesMap) VariablesMap {
var result = make(VariablesMap)

for _, m := range maps {
for key, val := range m {
result[key] = val
}
}

return result
}

type TransformValueFunc func(value any) (any, error)

type Variable interface {
Expand Down
55 changes: 55 additions & 0 deletions internal/config/variable/base_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package variable

import (
"github.com/stretchr/testify/assert"
"testing"
)

func TestMergeVariablesMaps(t *testing.T) {
tests := []struct {
name string
maps []VariablesMap
expected VariablesMap
}{
{
name: "merge two maps with different keys",
maps: []VariablesMap{
{"key1": MustCreateNewScalarVariable("value1")},
{"key2": MustCreateNewScalarVariable("value2")},
},
expected: VariablesMap{
"key1": MustCreateNewScalarVariable("value1"),
"key2": MustCreateNewScalarVariable("value2"),
},
},
{
name: "merge two maps with same keys",
maps: []VariablesMap{
{"key1": MustCreateNewScalarVariable("value1")},
{"key1": MustCreateNewScalarVariable("value2")},
},
expected: VariablesMap{
"key1": MustCreateNewScalarVariable("value2"),
},
},
{
name: "merge three maps with same keys",
maps: []VariablesMap{
{"key1": MustCreateNewScalarVariable("value1")},
{"key2": MustCreateNewScalarVariable("value2")},
{"key1": MustCreateNewScalarVariable("value3")},
},
expected: VariablesMap{
"key1": MustCreateNewScalarVariable("value3"),
"key2": MustCreateNewScalarVariable("value2"),
},
},
}

for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
result := MergeVariablesMaps(tc.maps...)
assert.Equal(t, tc.expected, result)
})
}
}
Loading

0 comments on commit f27f228

Please sign in to comment.