diff --git a/.changes/unreleased/Added-20240318-114635.yaml b/.changes/unreleased/Added-20240318-114635.yaml new file mode 100644 index 00000000..b25de79d --- /dev/null +++ b/.changes/unreleased/Added-20240318-114635.yaml @@ -0,0 +1,3 @@ +kind: Added +body: Moved hash storage out of terraform +time: 2024-03-18T11:46:35.79447595+01:00 diff --git a/Taskfile.yaml b/Taskfile.yaml index 2f8e0e50..0a4f3e57 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -23,6 +23,9 @@ tasks: test: cmd: go test -race ./... + test:integration: + cmd: go test -tags=integration ./... + cover: cmd: go test -race -coverprofile=coverage.out -covermode=atomic ./... diff --git a/docs/src/reference/cli/mach-composer_apply.md b/docs/src/reference/cli/mach-composer_apply.md index c26bdc7d..48db05d8 100644 --- a/docs/src/reference/cli/mach-composer_apply.md +++ b/docs/src/reference/cli/mach-composer_apply.md @@ -13,11 +13,11 @@ mach-composer apply [flags] -c, --component stringArray --destroy Destroy option is a convenient way to destroy all remote objects managed by this mach config -f, --file string YAML file to parse. (default "main.yml") + --force-init Force terraform initialization. By default mach-composer will reuse existing terraform resources -h, --help help for apply --ignore-change-detection Ignore change detection to run even if the components are considered up to date --ignore-version Skip MACH composer version check --output-path string Outputs path to store the generated files. (default "deployments") - --reuse Suppress a terraform init for improved speed (not recommended for production usage) -s, --site string Site to parse. If not set parse all sites. --var-file string Use a variable file to parse the configuration with. -w, --workers int The number of workers to use (default 1) diff --git a/docs/src/reference/cli/mach-composer_plan.md b/docs/src/reference/cli/mach-composer_plan.md index fd279180..a6572328 100644 --- a/docs/src/reference/cli/mach-composer_plan.md +++ b/docs/src/reference/cli/mach-composer_plan.md @@ -11,12 +11,12 @@ mach-composer plan [flags] ``` -c, --component stringArray -f, --file string YAML file to parse. (default "main.yml") + --force-init Force terraform initialization. By default mach-composer will reuse existing terraform resources -h, --help help for plan --ignore-change-detection Ignore change detection to run even if the components are considered up to date --ignore-version Skip MACH composer version check --lock Acquire a lock on the state file before running terraform plan (default true) --output-path string Outputs path to store the generated files. (default "deployments") - --reuse Suppress a terraform init for improved speed (not recommended for production usage) -s, --site string Site to parse. If not set parse all sites. --var-file string Use a variable file to parse the configuration with. -w, --workers int The number of workers to use (default 1) diff --git a/docs/src/reference/cli/mach-composer_show-plan.md b/docs/src/reference/cli/mach-composer_show-plan.md index 1560449d..425553aa 100644 --- a/docs/src/reference/cli/mach-composer_show-plan.md +++ b/docs/src/reference/cli/mach-composer_show-plan.md @@ -10,12 +10,12 @@ mach-composer show-plan [flags] ``` -f, --file string YAML file to parse. (default "main.yml") + --force-init Force terraform initialization. By default mach-composer will reuse existing terraform resources -h, --help help for show-plan --ignore-change-detection Ignore change detection to run even if the components are considered up to date --ignore-version Skip MACH composer version check --no-color Disable color output --output-path string Outputs path to store the generated files. (default "deployments") - --reuse Suppress a terraform init for improved speed (not recommended for production usage) -s, --site string Site to parse. If not set parse all sites. --var-file string Use a variable file to parse the configuration with. -w, --workers int The number of workers to use (default 1) diff --git a/docs/src/reference/cli/mach-composer_terraform.md b/docs/src/reference/cli/mach-composer_terraform.md index 300b2322..fdd740ff 100644 --- a/docs/src/reference/cli/mach-composer_terraform.md +++ b/docs/src/reference/cli/mach-composer_terraform.md @@ -14,7 +14,6 @@ mach-composer terraform [flags] --ignore-change-detection Ignore change detection to run even if the components are considered up to date. Per default the proxy will ignore change detection (default true) --ignore-version Skip MACH composer version check --output-path string Outputs path to store the generated files. (default "deployments") - --reuse Suppress a terraform init for improved speed (not recommended for production usage) -s, --site string Site to parse. If not set parse all sites. --var-file string Use a variable file to parse the configuration with. -w, --workers int The number of workers to use (default 1) diff --git a/internal/batcher/batcher.go b/internal/batcher/batcher.go new file mode 100644 index 00000000..4a021e39 --- /dev/null +++ b/internal/batcher/batcher.go @@ -0,0 +1,5 @@ +package batcher + +import "github.com/mach-composer/mach-composer-cli/internal/graph" + +type BatchFunc func(g *graph.Graph) map[int][]graph.Node diff --git a/internal/batcher/naive_batcher.go b/internal/batcher/naive_batcher.go new file mode 100644 index 00000000..04aabd63 --- /dev/null +++ b/internal/batcher/naive_batcher.go @@ -0,0 +1,29 @@ +package batcher + +import "github.com/mach-composer/mach-composer-cli/internal/graph" + +func NaiveBatchFunc() BatchFunc { + return func(g *graph.Graph) map[int][]graph.Node { + batches := map[int][]graph.Node{} + + var sets = map[string][]graph.Path{} + + for _, n := range g.Vertices() { + var route, _ = g.Routes(n.Path(), g.StartNode.Path()) + sets[n.Path()] = route + } + + for k, routes := range sets { + var mx int + for _, route := range routes { + if len(route) > mx { + mx = len(route) + } + } + n, _ := g.Vertex(k) + batches[mx] = append(batches[mx], n) + } + + return batches + } +} diff --git a/internal/batcher/naive_batcher_test.go b/internal/batcher/naive_batcher_test.go new file mode 100644 index 00000000..76497521 --- /dev/null +++ b/internal/batcher/naive_batcher_test.go @@ -0,0 +1,86 @@ +package batcher + +import ( + "github.com/dominikbraun/graph" + internalgraph "github.com/mach-composer/mach-composer-cli/internal/graph" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestBatchNodesDepth1(t *testing.T) { + ig := graph.New(func(n internalgraph.Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) + + start := new(internalgraph.NodeMock) + start.On("Path").Return("main/site-1") + + _ = ig.AddVertex(start) + + g := &internalgraph.Graph{Graph: ig, StartNode: start} + + batches := NaiveBatchFunc()(g) + + assert.Equal(t, 1, len(batches)) +} + +func TestBatchNodesDepth2(t *testing.T) { + ig := graph.New(func(n internalgraph.Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) + + site := new(internalgraph.NodeMock) + site.On("Path").Return("main/site-1") + + component1 := new(internalgraph.NodeMock) + component1.On("Path").Return("main/site-1/component-1") + + component2 := new(internalgraph.NodeMock) + component2.On("Path").Return("main/site-1/component-2") + + _ = ig.AddVertex(site) + _ = ig.AddVertex(component1) + _ = ig.AddVertex(component2) + + _ = ig.AddEdge("main/site-1", "main/site-1/component-1") + _ = ig.AddEdge("main/site-1", "main/site-1/component-2") + + g := &internalgraph.Graph{Graph: ig, StartNode: site} + + batches := NaiveBatchFunc()(g) + + assert.Equal(t, 2, len(batches)) + assert.Equal(t, 1, len(batches[0])) + assert.Equal(t, "main/site-1", batches[0][0].Path()) + assert.Equal(t, 2, len(batches[1])) + assert.Contains(t, batches[1][0].Path(), "component") + assert.Contains(t, batches[1][1].Path(), "component") +} + +func TestBatchNodesDepth3(t *testing.T) { + ig := graph.New(func(n internalgraph.Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) + + site := new(internalgraph.NodeMock) + site.On("Path").Return("main/site-1") + + component1 := new(internalgraph.NodeMock) + component1.On("Path").Return("main/site-1/component-1") + + component2 := new(internalgraph.NodeMock) + component2.On("Path").Return("main/site-1/component-2") + + _ = ig.AddVertex(site) + _ = ig.AddVertex(component1) + _ = ig.AddVertex(component2) + + _ = ig.AddEdge("main/site-1", "main/site-1/component-1") + _ = ig.AddEdge("main/site-1/component-1", "main/site-1/component-2") + + g := &internalgraph.Graph{Graph: ig, StartNode: site} + + batches := NaiveBatchFunc()(g) + + assert.Equal(t, 3, len(batches)) + assert.Equal(t, 1, len(batches[0])) + assert.Equal(t, "main/site-1", batches[0][0].Path()) + assert.Equal(t, 1, len(batches[1])) + assert.Contains(t, batches[1][0].Path(), "main/site-1/component-1") + assert.Equal(t, 1, len(batches[2])) + assert.Contains(t, batches[2][0].Path(), "main/site-1/component-2") +} diff --git a/internal/cmd/apply.go b/internal/cmd/apply.go index b0d11215..9c9acb9f 100644 --- a/internal/cmd/apply.go +++ b/internal/cmd/apply.go @@ -1,7 +1,9 @@ package cmd import ( + "github.com/mach-composer/mach-composer-cli/internal/batcher" "github.com/mach-composer/mach-composer-cli/internal/graph" + "github.com/mach-composer/mach-composer-cli/internal/hash" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -10,7 +12,7 @@ import ( ) var applyFlags struct { - reuse bool + forceInit bool autoApprove bool destroy bool components []string @@ -32,7 +34,7 @@ var applyCmd = &cobra.Command{ func init() { registerCommonFlags(applyCmd) - applyCmd.Flags().BoolVarP(&applyFlags.reuse, "reuse", "", false, "Suppress a terraform init for improved speed (not recommended for production usage)") + applyCmd.Flags().BoolVarP(&applyFlags.forceInit, "force-init", "", false, "Force terraform initialization. By default mach-composer will reuse existing terraform resources") applyCmd.Flags().BoolVarP(&applyFlags.autoApprove, "auto-approve", "", false, "Suppress a terraform init for improved speed (not recommended for production usage)") applyCmd.Flags().BoolVarP(&applyFlags.destroy, "destroy", "", false, "Destroy option is a convenient way to destroy all remote objects managed by this mach config") applyCmd.Flags().StringArrayVarP(&applyFlags.components, "component", "c", nil, "") @@ -61,13 +63,14 @@ func applyFunc(cmd *cobra.Command, _ []string) error { return err } - b := runner.NewGraphRunner(commonFlags.workers) + r := runner.NewGraphRunner( + batcher.NaiveBatchFunc(), + hash.Factory(cfg), + commonFlags.workers, + ) - if err = checkReuse(ctx, dg, b, applyFlags.reuse); err != nil { - return err - } - - return b.TerraformApply(ctx, dg, &runner.ApplyOptions{ + return r.TerraformApply(ctx, dg, &runner.ApplyOptions{ + ForceInit: applyFlags.forceInit, Destroy: applyFlags.destroy, AutoApprove: applyFlags.autoApprove, IgnoreChangeDetection: applyFlags.ignoreChangeDetection, diff --git a/internal/cmd/apply_test.go b/internal/cmd/apply_test.go new file mode 100644 index 00000000..b2a62d50 --- /dev/null +++ b/internal/cmd/apply_test.go @@ -0,0 +1,130 @@ +//go:build integration + +package cmd + +import ( + "github.com/stretchr/testify/assert" + "os" + "os/exec" + "path" + "testing" +) + +import ( + "github.com/stretchr/testify/suite" +) + +type ApplyTestSuite struct { + suite.Suite + tempDir string +} + +func TestExampleTestSuite(t *testing.T) { + suite.Run(t, new(ApplyTestSuite)) +} + +func (s *ApplyTestSuite) SetupSuite() { + _, err := exec.LookPath("terraform") + if err != nil { + s.T().Fatal("terraform command not found") + } + + tmpDir, _ := os.MkdirTemp("mach-composer", "test") + _ = os.Setenv("TF_PLUGIN_CACHE_DIR", tmpDir) + _ = os.Setenv("TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE", "1") + + s.tempDir = tmpDir +} + +func (s *ApplyTestSuite) TearDownSuite() { + _ = os.RemoveAll(s.tempDir) +} + +func cleanWorkingDir(workdir string) { + err := os.RemoveAll(path.Join(workdir, "deployments")) + if err != nil { + panic(err) + } + err = os.RemoveAll(path.Join(workdir, "states")) + if err != nil { + panic(err) + } + err = os.RemoveAll(path.Join(workdir, "hashes.json")) + if err != nil { + panic(err) + } +} + +func (s *ApplyTestSuite) TestApplySimple() { + pwd, _ := os.Getwd() + workdir := path.Join(pwd, "testdata/cases/apply/simple") + defer cleanWorkingDir(workdir) + + cmd := RootCmd + _ = os.Setenv("MC_HASH_FILE", path.Join(workdir, "hashes.json")) + cmd.SetArgs([]string{ + "apply", + "--output-path", path.Join(workdir, "deployments"), + "--file", path.Join(workdir, "main.yaml"), + "--auto-approve", + }) + err := cmd.Execute() + assert.NoError(s.T(), err) + + assert.FileExists(s.T(), path.Join(workdir, "hashes.json")) + assert.FileExists(s.T(), path.Join(workdir, "deployments/main/test-1/main.tf")) + assert.FileExists(s.T(), path.Join(workdir, "deployments/main/test-1/states/test-1.tfstate")) + assert.FileExists(s.T(), path.Join(workdir, "deployments/main/test-1/outputs/component-1.json")) +} + +func (s *ApplyTestSuite) TestApplySplitState() { + pwd, _ := os.Getwd() + workdir := path.Join(pwd, "testdata/cases/apply/split-state") + defer cleanWorkingDir(workdir) + + cmd := RootCmd + _ = os.Setenv("MC_HASH_FILE", path.Join(workdir, "hashes.json")) + _ = os.Setenv("STATES_PATH", path.Join(workdir, "states")) + cmd.SetArgs([]string{ + "apply", + "--output-path", path.Join(workdir, "deployments"), + "--file", path.Join(workdir, "main.yaml"), + "--auto-approve", + }) + err := cmd.Execute() + assert.NoError(s.T(), err) + + assert.FileExists(s.T(), path.Join(workdir, "hashes.json")) + assert.FileExists(s.T(), path.Join(workdir, "deployments/main/test-1/main.tf")) + assert.FileExists(s.T(), path.Join(workdir, "deployments/main/test-1/component-2/main.tf")) + assert.FileExists(s.T(), path.Join(workdir, "states/test-1.tfstate")) + assert.FileExists(s.T(), path.Join(workdir, "states/component-2.tfstate")) +} + +func (s *ApplyTestSuite) TestApplyNoHashesFile() { + pwd, _ := os.Getwd() + workdir := path.Join(pwd, "testdata/cases/apply/simple") + defer cleanWorkingDir(workdir) + + cmd := RootCmd + _ = os.Setenv("MC_HASH_FILE", path.Join(workdir, "hashes.json")) + cmd.SetArgs([]string{ + "apply", + "--output-path", path.Join(workdir, "deployments"), + "--file", path.Join(workdir, "main.yaml"), + "--auto-approve", + }) + err := cmd.Execute() + assert.NoError(s.T(), err) + + assert.FileExists(s.T(), path.Join(workdir, "hashes.json")) + + err = os.RemoveAll(path.Join(workdir, "hashes.json")) + if err != nil { + s.T().Fatal(err) + } + + err = cmd.Execute() + assert.NoError(s.T(), err) + assert.FileExists(s.T(), path.Join(workdir, "hashes.json")) +} diff --git a/internal/cmd/common.go b/internal/cmd/common.go index 799d4aaa..b589cec2 100644 --- a/internal/cmd/common.go +++ b/internal/cmd/common.go @@ -1,12 +1,9 @@ package cmd import ( - "context" "errors" "fmt" "github.com/mach-composer/mach-composer-cli/internal/cloud" - "github.com/mach-composer/mach-composer-cli/internal/graph" - "github.com/mach-composer/mach-composer-cli/internal/runner" "os" "path" "path/filepath" @@ -110,12 +107,3 @@ func loadConfig(cmd *cobra.Command, resolveVars bool) *config.MachConfig { return cfg } - -func checkReuse(ctx context.Context, dg *graph.Graph, b *runner.GraphRunner, reuse bool) error { - if reuse { - log.Info().Msgf("Reusing existing terraform state") - return nil - - } - return b.TerraformInit(ctx, dg) -} diff --git a/internal/cmd/init.go b/internal/cmd/init.go index 9fcfa208..6ac555d3 100644 --- a/internal/cmd/init.go +++ b/internal/cmd/init.go @@ -1,7 +1,9 @@ package cmd import ( + "github.com/mach-composer/mach-composer-cli/internal/batcher" "github.com/mach-composer/mach-composer-cli/internal/graph" + "github.com/mach-composer/mach-composer-cli/internal/hash" "github.com/spf13/cobra" "github.com/mach-composer/mach-composer-cli/internal/cli" @@ -9,10 +11,6 @@ import ( "github.com/mach-composer/mach-composer-cli/internal/runner" ) -var initFlags struct { - force bool -} - var initCmd = &cobra.Command{ Use: "init", Short: "Initialize site directories Terraform files.", @@ -30,7 +28,6 @@ var initCmd = &cobra.Command{ func init() { registerCommonFlags(initCmd) - initCmd.Flags().BoolVarP(&initFlags.force, "force", "", false, "Force the apply to run even if the components are considered up to date") } func initFunc(cmd *cobra.Command, _ []string) error { @@ -48,7 +45,11 @@ func initFunc(cmd *cobra.Command, _ []string) error { return err } - b := runner.NewGraphRunner(commonFlags.workers) + r := runner.NewGraphRunner( + batcher.NaiveBatchFunc(), + hash.Factory(cfg), + commonFlags.workers, + ) - return b.TerraformInit(ctx, dg) + return r.TerraformInit(ctx, dg) } diff --git a/internal/cmd/plan.go b/internal/cmd/plan.go index 36dd0bb3..a019cc20 100644 --- a/internal/cmd/plan.go +++ b/internal/cmd/plan.go @@ -1,7 +1,9 @@ package cmd import ( + "github.com/mach-composer/mach-composer-cli/internal/batcher" "github.com/mach-composer/mach-composer-cli/internal/graph" + "github.com/mach-composer/mach-composer-cli/internal/hash" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -10,7 +12,7 @@ import ( ) var planFlags struct { - reuse bool + forceInit bool components []string lock bool ignoreChangeDetection bool @@ -30,11 +32,9 @@ var planCmd = &cobra.Command{ func init() { registerCommonFlags(planCmd) - planCmd.Flags().BoolVarP(&planFlags.reuse, "reuse", "", false, - "Suppress a terraform init for improved speed (not recommended for production usage)") + planCmd.Flags().BoolVarP(&planFlags.forceInit, "force-init", "", false, "Force terraform initialization. By default mach-composer will reuse existing terraform resources") planCmd.Flags().StringArrayVarP(&planFlags.components, "component", "c", nil, "") - planCmd.Flags().BoolVarP(&planFlags.lock, "lock", "", true, - "Acquire a lock on the state file before running terraform plan") + planCmd.Flags().BoolVarP(&planFlags.lock, "lock", "", true, "Acquire a lock on the state file before running terraform plan") planCmd.Flags().BoolVarP(&planFlags.ignoreChangeDetection, "ignore-change-detection", "", false, "Ignore change detection to run even if the components are considered up to date") } @@ -57,13 +57,14 @@ func planFunc(cmd *cobra.Command, _ []string) error { return err } - b := runner.NewGraphRunner(commonFlags.workers) + r := runner.NewGraphRunner( + batcher.NaiveBatchFunc(), + hash.Factory(cfg), + commonFlags.workers, + ) - if err = checkReuse(ctx, dg, b, applyFlags.reuse); err != nil { - return err - } - - return b.TerraformPlan(ctx, dg, &runner.PlanOptions{ + return r.TerraformPlan(ctx, dg, &runner.PlanOptions{ + ForceInit: planFlags.forceInit, Lock: planFlags.lock, IgnoreChangeDetection: planFlags.ignoreChangeDetection, }) diff --git a/internal/cmd/show-plan.go b/internal/cmd/show-plan.go index fecd799d..2bd15dc9 100644 --- a/internal/cmd/show-plan.go +++ b/internal/cmd/show-plan.go @@ -1,14 +1,16 @@ package cmd import ( + "github.com/mach-composer/mach-composer-cli/internal/batcher" "github.com/mach-composer/mach-composer-cli/internal/graph" + "github.com/mach-composer/mach-composer-cli/internal/hash" "github.com/spf13/cobra" "github.com/mach-composer/mach-composer-cli/internal/runner" ) var showPlanFlags struct { - reuse bool + forceInit bool noColor bool ignoreChangeDetection bool } @@ -27,9 +29,8 @@ var showPlanCmd = &cobra.Command{ func init() { registerCommonFlags(showPlanCmd) + showPlanCmd.Flags().BoolVarP(&showPlanFlags.forceInit, "force-init", "", false, "Force terraform initialization. By default mach-composer will reuse existing terraform resources") showPlanCmd.Flags().BoolVarP(&showPlanFlags.noColor, "no-color", "", false, "Disable color output") - showPlanCmd.Flags().BoolVarP(&showPlanFlags.reuse, "reuse", "", false, - "Suppress a terraform init for improved speed (not recommended for production usage)") showPlanCmd.Flags().BoolVarP(&showPlanFlags.ignoreChangeDetection, "ignore-change-detection", "", false, "Ignore change detection to run even if the components are considered up to date") } @@ -44,13 +45,14 @@ func showPlanFunc(cmd *cobra.Command, _ []string) error { return err } - b := runner.NewGraphRunner(commonFlags.workers) + r := runner.NewGraphRunner( + batcher.NaiveBatchFunc(), + hash.Factory(cfg), + commonFlags.workers, + ) - if err = checkReuse(ctx, dg, b, applyFlags.reuse); err != nil { - return err - } - - return b.TerraformShow(ctx, dg, &runner.ShowPlanOptions{ + return r.TerraformShow(ctx, dg, &runner.ShowPlanOptions{ + ForceInit: showPlanFlags.forceInit, NoColor: showPlanFlags.noColor, IgnoreChangeDetection: showPlanFlags.ignoreChangeDetection, }) diff --git a/internal/cmd/suite_test.go b/internal/cmd/suite_test.go new file mode 100644 index 00000000..eb1640fa --- /dev/null +++ b/internal/cmd/suite_test.go @@ -0,0 +1,3 @@ +package cmd + +// Basic imports diff --git a/internal/cmd/terraform.go b/internal/cmd/terraform.go index 35f86224..9173c422 100644 --- a/internal/cmd/terraform.go +++ b/internal/cmd/terraform.go @@ -1,7 +1,9 @@ package cmd import ( + "github.com/mach-composer/mach-composer-cli/internal/batcher" "github.com/mach-composer/mach-composer-cli/internal/graph" + "github.com/mach-composer/mach-composer-cli/internal/hash" "github.com/spf13/cobra" "github.com/mach-composer/mach-composer-cli/internal/runner" @@ -25,11 +27,8 @@ var terraformCmd = &cobra.Command{ func init() { registerCommonFlags(terraformCmd) - terraformCmd.Flags().BoolVarP(&terraformFlags.reuse, "reuse", "", false, - "Suppress a terraform init for improved speed (not recommended for production usage)") terraformCmd.Flags().BoolVarP(&terraformFlags.ignoreChangeDetection, "ignore-change-detection", "", true, "Ignore change detection to run even if the components are considered up to date. Per default the proxy will ignore change detection") - } func terraformFunc(cmd *cobra.Command, args []string) error { @@ -42,13 +41,13 @@ func terraformFunc(cmd *cobra.Command, args []string) error { return err } - b := runner.NewGraphRunner(commonFlags.workers) - - if err = checkReuse(ctx, dg, b, applyFlags.reuse); err != nil { - return err - } + r := runner.NewGraphRunner( + batcher.NaiveBatchFunc(), + hash.Factory(cfg), + commonFlags.workers, + ) - return b.TerraformProxy(cmd.Context(), dg, &runner.ProxyOptions{ + return r.TerraformProxy(ctx, dg, &runner.ProxyOptions{ Command: args, IgnoreChangeDetection: terraformFlags.ignoreChangeDetection, }) diff --git a/internal/cmd/testdata/.gitignore b/internal/cmd/testdata/.gitignore new file mode 100644 index 00000000..51cd01b8 --- /dev/null +++ b/internal/cmd/testdata/.gitignore @@ -0,0 +1,3 @@ +cases/*/deployments +cases/*/hashes.json +cases/*/states diff --git a/internal/cmd/testdata/cases/apply/simple/main.yaml b/internal/cmd/testdata/cases/apply/simple/main.yaml new file mode 100644 index 00000000..b3529a64 --- /dev/null +++ b/internal/cmd/testdata/cases/apply/simple/main.yaml @@ -0,0 +1,30 @@ +mach_composer: + version: 1 + variables_file: variables.yaml + plugins: + aws: + source: mach-composer/aws + version: 0.1.0 + +global: + cloud: "" + environment: test + terraform_config: + remote_state: + plugin: local + path: ./states + +sites: + - identifier: test-1 + components: + - name: component-1 + variables: + sleep: 1s + random_value: ${env.RANDOM_VALUE} + parent_names: [ ] + +components: + - name: component-1 + source: ./testdata/modules/application + version: "test" + branch: main diff --git a/internal/cmd/testdata/cases/apply/simple/variables.yaml b/internal/cmd/testdata/cases/apply/simple/variables.yaml new file mode 100644 index 00000000..579b105c --- /dev/null +++ b/internal/cmd/testdata/cases/apply/simple/variables.yaml @@ -0,0 +1 @@ +random_value: "hello-world" diff --git a/internal/cmd/testdata/cases/apply/split-state/main.yaml b/internal/cmd/testdata/cases/apply/split-state/main.yaml new file mode 100644 index 00000000..c1e020ce --- /dev/null +++ b/internal/cmd/testdata/cases/apply/split-state/main.yaml @@ -0,0 +1,38 @@ +mach_composer: + version: 1 + variables_file: variables.yaml + plugins: + aws: + source: mach-composer/aws + version: 0.1.0 + +global: + cloud: "" + environment: test + terraform_config: + remote_state: + plugin: local + path: ${env.STATES_PATH} + +sites: + - identifier: test-1 + components: + - name: component-1 + variables: + parent_names: [ ] + - name: component-2 + deployment: + type: "site-component" + variables: + parent_names: + - ${component.component-1.name} + +components: + - name: component-1 + source: ./testdata/modules/application + version: "test" + branch: main + - name: component-2 + source: ./testdata/modules/application + version: "test" + branch: main diff --git a/internal/cmd/testdata/cases/apply/split-state/variables.yaml b/internal/cmd/testdata/cases/apply/split-state/variables.yaml new file mode 100644 index 00000000..e69de29b diff --git a/internal/cmd/testdata/modules/application/main.tf b/internal/cmd/testdata/modules/application/main.tf new file mode 100644 index 00000000..812824ac --- /dev/null +++ b/internal/cmd/testdata/modules/application/main.tf @@ -0,0 +1,47 @@ +locals { + module_path_elements = split("/", path.module) + module_name = element(local.module_path_elements, length(local.module_path_elements) - 1) +} + +resource "time_sleep" "sleep" { + create_duration = var.variables.sleep +} + +variable "variables" { + type = object({ + parent_names = list(string) + sleep = optional(string, "0s") + fail = optional(bool, false) + random_value = optional(string, "default") + }) +} + +data "http" "example" { + count = var.variables.fail == true ? 1 : 0 + url = "fails" +} + +resource "local_file" "child" { + content = jsonencode({ + name = local.module_name + parent_names = var.variables.parent_names + }) + filename = "${path.cwd}/outputs/${local.module_name}.json" + + depends_on = [ + time_sleep.sleep + ] +} + +output "name" { + value = local.module_name +} + +output "random_value" { + value = var.variables.random_value +} + + +output "array" { + value = [] +} diff --git a/internal/generator/component.go b/internal/generator/component.go index 0753e734..f1bd3f4c 100644 --- a/internal/generator/component.go +++ b/internal/generator/component.go @@ -70,16 +70,7 @@ func renderSiteComponent(ctx context.Context, cfg *config.MachConfig, n graph.No } result = append(result, val) - // Render hash output - val, err = renderHashOutput(n, []config.SiteComponentConfig{siteComponent}) - if err != nil { - return "", fmt.Errorf("failed to render hash output: %w", err) - } - result = append(result, val) - - content := strings.Join(result, "\n") - - return content, nil + return strings.Join(result, "\n"), nil } // renderSiteComponentTerraformConfig uses templates/terraform.tmpl to generate a terraform snippet for each component diff --git a/internal/generator/hash_output.go b/internal/generator/hash_output.go deleted file mode 100644 index 64b6ddb2..00000000 --- a/internal/generator/hash_output.go +++ /dev/null @@ -1,37 +0,0 @@ -package generator - -import ( - "embed" - "github.com/mach-composer/mach-composer-cli/internal/config" - "github.com/mach-composer/mach-composer-cli/internal/graph" - "github.com/mach-composer/mach-composer-cli/internal/utils" -) - -//go:embed templates/hash_output.tmpl -var hashOutputTmpl embed.FS - -// renderHashOutput uses templates/hash_output.tmpl to generate a terraform snippet for each node -func renderHashOutput(n graph.Node, siteComponents []config.SiteComponentConfig) (string, error) { - tpl, err := hashOutputTmpl.ReadFile("templates/hash_output.tmpl") - if err != nil { - return "", err - } - - hash, err := n.Hash() - if err != nil { - return "", err - } - - var componentNames []string - for _, component := range siteComponents { - componentNames = append(componentNames, component.Name) - } - - return utils.RenderGoTemplate(string(tpl), struct { - NodeHash string - ComponentNames []string - }{ - NodeHash: hash, - ComponentNames: componentNames, - }) -} diff --git a/internal/generator/site.go b/internal/generator/site.go index 2f6f0c6a..78cbf434 100644 --- a/internal/generator/site.go +++ b/internal/generator/site.go @@ -13,7 +13,7 @@ import ( // the main entrypoint for generating the terraform file for each site. func renderSite(ctx context.Context, cfg *config.MachConfig, n graph.Node) (string, error) { siteConfig := n.(*graph.Site).SiteConfig - nestedComponents := n.(*graph.Site).NestedSiteComponentConfigs + nestedNodes := n.(*graph.Site).NestedNodes result := []string{ "# This file is auto-generated by MACH composer", @@ -23,7 +23,7 @@ func renderSite(ctx context.Context, cfg *config.MachConfig, n graph.Node) (stri // Render the terraform config val, err := renderSiteTerraformConfig(cfg, &siteConfig) if err != nil { - return "", fmt.Errorf("renderTerraformConfig: %w", err) + return "", fmt.Errorf("failed to render terraform config: %w", err) } result = append(result, val) @@ -41,26 +41,18 @@ func renderSite(ctx context.Context, cfg *config.MachConfig, n graph.Node) (stri } result = append(result, val) - for _, component := range nestedComponents { - if component.Deployment.Type != config.DeploymentSite { + for _, component := range nestedNodes { + if component.SiteComponentConfig.Deployment.Type != config.DeploymentSite { continue } - val, err = renderComponentModule(ctx, cfg, &siteConfig, &component) + val, err = renderComponentModule(ctx, cfg, &siteConfig, &component.SiteComponentConfig) if err != nil { return "", fmt.Errorf("failed to render site component: %w", err) } result = append(result, val) } - // Render hash output - val, err = renderHashOutput(n, nestedComponents) - if err != nil { - return "", fmt.Errorf("failed to render hash output: %w", err) - } - result = append(result, val) - - content := strings.Join(result, "\n") - return content, nil + return strings.Join(result, "\n"), nil } func renderSiteTerraformConfig(cfg *config.MachConfig, site *config.SiteConfig) (string, error) { diff --git a/internal/generator/templates/hash_output.tmpl b/internal/generator/templates/hash_output.tmpl deleted file mode 100644 index be9b07d8..00000000 --- a/internal/generator/templates/hash_output.tmpl +++ /dev/null @@ -1,9 +0,0 @@ -output "hash" { - value = "{{ .NodeHash }}" - - depends_on = [ - {{range .ComponentNames}} - module.{{.}}, - {{end}} - ] -} diff --git a/internal/generator/writer.go b/internal/generator/writer.go index 0585181a..4048de0e 100644 --- a/internal/generator/writer.go +++ b/internal/generator/writer.go @@ -42,8 +42,8 @@ func Write(ctx context.Context, cfg *config.MachConfig, g *graph.Graph, _ *Gener } if site, ok := n.(*graph.Site); ok { - for _, c := range site.NestedSiteComponentConfigs { - cfg.StateRepository.Alias(n.Identifier(), c.Name) + for _, c := range site.NestedNodes { + cfg.StateRepository.Alias(n.Identifier(), c.SiteComponentConfig.Name) } } } @@ -62,11 +62,7 @@ func Write(ctx context.Context, cfg *config.MachConfig, g *graph.Graph, _ *Gener return err } - hash, err := n.Hash() - if err != nil { - return err - } - if err = writeContent(hash, n.Path(), body); err != nil { + if err = writeContent(n.Path(), body); err != nil { return err } break @@ -80,11 +76,7 @@ func Write(ctx context.Context, cfg *config.MachConfig, g *graph.Graph, _ *Gener return err } - hash, err := n.Hash() - if err != nil { - return err - } - if err = writeContent(hash, n.Path(), body); err != nil { + if err = writeContent(n.Path(), body); err != nil { return err } break @@ -97,7 +89,7 @@ func Write(ctx context.Context, cfg *config.MachConfig, g *graph.Graph, _ *Gener return nil } -func writeContent(hash, path, content string) error { +func writeContent(path, content string) error { filename := filepath.Join(path, "main.tf") log.Info().Msgf("Writing %s", filename) diff --git a/internal/gitutils/git.go b/internal/gitutils/git.go index 86525077..824ae1b0 100644 --- a/internal/gitutils/git.go +++ b/internal/gitutils/git.go @@ -292,7 +292,7 @@ func getGitCachePath(origin string) (string, error) { } base := strings.TrimSuffix(origin, filepath.Ext(origin)) - path := filepath.Join(cwd, ".mach", base) + path := filepath.Join(cwd, ".mach-composer", base) if err := os.MkdirAll(path, 0700); err != nil { return "", err } diff --git a/internal/graph/deployment.go b/internal/graph/deployment.go index b17fc858..9f504265 100644 --- a/internal/graph/deployment.go +++ b/internal/graph/deployment.go @@ -75,10 +75,7 @@ func reduceNodes(g *Graph) error { return true } - siteNode.NestedSiteComponentConfigs = append( - siteNode.NestedSiteComponentConfigs, - siteComponentNode.SiteComponentConfig, - ) + siteNode.NestedNodes = append(siteNode.NestedNodes, siteComponentNode) am, _ := g.Graph.AdjacencyMap() pm, _ := g.Graph.PredecessorMap() diff --git a/internal/graph/deployment_test.go b/internal/graph/deployment_test.go index 02462278..e3ed8404 100644 --- a/internal/graph/deployment_test.go +++ b/internal/graph/deployment_test.go @@ -55,8 +55,8 @@ func TestToDeploymentGraphSimple(t *testing.T) { assert.Equal(t, "main/site-1", siteNode.Path()) assert.IsType(t, &Site{}, siteNode) assert.Equal(t, cfg.Sites[0], siteNode.(*Site).SiteConfig) - assert.Equal(t, 1, len(siteNode.(*Site).NestedSiteComponentConfigs)) - assert.Equal(t, "site-component-1", siteNode.(*Site).NestedSiteComponentConfigs[0].Name) + assert.Equal(t, 1, len(siteNode.(*Site).NestedNodes)) + assert.Equal(t, "site-component-1", siteNode.(*Site).NestedNodes[0].SiteComponentConfig.Name) siteComponentNode, err := g.Vertex("main/site-1/site-component-2") assert.NoError(t, err) diff --git a/internal/graph/hash.go b/internal/graph/hash.go index 68b2042b..47dd1b29 100644 --- a/internal/graph/hash.go +++ b/internal/graph/hash.go @@ -6,7 +6,7 @@ import ( "github.com/mach-composer/mach-composer-cli/internal/utils" ) -func hashSiteComponentConfig(sc config.SiteComponentConfig) (string, error) { +func HashSiteComponentConfig(sc config.SiteComponentConfig) (string, error) { var err error var tfHash string diff --git a/internal/graph/hash_test.go b/internal/graph/hash_test.go index ed1d6b1a..546ad5e9 100644 --- a/internal/graph/hash_test.go +++ b/internal/graph/hash_test.go @@ -10,7 +10,7 @@ import ( func TestHashSiteComponentConfigOk(t *testing.T) { val, _ := variable.NewScalarVariable("value1") - h, err := hashSiteComponentConfig(config.SiteComponentConfig{ + h, err := HashSiteComponentConfig(config.SiteComponentConfig{ Name: "site-component-1", Variables: variable.VariablesMap{ "var1": val, @@ -40,11 +40,11 @@ func TestHashSiteComponentConfigChanged(t *testing.T) { }, } - h1, err := hashSiteComponentConfig(cfg) + h1, err := HashSiteComponentConfig(cfg) cfg.DependsOn = []string{"site-component-2"} - h2, err := hashSiteComponentConfig(cfg) + h2, err := HashSiteComponentConfig(cfg) assert.NoError(t, err) assert.NotEqual(t, h1, h2) @@ -64,7 +64,7 @@ func TestHashSiteComponentConfigGithubSource(t *testing.T) { }, } - h, err := hashSiteComponentConfig(cfg) + h, err := HashSiteComponentConfig(cfg) assert.NoError(t, err) assert.Equal(t, "de87afc8419dcd29e3e8cbe2e47b5026593ac0975555fe3d0f341eb3e0cf5785", h) diff --git a/internal/graph/mocks.go b/internal/graph/mocks.go index d1b5ea30..f513e309 100644 --- a/internal/graph/mocks.go +++ b/internal/graph/mocks.go @@ -3,12 +3,46 @@ package graph import ( "github.com/dominikbraun/graph" "github.com/stretchr/testify/mock" - "github.com/zclconf/go-cty/cty" ) +type EdgeMock struct { + Source string + Target string +} + +func CreateGraphMock( + vertices map[string]Node, + startNode Node, + edges ...EdgeMock, +) *Graph { + g := graph.New(func(n Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) + + for _, v := range vertices { + _ = g.AddVertex(v) + } + + for _, e := range edges { + _ = g.AddEdge(e.Source, e.Target) + } + + return &Graph{ + Graph: g, + StartNode: startNode, + } +} + type NodeMock struct { mock.Mock tainted bool + oldHash string +} + +func (n *NodeMock) SetOldHash(hash string) { + n.oldHash = hash +} + +func (n *NodeMock) GetOldHash() string { + return n.oldHash } func (n *NodeMock) Path() string { @@ -46,17 +80,8 @@ func (n *NodeMock) Tainted() bool { } func (n *NodeMock) Hash() (string, error) { - //TODO implement me - panic("implement me") -} - -func (n *NodeMock) Outputs() cty.Value { args := n.Called() - return args.Get(0).(cty.Value) -} - -func (n *NodeMock) SetOutputs(value cty.Value) { - _ = n.Called(value) + return args.String(0), args.Error(1) } func (n *NodeMock) SetTainted(tainted bool) { diff --git a/internal/graph/node.go b/internal/graph/node.go index 4e7619f4..1f97c523 100644 --- a/internal/graph/node.go +++ b/internal/graph/node.go @@ -3,7 +3,6 @@ package graph import ( "github.com/dominikbraun/graph" "github.com/mach-composer/mach-composer-cli/internal/config" - "github.com/zclconf/go-cty/cty" ) const ( @@ -42,21 +41,18 @@ type Node interface { //related components. This can be compared to other hashes to determine whether a node has changed Hash() (string, error) - //Outputs returns the outputs of the node - Outputs() cty.Value - - //SetOutputs sets the outputs of the node - SetOutputs(cty.Value) - //SetTainted sets the tainted status of the node SetTainted(tainted bool) - //HasChanges returns true if the node has changes, false otherwise - HasChanges() (bool, error) - //ResetGraph resets the graph of the node. If the graph the node belongs to the node graphs must also be reset, //as these are used to determine the parents of the node resetGraph(graph.Graph[string, Node]) + + //SetOldHash sets the old hash of the node. This is used to determine if the node has changed + SetOldHash(hash string) + + //GetOldHash returns the old hash of the node + GetOldHash() string } type baseNode struct { @@ -67,7 +63,7 @@ type baseNode struct { ancestor Node deploymentType config.DeploymentType tainted bool - outputs cty.Value + oldHash string } func newBaseNode(graph graph.Graph[string, Node], path string, identifier string, typ Type, ancestor Node, deploymentType config.DeploymentType) baseNode { @@ -78,7 +74,6 @@ func newBaseNode(graph graph.Graph[string, Node], path string, identifier string ancestor: ancestor, deploymentType: deploymentType, tainted: false, - outputs: cty.NilVal, } } @@ -86,14 +81,6 @@ func (n *baseNode) resetGraph(ng graph.Graph[string, Node]) { n.graph = ng } -func (n *baseNode) Outputs() cty.Value { - return n.outputs -} - -func (n *baseNode) SetOutputs(val cty.Value) { - n.outputs = val -} - func (n *baseNode) SetTainted(tainted bool) { n.tainted = tainted } @@ -151,3 +138,11 @@ func (n *baseNode) Independent() bool { return false } + +func (n *baseNode) SetOldHash(hash string) { + n.oldHash = hash +} + +func (n *baseNode) GetOldHash() string { + return n.oldHash +} diff --git a/internal/graph/outputs.go b/internal/graph/outputs.go deleted file mode 100644 index 3efc044d..00000000 --- a/internal/graph/outputs.go +++ /dev/null @@ -1,46 +0,0 @@ -package graph - -import ( - "context" - "fmt" - "github.com/mach-composer/mach-composer-cli/internal/cli" - "github.com/zclconf/go-cty/cty" - "sync" -) - -type ( - outputLoader func(ctx context.Context, path string) (cty.Value, error) -) - -// LoadOutputs loads the outputs for all nodes in the graph in parallel -func LoadOutputs(ctx context.Context, g *Graph, loader outputLoader) error { - wg := &sync.WaitGroup{} - errChan := make(chan error, len(g.Vertices())) - - for _, n := range g.Vertices() { - wg.Add(1) - - go func(ctx context.Context, n Node) { - defer wg.Done() - val, err := loader(ctx, n.Path()) - if err != nil { - errChan <- err - return - } - n.SetOutputs(val) - }(ctx, n) - } - wg.Wait() - close(errChan) - - if len(errChan) > 0 { - var errors []error - for err := range errChan { - errors = append(errors, err) - } - - return cli.NewGroupedError(fmt.Sprintf("failed loading outputs (%d errors)", len(errors)), errors) - } - - return nil -} diff --git a/internal/graph/outputs_test.go b/internal/graph/outputs_test.go deleted file mode 100644 index d22cd4b7..00000000 --- a/internal/graph/outputs_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package graph - -import ( - "context" - "errors" - "github.com/dominikbraun/graph" - "github.com/mach-composer/mach-composer-cli/internal/cli" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/zclconf/go-cty/cty" - "testing" -) - -type MockLoader struct { - mock.Mock -} - -func (m *MockLoader) Load(ctx context.Context, path string) (cty.Value, error) { - args := m.Called(ctx, path) - return args.Get(0).(cty.Value), args.Error(1) -} - -func TestLoadOutputsSingleError(t *testing.T) { - ctx := context.Background() - - ig := graph.New(func(n Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) - - site1 := new(NodeMock) - site1.On("Path").Return("main/site-1") - - mockLoader := new(MockLoader) - mockLoader.On("Load", ctx, "main/site-1").Return(cty.NilVal, errors.New("error")) - - err := ig.AddVertex(site1) - assert.NoError(t, err) - - g := &Graph{Graph: ig} - - err = LoadOutputs(ctx, g, mockLoader.Load) - - expected := &cli.GroupedError{} - - assert.ErrorAs(t, err, &expected) - - var gErr *cli.GroupedError - errors.As(err, &gErr) - assert.Equal(t, 1, len(gErr.Errors)) -} - -func TestLoadOutputsMultipleError(t *testing.T) { - ctx := context.Background() - - ig := graph.New(func(n Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) - - site1 := new(NodeMock) - site1.On("Path").Return("main/site-1") - site2 := new(NodeMock) - site2.On("Path").Return("main/site-2") - site2.On("SetOutputs", cty.StringVal("hello-world")).Return().Once() - - mockLoader := new(MockLoader) - mockLoader.On("Load", ctx, "main/site-1").Return(cty.NilVal, errors.New("error")).Once() - mockLoader.On("Load", ctx, "main/site-2").Return(cty.StringVal("hello-world"), nil).Once() - - _ = ig.AddVertex(site1) - _ = ig.AddVertex(site2) - - g := &Graph{Graph: ig} - - err := LoadOutputs(ctx, g, mockLoader.Load) - - var gErr *cli.GroupedError - errors.As(err, &gErr) - assert.Equal(t, 1, len(gErr.Errors)) -} diff --git a/internal/graph/project.go b/internal/graph/project.go index c4387ef3..7d0ca768 100644 --- a/internal/graph/project.go +++ b/internal/graph/project.go @@ -21,8 +21,3 @@ func NewProject(g graph.Graph[string, Node], path, identifier string, deployment func (p *Project) Hash() (string, error) { return "", nil } - -// HasChanges always returns false, as any change in the project config will be picked up at the site or site component level -func (p *Project) HasChanges() (bool, error) { - return false, nil -} diff --git a/internal/graph/site.go b/internal/graph/site.go index 3aec2909..4937ee2c 100644 --- a/internal/graph/site.go +++ b/internal/graph/site.go @@ -1,18 +1,15 @@ package graph import ( - "errors" "github.com/dominikbraun/graph" "github.com/mach-composer/mach-composer-cli/internal/config" "github.com/mach-composer/mach-composer-cli/internal/utils" - "github.com/rs/zerolog/log" - "sort" ) type Site struct { baseNode - NestedSiteComponentConfigs []config.SiteComponentConfig - SiteConfig config.SiteConfig + NestedNodes []*SiteComponent + SiteConfig config.SiteConfig } func NewSite(g graph.Graph[string, Node], path, identifier string, deploymentType config.DeploymentType, ancestor Node, @@ -24,40 +21,16 @@ func NewSite(g graph.Graph[string, Node], path, identifier string, deploymentTyp } func (s *Site) Hash() (string, error) { - sort.Slice(s.NestedSiteComponentConfigs, func(i, j int) bool { - return s.NestedSiteComponentConfigs[i].Name < s.NestedSiteComponentConfigs[j].Name - }) + SortSiteComponentNodes(s.NestedNodes) - var componentHashes []string - for _, component := range s.NestedSiteComponentConfigs { - hash, err := hashSiteComponentConfig(component) + var hashes []string + for _, component := range s.NestedNodes { + h, err := HashSiteComponentConfig(component.SiteComponentConfig) if err != nil { return "", err } - componentHashes = append(componentHashes, hash) + hashes = append(hashes, h) } - return utils.ComputeHash(componentHashes) -} - -func (s *Site) HasChanges() (bool, error) { - hash, err := s.Hash() - if err != nil { - return false, err - } - - tfHash, err := utils.ParseHashOutput(s.outputs) - if err != nil { - var serr *utils.MissingHashError - if errors.As(err, &serr) { - log.Warn().Msgf("Could not parse hash output: %s. This is "+ - "generally caused by incorrect output state, but will be updated "+ - "at the next succesful update, so can be ignored", serr) - return true, nil - } - - return false, err - } - - return hash != tfHash, nil + return utils.ComputeHash(hashes) } diff --git a/internal/graph/site_component.go b/internal/graph/site_component.go index ae8f567a..5d0058ce 100644 --- a/internal/graph/site_component.go +++ b/internal/graph/site_component.go @@ -1,11 +1,9 @@ package graph import ( - "errors" "github.com/dominikbraun/graph" "github.com/mach-composer/mach-composer-cli/internal/config" - "github.com/mach-composer/mach-composer-cli/internal/utils" - "github.com/rs/zerolog/log" + "sort" ) type SiteComponent struct { @@ -23,27 +21,11 @@ func NewSiteComponent(g graph.Graph[string, Node], path, identifier string, depl } func (sc *SiteComponent) Hash() (string, error) { - return hashSiteComponentConfig(sc.SiteComponentConfig) + return HashSiteComponentConfig(sc.SiteComponentConfig) } -func (sc *SiteComponent) HasChanges() (bool, error) { - hash, err := sc.Hash() - if err != nil { - return true, err - } - - tfHash, err := utils.ParseHashOutput(sc.outputs) - if err != nil { - var serr *utils.MissingHashError - if errors.As(err, &serr) { - log.Warn().Msgf("Could not parse hash output: %s. This is "+ - "generally caused by incorrect output state, but will be updated "+ - "at the next succesful update, so can be ignored", serr) - return true, nil - } - - return false, err - } - - return hash != tfHash, nil +func SortSiteComponentNodes(nodes []*SiteComponent) { + sort.Slice(nodes, func(i, j int) bool { + return nodes[i].SiteComponentConfig.Name < nodes[j].SiteComponentConfig.Name + }) } diff --git a/internal/graph/site_component_test.go b/internal/graph/site_component_test.go deleted file mode 100644 index 4bda76fa..00000000 --- a/internal/graph/site_component_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package graph - -import ( - "github.com/mach-composer/mach-composer-cli/internal/config" - "github.com/stretchr/testify/assert" - "github.com/zclconf/go-cty/cty" - "testing" -) - -func TestSiteComponent_Hash_Ok(t *testing.T) { - su := NewSiteComponent(nil, "", "", "", nil, - config.SiteConfig{}, - config.SiteComponentConfig{Name: "b", Definition: &config.ComponentConfig{Name: "b", Source: "testdata/dirhash"}}, - ) - hash, err := su.Hash() - - assert.NoError(t, err) - assert.Equal(t, "6c477d3042a9c36f088df7375cce8fed8e3a6c71d1c0da17d36e69593b8aafd7", hash, "Hashes should be equal") -} - -func TestSiteComponent_HasChanges_HashNotFound(t *testing.T) { - su := NewSiteComponent(nil, "", "", "", nil, - config.SiteConfig{}, - config.SiteComponentConfig{Name: "b", Definition: &config.ComponentConfig{Name: "b", Source: "testdata/dirhash"}}, - ) - su.outputs = cty.ObjectVal(map[string]cty.Value{}) - - changed, err := su.HasChanges() - assert.NoError(t, err) - assert.True(t, changed) -} - -func TestSiteComponent_HasChanges_Error(t *testing.T) { - su := NewSiteComponent(nil, "", "", "", nil, - config.SiteConfig{}, - config.SiteComponentConfig{Name: "b", Definition: &config.ComponentConfig{Name: "b", Source: "testdata/dirhash"}}, - ) - su.outputs = cty.ObjectVal(map[string]cty.Value{ - "hash": cty.StringVal("some-hash"), - }) - - _, err := su.HasChanges() - assert.Error(t, err) -} - -func TestSiteComponent_HasChanges_True(t *testing.T) { - su := NewSiteComponent(nil, "", "", "", nil, - config.SiteConfig{}, - config.SiteComponentConfig{Name: "b", Definition: &config.ComponentConfig{Name: "b", Source: "testdata/dirhash"}}, - ) - su.outputs = cty.ObjectVal(map[string]cty.Value{ - "hash": cty.ObjectVal(map[string]cty.Value{ - "sensitive": cty.BoolVal(false), - "value": cty.StringVal("different-hash"), - "type": cty.StringVal("some-type"), - }), - }) - - changed, err := su.HasChanges() - assert.NoError(t, err) - assert.True(t, changed) -} - -func TestSiteComponent_HasChanges_False(t *testing.T) { - su := NewSiteComponent(nil, "", "", "", nil, - config.SiteConfig{}, - config.SiteComponentConfig{Name: "b", Definition: &config.ComponentConfig{Name: "b", Source: "testdata/dirhash"}}, - ) - su.outputs = cty.ObjectVal(map[string]cty.Value{ - "hash": cty.ObjectVal(map[string]cty.Value{ - "sensitive": cty.BoolVal(false), - "value": cty.StringVal("6c477d3042a9c36f088df7375cce8fed8e3a6c71d1c0da17d36e69593b8aafd7"), - "type": cty.StringVal("some-type"), - }), - }) - - changed, err := su.HasChanges() - assert.NoError(t, err) - assert.False(t, changed) -} diff --git a/internal/graph/site_test.go b/internal/graph/site_test.go deleted file mode 100644 index f044381f..00000000 --- a/internal/graph/site_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package graph - -import ( - "github.com/mach-composer/mach-composer-cli/internal/config" - "github.com/stretchr/testify/assert" - "github.com/zclconf/go-cty/cty" - "testing" -) - -func TestSite_Hash_NestedComponentConfigSorted(t *testing.T) { - su := NewSite(nil, "", "", "", nil, config.SiteConfig{}) - su.NestedSiteComponentConfigs = []config.SiteComponentConfig{ - {Name: "b", Definition: &config.ComponentConfig{Name: "b", Source: "testdata/dirhash"}}, - {Name: "a", Definition: &config.ComponentConfig{Name: "a", Source: "testdata/dirhash"}}, - } - - unsortedHash, err := su.Hash() - - s := NewSite(nil, "", "", "", nil, config.SiteConfig{}) - s.NestedSiteComponentConfigs = []config.SiteComponentConfig{ - {Name: "a", Definition: &config.ComponentConfig{Name: "a", Source: "testdata/dirhash"}}, - {Name: "b", Definition: &config.ComponentConfig{Name: "b", Source: "testdata/dirhash"}}, - } - sortedHash, err := s.Hash() - - assert.NoError(t, err) - assert.Equal(t, unsortedHash, sortedHash, "Hashes should be equal") -} - -func TestSite_HasChanges_NoHash(t *testing.T) { - s := NewSite(nil, "", "", "", nil, config.SiteConfig{}) - s.NestedSiteComponentConfigs = []config.SiteComponentConfig{ - {Name: "b", Definition: &config.ComponentConfig{Name: "b", Source: "testdata/dirhash"}}, - {Name: "a", Definition: &config.ComponentConfig{Name: "a", Source: "testdata/dirhash"}}, - } - s.outputs = cty.ObjectVal(map[string]cty.Value{}) - - changed, err := s.HasChanges() - assert.NoError(t, err) - assert.True(t, changed) -} - -func TestSite_HasChanges_Error(t *testing.T) { - s := NewSite(nil, "", "", "", nil, config.SiteConfig{}) - s.outputs = cty.ObjectVal(map[string]cty.Value{ - "hash": cty.StringVal("some-hash"), - }) - - _, err := s.HasChanges() - assert.Error(t, err) -} - -func TestSite_HasChanges_True(t *testing.T) { - s := NewSite(nil, "", "", "", nil, config.SiteConfig{}) - s.NestedSiteComponentConfigs = []config.SiteComponentConfig{ - {Name: "b", Definition: &config.ComponentConfig{Name: "b", Source: "testdata/dirhash"}}, - {Name: "a", Definition: &config.ComponentConfig{Name: "a", Source: "testdata/dirhash"}}, - } - s.outputs = cty.ObjectVal(map[string]cty.Value{ - "hash": cty.ObjectVal(map[string]cty.Value{ - "sensitive": cty.BoolVal(false), - "value": cty.StringVal("different-hash"), - "type": cty.StringVal("some-type"), - }), - }) - - changed, err := s.HasChanges() - assert.NoError(t, err) - assert.True(t, changed) -} - -func TestSite_HasChanges_False(t *testing.T) { - s := NewSite(nil, "", "", "", nil, config.SiteConfig{}) - s.NestedSiteComponentConfigs = []config.SiteComponentConfig{ - {Name: "b", Definition: &config.ComponentConfig{Name: "b", Source: "testdata/dirhash"}}, - {Name: "a", Definition: &config.ComponentConfig{Name: "a", Source: "testdata/dirhash"}}, - } - - s.outputs = cty.ObjectVal(map[string]cty.Value{ - "hash": cty.ObjectVal(map[string]cty.Value{ - "sensitive": cty.BoolVal(false), - "value": cty.StringVal("6b70e89e46522af5c0dad35f403ed12c13e891cef0f1360024be3645754fa53e"), - "type": cty.StringVal("some-type"), - }), - }) - - changed, err := s.HasChanges() - assert.NoError(t, err) - assert.False(t, changed) -} diff --git a/internal/graph/taint.go b/internal/graph/taint.go deleted file mode 100644 index 82efccca..00000000 --- a/internal/graph/taint.go +++ /dev/null @@ -1,47 +0,0 @@ -package graph - -func determineTainted(n Node, parentTainted bool) (bool, error) { - // If a node has already been marked as tainted in a previous iteration, we don't need to check it again - if n.Tainted() { - return true, nil - } - - // If a parent has been marked as tainted the current node is also tainted - if parentTainted { - return true, nil - } - - // If a node has changes it is tainted - return n.HasChanges() -} - -func taintNode(g *Graph, path string, parentTainted bool) error { - n, err := g.Vertex(path) - if err != nil { - return err - } - - am, err := g.AdjacencyMap() - if err != nil { - return err - } - - isTainted, err := determineTainted(n, parentTainted) - if err != nil { - return err - } - n.SetTainted(isTainted) - - for _, child := range am[path] { - if err = taintNode(g, child.Target, isTainted); err != nil { - return err - } - } - - return nil -} - -// TaintNodes will mark all nodes as tainted that have changes or are dependent on a node with changes -func TaintNodes(g *Graph) error { - return taintNode(g, g.StartNode.Path(), false) -} diff --git a/internal/graph/taint_test.go b/internal/graph/taint_test.go deleted file mode 100644 index 73d47339..00000000 --- a/internal/graph/taint_test.go +++ /dev/null @@ -1,167 +0,0 @@ -package graph - -import ( - "fmt" - "github.com/dominikbraun/graph" - "github.com/stretchr/testify/assert" - "testing" -) - -func TestTaintNodesNoChanges(t *testing.T) { - ig := graph.New(func(n Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) - - site := new(NodeMock) - site.On("Path").Return("main") - site.On("HasChanges").Return(false, nil).Once() - - _ = ig.AddVertex(site) - - g := &Graph{Graph: ig, StartNode: site} - - err := TaintNodes(g) - assert.NoError(t, err) - assert.False(t, site.Tainted()) -} - -func TestTaintNodesHasChanges(t *testing.T) { - ig := graph.New(func(n Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) - - site := new(NodeMock) - site.On("Path").Return("site") - site.On("HasChanges").Return(true, nil).Once() - - _ = ig.AddVertex(site) - - g := &Graph{Graph: ig, StartNode: site} - - err := TaintNodes(g) - assert.NoError(t, err) - assert.True(t, site.Tainted()) -} - -func TestTaintNodesHasChanges_Error(t *testing.T) { - ig := graph.New(func(n Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) - - site := new(NodeMock) - site.On("Path").Return("site") - site.On("HasChanges").Return(false, fmt.Errorf("error")).Once() - - _ = ig.AddVertex(site) - - g := &Graph{Graph: ig, StartNode: site} - - err := TaintNodes(g) - assert.Error(t, err) -} - -func TestTaintNodesChildHasChanges(t *testing.T) { - ig := graph.New(func(n Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) - - site := new(NodeMock) - site.On("Path").Return("site").Times(2) - site.On("HasChanges").Return(false, nil).Once() - - component := new(NodeMock) - component.On("Path").Return("component").Times(2) - component.On("HasChanges").Return(true, nil).Once() - - _ = ig.AddVertex(site) - _ = ig.AddVertex(component) - - _ = ig.AddEdge("site", "component") - - g := &Graph{Graph: ig, StartNode: site} - - err := TaintNodes(g) - assert.NoError(t, err) - assert.False(t, site.Tainted()) - assert.True(t, component.Tainted()) -} - -func TestTaintNodesChildHasChanges_Error(t *testing.T) { - ig := graph.New(func(n Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) - - site := new(NodeMock) - site.On("Path").Return("site").Times(2) - site.On("HasChanges").Return(false, nil).Once() - - component := new(NodeMock) - component.On("Path").Return("component").Times(2) - component.On("HasChanges").Return(true, fmt.Errorf("error")).Once() - - _ = ig.AddVertex(site) - _ = ig.AddVertex(component) - - _ = ig.AddEdge("site", "component") - - g := &Graph{Graph: ig, StartNode: site} - - err := TaintNodes(g) - assert.Error(t, err) -} - -func TestTaintNodesParentHasChanges(t *testing.T) { - ig := graph.New(func(n Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) - - site := new(NodeMock) - site.On("Path").Return("site").Times(2) - site.On("HasChanges").Return(true, nil).Once() - - component := new(NodeMock) - component.On("Path").Return("component").Times(2) - component.On("HasChanges").Return(false, nil).Once() - - _ = ig.AddVertex(site) - _ = ig.AddVertex(component) - - _ = ig.AddEdge("site", "component") - - g := &Graph{Graph: ig, StartNode: site} - - err := TaintNodes(g) - assert.NoError(t, err) - assert.True(t, site.Tainted()) - assert.True(t, component.Tainted()) - - component.AssertNumberOfCalls(t, "HasChanges", 0) -} - -func TestTaintNodesOneParentHasChanges(t *testing.T) { - ig := graph.New(func(n Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) - - site := new(NodeMock) - site.On("Path").Return("site").Times(2) - site.On("HasChanges").Return(false, nil).Once() - - component1 := new(NodeMock) - component1.On("Path").Return("component-1").Times(2) - component1.On("HasChanges").Return(true, nil).Once() - - component2 := new(NodeMock) - component2.On("Path").Return("component-2").Times(2) - component2.On("HasChanges").Return(false, nil).Once() - - component3 := new(NodeMock) - component3.On("Path").Return("component-3").Times(2) - component3.On("HasChanges").Return(false, nil).Once() - - _ = ig.AddVertex(site) - _ = ig.AddVertex(component1) - _ = ig.AddVertex(component2) - _ = ig.AddVertex(component3) - - _ = ig.AddEdge("site", "component-1") - _ = ig.AddEdge("site", "component-2") - _ = ig.AddEdge("component-1", "component-3") - _ = ig.AddEdge("component-2", "component-3") - - g := &Graph{Graph: ig, StartNode: site} - - err := TaintNodes(g) - assert.NoError(t, err) - - assert.False(t, site.Tainted()) - assert.True(t, component1.Tainted()) - assert.False(t, component2.Tainted()) - assert.True(t, component3.Tainted()) -} diff --git a/internal/graph/utils.go b/internal/graph/utils.go index 51552ae2..fd587052 100644 --- a/internal/graph/utils.go +++ b/internal/graph/utils.go @@ -2,7 +2,6 @@ package graph import ( "github.com/dominikbraun/graph" - "github.com/zclconf/go-cty/cty" ) type Path []string @@ -26,22 +25,3 @@ func fetchPathsToTarget(source, target string, pm map[string]map[string]graph.Ed return paths } - -func HasMissingParentOutputs(n Node) (bool, error) { - if n.Type() != SiteComponentType { - return false, nil - } - - parents, err := n.Parents() - if err != nil { - return false, err - } - - for _, p := range parents { - if p.Outputs() == cty.NilVal { - return true, nil - } - } - - return false, nil -} diff --git a/internal/graph/utils_test.go b/internal/graph/utils_test.go deleted file mode 100644 index aa5c578f..00000000 --- a/internal/graph/utils_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package graph - -import ( - "fmt" - "github.com/stretchr/testify/assert" - "github.com/zclconf/go-cty/cty" - "testing" -) - -func TestHasMissingParentOutputsSiteType(t *testing.T) { - n := new(NodeMock) - n.On("Type").Return(SiteType).Once() - - missing, err := HasMissingParentOutputs(n) - assert.NoError(t, err) - assert.False(t, missing) -} - -func TestHasMissingParentOutputsNoParents(t *testing.T) { - n := new(NodeMock) - n.On("Type").Return(SiteComponentType).Once() - n.On("Parents").Return([]Node{}, fmt.Errorf("error")).Once() - - _, err := HasMissingParentOutputs(n) - assert.Error(t, err) -} - -func TestHasMissingParentOutputsParentWithNilVal(t *testing.T) { - parent := new(NodeMock) - parent.On("Outputs").Return(cty.NilVal).Once() - - n := new(NodeMock) - n.On("Type").Return(SiteComponentType).Once() - n.On("Parents").Return([]Node{parent}, nil).Once() - - missing, err := HasMissingParentOutputs(n) - assert.NoError(t, err) - assert.True(t, missing) -} - -func TestHasMissingParentOutputsSingle_False(t *testing.T) { - parent := new(NodeMock) - parent.On("Outputs").Return(cty.StringVal("value")).Once() - - n := new(NodeMock) - n.On("Type").Return(SiteComponentType).Once() - n.On("Parents").Return([]Node{parent}, nil).Once() - - missing, err := HasMissingParentOutputs(n) - assert.NoError(t, err) - assert.False(t, missing) -} - -func TestHasMissingParentOutputsMultiple_False(t *testing.T) { - parent1 := new(NodeMock) - parent1.On("Outputs").Return(cty.StringVal("value")).Once() - parent2 := new(NodeMock) - parent2.On("Outputs").Return(cty.StringVal("value")).Once() - - n := new(NodeMock) - n.On("Type").Return(SiteComponentType).Once() - n.On("Parents").Return([]Node{parent1, parent2}, nil).Once() - - missing, err := HasMissingParentOutputs(n) - assert.NoError(t, err) - assert.False(t, missing) -} - -func TestHasMissingParentOutputsMultiple_True(t *testing.T) { - parent1 := new(NodeMock) - parent1.On("Outputs").Return(cty.StringVal("value")).Once() - parent2 := new(NodeMock) - parent2.On("Outputs").Return(cty.NilVal).Once() - - n := new(NodeMock) - n.On("Type").Return(SiteComponentType).Once() - n.On("Parents").Return([]Node{parent1, parent2}, nil).Once() - - missing, err := HasMissingParentOutputs(n) - assert.NoError(t, err) - assert.True(t, missing) -} diff --git a/internal/hash/handler.go b/internal/hash/handler.go new file mode 100644 index 00000000..c93f3127 --- /dev/null +++ b/internal/hash/handler.go @@ -0,0 +1,24 @@ +package hash + +import ( + "context" + "github.com/mach-composer/mach-composer-cli/internal/config" + "github.com/mach-composer/mach-composer-cli/internal/graph" + "os" +) + +const defaultHashFile = ".mach-composer/hashes.json" + +type Handler interface { + Store(ctx context.Context, n graph.Node) error + Fetch(ctx context.Context, n graph.Node) (string, error) +} + +func Factory(_ *config.MachConfig) Handler { + hashFile := os.Getenv("MC_HASH_FILE") + if hashFile == "" { + hashFile = defaultHashFile + } + + return NewJsonFileHandler(hashFile) +} diff --git a/internal/hash/json_file.go b/internal/hash/json_file.go new file mode 100644 index 00000000..5a4ab7b1 --- /dev/null +++ b/internal/hash/json_file.go @@ -0,0 +1,126 @@ +package hash + +import ( + "context" + "encoding/json" + "fmt" + "github.com/mach-composer/mach-composer-cli/internal/graph" + "github.com/mach-composer/mach-composer-cli/internal/utils" + "github.com/rs/zerolog/log" + "os" + "path" + "sync" +) + +var mutex = &sync.RWMutex{} + +type Hashes map[string]string + +type JsonFileHandler struct { + file string +} + +func NewJsonFileHandler(file string) Handler { + if _, err := os.Stat(path.Dir(file)); os.IsNotExist(err) { + err = os.MkdirAll(path.Dir(file), 0777) + if err != nil { + log.Panic().Err(err).Msgf("Failed to create directory %s", path.Dir(file)) + } + } + + return &JsonFileHandler{ + file: file, + } +} + +func (h *JsonFileHandler) getHashes() (*Hashes, error) { + var hashes Hashes + + f, err := os.OpenFile(h.file, os.O_RDONLY|os.O_CREATE, 0777) + if err != nil { + return nil, err + } + defer f.Close() + + c, err := os.ReadFile(f.Name()) + if err != nil { + return nil, err + } + + if len(c) == 0 { + return &Hashes{}, nil + } + + err = json.Unmarshal(c, &hashes) + if err != nil { + return nil, err + } + + return &hashes, nil +} + +func (h *JsonFileHandler) Fetch(_ context.Context, n graph.Node) (string, error) { + mutex.RLock() + defer mutex.RUnlock() + + hashes, err := h.getHashes() + if err != nil { + return "", err + } + + switch n.Type() { + case graph.ProjectType: + return "", nil + case graph.SiteType: + s := n.(*graph.Site) + graph.SortSiteComponentNodes(s.NestedNodes) + + var componentHashes []string + for _, component := range s.NestedNodes { + h := (*hashes)[component.Identifier()] + componentHashes = append(componentHashes, h) + } + + return utils.ComputeHash(componentHashes) + case graph.SiteComponentType: + return (*hashes)[n.Identifier()], nil + default: + return "", fmt.Errorf("unknown node type %T", n) + } +} + +func (h *JsonFileHandler) Store(_ context.Context, n graph.Node) error { + mutex.Lock() + defer mutex.Unlock() + + hashes, err := h.getHashes() + if err != nil { + return err + } + + switch n.Type() { + case graph.ProjectType: + return nil + case graph.SiteType: + for _, nn := range n.(*graph.Site).NestedNodes { + (*hashes)[nn.Identifier()], err = nn.Hash() + if err != nil { + return err + } + } + case graph.SiteComponentType: + (*hashes)[n.Identifier()], err = n.Hash() + if err != nil { + return err + } + default: + return fmt.Errorf("unknown node type %T", n) + } + + c, err := json.Marshal(hashes) + if err != nil { + return err + } + + return os.WriteFile(h.file, c, 0777) +} diff --git a/internal/hash/memory.go b/internal/hash/memory.go new file mode 100644 index 00000000..6b3e80af --- /dev/null +++ b/internal/hash/memory.go @@ -0,0 +1,37 @@ +package hash + +import ( + "context" + "github.com/mach-composer/mach-composer-cli/internal/graph" +) + +type Entry struct { + Identifier string + Hash string +} + +type MemoryMap struct { + InternalMap map[string]string +} + +func NewMemoryMapHandler(entries ...Entry) Handler { + h := &MemoryMap{ + InternalMap: make(map[string]string), + } + + for _, e := range entries { + h.InternalMap[e.Identifier] = e.Hash + } + + return h +} + +func (h *MemoryMap) Fetch(_ context.Context, n graph.Node) (string, error) { + return h.InternalMap[n.Identifier()], nil +} +func (h *MemoryMap) Store(_ context.Context, n graph.Node) error { + var err error + h.InternalMap[n.Identifier()], err = n.Hash() + + return err +} diff --git a/internal/plugins/plugin.go b/internal/plugins/plugin.go index ca4cf48a..68f6d8cf 100644 --- a/internal/plugins/plugin.go +++ b/internal/plugins/plugin.go @@ -10,8 +10,6 @@ import ( schemav2 "github.com/mach-composer/mach-composer-plugin-sdk/v2/schema" "github.com/rs/zerolog/log" "hash/crc32" - "os" - "strings" ) type PluginHandler struct { @@ -40,9 +38,9 @@ func (p *PluginHandler) Start(ctx context.Context) error { } // Safety check to not run external handlers during test for now - if strings.HasSuffix(os.Args[0], ".test") { - panic(fmt.Sprintf("Not loading %s: invalid command: %s", p.Name, os.Args[0])) - } + //if strings.HasSuffix(os.Args[0], ".test") { + // panic(fmt.Sprintf("Not loading %s: invalid command: %s", p.Name, os.Args[0])) + //} logger := NewHCLogAdapter(log.Logger) diff --git a/internal/runner/graph.go b/internal/runner/graph.go index d77b3c41..a18e6dbc 100644 --- a/internal/runner/graph.go +++ b/internal/runner/graph.go @@ -3,8 +3,10 @@ package runner import ( "context" "fmt" + "github.com/mach-composer/mach-composer-cli/internal/batcher" "github.com/mach-composer/mach-composer-cli/internal/cli" "github.com/mach-composer/mach-composer-cli/internal/graph" + "github.com/mach-composer/mach-composer-cli/internal/hash" "github.com/mach-composer/mach-composer-cli/internal/terraform" "github.com/mach-composer/mach-composer-cli/internal/utils" "github.com/rs/zerolog/log" @@ -17,63 +19,26 @@ import ( type ( //executorFunc is a function that executes an arbitrary command on a node executorFunc func(ctx context.Context, node graph.Node) (string, error) - - //batchFunc is a function that batches nodes in groups that can run in parallel by some criteria - batchFunc func(g *graph.Graph) map[int][]graph.Node - - //taintFunc is a function that marks nodes as tainted if they have changes that need to be applied - taintFunc func(ctx context.Context, g *graph.Graph) error ) // GraphRunner will run a set of commands on a graph of nodes. Untainted nodes (no changes) will be skipped. // The nodes are batched based on a batching function, and all nodes in the same batch will be run in parallel. type GraphRunner struct { workers int - batch batchFunc - taint taintFunc -} - -// batchNodes will batch nodes based on the length of the longest route from the node to the root. -// This is a naive implementation that might break down for very complex graphs -func batchNodes(g *graph.Graph) map[int][]graph.Node { - batches := map[int][]graph.Node{} - - var sets = map[string][]graph.Path{} - - for _, n := range g.Vertices() { - var route, _ = g.Routes(n.Path(), g.StartNode.Path()) - sets[n.Path()] = route - } - - for k, routes := range sets { - var mx int - for _, route := range routes { - if len(route) > mx { - mx = len(route) - } - } - n, _ := g.Vertex(k) - batches[mx] = append(batches[mx], n) - } - - return batches + batch batcher.BatchFunc + hash hash.Handler } -func NewGraphRunner(workers int) *GraphRunner { +func NewGraphRunner(batcher batcher.BatchFunc, hashHandler hash.Handler, workers int) *GraphRunner { return &GraphRunner{ workers: workers, - batch: batchNodes, - taint: func(ctx context.Context, g *graph.Graph) error { - if err := graph.LoadOutputs(ctx, g, utils.GetTerraformOutputs); err != nil { - return err - } - - return graph.TaintNodes(g) - }} + batch: batcher, + hash: hashHandler, + } } func (gr *GraphRunner) run(ctx context.Context, g *graph.Graph, f executorFunc, ignoreChangeDetection bool) error { - if err := gr.taint(ctx, g); err != nil { + if err := taintGraph(ctx, g, gr.hash); err != nil { return err } @@ -134,7 +99,26 @@ func (gr *GraphRunner) run(ctx context.Context, g *graph.Graph, f executorFunc, func (gr *GraphRunner) TerraformApply(ctx context.Context, dg *graph.Graph, opts *ApplyOptions) error { if err := gr.run(ctx, dg, func(ctx context.Context, n graph.Node) (string, error) { - return terraform.Apply(ctx, n.Path(), opts.Destroy, opts.AutoApprove) + if !terraformIsInitialized(n.Path()) || opts.ForceInit { + log.Info().Msgf("Running terraform init for %s", n.Path()) + if out, err := terraform.Init(ctx, n.Path()); err != nil { + return out, err + } + } else { + log.Info().Msgf("Skipping terraform init for %s", n.Path()) + } + + out, err := terraform.Apply(ctx, n.Path(), opts.Destroy, opts.AutoApprove) + if err != nil { + return out, err + } + + log.Info().Msgf("Storing new hash for %s", n.Path()) + if err = gr.hash.Store(ctx, n); err != nil { + log.Warn().Err(err).Msgf("Failed to store hash for %s", n.Identifier()) + } + return out, nil + }, opts.IgnoreChangeDetection); err != nil { return err } @@ -144,12 +128,21 @@ func (gr *GraphRunner) TerraformApply(ctx context.Context, dg *graph.Graph, opts func (gr *GraphRunner) TerraformPlan(ctx context.Context, dg *graph.Graph, opts *PlanOptions) error { if err := gr.run(ctx, dg, func(ctx context.Context, n graph.Node) (string, error) { - missing, err := graph.HasMissingParentOutputs(n) + if !terraformIsInitialized(n.Path()) || opts.ForceInit { + log.Info().Msgf("Running terraform init for %s", n.Path()) + if out, err := terraform.Init(ctx, n.Path()); err != nil { + return out, err + } + } else { + log.Info().Msgf("Skipping terraform init for %s", n.Path()) + } + + canPlan, err := terraformCanPlan(ctx, n) if err != nil { return "", err } - if missing { + if !canPlan { log.Info().Msgf("Skipping planning %s because it has missing outputs", n.Path()) return "", nil } @@ -164,6 +157,10 @@ func (gr *GraphRunner) TerraformPlan(ctx context.Context, dg *graph.Graph, opts func (gr *GraphRunner) TerraformProxy(ctx context.Context, dg *graph.Graph, opts *ProxyOptions) error { if err := gr.run(ctx, dg, func(ctx context.Context, n graph.Node) (string, error) { + if !terraformIsInitialized(n.Path()) { + return "", fmt.Errorf("terraform is not initialized for %s. Please run init beforehand", n.Path()) + } + return utils.RunTerraform(ctx, n.Path(), false, opts.Command...) }, opts.IgnoreChangeDetection); err != nil { return err @@ -174,6 +171,15 @@ func (gr *GraphRunner) TerraformProxy(ctx context.Context, dg *graph.Graph, opts func (gr *GraphRunner) TerraformShow(ctx context.Context, dg *graph.Graph, opts *ShowPlanOptions) error { if err := gr.run(ctx, dg, func(ctx context.Context, n graph.Node) (string, error) { + if !terraformIsInitialized(n.Path()) || opts.ForceInit { + log.Info().Msgf("Running terraform init for %s", n.Path()) + if out, err := terraform.Init(ctx, n.Path()); err != nil { + return out, err + } + } else { + log.Info().Msgf("Skipping terraform init for %s", n.Path()) + } + return terraform.Show(ctx, n.Path(), opts.NoColor) }, opts.IgnoreChangeDetection); err != nil { return err @@ -182,63 +188,12 @@ func (gr *GraphRunner) TerraformShow(ctx context.Context, dg *graph.Graph, opts return nil } -// TerraformInit will run terraform init on all nodes in the graph. It will -// batch the nodes and run them in parallel. This is slightly different from the -// run command above, in that it skips the tainting of nodes. The only check it -// will do is see if terraform has already been initialized, and skip init if -// that is the case. -// -// TODO: move init into the graph runner after we have found a way to save and -// search hashes without using terraform as a storage func (gr *GraphRunner) TerraformInit(ctx context.Context, dg *graph.Graph) error { - batches := gr.batch(dg) - - keys := maps.Keys(batches) - sort.Ints(keys) - for i, k := range keys[1:] { - log.Info().Msgf("Running batch %d with %d nodes", i, len(batches[k])) - - errChan := make(chan error, len(batches[k])) - wg := &sync.WaitGroup{} - sem := semaphore.NewWeighted(int64(gr.workers)) - - for _, n := range batches[k] { - if err := sem.Acquire(ctx, 1); err != nil { - return err - } - wg.Add(1) - go func(ctx context.Context, n graph.Node) { - defer wg.Done() - defer sem.Release(1) - - log.Info().Msgf("Running init on %s", n.Identifier()) - - if n.Type() == graph.ProjectType { - return - } - err := terraform.Init(ctx, n.Path()) - if err != nil { - errChan <- err - return - } - }(ctx, n) - } - wg.Wait() - close(errChan) - - if len(errChan) > 0 { - var errors []error - for err := range errChan { - errors = append(errors, err) - } - - return cli.NewGroupedError(fmt.Sprintf("batch run %d failed (%d errors)", i, len(errors)), errors) - } - - log.Info().Msgf("Finished batch %d", i) + if err := gr.run(ctx, dg, func(ctx context.Context, n graph.Node) (string, error) { + return terraform.Init(ctx, n.Path()) + }, true); err != nil { + return err } - log.Info().Msgf("Finished all batches") - return nil } diff --git a/internal/runner/graph_test.go b/internal/runner/graph_test.go index 77ca3063..a591ea6a 100644 --- a/internal/runner/graph_test.go +++ b/internal/runner/graph_test.go @@ -3,127 +3,71 @@ package runner import ( "context" "errors" - "github.com/dominikbraun/graph" + "github.com/mach-composer/mach-composer-cli/internal/batcher" "github.com/mach-composer/mach-composer-cli/internal/cli" internalgraph "github.com/mach-composer/mach-composer-cli/internal/graph" + "github.com/mach-composer/mach-composer-cli/internal/hash" "github.com/stretchr/testify/assert" "testing" ) -func TestBatchNodesDepth1(t *testing.T) { - ig := graph.New(func(n internalgraph.Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) - - start := new(internalgraph.NodeMock) - start.On("Path").Return("main/site-1") - - _ = ig.AddVertex(start) - - g := &internalgraph.Graph{Graph: ig, StartNode: start} - - batches := batchNodes(g) - - assert.Equal(t, 1, len(batches)) -} - -func TestBatchNodesDepth2(t *testing.T) { - ig := graph.New(func(n internalgraph.Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) - - site := new(internalgraph.NodeMock) - site.On("Path").Return("main/site-1") - - component1 := new(internalgraph.NodeMock) - component1.On("Path").Return("main/site-1/component-1") - - component2 := new(internalgraph.NodeMock) - component2.On("Path").Return("main/site-1/component-2") - - _ = ig.AddVertex(site) - _ = ig.AddVertex(component1) - _ = ig.AddVertex(component2) - - _ = ig.AddEdge("main/site-1", "main/site-1/component-1") - _ = ig.AddEdge("main/site-1", "main/site-1/component-2") - - g := &internalgraph.Graph{Graph: ig, StartNode: site} - - batches := batchNodes(g) - - assert.Equal(t, 2, len(batches)) - assert.Equal(t, 1, len(batches[0])) - assert.Equal(t, "main/site-1", batches[0][0].Path()) - assert.Equal(t, 2, len(batches[1])) - assert.Contains(t, batches[1][0].Path(), "component") - assert.Contains(t, batches[1][1].Path(), "component") -} - -func TestBatchNodesDepth3(t *testing.T) { - ig := graph.New(func(n internalgraph.Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) - - site := new(internalgraph.NodeMock) - site.On("Path").Return("main/site-1") - - component1 := new(internalgraph.NodeMock) - component1.On("Path").Return("main/site-1/component-1") - - component2 := new(internalgraph.NodeMock) - component2.On("Path").Return("main/site-1/component-2") - - _ = ig.AddVertex(site) - _ = ig.AddVertex(component1) - _ = ig.AddVertex(component2) - - _ = ig.AddEdge("main/site-1", "main/site-1/component-1") - _ = ig.AddEdge("main/site-1/component-1", "main/site-1/component-2") - - g := &internalgraph.Graph{Graph: ig, StartNode: site} - - batches := batchNodes(g) - - assert.Equal(t, 3, len(batches)) - assert.Equal(t, 1, len(batches[0])) - assert.Equal(t, "main/site-1", batches[0][0].Path()) - assert.Equal(t, 1, len(batches[1])) - assert.Contains(t, batches[1][0].Path(), "main/site-1/component-1") - assert.Equal(t, 1, len(batches[2])) - assert.Contains(t, batches[2][0].Path(), "main/site-1/component-2") -} - func TestGraphRunnerMultipleLevels(t *testing.T) { project := new(internalgraph.NodeMock) project.On("Identifier").Return("main") + project.On("Path").Return("main") + project.On("Hash").Return("main", nil) + project.On("Type").Return(internalgraph.ProjectType) site := new(internalgraph.NodeMock) site.On("Identifier").Return("site-1") - site.SetTainted(false) + site.On("Path").Return("site-1") + site.On("Hash").Return("site-1", nil) + site.On("Type").Return(internalgraph.SiteType) component1 := new(internalgraph.NodeMock) component1.On("Identifier").Return("component-1") - component1.SetTainted(false) + component1.On("Path").Return("component-1") + component1.On("Hash").Return("component-1", nil) + component1.On("Type").Return(internalgraph.SiteComponentType) component2 := new(internalgraph.NodeMock) component2.On("Identifier").Return("component-2") - component2.SetTainted(true) + component2.On("Path").Return("component-2") + component2.On("Hash").Return("component-2", nil) + component2.On("Type").Return(internalgraph.SiteComponentType) component3 := new(internalgraph.NodeMock) component3.On("Identifier").Return("component-3") - component3.SetTainted(true) - - runner := NewGraphRunner(1) - runner.taint = func(ctx context.Context, g *internalgraph.Graph) error { - return nil - } - runner.batch = func(g *internalgraph.Graph) map[int][]internalgraph.Node { - return map[int][]internalgraph.Node{ - 0: {project}, - 1: {site}, - 2: {component1, component2}, - 3: {component3}, - } - } + component3.On("Path").Return("component-3") + component3.On("Hash").Return("component-3", nil) + component3.On("Type").Return(internalgraph.SiteComponentType) + + graph := internalgraph.CreateGraphMock( + map[string]internalgraph.Node{ + "main": project, + "site-1": site, + "component-1": component1, + "component-2": component2, + "component-3": component3, + }, + project, + internalgraph.EdgeMock{Source: "main", Target: "site-1"}, + internalgraph.EdgeMock{Source: "site-1", Target: "component-1"}, + internalgraph.EdgeMock{Source: "site-1", Target: "component-2"}, + internalgraph.EdgeMock{Source: "component-1", Target: "component-3"}, + internalgraph.EdgeMock{Source: "component-2", Target: "component-3"}, + ) + + runner := GraphRunner{workers: 1} + runner.hash = hash.NewMemoryMapHandler( + hash.Entry{Identifier: "site-1", Hash: "site-1"}, + hash.Entry{Identifier: "component-1", Hash: "component-1"}, + ) + runner.batch = batcher.NaiveBatchFunc() var called []string - err := runner.run(context.Background(), &internalgraph.Graph{}, func(ctx context.Context, node internalgraph.Node) (string, error) { + err := runner.run(context.Background(), graph, func(ctx context.Context, node internalgraph.Node) (string, error) { called = append(called, node.Identifier()) return "", nil }, false) @@ -135,32 +79,55 @@ func TestGraphRunnerMultipleLevels(t *testing.T) { func TestGraphRunnerError(t *testing.T) { project := new(internalgraph.NodeMock) project.On("Identifier").Return("main") + project.On("Path").Return("main") + project.On("Hash").Return("main", nil) + project.On("Type").Return(internalgraph.ProjectType) site := new(internalgraph.NodeMock) site.On("Identifier").Return("site-1") - site.SetTainted(true) + site.On("Path").Return("site-1") + site.On("Hash").Return("site-1", nil) + site.On("Type").Return(internalgraph.SiteType) component1 := new(internalgraph.NodeMock) component1.On("Identifier").Return("component-1") - component1.SetTainted(true) + component1.On("Path").Return("component-1") + component1.On("Hash").Return("component-1", nil) + component1.On("Type").Return(internalgraph.SiteComponentType) component2 := new(internalgraph.NodeMock) component2.On("Identifier").Return("component-2") - component2.SetTainted(true) - - runner := NewGraphRunner(1) - runner.taint = func(ctx context.Context, g *internalgraph.Graph) error { - return nil - } - runner.batch = func(g *internalgraph.Graph) map[int][]internalgraph.Node { - return map[int][]internalgraph.Node{ - 0: {project}, - 1: {site}, - 2: {component1, component2}, - } - } + component2.On("Path").Return("component-2") + component2.On("Hash").Return("component-2", nil) + component2.On("Type").Return(internalgraph.SiteComponentType) - err := runner.run(context.Background(), &internalgraph.Graph{}, func(ctx context.Context, node internalgraph.Node) (string, error) { + component3 := new(internalgraph.NodeMock) + component3.On("Identifier").Return("component-3") + component3.On("Path").Return("component-3") + component3.On("Hash").Return("component-3", nil) + component3.On("Type").Return(internalgraph.SiteComponentType) + + graph := internalgraph.CreateGraphMock( + map[string]internalgraph.Node{ + "main": project, + "site-1": site, + "component-1": component1, + "component-2": component2, + "component-3": component3, + }, + project, + internalgraph.EdgeMock{Source: "main", Target: "site-1"}, + internalgraph.EdgeMock{Source: "site-1", Target: "component-1"}, + internalgraph.EdgeMock{Source: "site-1", Target: "component-2"}, + internalgraph.EdgeMock{Source: "component-1", Target: "component-3"}, + internalgraph.EdgeMock{Source: "component-2", Target: "component-3"}, + ) + + runner := GraphRunner{workers: 1} + runner.hash = hash.NewMemoryMapHandler() + runner.batch = batcher.NaiveBatchFunc() + + err := runner.run(context.Background(), graph, func(ctx context.Context, node internalgraph.Node) (string, error) { if node.Identifier() == "component-2" { return "", assert.AnError } @@ -175,33 +142,3 @@ func TestGraphRunnerError(t *testing.T) { assert.Len(t, cliErr.Errors, 1) assert.Equal(t, assert.AnError, cliErr.Errors[0]) } - -func TestGraphRunnerForce(t *testing.T) { - project := new(internalgraph.NodeMock) - project.On("Identifier").Return("main") - - site := new(internalgraph.NodeMock) - site.On("Identifier").Return("site-1") - site.SetTainted(false) - - runner := NewGraphRunner(1) - runner.taint = func(ctx context.Context, g *internalgraph.Graph) error { - return nil - } - runner.batch = func(g *internalgraph.Graph) map[int][]internalgraph.Node { - return map[int][]internalgraph.Node{ - 0: {project}, - 1: {site}, - } - } - - var called []string - - err := runner.run(context.Background(), &internalgraph.Graph{}, func(ctx context.Context, node internalgraph.Node) (string, error) { - called = append(called, node.Identifier()) - return "", nil - }, true) - - assert.NoError(t, err) - assert.Equal(t, []string{"site-1"}, called) -} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 13471006..0ed317ce 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -6,12 +6,14 @@ import ( ) type ApplyOptions struct { + ForceInit bool IgnoreChangeDetection bool Destroy bool AutoApprove bool } type PlanOptions struct { + ForceInit bool IgnoreChangeDetection bool Lock bool } @@ -22,6 +24,7 @@ type ProxyOptions struct { } type ShowPlanOptions struct { + ForceInit bool IgnoreChangeDetection bool NoColor bool } diff --git a/internal/runner/taint.go b/internal/runner/taint.go new file mode 100644 index 00000000..b1530e1c --- /dev/null +++ b/internal/runner/taint.go @@ -0,0 +1,67 @@ +package runner + +import ( + "context" + "github.com/mach-composer/mach-composer-cli/internal/graph" + "github.com/mach-composer/mach-composer-cli/internal/hash" + "github.com/rs/zerolog/log" +) + +func determineTainted(oldHash string, n graph.Node, parentTainted bool) (bool, error) { + // If a node has already been marked as tainted in a previous iteration, we don't need to check it again + if n.Tainted() { + return true, nil + } + + // If a parent has been marked as tainted the current node is also tainted + if parentTainted { + return true, nil + } + + h, err := n.Hash() + if err != nil { + log.Warn().Err(err).Msgf("Failed to compute hash for %s", n.Path()) + return false, err + } + + return h != oldHash, nil +} + +func taintNode(ctx context.Context, hashFetcher hash.Handler, g *graph.Graph, path string, parentTainted bool) error { + n, err := g.Vertex(path) + if err != nil { + return err + } + + am, err := g.AdjacencyMap() + if err != nil { + return err + } + + oldHash, err := hashFetcher.Fetch(ctx, n) + if err != nil { + return err + } + n.SetOldHash(oldHash) + + var isTainted = false + if n.Type() != graph.ProjectType { + isTainted, err = determineTainted(oldHash, n, parentTainted) + if err != nil { + return err + } + } + n.SetTainted(isTainted) + + for _, child := range am[path] { + if err = taintNode(ctx, hashFetcher, g, child.Target, isTainted); err != nil { + return err + } + } + + return nil +} + +func taintGraph(ctx context.Context, g *graph.Graph, hashFetcher hash.Handler) error { + return taintNode(ctx, hashFetcher, g, g.StartNode.Path(), false) +} diff --git a/internal/runner/testdata/empty/test.tf b/internal/runner/testdata/empty/test.tf new file mode 100644 index 00000000..ee46aaaf --- /dev/null +++ b/internal/runner/testdata/empty/test.tf @@ -0,0 +1,3 @@ +locals { + name = "world" +} diff --git a/internal/runner/testdata/initialized/.terraform/terraform.tfstate b/internal/runner/testdata/initialized/.terraform/terraform.tfstate new file mode 100644 index 00000000..0153db98 --- /dev/null +++ b/internal/runner/testdata/initialized/.terraform/terraform.tfstate @@ -0,0 +1,23 @@ +{ + "version": 3, + "serial": 1, + "lineage": "e328d79c-f706-3a17-b70b-734f07eaaebf", + "backend": { + "type": "local", + "config": { + "path": "./terraform.tfstate", + "workspace_dir": null + }, + "hash": 3396623677 + }, + "modules": [ + { + "path": [ + "root" + ], + "outputs": {}, + "resources": {}, + "depends_on": [] + } + ] +} diff --git a/internal/runner/testdata/initialized/terraform.tfstate b/internal/runner/testdata/initialized/terraform.tfstate new file mode 100644 index 00000000..e9b8a30a --- /dev/null +++ b/internal/runner/testdata/initialized/terraform.tfstate @@ -0,0 +1,14 @@ +{ + "version": 4, + "terraform_version": "1.6.6", + "serial": 1, + "lineage": "1339094b-9299-b908-6558-7baad98e4bcc", + "outputs": { + "some-output": { + "value": "hello-world", + "type": "string" + } + }, + "resources": [], + "check_results": null +} diff --git a/internal/runner/testdata/initialized/test.tf b/internal/runner/testdata/initialized/test.tf new file mode 100644 index 00000000..3ac142a9 --- /dev/null +++ b/internal/runner/testdata/initialized/test.tf @@ -0,0 +1,10 @@ +terraform { + backend "local" { + path = "./terraform.tfstate" + } + required_providers {} +} + +output "some-output" { + value = "hello-world" +} diff --git a/internal/runner/utils.go b/internal/runner/utils.go new file mode 100644 index 00000000..8e07781a --- /dev/null +++ b/internal/runner/utils.go @@ -0,0 +1,45 @@ +package runner + +import ( + "context" + "github.com/mach-composer/mach-composer-cli/internal/graph" + "github.com/mach-composer/mach-composer-cli/internal/utils" + "github.com/rs/zerolog/log" + "os" + "path/filepath" +) + +func terraformIsInitialized(path string) bool { + tfLockFile := filepath.Join(path, ".terraform.lock.hcl") + if _, err := os.Stat(tfLockFile); err != nil { + if os.IsNotExist(err) { + return false + } + log.Fatal().Err(err) + } + return true +} + +func terraformCanPlan(ctx context.Context, n graph.Node) (bool, error) { + parents, err := n.Parents() + if err != nil { + return false, err + } + + // Sites can always plan, so no need to check + if n.Type() == graph.SiteType { + return true, nil + } + + for _, parent := range parents { + v, err := utils.GetTerraformOutputs(ctx, parent.Path()) + if err != nil { + return false, nil + } + a := v.Type().AttributeTypes() + if len(a) == 0 { + return false, nil + } + } + return true, nil +} diff --git a/internal/runner/utils_test.go b/internal/runner/utils_test.go new file mode 100644 index 00000000..363afc8a --- /dev/null +++ b/internal/runner/utils_test.go @@ -0,0 +1,65 @@ +package runner + +import ( + "context" + "github.com/mach-composer/mach-composer-cli/internal/graph" + "github.com/stretchr/testify/assert" + "os" + "path" + "testing" +) + +func TestTerraformCanPlanNoParents(t *testing.T) { + n := new(graph.NodeMock) + n.On("Type").Return(graph.ProjectType).Once() + n.On("Parents").Return([]graph.Node{}, nil).Once() + + canPlan, err := terraformCanPlan(context.Background(), n) + assert.NoError(t, err) + assert.True(t, canPlan) +} + +func TestTerraformCanPlanWithSite(t *testing.T) { + p := new(graph.NodeMock) + dir, _ := os.Getwd() + p.On("Path").Return(path.Join(dir, "testdata/empty")).Once() + p.On("Identifier").Return("main").Once() + + n := new(graph.NodeMock) + n.On("Parents").Return([]graph.Node{p}, nil).Once() + n.On("Type").Return(graph.SiteType).Once() + + canPlan, err := terraformCanPlan(context.Background(), n) + assert.NoError(t, err) + assert.True(t, canPlan) +} + +func TestTerraformCanPlanWithComponentParentEmptyOutput(t *testing.T) { + p := new(graph.NodeMock) + dir, _ := os.Getwd() + p.On("Path").Return(path.Join(dir, "testdata/empty")).Once() + p.On("Identifier").Return("main").Once() + + n := new(graph.NodeMock) + n.On("Parents").Return([]graph.Node{p}, nil).Once() + n.On("Type").Return(graph.SiteComponentType).Once() + + canPlan, err := terraformCanPlan(context.Background(), n) + assert.NoError(t, err) + assert.False(t, canPlan) +} + +func TestTerraformCanPlanWithParentOutput(t *testing.T) { + p := new(graph.NodeMock) + dir, _ := os.Getwd() + p.On("Path").Return(path.Join(dir, "testdata/initialized")).Once() + p.On("Identifier").Return("main").Once() + + n := new(graph.NodeMock) + n.On("Parents").Return([]graph.Node{p}, nil).Once() + n.On("Type").Return(graph.SiteComponentType).Once() + + canPlan, err := terraformCanPlan(context.Background(), n) + assert.NoError(t, err) + assert.True(t, canPlan) +} diff --git a/internal/terraform/init.go b/internal/terraform/init.go index 519fb94b..1d57216d 100644 --- a/internal/terraform/init.go +++ b/internal/terraform/init.go @@ -5,11 +5,8 @@ import ( "github.com/mach-composer/mach-composer-cli/internal/utils" ) -func Init(ctx context.Context, path string) error { - if !terraformIsInitialized(path) { - if _, err := utils.RunTerraform(ctx, path, false, "init"); err != nil { - return err - } - } - return nil +func Init(ctx context.Context, path string) (string, error) { + args := []string{"init"} + + return utils.RunTerraform(ctx, path, false, args...) } diff --git a/internal/terraform/utils.go b/internal/terraform/utils.go index caf76443..53e338c1 100644 --- a/internal/terraform/utils.go +++ b/internal/terraform/utils.go @@ -1,7 +1,6 @@ package terraform import ( - "github.com/rs/zerolog/log" "os" "path/filepath" ) @@ -15,14 +14,3 @@ func hasTerraformPlan(path string) (string, error) { } return "", nil } - -func terraformIsInitialized(path string) bool { - tfLockFile := filepath.Join(path, ".terraform.lock.hcl") - if _, err := os.Stat(tfLockFile); err != nil { - if os.IsNotExist(err) { - return false - } - log.Fatal().Err(err) - } - return true -} diff --git a/internal/utils/terraform.go b/internal/utils/terraform.go index a0c52893..3ec5d7c1 100644 --- a/internal/utils/terraform.go +++ b/internal/utils/terraform.go @@ -5,24 +5,11 @@ import ( "fmt" "github.com/rs/zerolog/log" "github.com/zclconf/go-cty/cty" - "github.com/zclconf/go-cty/cty/gocty" "github.com/zclconf/go-cty/cty/json" "os" "os/exec" ) -type MissingHashError struct { - message string -} - -func NewMissingHashError(message string) *MissingHashError { - return &MissingHashError{message: message} -} - -func (m *MissingHashError) Error() string { - return m.message -} - // RunTerraform will execute a terraform command with the given arguments in the given directory. func RunTerraform(ctx context.Context, cwd string, catchOutputs bool, args ...string) (string, error) { if _, err := os.Stat(cwd); err != nil { @@ -57,30 +44,3 @@ func GetTerraformOutputs(ctx context.Context, path string) (cty.Value, error) { return data.Value, nil } - -type HashOutput struct { - Sensitive bool `cty:"sensitive"` - Type string `cty:"type"` - Value *string `cty:"value"` -} - -// ParseHashOutput returns the hash output by the given key. -func ParseHashOutput(val cty.Value) (string, error) { - if !val.Type().HasAttribute("hash") { - return "", NewMissingHashError("no attribute with key hash found in terraform output") - } - - componentVal := val.GetAttr("hash") - - var hashOutput HashOutput - err := gocty.FromCtyValue(componentVal, &hashOutput) - if err != nil { - return "", err - } - - if hashOutput.Value == nil { - return "", NewMissingHashError("no value set for hash in terraform output") - } - - return *hashOutput.Value, nil -} diff --git a/internal/utils/terraform_test.go b/internal/utils/terraform_test.go deleted file mode 100644 index 69147f7d..00000000 --- a/internal/utils/terraform_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package utils - -import ( - "github.com/stretchr/testify/assert" - "github.com/zclconf/go-cty/cty" - "testing" -) - -func TestParseHashOutputNoHashAttribute(t *testing.T) { - val := cty.ObjectVal(map[string]cty.Value{}) - - _, err := ParseHashOutput(val) - assert.ErrorContains(t, err, "no attribute with key hash found in terraform output") -} - -func TestParseHashOutputInvalidResponse(t *testing.T) { - val := cty.ObjectVal(map[string]cty.Value{ - "hash": cty.StringVal("invalid"), - }) - - _, err := ParseHashOutput(val) - assert.ErrorAs(t, err, &cty.PathError{}) -} - -func TestParseHashOutputNilResponse(t *testing.T) { - val := cty.ObjectVal(map[string]cty.Value{ - "hash": cty.ObjectVal(map[string]cty.Value{ - "sensitive": cty.BoolVal(false), - "value": cty.NilVal, - "type": cty.StringVal("some-type"), - }), - }) - - _, err := ParseHashOutput(val) - assert.ErrorContains(t, err, "no value set for hash in terraform output") -} - -func TestParseHashOutputOk(t *testing.T) { - hash := "1234567890" - val := cty.ObjectVal(map[string]cty.Value{ - "hash": cty.ObjectVal(map[string]cty.Value{ - "sensitive": cty.BoolVal(false), - "value": cty.StringVal(hash), - "type": cty.StringVal("some-type"), - }), - }) - - parsedHash, err := ParseHashOutput(val) - assert.NoError(t, err) - assert.Equal(t, hash, parsedHash) -}