-
Notifications
You must be signed in to change notification settings - Fork 9
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: removed reliance on terraform state for config hashes (#384)
Removed storing the component config hash in terraform state, and instead store it in a `.mach/hashes.json` file. Because reading state requires a terraform init for every component before actually checking for changes the actual apply process is very slow. By storing the hashes into a file on disk we can speed up the process enormously as we can now skip many of the inits up front.
- Loading branch information
Showing
64 changed files
with
1,078 additions
and
1,217 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| kind: Added | ||
| body: Moved hash storage out of terraform | ||
| time: 2024-03-18T11:46:35.79447595+01:00 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| package batcher | ||
|
|
||
| import "github.com/mach-composer/mach-composer-cli/internal/graph" | ||
|
|
||
| type BatchFunc func(g *graph.Graph) map[int][]graph.Node |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| package batcher | ||
|
|
||
| import "github.com/mach-composer/mach-composer-cli/internal/graph" | ||
|
|
||
| func NaiveBatchFunc() BatchFunc { | ||
| return func(g *graph.Graph) map[int][]graph.Node { | ||
| batches := map[int][]graph.Node{} | ||
|
|
||
| var sets = map[string][]graph.Path{} | ||
|
|
||
| for _, n := range g.Vertices() { | ||
| var route, _ = g.Routes(n.Path(), g.StartNode.Path()) | ||
| sets[n.Path()] = route | ||
| } | ||
|
|
||
| for k, routes := range sets { | ||
| var mx int | ||
| for _, route := range routes { | ||
| if len(route) > mx { | ||
| mx = len(route) | ||
| } | ||
| } | ||
| n, _ := g.Vertex(k) | ||
| batches[mx] = append(batches[mx], n) | ||
| } | ||
|
|
||
| return batches | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| package batcher | ||
|
|
||
| import ( | ||
| "github.com/dominikbraun/graph" | ||
| internalgraph "github.com/mach-composer/mach-composer-cli/internal/graph" | ||
| "github.com/stretchr/testify/assert" | ||
| "testing" | ||
| ) | ||
|
|
||
| func TestBatchNodesDepth1(t *testing.T) { | ||
| ig := graph.New(func(n internalgraph.Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) | ||
|
|
||
| start := new(internalgraph.NodeMock) | ||
| start.On("Path").Return("main/site-1") | ||
|
|
||
| _ = ig.AddVertex(start) | ||
|
|
||
| g := &internalgraph.Graph{Graph: ig, StartNode: start} | ||
|
|
||
| batches := NaiveBatchFunc()(g) | ||
|
|
||
| assert.Equal(t, 1, len(batches)) | ||
| } | ||
|
|
||
| func TestBatchNodesDepth2(t *testing.T) { | ||
| ig := graph.New(func(n internalgraph.Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) | ||
|
|
||
| site := new(internalgraph.NodeMock) | ||
| site.On("Path").Return("main/site-1") | ||
|
|
||
| component1 := new(internalgraph.NodeMock) | ||
| component1.On("Path").Return("main/site-1/component-1") | ||
|
|
||
| component2 := new(internalgraph.NodeMock) | ||
| component2.On("Path").Return("main/site-1/component-2") | ||
|
|
||
| _ = ig.AddVertex(site) | ||
| _ = ig.AddVertex(component1) | ||
| _ = ig.AddVertex(component2) | ||
|
|
||
| _ = ig.AddEdge("main/site-1", "main/site-1/component-1") | ||
| _ = ig.AddEdge("main/site-1", "main/site-1/component-2") | ||
|
|
||
| g := &internalgraph.Graph{Graph: ig, StartNode: site} | ||
|
|
||
| batches := NaiveBatchFunc()(g) | ||
|
|
||
| assert.Equal(t, 2, len(batches)) | ||
| assert.Equal(t, 1, len(batches[0])) | ||
| assert.Equal(t, "main/site-1", batches[0][0].Path()) | ||
| assert.Equal(t, 2, len(batches[1])) | ||
| assert.Contains(t, batches[1][0].Path(), "component") | ||
| assert.Contains(t, batches[1][1].Path(), "component") | ||
| } | ||
|
|
||
| func TestBatchNodesDepth3(t *testing.T) { | ||
| ig := graph.New(func(n internalgraph.Node) string { return n.Path() }, graph.Directed(), graph.Tree(), graph.PreventCycles()) | ||
|
|
||
| site := new(internalgraph.NodeMock) | ||
| site.On("Path").Return("main/site-1") | ||
|
|
||
| component1 := new(internalgraph.NodeMock) | ||
| component1.On("Path").Return("main/site-1/component-1") | ||
|
|
||
| component2 := new(internalgraph.NodeMock) | ||
| component2.On("Path").Return("main/site-1/component-2") | ||
|
|
||
| _ = ig.AddVertex(site) | ||
| _ = ig.AddVertex(component1) | ||
| _ = ig.AddVertex(component2) | ||
|
|
||
| _ = ig.AddEdge("main/site-1", "main/site-1/component-1") | ||
| _ = ig.AddEdge("main/site-1/component-1", "main/site-1/component-2") | ||
|
|
||
| g := &internalgraph.Graph{Graph: ig, StartNode: site} | ||
|
|
||
| batches := NaiveBatchFunc()(g) | ||
|
|
||
| assert.Equal(t, 3, len(batches)) | ||
| assert.Equal(t, 1, len(batches[0])) | ||
| assert.Equal(t, "main/site-1", batches[0][0].Path()) | ||
| assert.Equal(t, 1, len(batches[1])) | ||
| assert.Contains(t, batches[1][0].Path(), "main/site-1/component-1") | ||
| assert.Equal(t, 1, len(batches[2])) | ||
| assert.Contains(t, batches[2][0].Path(), "main/site-1/component-2") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,130 @@ | ||
| //go:build integration | ||
|
|
||
| package cmd | ||
|
|
||
| import ( | ||
| "github.com/stretchr/testify/assert" | ||
| "os" | ||
| "os/exec" | ||
| "path" | ||
| "testing" | ||
| ) | ||
|
|
||
| import ( | ||
| "github.com/stretchr/testify/suite" | ||
| ) | ||
|
|
||
| type ApplyTestSuite struct { | ||
| suite.Suite | ||
| tempDir string | ||
| } | ||
|
|
||
| func TestExampleTestSuite(t *testing.T) { | ||
| suite.Run(t, new(ApplyTestSuite)) | ||
| } | ||
|
|
||
| func (s *ApplyTestSuite) SetupSuite() { | ||
| _, err := exec.LookPath("terraform") | ||
| if err != nil { | ||
| s.T().Fatal("terraform command not found") | ||
| } | ||
|
|
||
| tmpDir, _ := os.MkdirTemp("mach-composer", "test") | ||
| _ = os.Setenv("TF_PLUGIN_CACHE_DIR", tmpDir) | ||
| _ = os.Setenv("TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE", "1") | ||
|
|
||
| s.tempDir = tmpDir | ||
| } | ||
|
|
||
| func (s *ApplyTestSuite) TearDownSuite() { | ||
| _ = os.RemoveAll(s.tempDir) | ||
| } | ||
|
|
||
| func cleanWorkingDir(workdir string) { | ||
| err := os.RemoveAll(path.Join(workdir, "deployments")) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| err = os.RemoveAll(path.Join(workdir, "states")) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| err = os.RemoveAll(path.Join(workdir, "hashes.json")) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| } | ||
|
|
||
| func (s *ApplyTestSuite) TestApplySimple() { | ||
| pwd, _ := os.Getwd() | ||
| workdir := path.Join(pwd, "testdata/cases/apply/simple") | ||
| defer cleanWorkingDir(workdir) | ||
|
|
||
| cmd := RootCmd | ||
| _ = os.Setenv("MC_HASH_FILE", path.Join(workdir, "hashes.json")) | ||
| cmd.SetArgs([]string{ | ||
| "apply", | ||
| "--output-path", path.Join(workdir, "deployments"), | ||
| "--file", path.Join(workdir, "main.yaml"), | ||
| "--auto-approve", | ||
| }) | ||
| err := cmd.Execute() | ||
| assert.NoError(s.T(), err) | ||
|
|
||
| assert.FileExists(s.T(), path.Join(workdir, "hashes.json")) | ||
| assert.FileExists(s.T(), path.Join(workdir, "deployments/main/test-1/main.tf")) | ||
| assert.FileExists(s.T(), path.Join(workdir, "deployments/main/test-1/states/test-1.tfstate")) | ||
| assert.FileExists(s.T(), path.Join(workdir, "deployments/main/test-1/outputs/component-1.json")) | ||
| } | ||
|
|
||
| func (s *ApplyTestSuite) TestApplySplitState() { | ||
| pwd, _ := os.Getwd() | ||
| workdir := path.Join(pwd, "testdata/cases/apply/split-state") | ||
| defer cleanWorkingDir(workdir) | ||
|
|
||
| cmd := RootCmd | ||
| _ = os.Setenv("MC_HASH_FILE", path.Join(workdir, "hashes.json")) | ||
| _ = os.Setenv("STATES_PATH", path.Join(workdir, "states")) | ||
| cmd.SetArgs([]string{ | ||
| "apply", | ||
| "--output-path", path.Join(workdir, "deployments"), | ||
| "--file", path.Join(workdir, "main.yaml"), | ||
| "--auto-approve", | ||
| }) | ||
| err := cmd.Execute() | ||
| assert.NoError(s.T(), err) | ||
|
|
||
| assert.FileExists(s.T(), path.Join(workdir, "hashes.json")) | ||
| assert.FileExists(s.T(), path.Join(workdir, "deployments/main/test-1/main.tf")) | ||
| assert.FileExists(s.T(), path.Join(workdir, "deployments/main/test-1/component-2/main.tf")) | ||
| assert.FileExists(s.T(), path.Join(workdir, "states/test-1.tfstate")) | ||
| assert.FileExists(s.T(), path.Join(workdir, "states/component-2.tfstate")) | ||
| } | ||
|
|
||
| func (s *ApplyTestSuite) TestApplyNoHashesFile() { | ||
| pwd, _ := os.Getwd() | ||
| workdir := path.Join(pwd, "testdata/cases/apply/simple") | ||
| defer cleanWorkingDir(workdir) | ||
|
|
||
| cmd := RootCmd | ||
| _ = os.Setenv("MC_HASH_FILE", path.Join(workdir, "hashes.json")) | ||
| cmd.SetArgs([]string{ | ||
| "apply", | ||
| "--output-path", path.Join(workdir, "deployments"), | ||
| "--file", path.Join(workdir, "main.yaml"), | ||
| "--auto-approve", | ||
| }) | ||
| err := cmd.Execute() | ||
| assert.NoError(s.T(), err) | ||
|
|
||
| assert.FileExists(s.T(), path.Join(workdir, "hashes.json")) | ||
|
|
||
| err = os.RemoveAll(path.Join(workdir, "hashes.json")) | ||
| if err != nil { | ||
| s.T().Fatal(err) | ||
| } | ||
|
|
||
| err = cmd.Execute() | ||
| assert.NoError(s.T(), err) | ||
| assert.FileExists(s.T(), path.Join(workdir, "hashes.json")) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.