diff --git a/README.md b/README.md index 0d9db5e..45fa0c6 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ All `gdt` scenarios have the following fields: contents * `defaults`: (optional) is a map, keyed by a plugin name, of default options and configuration values for that plugin. -* `require`: (optional) list of strings indicating named fixtures that will be +* `fixtures`: (optional) list of strings indicating named fixtures that will be started before any of the tests in the file are run * `tests`: list of [`Spec`][basespec] specializations that represent the runnable test units in the test scenario. @@ -264,7 +264,7 @@ definition contained in a YAML file: ```yaml name: create-get-delete-pod description: create, get and delete a Pod -require: +fixtures: - kind tests: - name: create-pod @@ -284,7 +284,7 @@ definition using an inline YAML blob: ```yaml name: create-get-delete-pod description: create, get and delete a Pod -require: +fixtures: - kind tests: # "kube.create" is a shortcut for the longer object->field format @@ -314,7 +314,7 @@ connectivity to the Pod. ```yaml name: create-check-ssh description: create a Deployment then check SSH connectivity -require: +fixtures: - kind tests: - kube.create: manifests/deployment.yaml @@ -553,7 +553,7 @@ file `testdata/apply-deployment.yaml`: ```yaml name: apply-deployment description: create, get, apply a change, get, delete a Deployment -require: +fixtures: - kind tests: - name: create-deployment @@ -610,7 +610,7 @@ file: `testdata/matches.yaml`: ```yaml name: matches description: create a deployment and check the matches condition succeeds -require: +fixtures: - kind tests: - name: create-deployment @@ -751,11 +751,11 @@ func TestExample(t *testing.T) { } ``` -In your test file, you would list the "kind" fixture in the `requires` list: +In your test file, you would list the "kind" fixture in the `fixtures` list: ```yaml name: example-using-kind -require: +fixtures: - kind tests: - kube.get: pods/nginx diff --git a/errors.go b/errors.go index a0d53b0..969cf3d 100644 --- a/errors.go +++ b/errors.go @@ -111,6 +111,12 @@ var ( "%w: condition does not match expectation", gdterrors.ErrFailure, ) + // ErrConnect is returned when we failed to create a client config to + // connect to the Kubernetes API server. + ErrConnect = fmt.Errorf( + "%w: k8s connect failure", + gdterrors.RuntimeError, + ) ) // KubeConfigNotFound returns ErrKubeConfigNotFound for a given filepath @@ -184,3 +190,9 @@ func MatchesNotEqual(msg string) error { func ConditionDoesNotMatch(msg string) error { return fmt.Errorf("%w: %s", ErrConditionDoesNotMatch, msg) } + +// ConnectError returns ErrConnnect when an error is found trying to construct +// a Kubernetes client connection. +func ConnectError(err error) error { + return fmt.Errorf("%w: %s", ErrConnect, err) +} diff --git a/run.go b/eval.go similarity index 79% rename from run.go rename to eval.go index a5752dd..738c136 100644 --- a/run.go +++ b/eval.go @@ -8,6 +8,7 @@ import ( "bufio" "bytes" "context" + "fmt" "io" "os" "strings" @@ -16,6 +17,7 @@ import ( backoff "github.com/cenkalti/backoff/v4" "github.com/gdt-dev/gdt/debug" + gdterrors "github.com/gdt-dev/gdt/errors" "github.com/gdt-dev/gdt/parse" "github.com/gdt-dev/gdt/result" gdttypes "github.com/gdt-dev/gdt/types" @@ -37,38 +39,39 @@ const ( // Run executes the test described by the Kubernetes test. A new Kubernetes // client request is made during this call. -func (s *Spec) Run(ctx context.Context, t *testing.T) error { +func (s *Spec) Eval(ctx context.Context, t *testing.T) *result.Result { c, err := s.connect(ctx) if err != nil { - return err + return result.New( + result.WithRuntimeError(ConnectError(err)), + ) } + var res *result.Result t.Run(s.Title(), func(t *testing.T) { if s.Kube.Get != "" { - err = s.runGet(ctx, t, c) + res = s.get(ctx, t, c) } if s.Kube.Create != "" { - err = s.runCreate(ctx, t, c) + res = s.create(ctx, t, c) } if s.Kube.Delete != "" { - err = s.runDelete(ctx, t, c) + res = s.delete(ctx, t, c) } if s.Kube.Apply != "" { - err = s.runApply(ctx, t, c) + res = s.apply(ctx, t, c) } }) - return result.New( - result.WithError(err), - ) + return res } -// runGet executes either a List() or a Get() call against the Kubernetes API +// get executes either a List() or a Get() call against the Kubernetes API // server and evaluates any assertions that have been set for the returned // results. -func (s *Spec) runGet( +func (s *Spec) get( ctx context.Context, t *testing.T, c *connection, -) error { +) *result.Result { kind, name := splitKindName(s.Kube.Get) gvk := schema.GroupVersionKind{ Kind: kind, @@ -76,10 +79,7 @@ func (s *Spec) runGet( res, err := c.gvrFromGVK(gvk) a := newAssertions(s.Kube.Assert, err, nil) if !a.OK() { - for _, f := range a.Failures() { - t.Error(f) - } - return nil + return result.New(result.WithFailures(a.Failures()...)) } // if the Spec has no timeout, default it to a reasonable value @@ -95,7 +95,6 @@ func (s *Spec) runGet( bo := backoff.WithContext(backoff.NewExponentialBackOff(), ctx) ticker := backoff.NewTicker(bo) attempts := 0 - success := false start := time.Now().UTC() for tick := range ticker.C { attempts++ @@ -123,12 +122,7 @@ func (s *Spec) runGet( ) } } - if !success { - for _, f := range a.Failures() { - t.Error(f) - } - } - return nil + return result.New(result.WithFailures(a.Failures()...)) } // doList performs the List() call and assertion check for a supplied resource @@ -180,20 +174,23 @@ func splitKindName(subject string) (string, string) { return kind, name } -// runCreate executes a Create() call against the Kubernetes API server and +// create executes a Create() call against the Kubernetes API server and // evaluates any assertions that have been set for the returned results. -func (s *Spec) runCreate( +func (s *Spec) create( ctx context.Context, t *testing.T, c *connection, -) error { +) *result.Result { var err error var r io.Reader if probablyFilePath(s.Kube.Create) { path := s.Kube.Create f, err := os.Open(path) if err != nil { - return err + // This should never happen because we check during parse time + // whether the file can be opened. + rterr := fmt.Errorf("%w: %s", gdterrors.RuntimeError, err) + return result.New(result.WithRuntimeError(rterr)) } defer f.Close() r = f @@ -205,7 +202,8 @@ func (s *Spec) runCreate( objs, err := unstructuredFromReader(r) if err != nil { - return err + rterr := fmt.Errorf("%w: %s", gdterrors.RuntimeError, err) + return result.New(result.WithRuntimeError(rterr)) } for _, obj := range objs { gvk := obj.GetObjectKind().GroupVersionKind() @@ -216,10 +214,7 @@ func (s *Spec) runCreate( res, err := c.gvrFromGVK(gvk) a := newAssertions(s.Kube.Assert, err, nil) if !a.OK() { - for _, f := range a.Failures() { - t.Error(f) - } - return nil + return result.New(result.WithFailures(a.Failures()...)) } obj, err := c.client.Resource(res).Namespace(ns).Create( ctx, @@ -231,29 +226,28 @@ func (s *Spec) runCreate( // to the Assertions struct, I will modify this block to look for an // indexed set of error assertions. a = newAssertions(s.Kube.Assert, err, obj) - if !a.OK() { - for _, f := range a.Failures() { - t.Error(f) - } - } + return result.New(result.WithFailures(a.Failures()...)) } return nil } -// runApply executes an Apply() call against the Kubernetes API server and +// apply executes an Apply() call against the Kubernetes API server and // evaluates any assertions that have been set for the returned results. -func (s *Spec) runApply( +func (s *Spec) apply( ctx context.Context, t *testing.T, c *connection, -) error { +) *result.Result { var err error var r io.Reader if probablyFilePath(s.Kube.Apply) { path := s.Kube.Apply f, err := os.Open(path) if err != nil { - return err + // This should never happen because we check during parse time + // whether the file can be opened. + rterr := fmt.Errorf("%w: %s", gdterrors.RuntimeError, err) + return result.New(result.WithRuntimeError(rterr)) } defer f.Close() r = f @@ -265,7 +259,8 @@ func (s *Spec) runApply( objs, err := unstructuredFromReader(r) if err != nil { - return err + rterr := fmt.Errorf("%w: %s", gdterrors.RuntimeError, err) + return result.New(result.WithRuntimeError(rterr)) } for _, obj := range objs { gvk := obj.GetObjectKind().GroupVersionKind() @@ -276,10 +271,7 @@ func (s *Spec) runApply( res, err := c.gvrFromGVK(gvk) a := newAssertions(s.Kube.Assert, err, nil) if !a.OK() { - for _, f := range a.Failures() { - t.Error(f) - } - return nil + return result.New(result.WithFailures(a.Failures()...)) } obj, err := c.client.Resource(res).Namespace(ns).Apply( ctx, @@ -298,11 +290,7 @@ func (s *Spec) runApply( // to the Assertions struct, I will modify this block to look for an // indexed set of error assertions. a = newAssertions(s.Kube.Assert, err, obj) - if !a.OK() { - for _, f := range a.Failures() { - t.Error(f) - } - } + return result.New(result.WithFailures(a.Failures()...)) } return nil } @@ -340,33 +328,34 @@ func unstructuredFromReader( return objs, nil } -// runDelete executes either Delete() call against the Kubernetes API server +// delete executes either Delete() call against the Kubernetes API server // and evaluates any assertions that have been set for the returned results. -func (s *Spec) runDelete( +func (s *Spec) delete( ctx context.Context, t *testing.T, c *connection, -) error { +) *result.Result { if probablyFilePath(s.Kube.Delete) { path := s.Kube.Delete f, err := os.Open(path) if err != nil { - return err + // This should never happen because we check during parse time + // whether the file can be opened. + rterr := fmt.Errorf("%w: %s", gdterrors.RuntimeError, err) + return result.New(result.WithRuntimeError(rterr)) } defer f.Close() objs, err := unstructuredFromReader(f) if err != nil { - return err + rterr := fmt.Errorf("%w: %s", gdterrors.RuntimeError, err) + return result.New(result.WithRuntimeError(rterr)) } for _, obj := range objs { gvk := obj.GetObjectKind().GroupVersionKind() res, err := c.gvrFromGVK(gvk) a := newAssertions(s.Kube.Assert, err, nil) if !a.OK() { - for _, f := range a.Failures() { - t.Error(f) - } - return nil + return result.New(result.WithFailures(a.Failures()...)) } name := obj.GetName() ns := obj.GetNamespace() @@ -377,11 +366,12 @@ func (s *Spec) runDelete( // object that was deleted, which is wrong. When I add the polymorphism // to the Assertions struct, I will modify this block to look for an // indexed set of error assertions. - if err = s.doDelete(ctx, t, c, res, name, ns); err != nil { - return err + r := s.doDelete(ctx, t, c, res, name, ns) + if len(r.Failures()) > 0 { + return r } } - return nil + return result.New() } kind, name := splitKindName(s.Kube.Delete) @@ -391,10 +381,7 @@ func (s *Spec) runDelete( res, err := c.gvrFromGVK(gvk) a := newAssertions(s.Kube.Assert, err, nil) if !a.OK() { - for _, f := range a.Failures() { - t.Error(f) - } - return nil + return result.New(result.WithFailures(a.Failures()...)) } if name == "" { return s.doDeleteCollection(ctx, t, c, res, s.Namespace()) @@ -411,19 +398,14 @@ func (s *Spec) doDelete( res schema.GroupVersionResource, name string, namespace string, -) error { +) *result.Result { err := c.client.Resource(res).Namespace(namespace).Delete( ctx, name, metav1.DeleteOptions{}, ) a := newAssertions(s.Kube.Assert, err, nil) - if !a.OK() { - for _, f := range a.Failures() { - t.Error(f) - } - } - return nil + return result.New(result.WithFailures(a.Failures()...)) } // doDeleteCollection performs the DeleteCollection() call and assertion check @@ -434,17 +416,12 @@ func (s *Spec) doDeleteCollection( c *connection, res schema.GroupVersionResource, namespace string, -) error { +) *result.Result { err := c.client.Resource(res).Namespace(namespace).DeleteCollection( ctx, metav1.DeleteOptions{}, metav1.ListOptions{}, ) a := newAssertions(s.Kube.Assert, err, nil) - if !a.OK() { - for _, f := range a.Failures() { - t.Error(f) - } - } - return nil + return result.New(result.WithFailures(a.Failures()...)) } diff --git a/run_test.go b/eval_test.go similarity index 100% rename from run_test.go rename to eval_test.go diff --git a/go.mod b/go.mod index 191a0e6..ac43620 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( github.com/cenkalti/backoff/v4 v4.2.1 - github.com/gdt-dev/gdt v1.0.1 + github.com/gdt-dev/gdt v1.1.0 github.com/samber/lo v1.38.1 github.com/stretchr/testify v1.8.4 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 4bb6688..7d72a9f 100644 --- a/go.sum +++ b/go.sum @@ -65,8 +65,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/gdt-dev/gdt v1.0.1 h1:U7s/NrTobFmV0kL8rO7ZRX7XER9oxFqaWf6J6TvKeTU= -github.com/gdt-dev/gdt v1.0.1/go.mod h1:StnyGjC/67u59La2u6fh3HwW9MmodVhKdXcLlkgvNSY= +github.com/gdt-dev/gdt v1.1.0 h1:+eFYFSibOYCTFKqoACx2CefAbErmBmFWXG7kDioR5do= +github.com/gdt-dev/gdt v1.1.0/go.mod h1:StnyGjC/67u59La2u6fh3HwW9MmodVhKdXcLlkgvNSY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= diff --git a/parse_test.go b/parse_test.go index 4b43ceb..885af07 100644 --- a/parse_test.go +++ b/parse_test.go @@ -11,7 +11,6 @@ import ( "github.com/gdt-dev/gdt" "github.com/gdt-dev/gdt/errors" - "github.com/gdt-dev/gdt/scenario" gdttypes "github.com/gdt-dev/gdt/types" gdtkube "github.com/gdt-dev/kube" "github.com/stretchr/testify/assert" @@ -225,12 +224,12 @@ func TestParse(t *testing.T) { fp := filepath.Join("testdata", "parse.yaml") - s, err := gdt.From(fp) + suite, err := gdt.From(fp) require.Nil(err) - require.NotNil(s) + require.NotNil(suite) - assert.IsType(&scenario.Scenario{}, s) - sc := s.(*scenario.Scenario) + require.Len(suite.Scenarios, 1) + s := suite.Scenarios[0] podYAML := `apiVersion: v1 kind: Pod @@ -242,7 +241,7 @@ spec: image: nginx:1.7.9 ` - expTests := []gdttypes.TestUnit{ + expTests := []gdttypes.Evaluable{ &gdtkube.Spec{ Spec: gdttypes.Spec{ Index: 0, @@ -317,5 +316,5 @@ spec: }, }, } - assert.Equal(expTests, sc.Tests) + assert.Equal(expTests, s.Tests) } diff --git a/plugin.go b/plugin.go index accf185..26999df 100644 --- a/plugin.go +++ b/plugin.go @@ -30,8 +30,8 @@ func (p *plugin) Defaults() yaml.Unmarshaler { return &Defaults{} } -func (p *plugin) Specs() []gdttypes.TestUnit { - return []gdttypes.TestUnit{&Spec{}} +func (p *plugin) Specs() []gdttypes.Evaluable { + return []gdttypes.Evaluable{&Spec{}} } // Plugin returns the Kubernetes gdt plugin diff --git a/testdata/apply-deployment.yaml b/testdata/apply-deployment.yaml index e253ac7..00eada2 100644 --- a/testdata/apply-deployment.yaml +++ b/testdata/apply-deployment.yaml @@ -1,6 +1,6 @@ name: apply-deployment description: create, get, apply a change, get, delete a Deployment -require: +fixtures: - kind tests: - name: create-deployment diff --git a/testdata/conditions.yaml b/testdata/conditions.yaml index 1c219e5..5805611 100644 --- a/testdata/conditions.yaml +++ b/testdata/conditions.yaml @@ -1,6 +1,6 @@ name: conditions description: create a deployment and check the Ready condition eventually equals True -require: +fixtures: - kind tests: - name: create-deployment diff --git a/testdata/create-get-delete-pod.yaml b/testdata/create-get-delete-pod.yaml index 7856495..eed4fca 100644 --- a/testdata/create-get-delete-pod.yaml +++ b/testdata/create-get-delete-pod.yaml @@ -1,6 +1,6 @@ name: create-get-delete-pod description: create, get and delete a Pod -require: +fixtures: - kind tests: - name: create-pod diff --git a/testdata/create-unknown-resource.yaml b/testdata/create-unknown-resource.yaml index 6c26586..5cb67f1 100644 --- a/testdata/create-unknown-resource.yaml +++ b/testdata/create-unknown-resource.yaml @@ -1,6 +1,6 @@ name: create-unknown-resource description: create with YAML for unknown resource -require: +fixtures: - kind tests: - kube: diff --git a/testdata/delete-resource-not-found.yaml b/testdata/delete-resource-not-found.yaml index d20ef76..e6edf91 100644 --- a/testdata/delete-resource-not-found.yaml +++ b/testdata/delete-resource-not-found.yaml @@ -1,6 +1,6 @@ name: delete-resource-not-found description: delete resource not found -require: +fixtures: - kind tests: - kube: diff --git a/testdata/delete-unknown-resource.yaml b/testdata/delete-unknown-resource.yaml index 65e855d..977db67 100644 --- a/testdata/delete-unknown-resource.yaml +++ b/testdata/delete-unknown-resource.yaml @@ -1,6 +1,6 @@ name: delete-unknown-resource description: delete with YAML for unknown resource -require: +fixtures: - kind tests: - kube: diff --git a/testdata/envvar-substitution.yaml b/testdata/envvar-substitution.yaml index b505857..1f68b74 100644 --- a/testdata/envvar-substitution.yaml +++ b/testdata/envvar-substitution.yaml @@ -1,6 +1,6 @@ name: envvar-substitution description: create, get and delete a Pod with envvar substitutions -require: +fixtures: - kind tests: - name: create-${pod_name} diff --git a/testdata/get-pod-not-found.yaml b/testdata/get-pod-not-found.yaml index 3bc649e..4a36a55 100644 --- a/testdata/get-pod-not-found.yaml +++ b/testdata/get-pod-not-found.yaml @@ -1,6 +1,6 @@ name: get-pod-not-found description: test getting a single non-existent Pod resource -require: +fixtures: - kind tests: - name: assert-len-zero diff --git a/testdata/json.yaml b/testdata/json.yaml index c5bd6a2..3f9fdd0 100644 --- a/testdata/json.yaml +++ b/testdata/json.yaml @@ -1,6 +1,6 @@ name: json description: create a deployment and check the json condition succeeds -require: +fixtures: - kind tests: - name: create-deployment diff --git a/testdata/list-pods-empty.yaml b/testdata/list-pods-empty.yaml index d001724..bc5dce4 100644 --- a/testdata/list-pods-empty.yaml +++ b/testdata/list-pods-empty.yaml @@ -1,6 +1,6 @@ name: list-pods-empty description: test empty list of Pod resources -require: +fixtures: - kind tests: - name: verify-no-pods diff --git a/testdata/list-pods-with-labels.yaml b/testdata/list-pods-with-labels.yaml index ac60b5e..517bba5 100644 --- a/testdata/list-pods-with-labels.yaml +++ b/testdata/list-pods-with-labels.yaml @@ -1,6 +1,6 @@ name: list-pods-with-labels description: test list of Pod resources using label selector -require: +fixtures: - kind tests: - name: create-deployment diff --git a/testdata/matches.yaml b/testdata/matches.yaml index 4424e43..ddbd3b0 100644 --- a/testdata/matches.yaml +++ b/testdata/matches.yaml @@ -1,6 +1,6 @@ name: matches description: create a deployment and check the matches condition succeeds -require: +fixtures: - kind tests: - name: create-deployment diff --git a/testdata/parse/fail/create-file-not-found.yaml b/testdata/parse/fail/create-file-not-found.yaml index 9776e49..88167c6 100644 --- a/testdata/parse/fail/create-file-not-found.yaml +++ b/testdata/parse/fail/create-file-not-found.yaml @@ -1,6 +1,6 @@ name: create-file-not-found description: create filepath not found -require: +fixtures: - kind tests: - kube: diff --git a/testdata/parse/fail/delete-file-not-found.yaml b/testdata/parse/fail/delete-file-not-found.yaml index cc91609..87ccf46 100644 --- a/testdata/parse/fail/delete-file-not-found.yaml +++ b/testdata/parse/fail/delete-file-not-found.yaml @@ -1,6 +1,6 @@ name: delete-file-not-found description: delete filepath not found -require: +fixtures: - kind tests: - kube: diff --git a/testdata/parse/fail/invalid-delete-not-filepath-or-resource-specifier.yaml b/testdata/parse/fail/invalid-delete-not-filepath-or-resource-specifier.yaml index 4f3fe0d..aca49bd 100644 --- a/testdata/parse/fail/invalid-delete-not-filepath-or-resource-specifier.yaml +++ b/testdata/parse/fail/invalid-delete-not-filepath-or-resource-specifier.yaml @@ -1,6 +1,6 @@ name: delete-not-filepath-or-resource-specifier description: invalid spec contains YAML string for the delete resource specifier -require: +fixtures: - kind tests: - name: invalid-delete-not-filepath-or-resource-specifier diff --git a/testdata/parse/fail/invalid-resource-specifier-multiple-forward-slashes.yaml b/testdata/parse/fail/invalid-resource-specifier-multiple-forward-slashes.yaml index 1532520..db6db15 100644 --- a/testdata/parse/fail/invalid-resource-specifier-multiple-forward-slashes.yaml +++ b/testdata/parse/fail/invalid-resource-specifier-multiple-forward-slashes.yaml @@ -1,6 +1,6 @@ name: invalid-resource-specifier-multiple-resources description: invalid spec that shows an invalid resource specifier for `get` using spaces to indicate multiple resources -require: +fixtures: - kind tests: - kube: diff --git a/testdata/parse/fail/invalid-resource-specifier-multiple-resources.yaml b/testdata/parse/fail/invalid-resource-specifier-multiple-resources.yaml index 36b659e..c6a0d21 100644 --- a/testdata/parse/fail/invalid-resource-specifier-multiple-resources.yaml +++ b/testdata/parse/fail/invalid-resource-specifier-multiple-resources.yaml @@ -1,6 +1,6 @@ name: invalid-resource-specifier-multiple-resources description: invalid spec that shows an invalid resource specifier for `get` using spaces to indicate multiple resources -require: +fixtures: - kind tests: - kube: