Use testscript for e2e

testscript is a go package to test commands with isulated evrinonment,
using text based files.

This PR replaces the old preimitive CLI test, with testscript text
files, to get much more accurate cli test, with meaningful assertions
and reacher use-cases.

.github/workflows/sanity.yml

@@ -25,15 +25,16 @@ jobs:
         skip-pkg-cache: true
         args: --timeout=5m
 
-    - name: Build
-      run: make
-
-    - name: Test
+    - name: Unit Test and Coverage
       run: go test -covermode=count -coverprofile cover.out ./...
 
     - uses: shogo82148/actions-goveralls@v1
       with:
         path-to-profile: cover.out
 
-    - name: Functional Test
-      run: make test
+    - name: CLI Test
+      run: make test-cli
+
+    # make sure we can build the cli
+    - name: Build
+      run: make

.gitignore

@@ -1,2 +1,3 @@
 ginkgolinter
 bin/
+e2e

Makefile

@@ -5,7 +5,7 @@ HASH_FLAG := -X github.com/nunnatsa/ginkgolinter/version.gitHash=$(COMMIT_HASH)
 
 BUILD_ARGS := -ldflags "$(VERSION_FLAG) $(HASH_FLAG)"
 
-build: unit-test
+build:
 	go build $(BUILD_ARGS) -o ginkgolinter ./cmd/ginkgolinter
 
 unit-test:
@@ -23,5 +23,7 @@ build-for-linux:
 
 build-all: build build-for-linux build-for-mac build-for-windows
 
-test: build
-	./tests/e2e.sh
+test-cli:
+	cd tests; go test -v ./
+
+test: unit-test test-cli

cmd/ginkgolinter/cli.go

@@ -1,21 +1,7 @@
 package main
 
-import (
-	"fmt"
-	"github.com/nunnatsa/ginkgolinter"
-	"github.com/nunnatsa/ginkgolinter/version"
-	"golang.org/x/tools/go/analysis/singlechecker"
-	"os"
-	"runtime"
-)
+import "github.com/nunnatsa/ginkgolinter/cmd/ginkgolinter/cli"
 
 func main() {
-	if len(os.Args) == 2 && os.Args[1] == "version" {
-		fmt.Printf("ginkgolinter version: %s\n", version.Version())
-		fmt.Printf("git hash:             %s\n", version.GitHash())
-		fmt.Printf("go version:           %s\n", runtime.Version())
-		os.Exit(0)
-	}
-
-	singlechecker.Main(ginkgolinter.NewAnalyzer())
+	cli.Main()
 }

cmd/ginkgolinter/cli/cli.go

@@ -0,0 +1,23 @@
+package cli
+
+import (
+	"fmt"
+	"github.com/nunnatsa/ginkgolinter"
+	"github.com/nunnatsa/ginkgolinter/version"
+	"golang.org/x/tools/go/analysis/singlechecker"
+	"os"
+	"runtime"
+)
+
+func Main() int {
+	if len(os.Args) == 2 && os.Args[1] == "version" {
+		fmt.Printf("ginkgolinter version: %s\n", version.Version())
+		fmt.Printf("git hash:             %s\n", version.GitHash())
+		fmt.Printf("go version:           %s\n", runtime.Version())
+		os.Exit(0)
+	}
+
+	singlechecker.Main(ginkgolinter.NewAnalyzer())
+
+	return 0
+}

testdata/src/a/go.mod

@@ -1,6 +1,6 @@
 module a
 
-go 1.21
+go 1.22
 
 require (
 	github.com/onsi/ginkgo/v2 v2.13.2

tests/cli_test.go

@@ -0,0 +1,42 @@
+package tests_test
+
+import (
+	"os"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/rogpeppe/go-internal/testscript"
+
+	"github.com/nunnatsa/ginkgolinter/cmd/ginkgolinter/cli"
+)
+
+func TestMain(m *testing.M) {
+	pwd, err := os.Getwd()
+	if err != nil {
+		panic(err)
+	}
+
+	if err = os.Setenv("HOME", path.Join(pwd, "cli-test")); err != nil {
+		panic(err)
+	}
+
+	os.Exit(testscript.RunMain(m, map[string]func() int{
+		"ginkgolinter": cli.Main,
+	}))
+}
+
+func TestCli(t *testing.T) {
+	pwd, err := os.Getwd()
+	if err != nil {
+		panic(err)
+	}
+
+	if !strings.HasSuffix(pwd, "tests") {
+		pwd = path.Join(pwd, "tests")
+	}
+
+	testscript.Run(t, testscript.Params{
+		Dir: path.Join(pwd, "testdata"),
+	})
+}

tests/go.mod

@@ -0,0 +1,19 @@
+module github.com/nunnatsa/ginkgolinter/tests
+
+go 1.22.7
+
+replace github.com/nunnatsa/ginkgolinter => ..
+
+require (
+	github.com/nunnatsa/ginkgolinter v0.0.0-00010101000000-000000000000
+	github.com/rogpeppe/go-internal v1.13.1
+)
+
+require (
+	github.com/go-toolsmith/astcopy v1.1.0 // indirect
+	golang.org/x/exp/typeparams v0.0.0-20241009180824-f66d83c29e7c // indirect
+	golang.org/x/mod v0.21.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/tools v0.26.0 // indirect
+)

tests/go.sum

@@ -0,0 +1,24 @@
+github.com/go-toolsmith/astcopy v1.1.0 h1:YGwBN0WM+ekI/6SS6+52zLDEf8Yvp3n2seZITCUBt5s=
+github.com/go-toolsmith/astcopy v1.1.0/go.mod h1:hXM6gan18VA1T/daUEHCFcYiW8Ai1tIwIzHY6srfEAw=
+github.com/go-toolsmith/astequal v1.0.3/go.mod h1:9Ai4UglvtR+4up+bAD4+hCj7iTo4m/OXVTSLnCyTAx4=
+github.com/go-toolsmith/astequal v1.1.0 h1:kHKm1AWqClYn15R0K1KKE4RG614D46n+nqUQ06E1dTw=
+github.com/go-toolsmith/astequal v1.1.0/go.mod h1:sedf7VIdCL22LD8qIvv7Nn9MuWJruQA/ysswh64lffQ=
+github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
+github.com/go-toolsmith/strparse v1.1.0 h1:GAioeZUK9TGxnLS+qfdqNbA4z0SSm5zVNtCQiyP2Bvw=
+github.com/go-toolsmith/strparse v1.1.0/go.mod h1:7ksGy58fsaQkGQlY8WVoBFNyEPMGuJin1rfoPS4lBSQ=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
+golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
+golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
+golang.org/x/exp/typeparams v0.0.0-20241009180824-f66d83c29e7c h1:F/15/6p7LyGUSoP0GE5CB/U9+TNEER1foNOP5sWLLnI=
+golang.org/x/exp/typeparams v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
+golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0=
+golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ=
+golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0=

tests/testdata/asyncerr.txtar

@@ -0,0 +1,116 @@
+# run ginkgolinter to find wrong error assertions
+! exec ginkgolinter errassertion
+! stdout .
+stderr -count=1 'Success matcher only support a single error value, or function with Gomega as its first parameter'
+stderr -count=1 'wrong error assertion.'
+stderr -count=1 'Success matcher does not support multiple values'
+
+# also enable the force succeed rule
+! exec ginkgolinter --force-succeed=true errassertion
+! stdout .
+stderr -count=1 'Success matcher only support a single error value, or function with Gomega as its first parameter'
+stderr -count=1 'wrong error assertion.'
+stderr -count=1 'Success matcher does not support multiple values'
+stderr -count=1 'prefer using the HaveOccurred matcher for non-function error value, instead of Succeed'
+stderr -count=1 'prefer using the Succeed matcher for error function, instead of HaveOccurred.'
+
+# run with -fix, expect wrong error assertion errors
+! exec ginkgolinter --fix --force-succeed=true errassertion
+! stdout .
+stderr -count=1 'Success matcher only support a single error value, or function with Gomega as its first parameter'
+stderr -count=1 'wrong error assertion.'
+stderr -count=1 'Success matcher does not support multiple values'
+stderr -count=1 'prefer using the HaveOccurred matcher for non-function error value, instead of Succeed'
+stderr -count=1 'prefer using the Succeed matcher for error function, instead of HaveOccurred.'
+
+# run again after fix, expect only non-fixable errors
+! exec ginkgolinter errassertion
+! stdout .
+stderr -count=1 'Success matcher only support a single error value, or function with Gomega as its first parameter'
+! stderr 'wrong error assertion.'
+stderr -count=1 'Success matcher does not support multiple values'
+! stderr 'prefer using the HaveOccurred matcher for non-function error value, instead of Succeed'
+! stderr 'prefer using the Succeed matcher for error function, instead of HaveOccurred.'
+
+-- err.go --
+package errassertion
+
+import (
+	"errors"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("check error assertion", func() {
+	It("should not comparing ee to nil", func() {
+		err := errors.New("error")
+		Eventually(func() string{return "not error"}).Should(Succeed())
+		Eventually(func() error{return err}).Should(BeNil())
+		Eventually(func() (int, error) {return 42, nil}).Should(Succeed())
+		Eventually(func() error {return err}).Should(HaveOccurred())
+		Expect(err).ToNot(Succeed())
+		Expect(func() error {return err}()).To(HaveOccurred())
+	})
+})
+
+-- go.mod --
+module errassertion
+
+go 1.22
+
+require (
+	github.com/onsi/ginkgo/v2 v2.13.2
+	github.com/onsi/gomega v1.30.0
+)
+
+require (
+	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/pprof v0.0.0-20231212022811-ec68065c825e // indirect
+	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.16.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
+-- go.sum --
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
+github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/pprof v0.0.0-20231212022811-ec68065c825e h1:bwOy7hAFd0C91URzMIEBfr6BAz29yk7Qj0cy6S7DJlU=
+github.com/google/pprof v0.0.0-20231212022811-ec68065c825e/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
+github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs=
+github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM=
+github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
+github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA=
+golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=