Merge pull request #5 from Escape-Technologies/feat/major-refactor

feat: major refactor

.gitignore

@@ -8,7 +8,7 @@ testers
 !internal/utils/assets/wordlist.txt
 
 # ignore builds
-goctopus
+/goctopus
 
 dist/
 .env

cmd/goctopus/goctopus.go

@@ -3,25 +3,28 @@ package main
 import (
 	"os"
 
-	"github.com/Escape-Technologies/goctopus/internal/config"
 	"github.com/Escape-Technologies/goctopus/internal/utils"
-	"github.com/Escape-Technologies/goctopus/pkg/run"
+	"github.com/Escape-Technologies/goctopus/pkg/config"
+	"github.com/Escape-Technologies/goctopus/pkg/goctopus"
 
 	log "github.com/sirupsen/logrus"
 )
 
 func main() {
-	config.ParseFlags()
-	if !config.Conf.Silent {
+	config.LoadFromArgs()
+	if !config.Get().Silent {
 		utils.PrintASCII()
 	}
 
-	input, err := os.Open(config.Conf.InputFile)
-	if err != nil {
-		log.Error(err)
-		os.Exit(1)
+	if config.Get().InputFile != "" {
+		input, err := os.Open(config.Get().InputFile)
+		if err != nil {
+			log.Error(err)
+			os.Exit(1)
+		}
+		defer input.Close()
+		goctopus.FingerprintFromFile(input)
+	} else {
+		goctopus.FingerprintFromSlice(config.Get().Addresses)
 	}
-	defer input.Close()
-
-	run.RunFromFile(input)
 }

internal/test/helpers/http.go

@@ -1,6 +1,6 @@
 package helpers
 
-import "github.com/Escape-Technologies/goctopus/internal/http"
+import "github.com/Escape-Technologies/goctopus/pkg/http"
 
 func MockHttpResponse(statusCode int, body string) *http.Response {
 	bodyBytes := []byte(body)

internal/utils/usage.go

@@ -0,0 +1,14 @@
+package utils
+
+import (
+	"flag"
+	"fmt"
+)
+
+func PrintUsage() {
+	PrintASCII()
+	fmt.Println("Usage: goctopus [options] [addresses]")
+	fmt.Println("[addresses]: A list of addresses to fingerprint, comma separated.\nAddresses can be in the form of http://example.com/graphql or example.com.\n If an input file is specified, this argument is ignored.")
+	fmt.Println("[options]:")
+	flag.PrintDefaults()
+}

internal/config/config.go → pkg/config/config.go

@@ -1,18 +1,18 @@
 package config
 
 import (
+	"errors"
 	"flag"
 	"os"
+	"strings"
 
+	"github.com/Escape-Technologies/goctopus/internal/utils"
 	log "github.com/sirupsen/logrus"
 )
 
-var (
-	Conf *Config
-)
-
 type Config struct {
 	InputFile            string
+	Addresses            []string
 	OutputFile           string
 	MaxWorkers           int
 	Verbose              bool
@@ -24,16 +24,34 @@ type Config struct {
 	SubdomainEnumeration bool
 }
 
-func ParseFlags() {
+var (
+	c *Config
+)
+
+func Get() *Config {
+	if c == nil {
+		log.Panic("Config not initialized")
+	}
+	return c
+}
+
+func parseArgs() []string {
+	if flag.Arg(0) == "" {
+		log.Error("Invalid addresses argument")
+		utils.PrintUsage()
+		os.Exit(1)
+	}
+	input := flag.Arg(0)
+	return strings.Split(input, ",")
+}
+
+func LoadFromArgs() {
+	flag.Usage = utils.PrintUsage
 	config := Config{}
 	// -- INPUT --
-	flag.StringVar(&config.InputFile, "i", "", "Input file")
-	// @TODO
-	// flag.StringVar(&config.InputFile, "d", "", "Input domains (comma separated)")
-	// flag.StringVar(&config.InputFile, "u", "", "Input urls (comma separated)")
+	flag.StringVar(&config.InputFile, "f", "", "Input file")
 
 	// -- CONFIG --
-	// @todo make output file optional ?
 	flag.StringVar(&config.OutputFile, "o", "output.jsonl", "Output file (json-lines format)")
 	flag.StringVar(&config.WebhookUrl, "webhook", "", "Webhook URL")
 	flag.IntVar(&config.MaxWorkers, "w", 100, "Max workers")
@@ -46,6 +64,10 @@ func ParseFlags() {
 
 	flag.Parse()
 
+	if config.InputFile == "" {
+		config.Addresses = parseArgs()
+	}
+
 	if config.Verbose {
 		log.SetLevel(log.DebugLevel)
 	}
@@ -54,34 +76,40 @@ func ParseFlags() {
 		log.SetLevel(log.ErrorLevel)
 	}
 
-	ValidateConfig(&config)
-	Conf = &config
+	if err := ValidateConfig(&config); err != nil {
+		log.Error(err)
+		flag.PrintDefaults()
+		os.Exit(1)
+	}
+
+	c = &config
 }
 
-func ValidateConfig(conf *Config) {
+func ValidateConfig(conf *Config) error {
 	if conf.MaxWorkers < 1 {
-		log.Error("[Invalid args] Max workers must be greater than 0")
-		configError()
+		return errors.New("[Invalid config] Max workers must be greater than 0")
 	}
 
 	if conf.Timeout < 1 {
-		log.Error("[Invalid args] Timeout must be greater than 0")
-		configError()
+		return errors.New("[Invalid config] Timeout must be greater than 0")
 	}
 
 	if !conf.Introspection && conf.FieldSuggestion {
-		log.Error("[Invalid args] Introspection has to be enabled to use field suggestion fingerprinting")
-		configError()
+		return errors.New("[Invalid config] Introspection has to be enabled to use field suggestion fingerprinting")
 	}
 
-	if conf.InputFile == "" {
-		log.Error("[Invalid args] Please specify an input file")
-		configError()
+	if conf.InputFile == "" && len(conf.Addresses) == 0 {
+		return errors.New("[Invalid config] Please specify an input file or a list of addresses")
 	}
 
+	return nil
 }
 
-func configError() {
-	flag.PrintDefaults()
-	os.Exit(1)
+func Load(config *Config) {
+	if err := ValidateConfig(config); err != nil {
+		log.Error(err)
+		flag.PrintDefaults()
+		os.Exit(1)
+	}
+	c = config
 }

pkg/crawl/domain.go → pkg/domain/enumeration.go

@@ -1,15 +1,16 @@
-package crawl
+package domain
 
 import (
 	"io"
 
-	"github.com/Escape-Technologies/goctopus/internal/config"
+	"github.com/Escape-Technologies/goctopus/pkg/config"
 	"github.com/projectdiscovery/subfinder/v2/pkg/resolve"
 	"github.com/projectdiscovery/subfinder/v2/pkg/runner"
 )
 
-func CrawlDomain(domain string, subDomains chan string, c *config.Config) (err error) {
+func EnumerateSubdomains(domain string, subDomains chan string) (err error) {
 	subDomains <- domain
+	c := config.Get()
 
 	if !c.SubdomainEnumeration {
 		return nil

pkg/crawl/sub_domain.go → pkg/domain/fingerprint.go

@@ -1,43 +1,29 @@
-package crawl
+package domain
 
 import (
 	"errors"
-	"fmt"
 	"net"
 
-	"github.com/Escape-Technologies/goctopus/internal/config"
-	"github.com/Escape-Technologies/goctopus/pkg/fingerprint"
-	out "github.com/Escape-Technologies/goctopus/pkg/output"
+	"github.com/Escape-Technologies/goctopus/pkg/endpoint"
+	"github.com/Escape-Technologies/goctopus/pkg/output"
 	log "github.com/sirupsen/logrus"
 	"github.com/valyala/fasthttp"
 )
 
-func CrawlSubDomain(domain string) (*out.FingerprintOutput, error) {
-	routes := []string{
-		"",
-		"graphql",
-		"graphql/v2",
-		"graphql/v1",
-		"/api",
-		"api/graphql",
-		"api/v2/graphql",
-		"api/v1/graphql",
-		"appsync",
-		"altair",
-		"graph",
-	}
-	// @todo refactor this
-	for _, route := range routes {
-		url := fmt.Sprintf("https://%s/%s", domain, route)
-		fp := fingerprint.NewFingerprinter(url)
-		output, err := fingerprint.FingerprintUrl(url, fp, config.Conf)
-		fp.Close()
+// @todo test this
+func FingerprintSubDomain(domain string) (*output.FingerprintOutput, error) {
+	endpoints := endpoint.FuzzRoutes(domain)
+
+	for _, url := range endpoints {
+		output, err := endpoint.FingerprintEndpoint(url)
+		// @todo close client
+		// fp.Close()
 
 		// At the first timeout, drop the domain
 		// @todo number of tries in the config
 		if err != nil {
 			// If the domain is not a graphql endpoint, continue
-			if errors.Is(err, fingerprint.ErrNotGraphql) {
+			if errors.Is(err, endpoint.ErrNotGraphql) {
 				continue
 			}
 

pkg/endpoint/endpoint.go

@@ -0,0 +1,43 @@
+package endpoint
+
+import (
+	"github.com/Escape-Technologies/goctopus/pkg/graphql"
+	"github.com/Escape-Technologies/goctopus/pkg/http"
+	"github.com/Escape-Technologies/goctopus/pkg/introspection"
+	"github.com/Escape-Technologies/goctopus/pkg/suggestion"
+)
+
+type _endpointFingerprinter struct {
+	url    string
+	client http.Client
+}
+
+type endpointFingerprinter interface {
+	IsOpenGraphql() (bool, error)
+	IsAuthentifiedGraphql() (bool, error)
+	HasFieldSuggestion() (bool, error)
+	HasIntrospectionOpen() (bool, error)
+}
+
+func NewEndpointFingerprinter(url string, client http.Client) endpointFingerprinter {
+	return &_endpointFingerprinter{
+		url:    url,
+		client: client,
+	}
+}
+
+func (e *_endpointFingerprinter) IsOpenGraphql() (bool, error) {
+	return graphql.FingerprintOpenGraphql(e.url, e.client)
+}
+
+func (e *_endpointFingerprinter) IsAuthentifiedGraphql() (bool, error) {
+	return graphql.FingerprintAuthentifiedGraphql(e.url, e.client)
+}
+
+func (e *_endpointFingerprinter) HasFieldSuggestion() (bool, error) {
+	return suggestion.FingerprintFieldSuggestion(e.url, e.client), nil
+}
+
+func (e *_endpointFingerprinter) HasIntrospectionOpen() (bool, error) {
+	return introspection.FingerprintIntrospection(e.url, e.client)
+}