From 25bdf478b475d8797e32b8684ea4d08b750ca974 Mon Sep 17 00:00:00 2001 From: nohehf Date: Mon, 24 Apr 2023 15:38:31 +0200 Subject: [PATCH 1/2] chore: wording --- pkg/output/handler.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/output/handler.go b/pkg/output/handler.go index bf6d401..3b52a14 100644 --- a/pkg/output/handler.go +++ b/pkg/output/handler.go @@ -18,8 +18,8 @@ func openOutputFile(config *config.Config) (*os.File, error) { } func handleSingleOutput(output *FingerprintOutput, outputFile *os.File, wg *sync.WaitGroup, config *config.Config) { - isOutputFile := config.OutputFile != "" - isWebhook := config.WebhookUrl != "" + hasOutputFile := config.OutputFile != "" + hasWebhook := config.WebhookUrl != "" jsonOutput, err := json.Marshal(output) log.Infof("Found: %+v\n", string(jsonOutput)) @@ -27,14 +27,14 @@ func handleSingleOutput(output *FingerprintOutput, outputFile *os.File, wg *sync log.Error(err) } - if isOutputFile { + if hasOutputFile { content := append(jsonOutput, []byte("\n")...) if _, err := outputFile.Write(content); err != nil { log.Error(err) } } - if isWebhook { + if hasWebhook { wg.Add(1) go func() { if err := http.SendToWebhook(config.WebhookUrl, jsonOutput, wg); err != nil { From db35accef4455e6c57d80d5846f4824f60c75ee1 Mon Sep 17 00:00:00 2001 From: nohehf Date: Mon, 24 Apr 2023 17:04:32 +0200 Subject: [PATCH 2/2] feat: pass metadata & general improvements --- .gitignore | 5 ++++- pkg/address/address.go | 28 ++++++++++++++++++++++++---- pkg/config/config.go | 11 +++++++---- pkg/domain/enumeration.go | 4 +++- pkg/domain/fingerprint.go | 1 + pkg/goctopus/fingerprint.go | 9 ++++++--- pkg/goctopus/single_address.go | 20 ++++++++++++++++++++ pkg/output/output.go | 8 ++++++-- 8 files changed, 71 insertions(+), 15 deletions(-) create mode 100644 pkg/goctopus/single_address.go diff --git a/.gitignore b/.gitignore index e202fc9..35cabcd 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,7 @@ testers /goctopus dist/ -.env \ No newline at end of file +.env + +# root go files (often used for testing / prototyping) +/*.go \ No newline at end of file diff --git a/pkg/address/address.go b/pkg/address/address.go index 39bacec..a4f2042 100644 --- a/pkg/address/address.go +++ b/pkg/address/address.go @@ -1,14 +1,18 @@ package address +import "strings" + type Addr struct { - Address string - Source string + Address string + Source string + Metadata map[string]string } func New(address string) *Addr { return &Addr{ - Address: address, - Source: address, + Address: address, + Source: address, + Metadata: map[string]string{}, } } @@ -19,3 +23,19 @@ func NewSourced(address, source string) *Addr { Source: source, } } + +func (a *Addr) AddMetadata(key, value string) { + a.Metadata[key] = value +} + +func (a *Addr) Copy() *Addr { + metadataCopy := make(map[string]string) + for k, v := range a.Metadata { + metadataCopy[k] = v + } + return &Addr{ + Address: strings.Clone(a.Address), + Source: strings.Clone(a.Source), + Metadata: metadataCopy, + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 68d94bd..7745417 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -76,7 +76,7 @@ func LoadFromArgs() { log.SetLevel(log.ErrorLevel) } - if err := ValidateConfig(&config); err != nil { + if err := validateConfig(&config, true); err != nil { log.Error(err) flag.PrintDefaults() os.Exit(1) @@ -85,7 +85,10 @@ func LoadFromArgs() { c = &config } -func ValidateConfig(conf *Config) error { +// Validates the config. +// `cli` is used to determine if the config is loaded from the CLI or from a file. +// If cli is false, then the addresses check is skipped. +func validateConfig(conf *Config, isCli bool) error { if conf.MaxWorkers < 1 { return errors.New("[Invalid config] Max workers must be greater than 0") } @@ -98,7 +101,7 @@ func ValidateConfig(conf *Config) error { return errors.New("[Invalid config] Introspection has to be enabled to use field suggestion fingerprinting") } - if conf.InputFile == "" && len(conf.Addresses) == 0 { + if isCli && c.InputFile == "" && len(c.Addresses) == 0 { return errors.New("[Invalid config] Please specify an input file or a list of addresses") } @@ -106,7 +109,7 @@ func ValidateConfig(conf *Config) error { } func Load(config *Config) { - if err := ValidateConfig(config); err != nil { + if err := validateConfig(config, false); err != nil { log.Error(err) flag.PrintDefaults() os.Exit(1) diff --git a/pkg/domain/enumeration.go b/pkg/domain/enumeration.go index 68c1757..f31553f 100644 --- a/pkg/domain/enumeration.go +++ b/pkg/domain/enumeration.go @@ -11,7 +11,9 @@ import ( func makeCallback(domain *address.Addr, subDomains chan *address.Addr) func(s *resolve.HostEntry) { return func(s *resolve.HostEntry) { - subDomains <- address.NewSourced(s.Host, domain.Source) + addr := domain.Copy() + addr.Address = s.Host + subDomains <- addr } } diff --git a/pkg/domain/fingerprint.go b/pkg/domain/fingerprint.go index bbd958c..df21fe9 100644 --- a/pkg/domain/fingerprint.go +++ b/pkg/domain/fingerprint.go @@ -48,6 +48,7 @@ func FingerprintSubDomain(domain *address.Addr) (*output.FingerprintOutput, erro } output.Domain = domain.Address output.Source = domain.Source + output.Metadata = domain.Metadata return output, nil } return nil, errors.New("no graphql endpoint found") diff --git a/pkg/goctopus/fingerprint.go b/pkg/goctopus/fingerprint.go index 1f60d56..cb4784c 100644 --- a/pkg/goctopus/fingerprint.go +++ b/pkg/goctopus/fingerprint.go @@ -17,7 +17,7 @@ func worker(addresses chan *address.Addr, output chan *output.FingerprintOutput, log.Debugf("Worker %d instantiated", workerId) for address := range addresses { log.Debugf("Worker %d started on: %v", workerId, address) - res, err := FingerprintAddress(address) + res, err := fingerprintAddress(address) if err == nil { log.Debugf("Worker %d found endpoint: %v", workerId, res) output <- res @@ -26,7 +26,10 @@ func worker(addresses chan *address.Addr, output chan *output.FingerprintOutput, log.Debugf("Worker %d finished", workerId) } -func FingerprintAddress(address *address.Addr) (*output.FingerprintOutput, error) { +/** + * Fingerprint an address, without subdomain enumeration + */ +func fingerprintAddress(address *address.Addr) (*output.FingerprintOutput, error) { // If the domain is a url, we don't need to crawl it if utils.IsUrl(address.Address) { return endpoint.FingerprintEndpoint(address) @@ -65,6 +68,6 @@ func FingerprintAddresses(addresses chan *address.Addr, output chan *output.Fing close(enumeratedAddresses) log.Debugf("Waiting for workers to finish...") workersWg.Wait() - close(output) log.Debugf("All workers finished") + close(output) } diff --git a/pkg/goctopus/single_address.go b/pkg/goctopus/single_address.go new file mode 100644 index 0000000..9518701 --- /dev/null +++ b/pkg/goctopus/single_address.go @@ -0,0 +1,20 @@ +package goctopus + +import ( + "github.com/Escape-Technologies/goctopus/pkg/address" + out "github.com/Escape-Technologies/goctopus/pkg/output" +) + +// Fingerprints a single address and outputs a slice of FingerprintOutput. +func FingerprintAddress(addr *address.Addr) []*out.FingerprintOutput { + outputSlice := make([]*out.FingerprintOutput, 0) + output := make(chan *out.FingerprintOutput, 1) + addresses := make(chan *address.Addr, 1) + addresses <- addr + close(addresses) + go FingerprintAddresses(addresses, output) + for fingerprintOutput := range output { + outputSlice = append(outputSlice, fingerprintOutput) + } + return outputSlice +} diff --git a/pkg/output/output.go b/pkg/output/output.go index 703cef6..691a09a 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -12,7 +12,6 @@ type FingerprintResult string const ( ResultOpenGraphql FingerprintResult = "OPEN_GRAPHQL" ResultAuthentifiedGraphql FingerprintResult = "AUTHENTIFIED_GRAPHQL" - // ResultMaybeGraphql FingerprintResult = "MAYBE_GRAPHQL" ) type FingerprintOutput struct { @@ -21,7 +20,8 @@ type FingerprintOutput struct { Url string `json:"url"` Introspection bool `json:"introspection"` FieldSuggestion bool `json:"field_suggestion"` - Source string `json:"source"` // the original address used to fingerprint the endpoint + Source string `json:"source"` // the original address used to fingerprint the endpoint + Metadata map[string]string `json:"metadata"` // optional metadata } func (o *FingerprintOutput) MarshalJSON() ([]byte, error) { @@ -56,5 +56,9 @@ func marshalOutput(o *FingerprintOutput, c *config.Config) ([]byte, error) { outputMap["domain"] = utils.DomainFromUrl(o.Url) } + if o.Metadata == nil || len(o.Metadata) == 0 { + delete(outputMap, "metadata") + } + return json.Marshal(outputMap) }