diff --git a/pkg/operators/extractors/compile.go b/pkg/operators/extractors/compile.go index 2b55d374ab..60deae2487 100644 --- a/pkg/operators/extractors/compile.go +++ b/pkg/operators/extractors/compile.go @@ -8,6 +8,7 @@ import ( "github.com/Knetic/govaluate" "github.com/itchyny/gojq" "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions" ) // CompileExtractors performs the initial setup operation on an extractor @@ -20,6 +21,10 @@ func (e *Extractor) CompileExtractors() error { e.extractorType = computedType // Compile the regexes for _, regex := range e.Regex { + if varErr := expressions.ContainsUnresolvedVariables(regex); varErr != nil { + e.regexCompiled = append(e.regexCompiled, nil) + continue + } compiled, err := regexp.Compile(regex) if err != nil { return fmt.Errorf("could not compile regex: %s", regex) @@ -31,6 +36,10 @@ func (e *Extractor) CompileExtractors() error { } for _, query := range e.JSON { + if varErr := expressions.ContainsUnresolvedVariables(query); varErr != nil { + e.jsonCompiled = append(e.jsonCompiled, nil) + continue + } query, err := gojq.Parse(query) if err != nil { return fmt.Errorf("could not parse json: %s", query) diff --git a/pkg/operators/extractors/extract.go b/pkg/operators/extractors/extract.go index 61b2cc5a08..b0c7175de5 100644 --- a/pkg/operators/extractors/extract.go +++ b/pkg/operators/extractors/extract.go @@ -3,20 +3,37 @@ package extractors import ( "encoding/json" "fmt" + "regexp" "strings" "github.com/antchfx/htmlquery" "github.com/antchfx/xmlquery" + "github.com/itchyny/gojq" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v3/pkg/types" ) // ExtractRegex extracts text from a corpus and returns it -func (e *Extractor) ExtractRegex(corpus string) map[string]struct{} { +func (e *Extractor) ExtractRegex(corpus string, data map[string]interface{}) map[string]struct{} { results := make(map[string]struct{}) groupPlusOne := e.RegexGroup + 1 - for _, regex := range e.regexCompiled { + for i, regex := range e.regexCompiled { + if varErr := expressions.ContainsUnresolvedVariables(e.Regex[i]); varErr != nil { + regexStr, err := expressions.Evaluate(e.Regex[i], data) + if err != nil { + gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", e.Regex[i], err.Error()) + continue + } + regex, err = regexp.Compile(regexStr) + if err != nil { + gologger.Warning().Msgf("Could not compile regex: %s, error: %s", regexStr, err.Error()) + continue + } + } + matches := regex.FindAllStringSubmatch(corpus, -1) for _, match := range matches { @@ -128,7 +145,7 @@ func (e *Extractor) ExtractXML(corpus string) map[string]struct{} { } // ExtractJSON extracts text from a corpus using JQ queries and returns it -func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} { +func (e *Extractor) ExtractJSON(corpus string, data map[string]interface{}) map[string]struct{} { results := make(map[string]struct{}) var jsonObj interface{} @@ -137,7 +154,24 @@ func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} { return results } - for _, k := range e.jsonCompiled { + for i, k := range e.jsonCompiled { + if varErr := expressions.ContainsUnresolvedVariables(e.JSON[i]); varErr != nil { + jsonStr, err := expressions.Evaluate(e.JSON[i], data) + if err != nil { + gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", e.JSON[i], err.Error()) + continue + } + query, err := gojq.Parse(jsonStr) + if err != nil { + gologger.Warning().Msgf("Could not parse json: %s, error: %s", jsonStr, err.Error()) + continue + } + k, err = gojq.Compile(query) + if err != nil { + gologger.Warning().Msgf("Could not compile json: %s, error: %s", jsonStr, err.Error()) + continue + } + } iter := k.Run(jsonObj) for { v, ok := iter.Next() diff --git a/pkg/operators/extractors/extract_test.go b/pkg/operators/extractors/extract_test.go index ab6d5f6ff6..f5acd372d9 100644 --- a/pkg/operators/extractors/extract_test.go +++ b/pkg/operators/extractors/extract_test.go @@ -11,10 +11,10 @@ func TestExtractor_ExtractRegex(t *testing.T) { err := e.CompileExtractors() require.Nil(t, err) - got := e.ExtractRegex("RegEx") + got := e.ExtractRegex("RegEx", nil) require.Equal(t, map[string]struct{}{"RegEx": {}}, got) - got = e.ExtractRegex("regex") + got = e.ExtractRegex("regex", nil) require.Equal(t, map[string]struct{}{}, got) } @@ -70,10 +70,10 @@ func TestExtractor_ExtractJSON(t *testing.T) { err := e.CompileExtractors() require.Nil(t, err) - got := e.ExtractJSON(`[{"id": 1}]`) + got := e.ExtractJSON(`[{"id": 1}]`, nil) require.Equal(t, map[string]struct{}{"1": {}}, got) - got = e.ExtractJSON(`{"id": 1}`) + got = e.ExtractJSON(`{"id": 1}`, nil) require.Equal(t, map[string]struct{}{}, got) } diff --git a/pkg/protocols/dns/operators.go b/pkg/protocols/dns/operators.go index 0f18315303..2160146f03 100644 --- a/pkg/protocols/dns/operators.go +++ b/pkg/protocols/dns/operators.go @@ -57,7 +57,7 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto switch extractor.GetType() { case extractors.RegexExtractor: - return extractor.ExtractRegex(types.ToString(item)) + return extractor.ExtractRegex(types.ToString(item), data) case extractors.KValExtractor: return extractor.ExtractKval(data) case extractors.DSLExtractor: diff --git a/pkg/protocols/file/operators.go b/pkg/protocols/file/operators.go index 5dab75bb91..cc3e1229cf 100644 --- a/pkg/protocols/file/operators.go +++ b/pkg/protocols/file/operators.go @@ -45,11 +45,11 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto switch extractor.GetType() { case extractors.RegexExtractor: - return extractor.ExtractRegex(itemStr) + return extractor.ExtractRegex(itemStr, data) case extractors.KValExtractor: return extractor.ExtractKval(data) case extractors.JSONExtractor: - return extractor.ExtractJSON(itemStr) + return extractor.ExtractJSON(itemStr, data) case extractors.XPathExtractor: return extractor.ExtractXPath(itemStr) case extractors.DSLExtractor: diff --git a/pkg/protocols/headless/operators.go b/pkg/protocols/headless/operators.go index 3ad30e9c55..c99843db10 100644 --- a/pkg/protocols/headless/operators.go +++ b/pkg/protocols/headless/operators.go @@ -71,7 +71,7 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto switch extractor.GetType() { case extractors.RegexExtractor: - return extractor.ExtractRegex(itemStr) + return extractor.ExtractRegex(itemStr, data) case extractors.KValExtractor: return extractor.ExtractKval(data) case extractors.DSLExtractor: diff --git a/pkg/protocols/http/operators.go b/pkg/protocols/http/operators.go index d630bfd8b0..1ca4d547e2 100644 --- a/pkg/protocols/http/operators.go +++ b/pkg/protocols/http/operators.go @@ -67,13 +67,13 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto } switch extractor.GetType() { case extractors.RegexExtractor: - return extractor.ExtractRegex(item) + return extractor.ExtractRegex(item, data) case extractors.KValExtractor: return extractor.ExtractKval(data) case extractors.XPathExtractor: return extractor.ExtractXPath(item) case extractors.JSONExtractor: - return extractor.ExtractJSON(item) + return extractor.ExtractJSON(item, data) case extractors.DSLExtractor: return extractor.ExtractDSL(data) } diff --git a/pkg/protocols/network/operators.go b/pkg/protocols/network/operators.go index 2aa19e5b3b..f5da94d50f 100644 --- a/pkg/protocols/network/operators.go +++ b/pkg/protocols/network/operators.go @@ -46,7 +46,7 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto switch extractor.GetType() { case extractors.RegexExtractor: - return extractor.ExtractRegex(itemStr) + return extractor.ExtractRegex(itemStr, data) case extractors.KValExtractor: return extractor.ExtractKval(data) case extractors.DSLExtractor: diff --git a/pkg/protocols/offlinehttp/operators.go b/pkg/protocols/offlinehttp/operators.go index f69abc4412..db721d84dd 100644 --- a/pkg/protocols/offlinehttp/operators.go +++ b/pkg/protocols/offlinehttp/operators.go @@ -66,7 +66,7 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto } switch extractor.GetType() { case extractors.RegexExtractor: - return extractor.ExtractRegex(item) + return extractor.ExtractRegex(item, data) case extractors.KValExtractor: return extractor.ExtractKval(data) case extractors.DSLExtractor: diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index a9d50c1481..85c4eebc51 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -327,11 +327,11 @@ func MakeDefaultExtractFunc(data map[string]interface{}, extractor *extractors.E switch extractor.GetType() { case extractors.RegexExtractor: - return extractor.ExtractRegex(itemStr) + return extractor.ExtractRegex(itemStr, data) case extractors.KValExtractor: return extractor.ExtractKval(data) case extractors.JSONExtractor: - return extractor.ExtractJSON(itemStr) + return extractor.ExtractJSON(itemStr, data) case extractors.XPathExtractor: return extractor.ExtractXPath(itemStr) case extractors.DSLExtractor: