Extractor variable support

pkg/operators/extractors/compile.go

@@ -8,6 +8,7 @@ import (
 	"github.com/Knetic/govaluate"
 	"github.com/itchyny/gojq"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
 )
 
 // CompileExtractors performs the initial setup operation on an extractor
@@ -20,6 +21,10 @@ func (e *Extractor) CompileExtractors() error {
 	e.extractorType = computedType
 	// Compile the regexes
 	for _, regex := range e.Regex {
+		if varErr := expressions.ContainsUnresolvedVariables(regex); varErr != nil {
+			e.regexCompiled = append(e.regexCompiled, nil)
+			continue
+		}
 		compiled, err := regexp.Compile(regex)
 		if err != nil {
 			return fmt.Errorf("could not compile regex: %s", regex)
@@ -31,6 +36,10 @@ func (e *Extractor) CompileExtractors() error {
 	}
 
 	for _, query := range e.JSON {
+		if varErr := expressions.ContainsUnresolvedVariables(query); varErr != nil {
+			e.jsonCompiled = append(e.jsonCompiled, nil)
+			continue
+		}
 		query, err := gojq.Parse(query)
 		if err != nil {
 			return fmt.Errorf("could not parse json: %s", query)

pkg/operators/extractors/extract.go

@@ -3,20 +3,37 @@ package extractors
 import (
 	"encoding/json"
 	"fmt"
+	"regexp"
 	"strings"
 
 	"github.com/antchfx/htmlquery"
 	"github.com/antchfx/xmlquery"
+	"github.com/itchyny/gojq"
 
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 )
 
 // ExtractRegex extracts text from a corpus and returns it
-func (e *Extractor) ExtractRegex(corpus string) map[string]struct{} {
+func (e *Extractor) ExtractRegex(corpus string, data map[string]interface{}) map[string]struct{} {
 	results := make(map[string]struct{})
 
 	groupPlusOne := e.RegexGroup + 1
-	for _, regex := range e.regexCompiled {
+	for i, regex := range e.regexCompiled {
+		if varErr := expressions.ContainsUnresolvedVariables(e.Regex[i]); varErr != nil {
+			regexStr, err := expressions.Evaluate(e.Regex[i], data)
+			if err != nil {
+				gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", e.Regex[i], err.Error())
+				continue
+			}
+			regex, err = regexp.Compile(regexStr)
+			if err != nil {
+				gologger.Warning().Msgf("Could not compile regex: %s, error: %s", regexStr, err.Error())
+				continue
+			}
+		}
+
 		matches := regex.FindAllStringSubmatch(corpus, -1)
 
 		for _, match := range matches {
@@ -128,7 +145,7 @@ func (e *Extractor) ExtractXML(corpus string) map[string]struct{} {
 }
 
 // ExtractJSON extracts text from a corpus using JQ queries and returns it
-func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} {
+func (e *Extractor) ExtractJSON(corpus string, data map[string]interface{}) map[string]struct{} {
 	results := make(map[string]struct{})
 
 	var jsonObj interface{}
@@ -137,7 +154,24 @@ func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} {
 		return results
 	}
 
-	for _, k := range e.jsonCompiled {
+	for i, k := range e.jsonCompiled {
+		if varErr := expressions.ContainsUnresolvedVariables(e.JSON[i]); varErr != nil {
+			jsonStr, err := expressions.Evaluate(e.JSON[i], data)
+			if err != nil {
+				gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", e.JSON[i], err.Error())
+				continue
+			}
+			query, err := gojq.Parse(jsonStr)
+			if err != nil {
+				gologger.Warning().Msgf("Could not parse json: %s, error: %s", jsonStr, err.Error())
+				continue
+			}
+			k, err = gojq.Compile(query)
+			if err != nil {
+				gologger.Warning().Msgf("Could not compile json: %s, error: %s", jsonStr, err.Error())
+				continue
+			}
+		}
 		iter := k.Run(jsonObj)
 		for {
 			v, ok := iter.Next()

pkg/operators/extractors/extract_test.go

@@ -11,10 +11,10 @@ func TestExtractor_ExtractRegex(t *testing.T) {
 	err := e.CompileExtractors()
 	require.Nil(t, err)
 
-	got := e.ExtractRegex("RegEx")
+	got := e.ExtractRegex("RegEx", nil)
 	require.Equal(t, map[string]struct{}{"RegEx": {}}, got)
 
-	got = e.ExtractRegex("regex")
+	got = e.ExtractRegex("regex", nil)
 	require.Equal(t, map[string]struct{}{}, got)
 }
 
@@ -70,10 +70,10 @@ func TestExtractor_ExtractJSON(t *testing.T) {
 	err := e.CompileExtractors()
 	require.Nil(t, err)
 
-	got := e.ExtractJSON(`[{"id": 1}]`)
+	got := e.ExtractJSON(`[{"id": 1}]`, nil)
 	require.Equal(t, map[string]struct{}{"1": {}}, got)
 
-	got = e.ExtractJSON(`{"id": 1}`)
+	got = e.ExtractJSON(`{"id": 1}`, nil)
 	require.Equal(t, map[string]struct{}{}, got)
 }
 

pkg/protocols/dns/operators.go

@@ -57,7 +57,7 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto
 
 	switch extractor.GetType() {
 	case extractors.RegexExtractor:
-		return extractor.ExtractRegex(types.ToString(item))
+		return extractor.ExtractRegex(types.ToString(item), data)
 	case extractors.KValExtractor:
 		return extractor.ExtractKval(data)
 	case extractors.DSLExtractor:

pkg/protocols/file/operators.go

@@ -45,11 +45,11 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto
 
 	switch extractor.GetType() {
 	case extractors.RegexExtractor:
-		return extractor.ExtractRegex(itemStr)
+		return extractor.ExtractRegex(itemStr, data)
 	case extractors.KValExtractor:
 		return extractor.ExtractKval(data)
 	case extractors.JSONExtractor:
-		return extractor.ExtractJSON(itemStr)
+		return extractor.ExtractJSON(itemStr, data)
 	case extractors.XPathExtractor:
 		return extractor.ExtractXPath(itemStr)
 	case extractors.DSLExtractor:

pkg/protocols/headless/operators.go

@@ -71,7 +71,7 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto
 
 	switch extractor.GetType() {
 	case extractors.RegexExtractor:
-		return extractor.ExtractRegex(itemStr)
+		return extractor.ExtractRegex(itemStr, data)
 	case extractors.KValExtractor:
 		return extractor.ExtractKval(data)
 	case extractors.DSLExtractor:

pkg/protocols/http/operators.go

@@ -67,13 +67,13 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto
 	}
 	switch extractor.GetType() {
 	case extractors.RegexExtractor:
-		return extractor.ExtractRegex(item)
+		return extractor.ExtractRegex(item, data)
 	case extractors.KValExtractor:
 		return extractor.ExtractKval(data)
 	case extractors.XPathExtractor:
 		return extractor.ExtractXPath(item)
 	case extractors.JSONExtractor:
-		return extractor.ExtractJSON(item)
+		return extractor.ExtractJSON(item, data)
 	case extractors.DSLExtractor:
 		return extractor.ExtractDSL(data)
 	}

pkg/protocols/network/operators.go

@@ -46,7 +46,7 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto
 
 	switch extractor.GetType() {
 	case extractors.RegexExtractor:
-		return extractor.ExtractRegex(itemStr)
+		return extractor.ExtractRegex(itemStr, data)
 	case extractors.KValExtractor:
 		return extractor.ExtractKval(data)
 	case extractors.DSLExtractor:

pkg/protocols/offlinehttp/operators.go

@@ -66,7 +66,7 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto
 	}
 	switch extractor.GetType() {
 	case extractors.RegexExtractor:
-		return extractor.ExtractRegex(item)
+		return extractor.ExtractRegex(item, data)
 	case extractors.KValExtractor:
 		return extractor.ExtractKval(data)
 	case extractors.DSLExtractor:

pkg/protocols/protocols.go

@@ -327,11 +327,11 @@ func MakeDefaultExtractFunc(data map[string]interface{}, extractor *extractors.E
 
 	switch extractor.GetType() {
 	case extractors.RegexExtractor:
-		return extractor.ExtractRegex(itemStr)
+		return extractor.ExtractRegex(itemStr, data)
 	case extractors.KValExtractor:
 		return extractor.ExtractKval(data)
 	case extractors.JSONExtractor:
-		return extractor.ExtractJSON(itemStr)
+		return extractor.ExtractJSON(itemStr, data)
 	case extractors.XPathExtractor:
 		return extractor.ExtractXPath(itemStr)
 	case extractors.DSLExtractor: