diff --git a/apps/database-abstractor/src/main/java/com/akto/action/DbAction.java b/apps/database-abstractor/src/main/java/com/akto/action/DbAction.java index 6767c6db79..43a4ba4f2a 100644 --- a/apps/database-abstractor/src/main/java/com/akto/action/DbAction.java +++ b/apps/database-abstractor/src/main/java/com/akto/action/DbAction.java @@ -16,18 +16,8 @@ import com.akto.dto.test_editor.YamlTemplate; import com.akto.dto.test_run_findings.TestingIssuesId; import com.akto.dto.test_run_findings.TestingRunIssues; -import com.akto.dto.testing.AccessMatrixTaskInfo; -import com.akto.dto.testing.AccessMatrixUrlToRole; -import com.akto.dto.testing.CollectionWiseTestingEndpoints; -import com.akto.dto.testing.CustomTestingEndpoints; -import com.akto.dto.testing.EndpointLogicalGroup; -import com.akto.dto.testing.TestRoles; -import com.akto.dto.testing.TestingRun; -import com.akto.dto.testing.TestingRunConfig; -import com.akto.dto.testing.TestingRunResult; -import com.akto.dto.testing.TestingRunResultSummary; -import com.akto.dto.testing.WorkflowTest; -import com.akto.dto.testing.WorkflowTestResult; +import com.akto.dto.testing.*; +import com.akto.dto.testing.config.TestScript; import com.akto.dto.testing.sources.TestSourceConfig; import com.akto.dto.traffic.SampleData; import com.akto.dto.traffic.TrafficInfo; @@ -86,6 +76,7 @@ public class DbAction extends ActionSupport { List writesForTrafficMetrics; List writesForTestingRunIssues; List dependencyNodeList; + TestScript testScript; public List getWritesForTestingRunIssues() { return writesForTestingRunIssues; @@ -1698,6 +1689,16 @@ public String createCollectionForHostAndVpc() { return Action.SUCCESS.toUpperCase(); } + public String fetchTestScript() { + try { + testScript = DbLayer.fetchTestScript(); + return SUCCESS.toUpperCase(); + } catch (Exception e) { + System.out.println("Error in fetchTestScript " + e.toString()); + return Action.ERROR.toUpperCase(); + } + } + public String countTestingRunResultSummaries() { count = DbLayer.countTestingRunResultSummaries(filter); return Action.SUCCESS.toUpperCase(); @@ -2613,6 +2614,10 @@ public void setVpcId(String vpcId) { this.vpcId = vpcId; } + public TestScript getTestScript() { + return testScript; + } + public void setFilter(Bson filter) { this.filter = filter; } diff --git a/apps/database-abstractor/src/main/resources/struts.xml b/apps/database-abstractor/src/main/resources/struts.xml index e3ff876b55..882f817c67 100644 --- a/apps/database-abstractor/src/main/resources/struts.xml +++ b/apps/database-abstractor/src/main/resources/struts.xml @@ -1201,6 +1201,17 @@ + + + + + + 422 + false + ^actionErrors.* + + + diff --git a/libs/dao/src/main/java/com/akto/dao/testing/config/TestScriptsDao.java b/libs/dao/src/main/java/com/akto/dao/testing/config/TestScriptsDao.java new file mode 100644 index 0000000000..103a64a667 --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dao/testing/config/TestScriptsDao.java @@ -0,0 +1,26 @@ +package com.akto.dao.testing.config; + +import com.akto.dao.*; +import com.akto.dto.testing.config.TestScript; +import com.mongodb.BasicDBObject; + +public class TestScriptsDao extends AccountsContextDao { + + public static final TestScriptsDao instance = new TestScriptsDao(); + + private TestScriptsDao() {} + + public TestScript fetchTestScript() { + return TestScriptsDao.instance.findOne(new BasicDBObject()); + } + + @Override + public String getCollName() { + return "test_collection_properties"; + } + + @Override + public Class getClassT() { + return TestScript.class; + } +} diff --git a/libs/dao/src/main/java/com/akto/dto/OriginalHttpRequest.java b/libs/dao/src/main/java/com/akto/dto/OriginalHttpRequest.java index 114d2a0da2..6c81e0063f 100644 --- a/libs/dao/src/main/java/com/akto/dto/OriginalHttpRequest.java +++ b/libs/dao/src/main/java/com/akto/dto/OriginalHttpRequest.java @@ -346,11 +346,16 @@ public static String extractAktoUUid(String message) { } public String getPath(){ - String path = URI.create(this.url).getPath(); - if (path == null || path.isEmpty()) { - return "/"; + try { + String path = URI.create(this.url).getPath(); + if (path == null || path.isEmpty()) { + return "/"; + } + return path; + } catch (Exception e) { + String strippedUrl = this.url.replaceAll("^(https?://[^/]+)", ""); + return strippedUrl.isEmpty() ? "/" : strippedUrl; } - return path; } @Override diff --git a/libs/dao/src/main/java/com/akto/dto/testing/config/TestScript.java b/libs/dao/src/main/java/com/akto/dto/testing/config/TestScript.java new file mode 100644 index 0000000000..391829043c --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dto/testing/config/TestScript.java @@ -0,0 +1,122 @@ +package com.akto.dto.testing.config; +import java.util.Objects; + +import org.bson.codecs.pojo.annotations.BsonId; + +import com.akto.dao.context.Context; + +public class TestScript { + + + public enum Type { + PRE_TEST, PRE_REQUEST, POST_REQUEST, POST_TEST + } + + public static final String ID = "_id"; + @BsonId + private String id; + + public static final String JAVASCRIPT = "javascript"; + private String javascript; + + private Type type; + + public static final String AUTHOR = "author"; + private String author; + + private int lastCreatedAt; + + public static final String LAST_UPDATED_AT = "lastUpdatedAt"; + private int lastUpdatedAt; + + + public TestScript() { + } + + public TestScript(String id, String javascript, Type type, String author, int lastUpdatedAt) { + this.id = id; + this.javascript = javascript; + this.type = type; + this.author = author; + this.lastCreatedAt = Context.now(); + this.lastUpdatedAt = lastUpdatedAt; + } + + public String getJavascript() { + return this.javascript; + } + + public void setJavascript(String javascript) { + this.javascript = javascript; + } + + public Type getType() { + return this.type; + } + + public void setType(Type type) { + this.type = type; + } + + public String getAuthor() { + return this.author; + } + + public void setAuthor(String author) { + this.author = author; + } + + public int getLastCreatedAt() { + return this.lastCreatedAt; + } + + public void setLastCreatedAt(int lastCreatedAt) { + this.lastCreatedAt = lastCreatedAt; + } + + public int getLastUpdatedAt() { + return this.lastUpdatedAt; + } + + public void setLastUpdatedAt(int lastUpdatedAt) { + this.lastUpdatedAt = lastUpdatedAt; + } + + public String getId() { + return this.id; + } + + public void setId(String id) { + this.id = id; + } + + @Override + public boolean equals(Object o) { + if (o == this) + return true; + if (!(o instanceof TestScript)) { + return false; + } + TestScript testScript = (TestScript) o; + return Objects.equals(javascript, testScript.javascript) && Objects.equals(type, testScript.type) && Objects.equals(author, testScript.author) && lastCreatedAt == testScript.lastCreatedAt && lastUpdatedAt == testScript.lastUpdatedAt; + } + + @Override + public int hashCode() { + return Objects.hash(javascript, type, author, lastCreatedAt, lastUpdatedAt); + } + + @Override + public String toString() { + return "{" + + " id='" + getId() + "'" + + ", javascript='" + getJavascript() + "'" + + ", type='" + getType() + "'" + + ", author='" + getAuthor() + "'" + + ", lastCreatedAt='" + getLastCreatedAt() + "'" + + ", lastUpdatedAt='" + getLastUpdatedAt() + "'" + + "}"; + } + + +} diff --git a/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java b/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java index a14b3ff087..df1ef83682 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java +++ b/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java @@ -46,6 +46,7 @@ import com.akto.dto.testing.TestingRunResultSummary; import com.akto.dto.testing.WorkflowTest; import com.akto.dto.testing.WorkflowTestResult; +import com.akto.dto.testing.config.TestScript; import com.akto.dto.testing.sources.TestSourceConfig; import com.akto.dto.traffic.SampleData; import com.akto.dto.type.SingleTypeInfo; @@ -3442,7 +3443,7 @@ public long countTestingRunResultSummaries(Bson filter) { Map> headers = buildHeaders(); OriginalHttpRequest request = new OriginalHttpRequest(url + "/countTestingRunResultSummaries", "", "POST", obj.toString(), headers, ""); try { - OriginalHttpResponse response = ApiExecutor.sendRequestBackOff(request, true, null, false, null); + OriginalHttpResponse response = ApiExecutor.sendRequest(request, true, null, false, null); String responsePayload = response.getBody(); if (response.getStatusCode() != 200 || responsePayload == null) { loggerMaker.errorAndAddToDb("non 2xx response in countTestingRunResultSummaries", LoggerMaker.LogDb.TESTING); @@ -3462,4 +3463,31 @@ public long countTestingRunResultSummaries(Bson filter) { } } + public TestScript fetchTestScript(){ + TestScript testScript = null; + + Map> headers = buildHeaders(); + OriginalHttpRequest request = new OriginalHttpRequest(url + "/fetchTestScript", "", "GET", null, headers, ""); + try { + OriginalHttpResponse response = ApiExecutor.sendRequest(request, true, null, false, null); + String responsePayload = response.getBody(); + if (response.getStatusCode() != 200 || responsePayload == null) { + loggerMaker.errorAndAddToDb("invalid response in fetchTestScript", LoggerMaker.LogDb.RUNTIME); + return testScript; + } + BasicDBObject payloadObj; + + try { + payloadObj = BasicDBObject.parse(responsePayload); + BasicDBObject testScriptObj = (BasicDBObject) payloadObj.get("testScript"); + testScript = objectMapper.readValue(testScriptObj.toJson(), TestScript.class); + } catch (Exception e) { + loggerMaker.errorAndAddToDb("error extracting response in fetchTestScript" + e, LoggerMaker.LogDb.RUNTIME); + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb("error in fetchTestScript" + e, LoggerMaker.LogDb.RUNTIME); + } + return testScript; + } + } diff --git a/libs/utils/src/main/java/com/akto/data_actor/DataActor.java b/libs/utils/src/main/java/com/akto/data_actor/DataActor.java index 7534ccd55b..da5b9c1571 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/DataActor.java +++ b/libs/utils/src/main/java/com/akto/data_actor/DataActor.java @@ -22,6 +22,7 @@ import com.akto.dto.testing.TestingRunResultSummary; import com.akto.dto.testing.WorkflowTest; import com.akto.dto.testing.WorkflowTestResult; +import com.akto.dto.testing.config.TestScript; import com.akto.dto.testing.sources.TestSourceConfig; import com.akto.dto.traffic.SampleData; import com.akto.dto.type.SingleTypeInfo; @@ -261,4 +262,6 @@ public abstract class DataActor { public abstract long countTestingRunResultSummaries(Bson filter); + public abstract TestScript fetchTestScript(); + } diff --git a/libs/utils/src/main/java/com/akto/data_actor/DbActor.java b/libs/utils/src/main/java/com/akto/data_actor/DbActor.java index 18bb404560..4fa537e212 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/DbActor.java +++ b/libs/utils/src/main/java/com/akto/data_actor/DbActor.java @@ -22,6 +22,7 @@ import com.akto.dto.testing.TestRoles; import com.akto.dto.testing.TestingRun; import com.akto.dto.testing.TestingRun.State; +import com.akto.dto.testing.config.TestScript; import com.akto.dto.testing.TestingRunConfig; import com.akto.dto.testing.TestingRunResult; import com.akto.dto.testing.TestingRunResultSummary; @@ -542,4 +543,8 @@ public long countTestingRunResultSummaries(Bson filter){ return DbLayer.countTestingRunResultSummaries(filter); } + public TestScript fetchTestScript(){ + return DbLayer.fetchTestScript(); + } + } \ No newline at end of file diff --git a/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java b/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java index 0de6851ed3..aca7db81a0 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java +++ b/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java @@ -38,6 +38,8 @@ import com.akto.dao.testing.TestingRunResultSummariesDao; import com.akto.dao.testing.WorkflowTestResultsDao; import com.akto.dao.testing.WorkflowTestsDao; +import com.akto.dao.testing.config.TestCollectionPropertiesDao; +import com.akto.dao.testing.config.TestScriptsDao; import com.akto.dao.testing.sources.TestSourceConfigsDao; import com.akto.dao.testing_run_findings.TestingRunIssuesDao; import com.akto.dao.traffic_metrics.TrafficMetricsDao; @@ -62,6 +64,7 @@ import com.akto.dto.testing.WorkflowTest; import com.akto.dto.testing.WorkflowTestResult; import com.akto.dto.testing.TestingRun.State; +import com.akto.dto.testing.config.TestScript; import com.akto.dto.testing.sources.TestSourceConfig; import com.akto.dto.traffic.SampleData; import com.akto.dto.traffic.TrafficInfo; @@ -1014,4 +1017,9 @@ public static long countTestingRunResultSummaries(Bson filter){ return TestingRunResultSummariesDao.instance.count(filter); } + + public static TestScript fetchTestScript(){ + return TestScriptsDao.instance.fetchTestScript(); + } + } diff --git a/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java b/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java index 0fb37d504d..d803334edf 100644 --- a/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java +++ b/libs/utils/src/main/java/com/akto/testing/ApiExecutor.java @@ -1,10 +1,14 @@ package com.akto.testing; import com.akto.dao.context.Context; +import com.akto.dao.testing.config.TestScriptsDao; +import com.akto.data_actor.DataActor; +import com.akto.data_actor.DataActorFactory; import com.akto.dto.OriginalHttpRequest; import com.akto.dto.OriginalHttpResponse; import com.akto.dto.testing.TestingRunConfig; import com.akto.dto.testing.TestingRunResult; +import com.akto.dto.testing.config.TestScript; import com.akto.dto.testing.rate_limit.RateLimitHandler; import com.akto.dto.type.URLMethods; import com.akto.log.LoggerMaker; @@ -27,12 +31,22 @@ import java.net.URL; import java.util.*; +import javax.script.ScriptContext; +import javax.script.ScriptEngine; +import javax.script.ScriptEngineManager; +import javax.script.SimpleScriptContext; +import jdk.nashorn.api.scripting.ScriptObjectMirror; + public class ApiExecutor { private static final LoggerMaker loggerMaker = new LoggerMaker(ApiExecutor.class); private static final Logger logger = LoggerFactory.getLogger(ApiExecutor.class); // Load only first 1 MiB of response body into memory. private static final int MAX_RESPONSE_SIZE = 1024*1024; + private static Map lastFetchedMap = new HashMap<>(); + private static Map testScriptMap = new HashMap<>(); + + private static final DataActor dataActor = DataActorFactory.fetchInstance(); private static OriginalHttpResponse common(Request request, boolean followRedirects, boolean debug, List testLogs, boolean skipSSRFCheck, boolean nonTestingContext) throws Exception { @@ -268,6 +282,8 @@ public static OriginalHttpResponse sendRequest(OriginalHttpRequest request, bool URLMethods.Method method = URLMethods.Method.fromString(request.getMethod()); builder = builder.url(request.getFullUrlWithParams()); + boolean executeScript = testingRunConfig != null; + calculateHashAndAddAuth(request, executeScript); boolean nonTestingContext = false; if (testingRunConfig == null) { @@ -376,6 +392,74 @@ public void writeTo(BufferedSink sink) throws IOException { } + private static void calculateHashAndAddAuth(OriginalHttpRequest originalHttpRequest, boolean executeScript) { + if (!executeScript) { + return; + } + int accountId = Context.accountId.get(); + try { + String script; + TestScript testScript = testScriptMap.getOrDefault(accountId, null); + int lastTestScriptFetched = lastFetchedMap.getOrDefault(accountId, 0); + if (Context.now() - lastTestScriptFetched > 5 * 60) { + testScript = dataActor.fetchTestScript(); + lastTestScriptFetched = Context.now(); + testScriptMap.put(accountId, testScript); + lastFetchedMap.put(accountId, Context.now()); + } + if (testScript != null && testScript.getJavascript() != null) { + script = testScript.getJavascript(); + } else { + // loggerMaker.infoAndAddToDb("returning from calculateHashAndAddAuth, no test script present"); + return; + } + loggerMaker.infoAndAddToDb("Starting calculateHashAndAddAuth"); + + ScriptEngineManager manager = new ScriptEngineManager(); + ScriptEngine engine = manager.getEngineByName("nashorn"); + + SimpleScriptContext sctx = ((SimpleScriptContext) engine.get("context")); + sctx.setAttribute("method", originalHttpRequest.getMethod(), ScriptContext.ENGINE_SCOPE); + sctx.setAttribute("headers", originalHttpRequest.getHeaders(), ScriptContext.ENGINE_SCOPE); + sctx.setAttribute("url", originalHttpRequest.getPath(), ScriptContext.ENGINE_SCOPE); + sctx.setAttribute("payload", originalHttpRequest.getBody(), ScriptContext.ENGINE_SCOPE); + sctx.setAttribute("queryParams", originalHttpRequest.getQueryParams(), ScriptContext.ENGINE_SCOPE); + engine.eval(script); + + String method = (String) sctx.getAttribute("method"); + Map headers = (Map) sctx.getAttribute("headers"); + String url = (String) sctx.getAttribute("url"); + String payload = (String) sctx.getAttribute("payload"); + String queryParams = (String) sctx.getAttribute("queryParams"); + + Map> hs = new HashMap<>(); + for (String key: headers.keySet()) { + try { + ScriptObjectMirror scm = ((ScriptObjectMirror) headers.get(key)); + List val = new ArrayList<>(); + for (int i = 0; i < scm.size(); i++) { + val.add((String) scm.get(Integer.toString(i))); + } + hs.put(key, val); + } catch (Exception e) { + hs.put(key, (List) headers.get(key)); + } + } + + originalHttpRequest.setBody(payload); + originalHttpRequest.setMethod(method); + originalHttpRequest.setUrl(url); + originalHttpRequest.setHeaders(hs); + originalHttpRequest.setQueryParams(queryParams); + + } catch (Exception e) { + loggerMaker.errorAndAddToDb("error in calculateHashAndAddAuth " + e.getMessage() + " url " + originalHttpRequest.getUrl()); + e.printStackTrace(); + return; + } + } + + private static OriginalHttpResponse sendWithRequestBody(OriginalHttpRequest request, Request.Builder builder, boolean followRedirects, boolean debug, List testLogs, boolean skipSSRFCheck, boolean nonTestingContext) throws Exception {