From fb70eba432abb34ae4fb1b440114c61586d6909c Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Thu, 10 Oct 2024 15:15:13 +0530 Subject: [PATCH 1/3] added debug logs for hosts --- .../java/com/akto/parsers/HttpCallParser.java | 7 +++++ .../src/main/java/com/akto/runtime/Main.java | 5 ++++ .../akto/hybrid_parsers/HttpCallParser.java | 7 +++++ .../java/com/akto/hybrid_runtime/Main.java | 5 ++++ .../java/com/akto/runtime/utils/Utils.java | 30 +++++++++++++++---- 5 files changed, 49 insertions(+), 5 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java index a3680ade2c..9b77062471 100644 --- a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java +++ b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java @@ -627,6 +627,13 @@ public static boolean aggregate(List responses, Map urlSet= new HashSet<>(); for (HttpResponseParams responseParams: responses) { + + String debugHost = Utils.printDebugLog(responseParams); + if (debugHost != null) { + HttpRequestParams requestParams = responseParams.getRequestParams(); + loggerMaker.infoAndAddToDb("Found host: " + debugHost + " in HttpCallParser.java for url: " + requestParams.getMethod() + " " + requestParams.getURL()); + } + if (responseParams.getSource() == HttpResponseParams.Source.HAR || responseParams.getSource() == HttpResponseParams.Source.PCAP) { ret = true; } diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/Main.java b/apps/api-runtime/src/main/java/com/akto/runtime/Main.java index e0644e755e..d02109f277 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/Main.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/Main.java @@ -228,6 +228,11 @@ public void run() { } httpResponseParams = HttpCallParser.parseKafkaMessage(r.value()); + String debugHost = Utils.printDebugLog(httpResponseParams); + if (debugHost != null) { + HttpRequestParams requestParams = httpResponseParams.getRequestParams(); + loggerMaker.infoAndAddToDb("Found host: " + debugHost + " in Main.java for url: " + requestParams.getMethod() + " " + requestParams.getURL()); + } } catch (Exception e) { loggerMaker.errorAndAddToDb(e, "Error while parsing kafka message " + e, LogDb.RUNTIME); continue; diff --git a/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java b/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java index 3ec1d7ea94..907138645c 100644 --- a/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java +++ b/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java @@ -20,6 +20,7 @@ import com.akto.hybrid_runtime.Main; import com.akto.hybrid_runtime.URLAggregator; import com.akto.runtime.RuntimeUtil; +import com.akto.runtime.utils.Utils; import com.akto.util.JSONUtils; import com.akto.util.Constants; import com.akto.util.HttpRequestResponseUtils; @@ -546,6 +547,12 @@ public static boolean aggregate(List responses, Map urlSet= new HashSet<>(); for (HttpResponseParams responseParams: responses) { + String debugHost = Utils.printDebugLog(responseParams); + if (debugHost != null) { + HttpRequestParams requestParams = responseParams.getRequestParams(); + loggerMaker.infoAndAddToDb("Found host: " + debugHost + " in HttpCallParser.java for url: " + requestParams.getMethod() + " " + requestParams.getURL()); + } + if (responseParams.getSource() == HttpResponseParams.Source.HAR || responseParams.getSource() == HttpResponseParams.Source.PCAP) { ret = true; } diff --git a/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/Main.java b/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/Main.java index 5ecc98741d..1ce05503ed 100644 --- a/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/Main.java +++ b/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/Main.java @@ -234,6 +234,11 @@ public void run() { } httpResponseParams = HttpCallParser.parseKafkaMessage(r.value()); + String debugHost = Utils.printDebugLog(httpResponseParams); + if (debugHost != null) { + HttpRequestParams requestParams = httpResponseParams.getRequestParams(); + loggerMaker.infoAndAddToDb("Found host: " + debugHost + " in url: " + requestParams.getMethod() + " " + requestParams.getURL()); + } } catch (Exception e) { loggerMaker.errorAndAddToDb(e, "Error while parsing kafka message " + e, LogDb.RUNTIME); continue; diff --git a/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java b/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java index e8fd30c157..4b13396031 100644 --- a/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java +++ b/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java @@ -1,10 +1,6 @@ package com.akto.runtime.utils; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.StringJoiner; +import java.util.*; import java.util.regex.Pattern; import org.apache.commons.lang3.math.NumberUtils; @@ -98,6 +94,30 @@ public static Map parseCookie(List cookieList){ private static int GRPC_DEBUG_COUNTER = 50; + private static final Set DEBUG_HOSTS_SET = initializeDebugHostsSet(); + + private static Set initializeDebugHostsSet() { + String debugHosts = System.getenv("DEBUG_HOSTS"); + if (debugHosts == null || debugHosts.isEmpty()) { + return new HashSet<>(); + } + return new HashSet<>(Arrays.asList(debugHosts.split(","))); + } + + public static String printDebugLog(HttpResponseParams httpResponseParams) { + if (DEBUG_HOSTS_SET.isEmpty()) return null; + Map> headers = httpResponseParams.getRequestParams().getHeaders(); + List hosts = headers.get("host"); + if (hosts == null || hosts.isEmpty()) return null; + String host = hosts.get(0); + + return printDebugLog(host) ? host : null; + } + + public static boolean printDebugLog(String host) { + return DEBUG_HOSTS_SET.contains(host); + } + public static HttpResponseParams parseKafkaMessage(String message) throws Exception { //convert java object to JSON format From 3d168f1c627c981d44798558387b426c209ba070 Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Thu, 10 Oct 2024 16:11:13 +0530 Subject: [PATCH 2/3] added debug logs for urls --- .../java/com/akto/parsers/HttpCallParser.java | 2 +- .../java/com/akto/runtime/APICatalogSync.java | 6 ++++++ .../src/main/java/com/akto/runtime/Main.java | 13 +++++++++---- .../akto/hybrid_parsers/HttpCallParser.java | 5 +---- .../akto/hybrid_runtime/APICatalogSync.java | 5 +++++ .../java/com/akto/hybrid_runtime/Main.java | 18 +++++++++--------- .../java/com/akto/runtime/utils/Utils.java | 19 +++++++++++++++---- 7 files changed, 46 insertions(+), 22 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java index 9b77062471..cdaede34da 100644 --- a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java +++ b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java @@ -628,7 +628,7 @@ public static boolean aggregate(List responses, Map urlSet= new HashSet<>(); for (HttpResponseParams responseParams: responses) { - String debugHost = Utils.printDebugLog(responseParams); + String debugHost = Utils.printDebugHostLog(responseParams); if (debugHost != null) { HttpRequestParams requestParams = responseParams.getRequestParams(); loggerMaker.infoAndAddToDb("Found host: " + debugHost + " in HttpCallParser.java for url: " + requestParams.getMethod() + " " + requestParams.getURL()); diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index eeb63d5646..f91caa05b1 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -29,6 +29,7 @@ import com.akto.dto.usage.MetricTypes; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; +import com.akto.runtime.utils.Utils; import com.akto.util.filter.DictionaryFilter; import com.akto.runtime.merge.MergeOnHostOnly; import com.akto.runtime.policies.AktoPolicyNew; @@ -1258,6 +1259,11 @@ public DbUpdateReturn getDBUpdatesForParams(APICatalog currentDelta, APICatalog for(String key: deltaInfoMap.keySet()) { SingleTypeInfo dbInfo = dbInfoMap.get(key); SingleTypeInfo deltaInfo = deltaInfoMap.get(key); + + if (Utils.printDebugUrlLog(deltaInfo.getUrl())) { + loggerMaker.infoAndAddToDb("Found debug url in getDBUpdatesForParams " + deltaInfo.getUrl()); + } + Bson update; int inc = deltaInfo.getCount() - (dbInfo == null ? 0 : dbInfo.getCount()); diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/Main.java b/apps/api-runtime/src/main/java/com/akto/runtime/Main.java index d02109f277..f3b2048566 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/Main.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/Main.java @@ -29,7 +29,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.kafka.clients.consumer.*; import org.apache.kafka.common.errors.WakeupException; -import org.apache.kafka.common.serialization.StringDeserializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.concurrent.atomic.AtomicBoolean; @@ -228,11 +227,17 @@ public void run() { } httpResponseParams = HttpCallParser.parseKafkaMessage(r.value()); - String debugHost = Utils.printDebugLog(httpResponseParams); + HttpRequestParams requestParams = httpResponseParams.getRequestParams(); + + String debugHost = Utils.printDebugHostLog(httpResponseParams); if (debugHost != null) { - HttpRequestParams requestParams = httpResponseParams.getRequestParams(); - loggerMaker.infoAndAddToDb("Found host: " + debugHost + " in Main.java for url: " + requestParams.getMethod() + " " + requestParams.getURL()); + loggerMaker.infoAndAddToDb("Found debug host: " + debugHost + " in url: " + requestParams.getMethod() + " " + requestParams.getURL()); + } + + if (Utils.printDebugUrlLog(requestParams.getURL())) { + loggerMaker.infoAndAddToDb("Found debug url in Main.java: " + requestParams.getURL()); } + } catch (Exception e) { loggerMaker.errorAndAddToDb(e, "Error while parsing kafka message " + e, LogDb.RUNTIME); continue; diff --git a/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java b/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java index 907138645c..a8981d50bb 100644 --- a/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java +++ b/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java @@ -1,7 +1,5 @@ package com.akto.hybrid_parsers; -import com.akto.dao.ApiCollectionsDao; -import com.akto.dao.billing.OrganizationsDao; import com.akto.dao.context.Context; import com.akto.dao.traffic_metrics.TrafficMetricsDao; import com.akto.hybrid_dependency.DependencyAnalyser; @@ -30,7 +28,6 @@ import com.mongodb.client.model.*; import okhttp3.*; import org.apache.commons.lang3.math.NumberUtils; -import org.bson.conversions.Bson; import com.alibaba.fastjson2.*; import java.io.IOException; @@ -547,7 +544,7 @@ public static boolean aggregate(List responses, Map urlSet= new HashSet<>(); for (HttpResponseParams responseParams: responses) { - String debugHost = Utils.printDebugLog(responseParams); + String debugHost = Utils.printDebugHostLog(responseParams); if (debugHost != null) { HttpRequestParams requestParams = responseParams.getRequestParams(); loggerMaker.infoAndAddToDb("Found host: " + debugHost + " in HttpCallParser.java for url: " + requestParams.getMethod() + " " + requestParams.getURL()); diff --git a/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java b/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java index c1ba13af1b..c862d43d7b 100644 --- a/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java +++ b/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java @@ -24,6 +24,7 @@ import com.akto.data_actor.DataActor; import com.akto.data_actor.DataActorFactory; import com.akto.hybrid_runtime.policies.AktoPolicyNew; +import com.akto.runtime.utils.Utils; import com.akto.util.filter.DictionaryFilter; import com.akto.types.CappedSet; import com.akto.util.JSONUtils; @@ -1412,6 +1413,10 @@ public DbUpdateReturnHybrid getDBUpdatesForParamsHybrid(APICatalog currentDelta, SingleTypeInfo dbInfo = dbInfoMap.get(key); SingleTypeInfo deltaInfo = deltaInfoMap.get(key); + if (Utils.printDebugUrlLog(deltaInfo.getUrl())) { + loggerMaker.infoAndAddToDb("Found debug url in getDBUpdatesForParamsHybrid " + deltaInfo.getUrl()); + } + int inc = deltaInfo.getCount() - (dbInfo == null ? 0 : dbInfo.getCount()); long lastSeenDiff = deltaInfo.getLastSeen() - (dbInfo == null ? 0 : dbInfo.getLastSeen()); boolean minMaxChanged = (dbInfo == null) || (dbInfo.getMinValue() != deltaInfo.getMinValue()) || (dbInfo.getMaxValue() != deltaInfo.getMaxValue()); diff --git a/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/Main.java b/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/Main.java index 1ce05503ed..c6569112ff 100644 --- a/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/Main.java +++ b/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/Main.java @@ -4,10 +4,8 @@ import java.util.*; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; -import com.akto.DaoInit; import com.akto.RuntimeMode; import com.akto.billing.UsageMetricUtils; import com.akto.dao.*; @@ -21,17 +19,13 @@ import com.akto.hybrid_parsers.HttpCallParser; import com.akto.data_actor.DataActor; import com.akto.data_actor.DataActorFactory; -import com.akto.database_abstractor_authenticator.JwtAuthenticator; import com.akto.util.DashboardMode; import com.akto.util.filter.DictionaryFilter; import com.google.gson.Gson; -import com.mongodb.ConnectionString; -import com.mongodb.client.model.Filters; import org.apache.commons.lang3.StringUtils; import org.apache.kafka.clients.consumer.*; import org.apache.kafka.common.errors.WakeupException; -import org.apache.kafka.common.serialization.StringDeserializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.concurrent.atomic.AtomicBoolean; @@ -234,11 +228,17 @@ public void run() { } httpResponseParams = HttpCallParser.parseKafkaMessage(r.value()); - String debugHost = Utils.printDebugLog(httpResponseParams); + HttpRequestParams requestParams = httpResponseParams.getRequestParams(); + + String debugHost = Utils.printDebugHostLog(httpResponseParams); if (debugHost != null) { - HttpRequestParams requestParams = httpResponseParams.getRequestParams(); - loggerMaker.infoAndAddToDb("Found host: " + debugHost + " in url: " + requestParams.getMethod() + " " + requestParams.getURL()); + loggerMaker.infoAndAddToDb("Found debug host: " + debugHost + " in url: " + requestParams.getMethod() + " " + requestParams.getURL()); + } + + if (Utils.printDebugUrlLog(requestParams.getURL())) { + loggerMaker.infoAndAddToDb("Found debug url in Main.java: " + requestParams.getURL()); } + } catch (Exception e) { loggerMaker.errorAndAddToDb(e, "Error while parsing kafka message " + e, LogDb.RUNTIME); continue; diff --git a/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java b/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java index 4b13396031..f41b006f6a 100644 --- a/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java +++ b/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java @@ -95,6 +95,7 @@ public static Map parseCookie(List cookieList){ private static int GRPC_DEBUG_COUNTER = 50; private static final Set DEBUG_HOSTS_SET = initializeDebugHostsSet(); + private static final Set DEBUG_URLS_SET = initializeDebugUrlsSet(); private static Set initializeDebugHostsSet() { String debugHosts = System.getenv("DEBUG_HOSTS"); @@ -104,20 +105,30 @@ private static Set initializeDebugHostsSet() { return new HashSet<>(Arrays.asList(debugHosts.split(","))); } - public static String printDebugLog(HttpResponseParams httpResponseParams) { + public static String printDebugHostLog(HttpResponseParams httpResponseParams) { if (DEBUG_HOSTS_SET.isEmpty()) return null; Map> headers = httpResponseParams.getRequestParams().getHeaders(); List hosts = headers.get("host"); if (hosts == null || hosts.isEmpty()) return null; String host = hosts.get(0); - return printDebugLog(host) ? host : null; + return DEBUG_HOSTS_SET.contains(host) ? host : null; } - public static boolean printDebugLog(String host) { - return DEBUG_HOSTS_SET.contains(host); + private static Set initializeDebugUrlsSet() { + String debugUrls = System.getenv("DEBUG_URLS"); + if (debugUrls == null || debugUrls.isEmpty()) { + return new HashSet<>(); + } + return new HashSet<>(Arrays.asList(debugUrls.split(","))); } + public static boolean printDebugUrlLog(String url) { + if (DEBUG_URLS_SET.isEmpty()) return false; + return DEBUG_HOSTS_SET.contains(url); + } + + public static HttpResponseParams parseKafkaMessage(String message) throws Exception { //convert java object to JSON format From 108d10489e352462a19d4ae37291e4b316b94ee3 Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Thu, 10 Oct 2024 16:29:05 +0530 Subject: [PATCH 3/3] fixed logs --- .../src/main/java/com/akto/parsers/HttpCallParser.java | 2 +- apps/api-runtime/src/main/java/com/akto/runtime/Main.java | 4 ++-- .../src/main/java/com/akto/hybrid_parsers/HttpCallParser.java | 2 +- .../src/main/java/com/akto/hybrid_runtime/Main.java | 4 ++-- libs/utils/src/main/java/com/akto/runtime/utils/Utils.java | 3 +++ 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java index cdaede34da..e28512e60c 100644 --- a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java +++ b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java @@ -631,7 +631,7 @@ public static boolean aggregate(List responses, Map responses, Map initializeDebugHostsSet() { public static String printDebugHostLog(HttpResponseParams httpResponseParams) { if (DEBUG_HOSTS_SET.isEmpty()) return null; + if (httpResponseParams == null || httpResponseParams.requestParams == null || httpResponseParams.requestParams.getHeaders() == null) { + return null; + } Map> headers = httpResponseParams.getRequestParams().getHeaders(); List hosts = headers.get("host"); if (hosts == null || hosts.isEmpty()) return null;