From c2b21d18fc8d8dc537bed181d09ac84e29023304 Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Wed, 4 Sep 2024 22:57:08 -0700 Subject: [PATCH 01/16] dry run for removing bad APIs --- .../java/com/akto/parsers/HttpCallParser.java | 81 +-------- .../java/com/akto/parsers/TestDBSync.java | 11 +- .../akto/listener/InitializerListener.java | 167 +++++++++++++++++- .../java/com/akto/runtime/RuntimeUtil.java | 104 +++++++++++ 4 files changed, 277 insertions(+), 86 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java index 2746261226..7e32d4e5d6 100644 --- a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java +++ b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java @@ -394,53 +394,6 @@ public void incTrafficMetrics(TrafficMetrics.Key key, int value) { trafficMetrics.inc(value); } - public static final String CONTENT_TYPE = "CONTENT-TYPE"; - - public boolean isRedundantEndpoint(String url, List discardedUrlList){ - StringJoiner joiner = new StringJoiner("|", ".*\\.(", ")(\\?.*)?"); - for (String extension : discardedUrlList) { - if(extension.startsWith(CONTENT_TYPE)){ - continue; - } - joiner.add(extension); - } - String regex = joiner.toString(); - - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(url); - return matcher.matches(); - } - - private boolean isInvalidContentType(String contentType){ - boolean res = false; - if(contentType == null || contentType.length() == 0) return res; - - res = contentType.contains("javascript") || contentType.contains("png"); - return res; - } - - private boolean isBlankResponseBodyForGET(String method, String contentType, String matchContentType, - String responseBody) { - boolean res = true; - if (contentType == null || contentType.length() == 0) - return false; - res &= contentType.contains(matchContentType); - res &= "GET".equals(method.toUpperCase()); - - /* - * To be sure that the content type - * header matches the actual payload. - * - * We will need to add more type validation as needed. - */ - if (matchContentType.contains("html")) { - res &= responseBody.startsWith("<") && responseBody.endsWith(">"); - } else { - res &= false; - } - return res; - } - public List filterHttpResponseParams(List httpResponseParamsList, AccountSettings accountSettings) { List filteredResponseParams = new ArrayList<>(); int originalSize = httpResponseParamsList.size(); @@ -463,41 +416,9 @@ public List filterHttpResponseParams(List contentTypeList = (List) httpResponseParam.getRequestParams().getHeaders().getOrDefault("content-type", new ArrayList<>()); - String contentType = null; - if(!contentTypeList.isEmpty()){ - contentType = contentTypeList.get(0); - } - if(isInvalidContentType(contentType)){ + if (RuntimeUtil.shouldIgnore(httpResponseParam, accountSettings.getAllowRedundantEndpointsList())) { continue; } - - try { - List responseContentTypeList = (List) httpResponseParam.getHeaders().getOrDefault("content-type", new ArrayList<>()); - String allContentTypes = responseContentTypeList.toString(); - String method = httpResponseParam.getRequestParams().getMethod(); - String responseBody = httpResponseParam.getPayload(); - boolean ignore = false; - for (String extension : accountSettings.getAllowRedundantEndpointsList()) { - if(extension.startsWith(CONTENT_TYPE)){ - String matchContentType = extension.split(" ")[1]; - if(isBlankResponseBodyForGET(method, allContentTypes, matchContentType, responseBody)){ - ignore = true; - break; - } - } - } - if(ignore){ - continue; - } - - } catch(Exception e){ - loggerMaker.errorAndAddToDb(e, "Error while ignoring content-type redundant samples " + e.toString(), LogDb.RUNTIME); - } - } Map> reqHeaders = httpResponseParam.getRequestParams().getHeaders(); diff --git a/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java b/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java index 416e766186..98ac0ec477 100644 --- a/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java +++ b/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java @@ -24,6 +24,7 @@ import com.akto.dto.HttpResponseParams.Source; import com.akto.runtime.APICatalogSync; import com.akto.runtime.Main; +import com.akto.runtime.RuntimeUtil; import com.akto.runtime.URLAggregator; import com.mongodb.BasicDBObject; import com.mongodb.client.model.Filters; @@ -589,10 +590,10 @@ public void testRedundantUrlCheck() { String url5 = "test5.js.js"; List allowedUrlType = Arrays.asList("js"); - assertEquals(httpCallParser.isRedundantEndpoint(url1, allowedUrlType), true); - assertEquals(httpCallParser.isRedundantEndpoint(url2, allowedUrlType), false); - assertEquals(httpCallParser.isRedundantEndpoint(url3, allowedUrlType), false); - assertEquals(httpCallParser.isRedundantEndpoint(url4, allowedUrlType), true); - assertEquals(httpCallParser.isRedundantEndpoint(url5, allowedUrlType), true); + assertEquals(RuntimeUtil.isRedundantEndpoint(url1, allowedUrlType), true); + assertEquals(RuntimeUtil.isRedundantEndpoint(url2, allowedUrlType), false); + assertEquals(RuntimeUtil.isRedundantEndpoint(url3, allowedUrlType), false); + assertEquals(RuntimeUtil.isRedundantEndpoint(url4, allowedUrlType), true); + assertEquals(RuntimeUtil.isRedundantEndpoint(url5, allowedUrlType), true); } } diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 11068a696f..70e9deb778 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -620,6 +620,152 @@ private static CustomDataType getCustomDataTypeFromPiiType(PIISource piiSource, return ret; } + private void setupBadApisRemover() { + scheduler.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + Integer[] accountIds = new Integer[]{1724877069, 1714700875}; + + for (int accountId: accountIds) { + Account account = AccountsDao.instance.findOne(eq("_id", accountId)); + if (account == null) { + continue; + } + + try { + Context.accountId.set(accountId); + List apiCollections = ApiCollectionsDao.instance.getMetaAll(); + Map apiCollectionMap = apiCollections.stream().collect(Collectors.toMap(ApiCollection::getId, Function.identity())); + + List sampleDataList = new ArrayList<>(); + Bson filters = Filters.empty(); + int skip = 0; + int limit = 100; + Bson sort = Sorts.ascending("_id.apiCollectionId", "_id.url", "_id.method"); + AccountSettings accountSettings = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter()); + + do { + sampleDataList = SampleDataDao.instance.findAll(filters, skip, limit, sort); + skip += limit; + List toBeDeleted = new ArrayList<>(); + for(SampleData sampleData: sampleDataList) { + try { + List samples = sampleData.getSamples(); + if (samples == null || samples.isEmpty()) { + logger.info("[BadApisRemover] No samples found for : " + sampleData.getId()); + continue; + } + + ApiCollection apiCollection = apiCollectionMap.get(sampleData.getId().getApiCollectionId()); + if (apiCollection == null) { + logger.info("[BadApisRemover] No apiCollection found for : " + sampleData.getId()); + continue; + } + + + boolean allMatchDefault = true; + + for (String sample : samples) { + HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); + if(accountSettings != null && accountSettings.getAllowRedundantEndpointsList() != null){ + if (!RuntimeUtil.shouldIgnore(httpResponseParams, accountSettings.getAllowRedundantEndpointsList())) { + allMatchDefault = false; + break; + } + } + } + + if (allMatchDefault) { + logger.info("[BadApisRemover] Deleting bad API: " + toBeDeleted, LogDb.DASHBOARD); + toBeDeleted.add(sampleData.getId()); + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb("[BadApisRemover] Couldn't delete an api for default payload: " + sampleData.getId() + e.getMessage(), LogDb.DASHBOARD); + } + } + + // deleteApis(toBeDeleted); + + } while (!sampleDataList.isEmpty()); + + } catch (Exception e) { + loggerMaker.errorAndAddToDb("Couldn't complete scan for APIs remover: " + e.getMessage(), LogDb.DASHBOARD); + } + + } + + AccountTask.instance.executeTask(new Consumer() { + @Override + public void accept(Account account) { + AccountSettings accountSettings = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter()); + Map defaultPayloadMap = accountSettings.getDefaultPayloads(); + if (defaultPayloadMap == null || defaultPayloadMap.isEmpty()) { + return; + } + + List apiCollections = ApiCollectionsDao.instance.getMetaAll(); + Map apiCollectionMap = apiCollections.stream().collect(Collectors.toMap(ApiCollection::getId, Function.identity())); + + for(Map.Entry entry: defaultPayloadMap.entrySet()) { + DefaultPayload dp = entry.getValue(); + String base64Url = entry.getKey(); + if (!dp.getScannedExistingData()) { + try { + List sampleDataList = new ArrayList<>(); + Bson filters = Filters.empty(); + int skip = 0; + int limit = 100; + Bson sort = Sorts.ascending("_id.apiCollectionId", "_id.url", "_id.method"); + do { + sampleDataList = SampleDataDao.instance.findAll(filters, skip, limit, sort); + skip += limit; + List toBeDeleted = new ArrayList<>(); + for(SampleData sampleData: sampleDataList) { + try { + List samples = sampleData.getSamples(); + ApiCollection apiCollection = apiCollectionMap.get(sampleData.getId().getApiCollectionId()); + if (apiCollection == null) continue; + if (apiCollection.getHostName() != null && !dp.getId().equalsIgnoreCase(apiCollection.getHostName())) continue; + if (samples == null || samples.isEmpty()) continue; + + boolean allMatchDefault = true; + + for (String sample : samples) { + HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); + if (!matchesDefaultPayload(httpResponseParams, defaultPayloadMap)) { + allMatchDefault = false; + break; + } + } + + if (allMatchDefault) { + loggerMaker.errorAndAddToDb("Deleting API that matches default payload: " + toBeDeleted, LogDb.DASHBOARD); + toBeDeleted.add(sampleData.getId()); + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb("Couldn't delete an api for default payload: " + sampleData.getId() + e.getMessage(), LogDb.DASHBOARD); + } + } + + deleteApis(toBeDeleted); + + } while (!sampleDataList.isEmpty()); + + Bson completedScan = Updates.set(AccountSettings.DEFAULT_PAYLOADS+"."+base64Url+"."+DefaultPayload.SCANNED_EXISTING_DATA, true); + AccountSettingsDao.instance.updateOne(AccountSettingsDao.generateFilter(), completedScan); + + } catch (Exception e) { + + loggerMaker.errorAndAddToDb("Couldn't complete scan for default payload: " + e.getMessage(), LogDb.DASHBOARD); + } + } + } + } + }, "setUpDefaultPayloadRemover"); + } + }, 0, 5, TimeUnit.MINUTES); + } + private void setUpDefaultPayloadRemover() { scheduler.scheduleAtFixedRate(new Runnable() { @Override @@ -700,15 +846,34 @@ private void deleteApisPerDao(List toBeDeleted, AccountsContextDao d if (toBeDeleted == null || toBeDeleted.isEmpty()) return; List> stiList = new ArrayList<>(); + int counter = 0; for(Key key: toBeDeleted) { stiList.add(new DeleteManyModel<>(Filters.and( Filters.eq(prefix + "apiCollectionId", key.getApiCollectionId()), Filters.eq(prefix + "method", key.getMethod()), Filters.eq(prefix + "url", key.getUrl()) ))); + counter ++; + + if (counter%20 == 0) { + try { + dao.bulkWrite(stiList, new BulkWriteOptions().ordered(false)); + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e, "Error while deleting apis: " + e.getMessage(), LogDb.DASHBOARD); + } + + stiList = new ArrayList<>(); + } + } + + if (!stiList.isEmpty()) { + try { + dao.bulkWrite(stiList, new BulkWriteOptions().ordered(false)); + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e, "Error while deleting apis: " + e.getMessage(), LogDb.DASHBOARD); + } } - dao.bulkWrite(stiList, new BulkWriteOptions().ordered(false)); } private void deleteApis(List toBeDeleted) { diff --git a/libs/utils/src/main/java/com/akto/runtime/RuntimeUtil.java b/libs/utils/src/main/java/com/akto/runtime/RuntimeUtil.java index b451aa4fa1..13b23a56b1 100644 --- a/libs/utils/src/main/java/com/akto/runtime/RuntimeUtil.java +++ b/libs/utils/src/main/java/com/akto/runtime/RuntimeUtil.java @@ -3,15 +3,21 @@ import com.akto.dto.HttpResponseParams; import com.akto.dto.settings.DefaultPayload; import com.akto.log.LoggerMaker; +import com.akto.log.LoggerMaker.LogDb; import java.net.URL; import java.util.ArrayList; import java.util.Base64; import java.util.List; import java.util.Map; +import java.util.StringJoiner; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class RuntimeUtil { private static final LoggerMaker loggerMaker = new LoggerMaker(RuntimeUtil.class); + public static final String CONTENT_TYPE = "CONTENT-TYPE"; + public static boolean matchesDefaultPayload(HttpResponseParams httpResponseParams, Map defaultPayloadMap) { try { Map> reqHeaders = httpResponseParams.getRequestParams().getHeaders(); @@ -40,6 +46,18 @@ public static boolean matchesDefaultPayload(HttpResponseParams httpResponseParam return false; } + public static boolean looksLikeAttack(HttpResponseParams httpResponseParams) { + try { + Map> reqHeaders = httpResponseParams.getRequestParams().getHeaders(); + List host = reqHeaders.getOrDefault("host", new ArrayList<>()); + + } catch (Exception e) { + loggerMaker.errorAndAddToDb("Error while filtering default payloads: " + e.getMessage(), LoggerMaker.LogDb.RUNTIME); + } + + return false; + } + public static boolean hasSpecialCharacters(String input) { // Define the special characters String specialCharacters = "<>%/?#[]@!$&'()*+,;="; @@ -51,4 +69,90 @@ public static boolean hasSpecialCharacters(String input) { return false; } + + public static boolean isRedundantEndpoint(String url, List discardedUrlList){ + StringJoiner joiner = new StringJoiner("|", ".*\\.(", ")(\\?.*)?"); + for (String extension : discardedUrlList) { + if(extension.startsWith(RuntimeUtil.CONTENT_TYPE)){ + continue; + } + joiner.add(extension); + } + String regex = joiner.toString(); + + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(url); + return matcher.matches(); + } + + private static boolean isInvalidContentType(String contentType){ + boolean res = false; + if(contentType == null || contentType.length() == 0) return res; + + res = contentType.contains("javascript") || contentType.contains("png"); + return res; + } + + private static boolean isBlankResponseBodyForGET(String method, String contentType, String matchContentType, + String responseBody) { + boolean res = true; + if (contentType == null || contentType.length() == 0) + return false; + res &= contentType.contains(matchContentType); + res &= "GET".equals(method.toUpperCase()); + + /* + * To be sure that the content type + * header matches the actual payload. + * + * We will need to add more type validation as needed. + */ + if (matchContentType.contains("html")) { + res &= responseBody.startsWith("<") && responseBody.endsWith(">"); + } else { + res &= false; + } + return res; + } + + + public static boolean shouldIgnore(HttpResponseParams httpResponseParam, List allowedRedundantList) { + if(isRedundantEndpoint(httpResponseParam.getRequestParams().getURL(), allowedRedundantList)){ + return true; + } + List contentTypeList = (List) httpResponseParam.getRequestParams().getHeaders().getOrDefault("content-type", new ArrayList<>()); + String contentType = null; + if(!contentTypeList.isEmpty()){ + contentType = contentTypeList.get(0); + } + if(isInvalidContentType(contentType)){ + return true; + } + + try { + List responseContentTypeList = (List) httpResponseParam.getHeaders().getOrDefault("content-type", new ArrayList<>()); + String allContentTypes = responseContentTypeList.toString(); + String method = httpResponseParam.getRequestParams().getMethod(); + String responseBody = httpResponseParam.getPayload(); + boolean ignore = false; + for (String extension : allowedRedundantList) { + if(extension.startsWith(CONTENT_TYPE)){ + String matchContentType = extension.split(" ")[1]; + if(isBlankResponseBodyForGET(method, allContentTypes, matchContentType, responseBody)){ + ignore = true; + break; + } + } + } + if(ignore){ + return true; + } + + } catch(Exception e){ + loggerMaker.errorAndAddToDb(e, "Error while ignoring content-type redundant samples " + e.toString(), LogDb.RUNTIME); + } + + return false; + + } } From b23b6613b8f7f6a5ee7dc6d9d7c64c34a852a9bf Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Wed, 4 Sep 2024 23:15:13 -0700 Subject: [PATCH 02/16] call the remover function --- .../akto/listener/InitializerListener.java | 184 ++++++------------ 1 file changed, 55 insertions(+), 129 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 70e9deb778..7dae4b2b87 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -621,149 +621,75 @@ private static CustomDataType getCustomDataTypeFromPiiType(PIISource piiSource, } private void setupBadApisRemover() { - scheduler.scheduleAtFixedRate(new Runnable() { - @Override - public void run() { - Integer[] accountIds = new Integer[]{1724877069, 1714700875}; + Integer[] accountIds = new Integer[]{1724877069, 1714700875}; - for (int accountId: accountIds) { - Account account = AccountsDao.instance.findOne(eq("_id", accountId)); - if (account == null) { - continue; - } - - try { - Context.accountId.set(accountId); - List apiCollections = ApiCollectionsDao.instance.getMetaAll(); - Map apiCollectionMap = apiCollections.stream().collect(Collectors.toMap(ApiCollection::getId, Function.identity())); - - List sampleDataList = new ArrayList<>(); - Bson filters = Filters.empty(); - int skip = 0; - int limit = 100; - Bson sort = Sorts.ascending("_id.apiCollectionId", "_id.url", "_id.method"); - AccountSettings accountSettings = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter()); + for (int accountId: accountIds) { + Account account = AccountsDao.instance.findOne(eq("_id", accountId)); + if (account == null) { + continue; + } - do { - sampleDataList = SampleDataDao.instance.findAll(filters, skip, limit, sort); - skip += limit; - List toBeDeleted = new ArrayList<>(); - for(SampleData sampleData: sampleDataList) { - try { - List samples = sampleData.getSamples(); - if (samples == null || samples.isEmpty()) { - logger.info("[BadApisRemover] No samples found for : " + sampleData.getId()); - continue; - } + try { + Context.accountId.set(accountId); + List apiCollections = ApiCollectionsDao.instance.getMetaAll(); + Map apiCollectionMap = apiCollections.stream().collect(Collectors.toMap(ApiCollection::getId, Function.identity())); + + List sampleDataList = new ArrayList<>(); + Bson filters = Filters.empty(); + int skip = 0; + int limit = 100; + Bson sort = Sorts.ascending("_id.apiCollectionId", "_id.url", "_id.method"); + AccountSettings accountSettings = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter()); - ApiCollection apiCollection = apiCollectionMap.get(sampleData.getId().getApiCollectionId()); - if (apiCollection == null) { - logger.info("[BadApisRemover] No apiCollection found for : " + sampleData.getId()); - continue; - } + do { + sampleDataList = SampleDataDao.instance.findAll(filters, skip, limit, sort); + skip += limit; + List toBeDeleted = new ArrayList<>(); + for(SampleData sampleData: sampleDataList) { + try { + List samples = sampleData.getSamples(); + if (samples == null || samples.isEmpty()) { + logger.info("[BadApisRemover] No samples found for : " + sampleData.getId()); + continue; + } - - boolean allMatchDefault = true; + ApiCollection apiCollection = apiCollectionMap.get(sampleData.getId().getApiCollectionId()); + if (apiCollection == null) { + logger.info("[BadApisRemover] No apiCollection found for : " + sampleData.getId()); + continue; + } - for (String sample : samples) { - HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); - if(accountSettings != null && accountSettings.getAllowRedundantEndpointsList() != null){ - if (!RuntimeUtil.shouldIgnore(httpResponseParams, accountSettings.getAllowRedundantEndpointsList())) { - allMatchDefault = false; - break; - } - } - } + + boolean allMatchDefault = true; - if (allMatchDefault) { - logger.info("[BadApisRemover] Deleting bad API: " + toBeDeleted, LogDb.DASHBOARD); - toBeDeleted.add(sampleData.getId()); + for (String sample : samples) { + HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); + if(accountSettings != null && accountSettings.getAllowRedundantEndpointsList() != null){ + if (!RuntimeUtil.shouldIgnore(httpResponseParams, accountSettings.getAllowRedundantEndpointsList())) { + allMatchDefault = false; + break; } - } catch (Exception e) { - loggerMaker.errorAndAddToDb("[BadApisRemover] Couldn't delete an api for default payload: " + sampleData.getId() + e.getMessage(), LogDb.DASHBOARD); } } - // deleteApis(toBeDeleted); - - } while (!sampleDataList.isEmpty()); - - } catch (Exception e) { - loggerMaker.errorAndAddToDb("Couldn't complete scan for APIs remover: " + e.getMessage(), LogDb.DASHBOARD); - } - - } - - AccountTask.instance.executeTask(new Consumer() { - @Override - public void accept(Account account) { - AccountSettings accountSettings = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter()); - Map defaultPayloadMap = accountSettings.getDefaultPayloads(); - if (defaultPayloadMap == null || defaultPayloadMap.isEmpty()) { - return; + if (allMatchDefault) { + logger.info("[BadApisRemover] Deleting bad API: " + toBeDeleted, LogDb.DASHBOARD); + toBeDeleted.add(sampleData.getId()); + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb("[BadApisRemover] Couldn't delete an api for default payload: " + sampleData.getId() + e.getMessage(), LogDb.DASHBOARD); } + } - List apiCollections = ApiCollectionsDao.instance.getMetaAll(); - Map apiCollectionMap = apiCollections.stream().collect(Collectors.toMap(ApiCollection::getId, Function.identity())); - - for(Map.Entry entry: defaultPayloadMap.entrySet()) { - DefaultPayload dp = entry.getValue(); - String base64Url = entry.getKey(); - if (!dp.getScannedExistingData()) { - try { - List sampleDataList = new ArrayList<>(); - Bson filters = Filters.empty(); - int skip = 0; - int limit = 100; - Bson sort = Sorts.ascending("_id.apiCollectionId", "_id.url", "_id.method"); - do { - sampleDataList = SampleDataDao.instance.findAll(filters, skip, limit, sort); - skip += limit; - List toBeDeleted = new ArrayList<>(); - for(SampleData sampleData: sampleDataList) { - try { - List samples = sampleData.getSamples(); - ApiCollection apiCollection = apiCollectionMap.get(sampleData.getId().getApiCollectionId()); - if (apiCollection == null) continue; - if (apiCollection.getHostName() != null && !dp.getId().equalsIgnoreCase(apiCollection.getHostName())) continue; - if (samples == null || samples.isEmpty()) continue; - - boolean allMatchDefault = true; - - for (String sample : samples) { - HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); - if (!matchesDefaultPayload(httpResponseParams, defaultPayloadMap)) { - allMatchDefault = false; - break; - } - } - - if (allMatchDefault) { - loggerMaker.errorAndAddToDb("Deleting API that matches default payload: " + toBeDeleted, LogDb.DASHBOARD); - toBeDeleted.add(sampleData.getId()); - } - } catch (Exception e) { - loggerMaker.errorAndAddToDb("Couldn't delete an api for default payload: " + sampleData.getId() + e.getMessage(), LogDb.DASHBOARD); - } - } - - deleteApis(toBeDeleted); - - } while (!sampleDataList.isEmpty()); + // deleteApis(toBeDeleted); - Bson completedScan = Updates.set(AccountSettings.DEFAULT_PAYLOADS+"."+base64Url+"."+DefaultPayload.SCANNED_EXISTING_DATA, true); - AccountSettingsDao.instance.updateOne(AccountSettingsDao.generateFilter(), completedScan); + } while (!sampleDataList.isEmpty()); - } catch (Exception e) { - - loggerMaker.errorAndAddToDb("Couldn't complete scan for default payload: " + e.getMessage(), LogDb.DASHBOARD); - } - } - } - } - }, "setUpDefaultPayloadRemover"); + } catch (Exception e) { + loggerMaker.errorAndAddToDb("Couldn't complete scan for APIs remover: " + e.getMessage(), LogDb.DASHBOARD); } - }, 0, 5, TimeUnit.MINUTES); + + } } private void setUpDefaultPayloadRemover() { @@ -2121,7 +2047,7 @@ public void accept(Account account) { runInitializerFunctions(); } }, "context-initializer-secondary"); - + setupBadApisRemover(); crons.trafficAlertsScheduler(); if (DashboardMode.isMetered()) { setupUsageScheduler(); From 9d715da85e33495f5bf093bcfebdf1f09836c2b0 Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Thu, 5 Sep 2024 00:00:34 -0700 Subject: [PATCH 03/16] move up --- .../src/main/java/com/akto/listener/InitializerListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 7dae4b2b87..74600671ea 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -2040,6 +2040,7 @@ public void accept(Account account) { if (runJobFunctions || runJobFunctionsAnyway) { logger.info("Starting init functions and scheduling jobs at " + now); + setupBadApisRemover(); AccountTask.instance.executeTask(new Consumer() { @Override @@ -2047,7 +2048,6 @@ public void accept(Account account) { runInitializerFunctions(); } }, "context-initializer-secondary"); - setupBadApisRemover(); crons.trafficAlertsScheduler(); if (DashboardMode.isMetered()) { setupUsageScheduler(); From eb856f66300b4edbedac415c3d26cfb7dce62d0b Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Thu, 5 Sep 2024 00:13:27 -0700 Subject: [PATCH 04/16] skip tets --- .github/workflows/staging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 3f0d8a8703..854af3bc60 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -45,7 +45,7 @@ jobs: - name: Deploy polaris site to S3 bucket run: aws s3 sync ./apps/dashboard/web/polaris_web/web/dist s3://dashboard-on-cdn/polaris_web/${{steps.docker_tag.outputs.IMAGE_TAG}}/dist --delete - - run: mvn package -Dakto-image-tag=${{ github.event.inputs.Tag }} -Dakto-build-time=$(eval "date +%s") -Dakto-release-version=${{steps.docker_tag.outputs.IMAGE_TAG}} + - run: mvn package -Dakto-image-tag=${{ github.event.inputs.Tag }} -Dakto-build-time=$(eval "date +%s") -Dakto-release-version=${{steps.docker_tag.outputs.IMAGE_TAG}} -DskipTests=true - name: DockerHub login env: DOCKER_USERNAME: ${{secrets.DOCKER_USERNAME}} From 00c59e5dd7eb28e8731fe7d25cab8034fa126efb Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Thu, 5 Sep 2024 00:44:10 -0700 Subject: [PATCH 05/16] print all urls --- .../src/main/java/com/akto/listener/InitializerListener.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 74600671ea..dde6552fe3 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -673,8 +673,10 @@ private void setupBadApisRemover() { } if (allMatchDefault) { - logger.info("[BadApisRemover] Deleting bad API: " + toBeDeleted, LogDb.DASHBOARD); toBeDeleted.add(sampleData.getId()); + logger.info("[BadApisRemover] Deleting bad API: " + sampleData.getId(), LogDb.DASHBOARD); + } else { + logger.info("[BadApisRemover] Keeping bad API: " + sampleData.getId(), LogDb.DASHBOARD); } } catch (Exception e) { loggerMaker.errorAndAddToDb("[BadApisRemover] Couldn't delete an api for default payload: " + sampleData.getId() + e.getMessage(), LogDb.DASHBOARD); From 0277e7dac19e1d8913adcbd5877ed8fced472508 Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Thu, 5 Sep 2024 01:02:21 -0700 Subject: [PATCH 06/16] remove html check --- libs/utils/src/main/java/com/akto/runtime/RuntimeUtil.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/libs/utils/src/main/java/com/akto/runtime/RuntimeUtil.java b/libs/utils/src/main/java/com/akto/runtime/RuntimeUtil.java index 13b23a56b1..60ab29345f 100644 --- a/libs/utils/src/main/java/com/akto/runtime/RuntimeUtil.java +++ b/libs/utils/src/main/java/com/akto/runtime/RuntimeUtil.java @@ -107,11 +107,6 @@ private static boolean isBlankResponseBodyForGET(String method, String contentTy * * We will need to add more type validation as needed. */ - if (matchContentType.contains("html")) { - res &= responseBody.startsWith("<") && responseBody.endsWith(">"); - } else { - res &= false; - } return res; } From 544ca60dbac3909395700ad20e5812c0764c207c Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Thu, 5 Sep 2024 02:34:57 -0700 Subject: [PATCH 07/16] remove html check --- .../main/java/com/akto/listener/InitializerListener.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index dde6552fe3..c1c6ca97ca 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -661,9 +661,10 @@ private void setupBadApisRemover() { boolean allMatchDefault = true; - + boolean isNetsparkerPresent = false; for (String sample : samples) { HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); + isNetsparkerPresent |= sample.toLowerCase().contains("netsparker"); if(accountSettings != null && accountSettings.getAllowRedundantEndpointsList() != null){ if (!RuntimeUtil.shouldIgnore(httpResponseParams, accountSettings.getAllowRedundantEndpointsList())) { allMatchDefault = false; @@ -674,9 +675,10 @@ private void setupBadApisRemover() { if (allMatchDefault) { toBeDeleted.add(sampleData.getId()); - logger.info("[BadApisRemover] Deleting bad API: " + sampleData.getId(), LogDb.DASHBOARD); + + logger.info("[BadApisRemover] " + isNetsparkerPresent + " Deleting bad API: " + sampleData.getId(), LogDb.DASHBOARD); } else { - logger.info("[BadApisRemover] Keeping bad API: " + sampleData.getId(), LogDb.DASHBOARD); + logger.info("[BadApisRemover] " + isNetsparkerPresent + " Keeping bad API: " + sampleData.getId(), LogDb.DASHBOARD); } } catch (Exception e) { loggerMaker.errorAndAddToDb("[BadApisRemover] Couldn't delete an api for default payload: " + sampleData.getId() + e.getMessage(), LogDb.DASHBOARD); From d963191f4df33d95cd55d958ec0916dc3257bc03 Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Thu, 5 Sep 2024 03:10:46 -0700 Subject: [PATCH 08/16] remove redundant api --- .../java/com/akto/listener/InitializerListener.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index c1c6ca97ca..03acfcedf6 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -633,7 +633,8 @@ private void setupBadApisRemover() { Context.accountId.set(accountId); List apiCollections = ApiCollectionsDao.instance.getMetaAll(); Map apiCollectionMap = apiCollections.stream().collect(Collectors.toMap(ApiCollection::getId, Function.identity())); - + String filePath = "./samples_"+accountId+".txt"; + List sampleDataList = new ArrayList<>(); Bson filters = Filters.empty(); int skip = 0; @@ -673,7 +674,11 @@ private void setupBadApisRemover() { } } - if (allMatchDefault) { + if (allMatchDefault) { + try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(filePath)))) { + writer.write(sampleData.toString()); + } + toBeDeleted.add(sampleData.getId()); logger.info("[BadApisRemover] " + isNetsparkerPresent + " Deleting bad API: " + sampleData.getId(), LogDb.DASHBOARD); @@ -685,7 +690,9 @@ private void setupBadApisRemover() { } } - // deleteApis(toBeDeleted); + if (System.getenv("DELETE_REDUNDANT_APIS").equalsIgnoreCase("true")) { + deleteApis(toBeDeleted); + } } while (!sampleDataList.isEmpty()); From 525b5bc2971726199485616c60313731ce4c72e6 Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Thu, 5 Sep 2024 03:14:02 -0700 Subject: [PATCH 09/16] open file once --- .../java/com/akto/listener/InitializerListener.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 03acfcedf6..7da7b790b3 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -634,7 +634,7 @@ private void setupBadApisRemover() { List apiCollections = ApiCollectionsDao.instance.getMetaAll(); Map apiCollectionMap = apiCollections.stream().collect(Collectors.toMap(ApiCollection::getId, Function.identity())); String filePath = "./samples_"+accountId+".txt"; - + BufferedWriter writer = new BufferedWriter(new FileWriter(new File(filePath))); List sampleDataList = new ArrayList<>(); Bson filters = Filters.empty(); int skip = 0; @@ -675,12 +675,8 @@ private void setupBadApisRemover() { } if (allMatchDefault) { - try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(filePath)))) { - writer.write(sampleData.toString()); - } - - toBeDeleted.add(sampleData.getId()); - + writer.write(sampleData.toString()); + toBeDeleted.add(sampleData.getId()); logger.info("[BadApisRemover] " + isNetsparkerPresent + " Deleting bad API: " + sampleData.getId(), LogDb.DASHBOARD); } else { logger.info("[BadApisRemover] " + isNetsparkerPresent + " Keeping bad API: " + sampleData.getId(), LogDb.DASHBOARD); @@ -696,6 +692,9 @@ private void setupBadApisRemover() { } while (!sampleDataList.isEmpty()); + writer.flush(); + writer.close(); + } catch (Exception e) { loggerMaker.errorAndAddToDb("Couldn't complete scan for APIs remover: " + e.getMessage(), LogDb.DASHBOARD); } From 7fa64376f60a5671ead429a8e1afccd60c445340 Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Thu, 5 Sep 2024 04:34:30 -0700 Subject: [PATCH 10/16] check null for env var --- .../src/main/java/com/akto/listener/InitializerListener.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 7da7b790b3..9e194497aa 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -686,7 +686,8 @@ private void setupBadApisRemover() { } } - if (System.getenv("DELETE_REDUNDANT_APIS").equalsIgnoreCase("true")) { + String shouldDelete = System.getenv("DELETE_REDUNDANT_APIS"); + if ( shouldDelete != null && shouldDelete.equalsIgnoreCase("true")) { deleteApis(toBeDeleted); } @@ -697,6 +698,7 @@ private void setupBadApisRemover() { } catch (Exception e) { loggerMaker.errorAndAddToDb("Couldn't complete scan for APIs remover: " + e.getMessage(), LogDb.DASHBOARD); + e.printStackTrace(); } } From 90264d13b999a66b149190f4b0125c7ae07c136c Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Thu, 5 Sep 2024 12:08:02 -0700 Subject: [PATCH 11/16] remove OPTIONS and no-sample APIs --- .../akto/listener/InitializerListener.java | 151 ++++++++++++------ 1 file changed, 102 insertions(+), 49 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 9e194497aa..ad5bbd135f 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -58,6 +58,7 @@ import com.akto.dto.type.SingleTypeInfo; import com.akto.dto.upload.FileUpload; import com.akto.dto.type.URLMethods; +import com.akto.dto.type.URLMethods.Method; import com.akto.dto.usage.UsageMetric; import com.akto.log.CacheLoggerMaker; import com.akto.log.LoggerMaker; @@ -630,59 +631,50 @@ private void setupBadApisRemover() { } try { + logger.info("Starting setupBadApisRemover for account: " + accountId, LogDb.DASHBOARD); Context.accountId.set(accountId); List apiCollections = ApiCollectionsDao.instance.getMetaAll(); - Map apiCollectionMap = apiCollections.stream().collect(Collectors.toMap(ApiCollection::getId, Function.identity())); - String filePath = "./samples_"+accountId+".txt"; - BufferedWriter writer = new BufferedWriter(new FileWriter(new File(filePath))); - List sampleDataList = new ArrayList<>(); - Bson filters = Filters.empty(); - int skip = 0; - int limit = 100; - Bson sort = Sorts.ascending("_id.apiCollectionId", "_id.url", "_id.method"); - AccountSettings accountSettings = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter()); - do { - sampleDataList = SampleDataDao.instance.findAll(filters, skip, limit, sort); - skip += limit; + for (ApiCollection apiCollection: apiCollections) { List toBeDeleted = new ArrayList<>(); - for(SampleData sampleData: sampleDataList) { - try { - List samples = sampleData.getSamples(); - if (samples == null || samples.isEmpty()) { - logger.info("[BadApisRemover] No samples found for : " + sampleData.getId()); - continue; - } - ApiCollection apiCollection = apiCollectionMap.get(sampleData.getId().getApiCollectionId()); - if (apiCollection == null) { - logger.info("[BadApisRemover] No apiCollection found for : " + sampleData.getId()); - continue; - } + if (apiCollection.getHostName() == null) { + continue; + } - - boolean allMatchDefault = true; - boolean isNetsparkerPresent = false; - for (String sample : samples) { - HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); - isNetsparkerPresent |= sample.toLowerCase().contains("netsparker"); - if(accountSettings != null && accountSettings.getAllowRedundantEndpointsList() != null){ - if (!RuntimeUtil.shouldIgnore(httpResponseParams, accountSettings.getAllowRedundantEndpointsList())) { - allMatchDefault = false; - break; - } - } - } + List endpoints = com.akto.action.observe.Utils.fetchEndpointsInCollectionUsingHost(apiCollection.getId(), 0); - if (allMatchDefault) { - writer.write(sampleData.toString()); - toBeDeleted.add(sampleData.getId()); - logger.info("[BadApisRemover] " + isNetsparkerPresent + " Deleting bad API: " + sampleData.getId(), LogDb.DASHBOARD); - } else { - logger.info("[BadApisRemover] " + isNetsparkerPresent + " Keeping bad API: " + sampleData.getId(), LogDb.DASHBOARD); - } - } catch (Exception e) { - loggerMaker.errorAndAddToDb("[BadApisRemover] Couldn't delete an api for default payload: " + sampleData.getId() + e.getMessage(), LogDb.DASHBOARD); + if (endpoints == null || endpoints.isEmpty()) { + continue; + } + + logger.info("[BadApisRemover] Starting for APICollection: " + apiCollection.getId(), LogDb.DASHBOARD); + for (BasicDBObject singleTypeInfo: endpoints) { + singleTypeInfo = (BasicDBObject) (singleTypeInfo.getOrDefault("_id", new BasicDBObject())); + int apiCollectionId = singleTypeInfo.getInt("apiCollectionId"); + String url = singleTypeInfo.getString("url"); + String method = singleTypeInfo.getString("method"); + + Key key = new Key(apiCollectionId, url, Method.fromString(method), -1, 0, 0); + + if (method.equalsIgnoreCase("OPTIONS")) { + logger.info("[BadApisRemover] OPTIONS Deleting bad API: " + key, LogDb.DASHBOARD); + toBeDeleted.add(key); + continue; + } + + Bson filter = Filters.and( + Filters.eq("_id.apiCollectionId", apiCollectionId), + Filters.eq("_id.url", url), + Filters.eq("_id.method", method) + ); + + SampleData sampleData = SampleDataDao.instance.findOne(filter); + if (sampleData == null || sampleData.getSamples() == null || sampleData.getSamples().isEmpty()) { + logger.info("[BadApisRemover] no-sample Deleting bad API: " + key, LogDb.DASHBOARD); + toBeDeleted.add(key); + } else { + logger.info("[BadApisRemover] yes-sample Deleting bad API: " + key, LogDb.DASHBOARD); } } @@ -690,11 +682,72 @@ private void setupBadApisRemover() { if ( shouldDelete != null && shouldDelete.equalsIgnoreCase("true")) { deleteApis(toBeDeleted); } + } + - } while (!sampleDataList.isEmpty()); + // Map apiCollectionMap = apiCollections.stream().collect(Collectors.toMap(ApiCollection::getId, Function.identity())); + // String filePath = "./samples_"+accountId+".txt"; + // BufferedWriter writer = new BufferedWriter(new FileWriter(new File(filePath))); + // List sampleDataList = new ArrayList<>(); + // Bson filters = Filters.empty(); + // int skip = 0; + // int limit = 100; + // Bson sort = Sorts.ascending("_id.apiCollectionId", "_id.url", "_id.method"); + // AccountSettings accountSettings = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter()); + + // do { + // sampleDataList = SampleDataDao.instance.findAll(filters, skip, limit, sort); + // skip += limit; + // List toBeDeleted = new ArrayList<>(); + // for(SampleData sampleData: sampleDataList) { + // try { + // List samples = sampleData.getSamples(); + // if (samples == null || samples.isEmpty()) { + // logger.info("[BadApisRemover] No samples found for : " + sampleData.getId()); + // continue; + // } + + // ApiCollection apiCollection = apiCollectionMap.get(sampleData.getId().getApiCollectionId()); + // if (apiCollection == null) { + // logger.info("[BadApisRemover] No apiCollection found for : " + sampleData.getId()); + // continue; + // } - writer.flush(); - writer.close(); + + // boolean allMatchDefault = true; + // boolean isNetsparkerPresent = false; + // for (String sample : samples) { + // HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); + // isNetsparkerPresent |= sample.toLowerCase().contains("netsparker"); + // if(accountSettings != null && accountSettings.getAllowRedundantEndpointsList() != null){ + // if (!RuntimeUtil.shouldIgnore(httpResponseParams, accountSettings.getAllowRedundantEndpointsList())) { + // allMatchDefault = false; + // break; + // } + // } + // } + + // if (allMatchDefault) { + // writer.write(sampleData.toString()); + // toBeDeleted.add(sampleData.getId()); + // logger.info("[BadApisRemover] " + isNetsparkerPresent + " Deleting bad API: " + sampleData.getId(), LogDb.DASHBOARD); + // } else { + // logger.info("[BadApisRemover] " + isNetsparkerPresent + " Keeping bad API: " + sampleData.getId(), LogDb.DASHBOARD); + // } + // } catch (Exception e) { + // loggerMaker.errorAndAddToDb("[BadApisRemover] Couldn't delete an api for default payload: " + sampleData.getId() + e.getMessage(), LogDb.DASHBOARD); + // } + // } + + // String shouldDelete = System.getenv("DELETE_REDUNDANT_APIS"); + // if ( shouldDelete != null && shouldDelete.equalsIgnoreCase("true")) { + // deleteApis(toBeDeleted); + // } + + // } while (!sampleDataList.isEmpty()); + + // writer.flush(); + // writer.close(); } catch (Exception e) { loggerMaker.errorAndAddToDb("Couldn't complete scan for APIs remover: " + e.getMessage(), LogDb.DASHBOARD); From 2f47aff095ecaced49e4080d9dd79977ab647e0a Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Thu, 5 Sep 2024 13:26:53 -0700 Subject: [PATCH 12/16] use single type info to filter --- .../com/akto/listener/InitializerListener.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index ad5bbd135f..7b41de3f92 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -671,8 +671,21 @@ private void setupBadApisRemover() { SampleData sampleData = SampleDataDao.instance.findOne(filter); if (sampleData == null || sampleData.getSamples() == null || sampleData.getSamples().isEmpty()) { - logger.info("[BadApisRemover] no-sample Deleting bad API: " + key, LogDb.DASHBOARD); - toBeDeleted.add(key); + Bson stiFilterReq = Filters.and( + Filters.eq("url", url), + Filters.eq("method", method), + Filters.in("responseCode", new Integer[]{-1, 200, 201, 204, 302}), + Filters.eq("isHeader", false), + Filters.eq("isUrlParam", false), + Filters.eq("apiCollectionId", apiCollectionId) + ); + SingleTypeInfo singleTypeInfoForApi = SingleTypeInfoDao.instance.findOne(stiFilterReq); + if (singleTypeInfoForApi == null) { + logger.info("[BadApisRemover] no-sample Deleting bad API: " + key, LogDb.DASHBOARD); + toBeDeleted.add(key); + } else { + logger.info("[BadApisRemover] yes-sti Deleting bad API: " + key + " " + singleTypeInfoForApi.composeKey(), LogDb.DASHBOARD); + } } else { logger.info("[BadApisRemover] yes-sample Deleting bad API: " + key, LogDb.DASHBOARD); } From ae3db0df4e77f2484c37f15d629fbc2499b014ca Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Thu, 5 Sep 2024 14:08:01 -0700 Subject: [PATCH 13/16] fix one filter --- .../src/main/java/com/akto/listener/InitializerListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 7b41de3f92..7c7c6681c5 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -676,7 +676,7 @@ private void setupBadApisRemover() { Filters.eq("method", method), Filters.in("responseCode", new Integer[]{-1, 200, 201, 204, 302}), Filters.eq("isHeader", false), - Filters.eq("isUrlParam", false), + Filters.or(Filters.eq("isUrlParam", false), Filters.exists("isUrlParam", false)), Filters.eq("apiCollectionId", apiCollectionId) ); SingleTypeInfo singleTypeInfoForApi = SingleTypeInfoDao.instance.findOne(stiFilterReq); From 8977498297febff0648b1a7d5b23eae28fef5a9e Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Thu, 5 Sep 2024 14:34:31 -0700 Subject: [PATCH 14/16] skip non-GET APIs --- .../main/java/com/akto/listener/InitializerListener.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 7c7c6681c5..dfe011665f 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -657,12 +657,17 @@ private void setupBadApisRemover() { Key key = new Key(apiCollectionId, url, Method.fromString(method), -1, 0, 0); - if (method.equalsIgnoreCase("OPTIONS")) { + if (method.equalsIgnoreCase("options")) { logger.info("[BadApisRemover] OPTIONS Deleting bad API: " + key, LogDb.DASHBOARD); toBeDeleted.add(key); continue; } + if (!method.equalsIgnoreCase("get")) { + logger.info("[BadApisRemover] Non-get Deleting bad API: " + key, LogDb.DASHBOARD); + continue; + } + Bson filter = Filters.and( Filters.eq("_id.apiCollectionId", apiCollectionId), Filters.eq("_id.url", url), From c3380a3b12c50eec43811f98b3d493a61f372bfb Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Tue, 10 Sep 2024 01:58:09 -0700 Subject: [PATCH 15/16] run fixer for great learning --- .../akto/listener/InitializerListener.java | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index dfe011665f..656daf8259 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -622,7 +622,7 @@ private static CustomDataType getCustomDataTypeFromPiiType(PIISource piiSource, } private void setupBadApisRemover() { - Integer[] accountIds = new Integer[]{1724877069, 1714700875}; + Integer[] accountIds = new Integer[]{1714700875}; for (int accountId: accountIds) { Account account = AccountsDao.instance.findOne(eq("_id", accountId)); @@ -2105,25 +2105,33 @@ public void run() { } } while (!connectedToMongo); - setDashboardMode(); - updateGlobalAktoVersion(); + // setDashboardMode(); + // updateGlobalAktoVersion(); - AccountTask.instance.executeTask(new Consumer() { - @Override - public void accept(Account account) { - AccountSettingsDao.instance.getStats(); - Intercom.setToken(System.getenv("INTERCOM_TOKEN")); - setDashboardVersionForAccount(); - } - }, "context-initializer"); + // AccountTask.instance.executeTask(new Consumer() { + // @Override + // public void accept(Account account) { + // AccountSettingsDao.instance.getStats(); + // Intercom.setToken(System.getenv("INTERCOM_TOKEN")); + // setDashboardVersionForAccount(); + // } + // }, "context-initializer"); - SingleTypeInfo.init(); + // SingleTypeInfo.init(); int now = Context.now(); if (runJobFunctions || runJobFunctionsAnyway) { logger.info("Starting init functions and scheduling jobs at " + now); setupBadApisRemover(); + while ((1+2)>1) { + try { + Thread.sleep (10000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + AccountTask.instance.executeTask(new Consumer() { @Override From 29ab32b3ee1f732dc09437ab49c2017b529f65f0 Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Tue, 10 Sep 2024 02:09:23 -0700 Subject: [PATCH 16/16] pass after 1 day --- .../java/com/akto/listener/InitializerListener.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 656daf8259..4fa76fb1ae 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -2124,12 +2124,18 @@ public void run() { logger.info("Starting init functions and scheduling jobs at " + now); setupBadApisRemover(); - while ((1+2)>1) { + int counter = 0; + while (true) { try { - Thread.sleep (10000); + Thread.sleep (60000); + counter++; } catch (InterruptedException e) { e.printStackTrace(); } + + if (counter > 1440) { + break; + } }