diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index 6c93a80b27..75b07dfc7f 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -361,6 +361,26 @@ public static ApiMergerResult tryMergeURLsInCollection(int apiCollectionId, Bool finalResult.templateToStaticURLs.putAll(result.templateToStaticURLs); } + for (String s: templateUrls) { + if (finalResult.deleteStaticUrls.contains(s)) continue; + String staticEndpoint = s.split(" ")[1]; + String staticMethodStr = s.split(" ")[0]; + Method staticMethod = Method.fromString(staticMethodStr); + for (String t: templateUrls) { + if (finalResult.deleteStaticUrls.contains(t) || s.equals(t)) continue; + String[] tSplit = t.split(" "); + String tUrl = tSplit[1]; + String tMethodStr = tSplit[0]; + Method tMethod= Method.fromString(tMethodStr); + URLTemplate urlTemplate = createUrlTemplate(tUrl, tMethod); + + if (urlTemplate.match(staticEndpoint, staticMethod)) { + finalResult.deleteStaticUrls.add(s); + break; + } + } + } + offset += limit; } while (!singleTypeInfos.isEmpty()); diff --git a/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java b/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java index 0fbef78c04..a4b424123e 100644 --- a/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java +++ b/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java @@ -100,6 +100,22 @@ public int getStaticURLsSize(HttpCallParser parser) { return urlStaticMap.size(); } + @Test + public void test() throws Exception { + HttpCallParser httpCallParser = new HttpCallParser("", 0, 0, 0, true); + String p1 = "{\"method\":\"GET\",\"requestPayload\":\"{}\",\"responsePayload\":\"{}\",\"ip\":\"\",\"source\":\"MIRRORING\",\"type\":\"HTTP/1.1\",\"akto_vxlan_id\":0,\"path\":\"/service/mirror/serve/401f58ec-7743-4dca-bcf3-3e402619bad4/b2Xe4GGuDkiPBhymzYBwQmZrapIvuRQyGSiZCchTezv\",\"requestHeaders\":\"{\\\"referer\\\":\\\"https://uataz.quickolabs.com/service/mirror/serve/401f58ec-7743-4dca-bcf3-3e402619bad4/Phf6i8ypcVPzIVYF9qMDHKw35Dhz6nZfk7wACnEGEvt?openeditor=true\\\",\\\"cf-ipcountry\\\":\\\"IN\\\",\\\"sec-fetch-site\\\":\\\"same-origin\\\",\\\"true-client-ip\\\":\\\"117.215.69.146\\\",\\\"sec-ch-ua-mobile\\\":\\\"?0\\\",\\\"cf-visitor\\\":\\\"{\\\\\\\"scheme\\\\\\\":\\\\\\\"https\\\\\\\"}\\\",\\\"host\\\":\\\"mamauat.quickolabs.com\\\",\\\"cf-connecting-ip\\\":\\\"117.215.69.146\\\",\\\"connection\\\":\\\"close\\\",\\\"sec-fetch-mode\\\":\\\"no-cors\\\",\\\"cdn-loop\\\":\\\"cloudflare\\\",\\\"cf-ray\\\":\\\"8a41e41a7c365975-DEL\\\",\\\"unique_id\\\":\\\"4ad0fed5da381e3186b063fb783edfc8\\\",\\\"accept-language\\\":\\\"en-GB,en-US;q=0.9,en;q=0.8\\\",\\\"cookie\\\":\\\"wfx_unq=l1w4x9CFuu1fsIr7; region=US; __zlcmid=1Mfmr4uSrrbIeg2; sid=ofvszol90ew8uk23zum6xysxak1qyn; XSRF-TOKEN=ot4klrkqerp7tczzc; uid=2057f513-d773-405e-9f77-359f3e4834e1; loggedin=db; eidg=; eid=401f58ec-7743-4dca-bcf3-3e402619bad4; wfx_unq=l1w4x9CFuu1fsIr7\\\",\\\"x-forwarded-proto\\\":\\\"https\\\",\\\"x-forwarded-for\\\":\\\"117.215.69.146\\\",\\\"priority\\\":\\\"i\\\",\\\"x-host\\\":\\\"uataz.quickolabs.com\\\",\\\"accept\\\":\\\"image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8\\\",\\\"sec-ch-ua\\\":\\\"\\\\\\\"Not/A)Brand\\\\\\\";v=\\\\\\\"8\\\\\\\", \\\\\\\"Chromium\\\\\\\";v=\\\\\\\"126\\\\\\\", \\\\\\\"Google Chrome\\\\\\\";v=\\\\\\\"126\\\\\\\"\\\",\\\"sec-ch-ua-platform\\\":\\\"\\\\\\\"macOS\\\\\\\"\\\",\\\"accept-encoding\\\":\\\"gzip, br\\\",\\\"sec-fetch-dest\\\":\\\"image\\\",\\\"user-agent\\\":\\\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36\\\"}\",\"responseHeaders\":\"{\\\"date\\\":\\\"Tue, 16 Jul 2024 12:10:46 GMT\\\",\\\"content-length\\\":\\\"20958\\\",\\\"server\\\":\\\"istio-envoy\\\",\\\"expires\\\":\\\"Thu, 15 Aug 2024 12:10:47 GMT\\\",\\\"x-envoy-upstream-service-time\\\":\\\"245\\\",\\\"content-type\\\":\\\"image/jpeg\\\",\\\"accept-ranges\\\":\\\"bytes\\\",\\\"cache-control\\\":\\\"private, no-store, max-age=0\\\",\\\"surrogate-control\\\":\\\"max-age=2592000, immutable\\\"}\",\"time\":\"1721131847\",\"statusCode\":\"200\",\"status\":\"200 OK\",\"akto_account_id\":\"1710118493\",\"is_pending\":\"false\"}"; + HttpResponseParams http1= HttpCallParser.parseKafkaMessage(p1); + String p2 = "{\"destIp\":\"10.103.0.4:20480\",\"method\":\"GET\",\"requestPayload\":\"{}\",\"responsePayload\":\"{}\",\"ip\":\"10.103.1.66:34000\",\"source\":\"MIRRORING\",\"type\":\"HTTP/1.1\",\"akto_vxlan_id\":0,\"path\":\"/service/mirror/serve/c9eae1f3-8b53-45a7-a707-c0d5edef889a/TetofKemjVFDhEsbBnrTHnoRsNpXNbAbJ6J1FpFDIXW\",\"requestHeaders\":\"{\\\"referer\\\":\\\"https://uataz.quickolabs.com/service/mirror/serve/c9eae1f3-8b53-45a7-a707-c0d5edef889a/lZyenTHK7MVazWdpWhErb0ITvHSjAVDfT49JtvQeXfA\\\",\\\"cf-ipcountry\\\":\\\"IN\\\",\\\"sec-fetch-site\\\":\\\"same-origin\\\",\\\"true-client-ip\\\":\\\"61.246.82.230\\\",\\\"sec-ch-ua-mobile\\\":\\\"?0\\\",\\\"cf-visitor\\\":\\\"{\\\\\\\"scheme\\\\\\\":\\\\\\\"https\\\\\\\"}\\\",\\\"host\\\":\\\"mamauat.quickolabs.com\\\",\\\"cf-connecting-ip\\\":\\\"61.246.82.230\\\",\\\"connection\\\":\\\"close\\\",\\\"sec-fetch-mode\\\":\\\"no-cors\\\",\\\"cdn-loop\\\":\\\"cloudflare\\\",\\\"cf-ray\\\":\\\"8aab1f404cb0c188-BOM\\\",\\\"unique_id\\\":\\\"155588aecf459182ad6402d654470a9c\\\",\\\"accept-language\\\":\\\"en,zh;q=0.9,fr;q=0.8,en-US;q=0.7,it;q=0.6\\\",\\\"cookie\\\":\\\"wfx_unq=fJhDntmD5cDVX6uU; region=US; loggedin=db; uid=4eb3b287-47c5-4f5b-b1c8-86d6b9ca3a68; eidg=; wfx_unq=fJhDntmD5cDVX6uU; eid=c9eae1f3-8b53-45a7-a707-c0d5edef889a; sid=sso-qh018013tt8rorw8mx8a9100\\\",\\\"x-forwarded-proto\\\":\\\"https\\\",\\\"x-forwarded-for\\\":\\\"61.246.82.230\\\",\\\"priority\\\":\\\"i\\\",\\\"x-host\\\":\\\"uataz.quickolabs.com\\\",\\\"accept\\\":\\\"image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8\\\",\\\"sec-ch-ua\\\":\\\"\\\\\\\"Not/A)Brand\\\\\\\";v=\\\\\\\"8\\\\\\\", \\\\\\\"Chromium\\\\\\\";v=\\\\\\\"126\\\\\\\", \\\\\\\"Google Chrome\\\\\\\";v=\\\\\\\"126\\\\\\\"\\\",\\\"sec-ch-ua-platform\\\":\\\"\\\\\\\"macOS\\\\\\\"\\\",\\\"accept-encoding\\\":\\\"gzip, br\\\",\\\"sec-fetch-dest\\\":\\\"image\\\",\\\"user-agent\\\":\\\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36\\\"}\",\"responseHeaders\":\"{\\\"date\\\":\\\"Mon, 29 Jul 2024 06:41:14 GMT\\\",\\\"content-length\\\":\\\"1206\\\",\\\"server\\\":\\\"istio-envoy\\\",\\\"expires\\\":\\\"Wed, 28 Aug 2024 06:41:14 GMT\\\",\\\"x-envoy-upstream-service-time\\\":\\\"129\\\",\\\"content-type\\\":\\\"image/svg+xml\\\",\\\"accept-ranges\\\":\\\"bytes\\\",\\\"cache-control\\\":\\\"private, no-store, max-age=0\\\",\\\"surrogate-control\\\":\\\"max-age=2592000, immutable\\\"}\",\"time\":\"1722235275\",\"statusCode\":\"200\",\"status\":\"200 OK\",\"akto_account_id\":\"1000000\",\"is_pending\":\"false\"}"; + HttpResponseParams http2= HttpCallParser.parseKafkaMessage(p2); + + String p3 = "{\"destIp\":\"10.103.0.4:20480\",\"method\":\"GET\",\"requestPayload\":\"{}\",\"responsePayload\":\"{}\",\"ip\":\"10.103.1.66:34000\",\"source\":\"MIRRORING\",\"type\":\"HTTP/1.1\",\"akto_vxlan_id\":0,\"path\":\"/service/mirror/serve/STRING/STRING\",\"requestHeaders\":\"{\\\"referer\\\":\\\"https://uataz.quickolabs.com/service/mirror/serve/c9eae1f3-8b53-45a7-a707-c0d5edef889a/lZyenTHK7MVazWdpWhErb0ITvHSjAVDfT49JtvQeXfA\\\",\\\"cf-ipcountry\\\":\\\"IN\\\",\\\"sec-fetch-site\\\":\\\"same-origin\\\",\\\"true-client-ip\\\":\\\"61.246.82.230\\\",\\\"sec-ch-ua-mobile\\\":\\\"?0\\\",\\\"cf-visitor\\\":\\\"{\\\\\\\"scheme\\\\\\\":\\\\\\\"https\\\\\\\"}\\\",\\\"host\\\":\\\"mamauat.quickolabs.com\\\",\\\"cf-connecting-ip\\\":\\\"61.246.82.230\\\",\\\"connection\\\":\\\"close\\\",\\\"sec-fetch-mode\\\":\\\"no-cors\\\",\\\"cdn-loop\\\":\\\"cloudflare\\\",\\\"cf-ray\\\":\\\"8aab1f404cb0c188-BOM\\\",\\\"unique_id\\\":\\\"155588aecf459182ad6402d654470a9c\\\",\\\"accept-language\\\":\\\"en,zh;q=0.9,fr;q=0.8,en-US;q=0.7,it;q=0.6\\\",\\\"cookie\\\":\\\"wfx_unq=fJhDntmD5cDVX6uU; region=US; loggedin=db; uid=4eb3b287-47c5-4f5b-b1c8-86d6b9ca3a68; eidg=; wfx_unq=fJhDntmD5cDVX6uU; eid=c9eae1f3-8b53-45a7-a707-c0d5edef889a; sid=sso-qh018013tt8rorw8mx8a9100\\\",\\\"x-forwarded-proto\\\":\\\"https\\\",\\\"x-forwarded-for\\\":\\\"61.246.82.230\\\",\\\"priority\\\":\\\"i\\\",\\\"x-host\\\":\\\"uataz.quickolabs.com\\\",\\\"accept\\\":\\\"image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8\\\",\\\"sec-ch-ua\\\":\\\"\\\\\\\"Not/A)Brand\\\\\\\";v=\\\\\\\"8\\\\\\\", \\\\\\\"Chromium\\\\\\\";v=\\\\\\\"126\\\\\\\", \\\\\\\"Google Chrome\\\\\\\";v=\\\\\\\"126\\\\\\\"\\\",\\\"sec-ch-ua-platform\\\":\\\"\\\\\\\"macOS\\\\\\\"\\\",\\\"accept-encoding\\\":\\\"gzip, br\\\",\\\"sec-fetch-dest\\\":\\\"image\\\",\\\"user-agent\\\":\\\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36\\\"}\",\"responseHeaders\":\"{\\\"date\\\":\\\"Mon, 29 Jul 2024 06:41:14 GMT\\\",\\\"content-length\\\":\\\"1206\\\",\\\"server\\\":\\\"istio-envoy\\\",\\\"expires\\\":\\\"Wed, 28 Aug 2024 06:41:14 GMT\\\",\\\"x-envoy-upstream-service-time\\\":\\\"129\\\",\\\"content-type\\\":\\\"image/svg+xml\\\",\\\"accept-ranges\\\":\\\"bytes\\\",\\\"cache-control\\\":\\\"private, no-store, max-age=0\\\",\\\"surrogate-control\\\":\\\"max-age=2592000, immutable\\\"}\",\"time\":\"1722235275\",\"statusCode\":\"200\",\"status\":\"200 OK\",\"akto_account_id\":\"1000000\",\"is_pending\":\"false\"}"; + HttpResponseParams http3= HttpCallParser.parseKafkaMessage(p3); + httpCallParser.syncFunction(Arrays.asList(http1, http2), true, true, null); +// httpCallParser.syncFunction(Arrays.asList(http3), true, true, null); +// APICatalogSync.mergeUrlsAndSave(-1103003223, true, true, httpCallParser.apiCatalogSync.existingAPIsInDb); + System.out.println(""); + } + @Test public void testmultipleUUIDForceMerge(){