From ce664b1178394c1f5bf7c2ba4031683c06f68ea4 Mon Sep 17 00:00:00 2001 From: SHIVAM RAWAT Date: Tue, 1 Oct 2024 02:10:41 +0530 Subject: [PATCH 1/6] removing 1000000 account check and staging.yml changes for source-code-analyser build --- .github/workflows/staging.yml | 68 +++++++++---------- .../java/com/akto/data_actor/ClientActor.java | 4 +- 2 files changed, 35 insertions(+), 37 deletions(-) diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 580b82e492..ce37cb654c 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -25,27 +25,27 @@ jobs: - name: Convert github branch name to be compatible with docker tag name convention and generate tag name id: docker_tag run: echo "IMAGE_TAG=a-$(echo ${{ github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')" >> $GITHUB_OUTPUT - - name: Download Akto templates zip and PII files - working-directory: ./apps/dashboard/src/main/resources - run: | - wget -O tests-library-master.zip https://github.com/akto-api-security/tests-library/archive/refs/heads/master.zip - wget -O general.json https://raw.githubusercontent.com/akto-api-security/pii-types/master/general.json - wget -O fintech.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/fintech.json - wget -O filetypes.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/filetypes.json - - name: Prepare Dashboard polaris UI - working-directory: ./apps/dashboard/web/polaris_web - run: npm install && export RELEASE_VERSION=${{steps.docker_tag.outputs.IMAGE_TAG}} && npm run build - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{secrets.AWS_ACCESS_KEY_ID}} - aws-secret-access-key: ${{secrets.AWS_SECRET_ACCESS_KEY}} - aws-region: ap-south-1 - - - name: Deploy polaris site to S3 bucket - run: aws s3 sync ./apps/dashboard/web/polaris_web/web/dist s3://dashboard-on-cdn/polaris_web/${{steps.docker_tag.outputs.IMAGE_TAG}}/dist --delete - - - run: mvn package -Dakto-image-tag=${{ github.event.inputs.Tag }} -Dakto-build-time=$(eval "date +%s") -Dakto-release-version=${{steps.docker_tag.outputs.IMAGE_TAG}} +# - name: Download Akto templates zip and PII files +# working-directory: ./apps/dashboard/src/main/resources +# run: | +# wget -O tests-library-master.zip https://github.com/akto-api-security/tests-library/archive/refs/heads/master.zip +# wget -O general.json https://raw.githubusercontent.com/akto-api-security/pii-types/master/general.json +# wget -O fintech.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/fintech.json +# wget -O filetypes.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/filetypes.json +# - name: Prepare Dashboard polaris UI +# working-directory: ./apps/dashboard/web/polaris_web +# run: npm install && export RELEASE_VERSION=${{steps.docker_tag.outputs.IMAGE_TAG}} && npm run build +# - name: Configure AWS Credentials +# uses: aws-actions/configure-aws-credentials@v1 +# with: +# aws-access-key-id: ${{secrets.AWS_ACCESS_KEY_ID}} +# aws-secret-access-key: ${{secrets.AWS_SECRET_ACCESS_KEY}} +# aws-region: ap-south-1 +# +# - name: Deploy polaris site to S3 bucket +# run: aws s3 sync ./apps/dashboard/web/polaris_web/web/dist s3://dashboard-on-cdn/polaris_web/${{steps.docker_tag.outputs.IMAGE_TAG}}/dist --delete +# + - run: mvn package -Dakto-image-tag=${{ github.event.inputs.Tag }} -Dakto-build-time=$(eval "date +%s") -Dakto-release-version=${{steps.docker_tag.outputs.IMAGE_TAG}} -DskipTests=true - name: DockerHub login env: DOCKER_USERNAME: ${{secrets.DOCKER_USERNAME}} @@ -65,19 +65,19 @@ jobs: echo $IMAGE_TAG >> $GITHUB_STEP_SUMMARY docker buildx create --use # Build a docker container and push it to DockerHub - cd apps/dashboard - docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/$ECR_REPOSITORY-dashboard:$IMAGE_TAG $IMAGE_TAG_DASHBOARD . --push - cd ../testing - docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-testing:$IMAGE_TAG $IMAGE_TAG_TESTING . --push - cd ../testing-cli - docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-testing-cli:$IMAGE_TAG $IMAGE_TAG_TESTING_CLI . --push - cd ../billing - docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-billing:$IMAGE_TAG . --push - cd ../internal - docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-internal:$IMAGE_TAG . --push - cd ../api-threat-detection - docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-protection:$IMAGE_TAG . --push - cd ../source-code-analyser +# cd apps/dashboard +# docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/$ECR_REPOSITORY-dashboard:$IMAGE_TAG $IMAGE_TAG_DASHBOARD . --push +# cd ../testing +# docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-testing:$IMAGE_TAG $IMAGE_TAG_TESTING . --push +# cd ../testing-cli +# docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-testing-cli:$IMAGE_TAG $IMAGE_TAG_TESTING_CLI . --push +# cd ../billing +# docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-billing:$IMAGE_TAG . --push +# cd ../internal +# docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-internal:$IMAGE_TAG . --push +# cd ../api-threat-detection +# docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-protection:$IMAGE_TAG . --push + cd apps/source-code-analyser docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/source-code-analyser:$IMAGE_TAG . --push - name: Set up JDK 11 diff --git a/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java b/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java index bbd3b5c454..54cef93837 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java +++ b/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java @@ -59,9 +59,7 @@ public class ClientActor extends DataActor { public static String buildDbAbstractorUrl() { String dbAbsHost = CYBORG_URL; - if (checkAccount()) { - dbAbsHost = System.getenv("DATABASE_ABSTRACTOR_SERVICE_URL"); - } + dbAbsHost = System.getenv("DATABASE_ABSTRACTOR_SERVICE_URL"); System.out.println("dbHost value " + dbAbsHost); if (dbAbsHost.endsWith("/")) { dbAbsHost = dbAbsHost.substring(0, dbAbsHost.length() - 1); From cb54622b315f8a0521ce40950c49e5db419edbe6 Mon Sep 17 00:00:00 2001 From: SHIVAM RAWAT Date: Tue, 1 Oct 2024 02:12:30 +0530 Subject: [PATCH 2/6] removing comments --- .github/workflows/staging.yml | 33 +-------------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index ce37cb654c..03a6a4e653 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -25,26 +25,7 @@ jobs: - name: Convert github branch name to be compatible with docker tag name convention and generate tag name id: docker_tag run: echo "IMAGE_TAG=a-$(echo ${{ github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')" >> $GITHUB_OUTPUT -# - name: Download Akto templates zip and PII files -# working-directory: ./apps/dashboard/src/main/resources -# run: | -# wget -O tests-library-master.zip https://github.com/akto-api-security/tests-library/archive/refs/heads/master.zip -# wget -O general.json https://raw.githubusercontent.com/akto-api-security/pii-types/master/general.json -# wget -O fintech.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/fintech.json -# wget -O filetypes.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/filetypes.json -# - name: Prepare Dashboard polaris UI -# working-directory: ./apps/dashboard/web/polaris_web -# run: npm install && export RELEASE_VERSION=${{steps.docker_tag.outputs.IMAGE_TAG}} && npm run build -# - name: Configure AWS Credentials -# uses: aws-actions/configure-aws-credentials@v1 -# with: -# aws-access-key-id: ${{secrets.AWS_ACCESS_KEY_ID}} -# aws-secret-access-key: ${{secrets.AWS_SECRET_ACCESS_KEY}} -# aws-region: ap-south-1 -# -# - name: Deploy polaris site to S3 bucket -# run: aws s3 sync ./apps/dashboard/web/polaris_web/web/dist s3://dashboard-on-cdn/polaris_web/${{steps.docker_tag.outputs.IMAGE_TAG}}/dist --delete -# + - run: mvn package -Dakto-image-tag=${{ github.event.inputs.Tag }} -Dakto-build-time=$(eval "date +%s") -Dakto-release-version=${{steps.docker_tag.outputs.IMAGE_TAG}} -DskipTests=true - name: DockerHub login env: @@ -65,18 +46,6 @@ jobs: echo $IMAGE_TAG >> $GITHUB_STEP_SUMMARY docker buildx create --use # Build a docker container and push it to DockerHub -# cd apps/dashboard -# docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/$ECR_REPOSITORY-dashboard:$IMAGE_TAG $IMAGE_TAG_DASHBOARD . --push -# cd ../testing -# docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-testing:$IMAGE_TAG $IMAGE_TAG_TESTING . --push -# cd ../testing-cli -# docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-testing-cli:$IMAGE_TAG $IMAGE_TAG_TESTING_CLI . --push -# cd ../billing -# docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-billing:$IMAGE_TAG . --push -# cd ../internal -# docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-internal:$IMAGE_TAG . --push -# cd ../api-threat-detection -# docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-protection:$IMAGE_TAG . --push cd apps/source-code-analyser docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/source-code-analyser:$IMAGE_TAG . --push From 7b78cc5e54bcbaec7b6c580934ff6fa8f2a16e9c Mon Sep 17 00:00:00 2001 From: SHIVAM RAWAT Date: Tue, 1 Oct 2024 03:15:34 +0530 Subject: [PATCH 3/6] docker volume --- .../src/main/java/com/akto/SourceCodeAnalyserRepo.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/source-code-analyser/src/main/java/com/akto/SourceCodeAnalyserRepo.java b/apps/source-code-analyser/src/main/java/com/akto/SourceCodeAnalyserRepo.java index 398abfc48a..035e69748e 100644 --- a/apps/source-code-analyser/src/main/java/com/akto/SourceCodeAnalyserRepo.java +++ b/apps/source-code-analyser/src/main/java/com/akto/SourceCodeAnalyserRepo.java @@ -61,7 +61,7 @@ public String downloadRepository () { if (finalUrl == null || token == null) { return null; } - String outputFilePath = repoToBeAnalysed.getRepoName()+ ".zip"; // The local file where the repository will be saved + String outputFilePath = System.getenv("DOCKER_VOLUME") + repoToBeAnalysed.getRepoName()+ ".zip"; // The local file where the repository will be saved File file = new File(outputFilePath); Request.Builder builder = new Request.Builder(); From 1f4ceb138f0912029b51c182210ad44a67106aba Mon Sep 17 00:00:00 2001 From: SHIVAM RAWAT Date: Tue, 5 Nov 2024 13:57:55 +0530 Subject: [PATCH 4/6] fixing getter and setter for isLastBatch in CodeAnalysisAction --- .../src/main/java/com/akto/action/CodeAnalysisAction.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/database-abstractor/src/main/java/com/akto/action/CodeAnalysisAction.java b/apps/database-abstractor/src/main/java/com/akto/action/CodeAnalysisAction.java index 1662d13840..a7efc9bb47 100644 --- a/apps/database-abstractor/src/main/java/com/akto/action/CodeAnalysisAction.java +++ b/apps/database-abstractor/src/main/java/com/akto/action/CodeAnalysisAction.java @@ -368,12 +368,12 @@ public void setProjectName(String projectName) { this.projectName = projectName; } - public boolean isLastBatch() { + public boolean getIsLastBatch() { return isLastBatch; } - public void setLastBatch(boolean lastBatch) { - isLastBatch = lastBatch; + public void setIsLastBatch(boolean isLastBatch) { + this.isLastBatch = isLastBatch; } public CodeAnalysisRepo getCodeAnalysisRepo() { From 206e560c7449fddaf333b2cca97a2efcb215cc86 Mon Sep 17 00:00:00 2001 From: SHIVAM RAWAT Date: Fri, 8 Nov 2024 09:08:36 +0530 Subject: [PATCH 5/6] dbactor changes for codeanalysis and handling for multiple accounts given connected to mongo --- .../src/main/java/com/akto/Main.java | 97 ++++-- .../java/com/akto/SourceCodeAnalyserRepo.java | 9 +- .../java/com/akto/data_actor/DbActor.java | 293 +++++++++++++++++- 3 files changed, 366 insertions(+), 33 deletions(-) diff --git a/apps/source-code-analyser/src/main/java/com/akto/Main.java b/apps/source-code-analyser/src/main/java/com/akto/Main.java index 004e30962b..6543aebf0c 100644 --- a/apps/source-code-analyser/src/main/java/com/akto/Main.java +++ b/apps/source-code-analyser/src/main/java/com/akto/Main.java @@ -1,9 +1,16 @@ package com.akto; +import com.akto.dao.AccountsDao; +import com.akto.dao.context.Context; import com.akto.data_actor.DataActor; import com.akto.data_actor.DataActorFactory; import com.akto.dto.CodeAnalysisRepo; import com.akto.log.LoggerMaker; +import com.akto.util.AccountTask; +import com.mongodb.ConnectionString; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; @@ -12,6 +19,39 @@ public class Main { private static final LoggerMaker loggerMaker = new LoggerMaker(Main.class, LoggerMaker.LogDb.RUNTIME); private static final DataActor dataActor = DataActorFactory.fetchInstance(); private static final int SLEEP_TIME = 10 * 1000; + private static final Logger logger = LoggerFactory.getLogger(Main.class); + + private static boolean connectToMongo() { + + String mongoURI = System.getenv("AKTO_MONGO_CONN"); + if (StringUtils.isEmpty(mongoURI)) { + return false; + } + boolean connectedToMongo = false; + boolean calledOnce = false; + do { + try { + + if (!calledOnce) { + DaoInit.init(new ConnectionString(mongoURI)); + calledOnce = true; + } + AccountsDao.instance.getStats(); + connectedToMongo = true; + + logger.info("triggering merging cron for db abstractor " + Context.now()); + } catch (Exception e) { + logger.error("error running initializer method for db abstractor", e); + } finally { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + } + } while (!connectedToMongo); + return connectedToMongo; + } + private static List fetchReposToSync() { List repos = dataActor.findReposToRun(); @@ -21,39 +61,48 @@ private static List fetchReposToSync() { return null; } + private static void runForRepo(List repos) { + if (repos == null) { + loggerMaker.infoAndAddToDb("No repos to run, skipping"); + return; + } + if (BitbucketRepo.doesEnvVariablesExists()) { + BitbucketRepo.fetchAllProjectKeys(); + } + for (CodeAnalysisRepo repo : repos) { + SourceCodeAnalyserRepo sourceCodeAnalyserRepo; + if (repo.getSourceCodeType() == CodeAnalysisRepo.SourceCodeType.BITBUCKET) { + sourceCodeAnalyserRepo = new BitbucketRepo(repo); + } else { + sourceCodeAnalyserRepo = new GithubRepo(repo); + } + try { + sourceCodeAnalyserRepo.fetchEndpointsUsingAnalyser(); + } catch (Exception e) { + loggerMaker.errorAndAddToDb("Error while fetching endpoints:" + e.getMessage()); + } + } + } + public static void main(String[] args) throws InterruptedException { + boolean isConnectedToMongo = connectToMongo();//When mongo connection, fetch for all accounts + while (true) { if (!BitbucketRepo.doesEnvVariablesExists() && !GithubRepo.doesEnvVariablesExists()) { loggerMaker.infoAndAddToDb("No tokens found"); Thread.sleep(SLEEP_TIME); continue; } - List repos = fetchReposToSync(); - if (repos == null) { - loggerMaker.infoAndAddToDb("No repos to run, skipping"); - Thread.sleep(SLEEP_TIME); - continue; - } - if (BitbucketRepo.doesEnvVariablesExists()) { - BitbucketRepo.fetchAllProjectKeys(); + if (isConnectedToMongo) { + AccountTask.instance.executeTask(t -> { + List repos = fetchReposToSync(); + runForRepo(repos); + }, "initialize-runtime-task"); + } else { + List repos = fetchReposToSync(); + runForRepo(repos); } - for (CodeAnalysisRepo repo : repos) { - SourceCodeAnalyserRepo sourceCodeAnalyserRepo; - if (repo.getSourceCodeType() == CodeAnalysisRepo.SourceCodeType.BITBUCKET) { - sourceCodeAnalyserRepo = new BitbucketRepo(repo); - } else { - sourceCodeAnalyserRepo = new GithubRepo(repo); - } - try { - sourceCodeAnalyserRepo.fetchEndpointsUsingAnalyser(); - } catch (Exception e) { - loggerMaker.errorAndAddToDb("Error while fetching endpoints:" + e.getMessage()); - } - } - Thread.sleep(SLEEP_TIME); } - } - } \ No newline at end of file diff --git a/apps/source-code-analyser/src/main/java/com/akto/SourceCodeAnalyserRepo.java b/apps/source-code-analyser/src/main/java/com/akto/SourceCodeAnalyserRepo.java index 035e69748e..e517361982 100644 --- a/apps/source-code-analyser/src/main/java/com/akto/SourceCodeAnalyserRepo.java +++ b/apps/source-code-analyser/src/main/java/com/akto/SourceCodeAnalyserRepo.java @@ -67,7 +67,7 @@ public String downloadRepository () { Request.Builder builder = new Request.Builder(); builder.url(finalUrl); builder.get(); - builder.addHeader("Authorization", "Bearer " + token); +// builder.addHeader("Authorization", "Bearer " + token); Request request = builder.build(); try { @@ -158,6 +158,13 @@ public void fetchEndpointsUsingAnalyser() { syncRepoToDashboard(originalHttpResponse.getBody(), repoToBeAnalysed); } catch (Exception e) { loggerMaker.errorAndAddToDb("Error while fetching api's from code-analysis for repository:" + repoToBeAnalysed); + } finally { + if (repositoryPath != null) { + File file = new File(repositoryPath); + if(file.delete()) { + loggerMaker.infoAndAddToDb("successfully deleted the zip file", LoggerMaker.LogDb.RUNTIME); + } + } } } diff --git a/libs/utils/src/main/java/com/akto/data_actor/DbActor.java b/libs/utils/src/main/java/com/akto/data_actor/DbActor.java index 93d8652c80..3295316d23 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/DbActor.java +++ b/libs/utils/src/main/java/com/akto/data_actor/DbActor.java @@ -1,5 +1,7 @@ package com.akto.data_actor; +import com.akto.dao.*; +import com.akto.dao.context.Context; import com.akto.dto.*; import com.akto.dto.billing.Organization; import com.akto.dto.filter.MergedUrls; @@ -10,12 +12,16 @@ import com.akto.dto.traffic.TrafficInfo; import com.akto.dto.traffic_metrics.TrafficMetrics; import com.akto.dto.type.SingleTypeInfo; -import com.mongodb.client.model.WriteModel; +import com.mongodb.BasicDBObject; +import com.mongodb.client.model.*; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.bson.types.ObjectId; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; +import java.net.URI; +import java.util.*; + +import static com.akto.util.HttpRequestResponseUtils.generateSTIsFromPayload; public class DbActor extends DataActor { @@ -208,17 +214,288 @@ public void insertProtectionLog(Log log) { } @Override public List findReposToRun() { - return Collections.emptyList(); + return CodeAnalysisRepoDao.instance.findAll( + Filters.expr( + Document.parse("{ $gt: [ \"$" + CodeAnalysisRepo.SCHEDULE_TIME + "\", \"$" + CodeAnalysisRepo.LAST_RUN + "\" ] }") + ) + ); } + public static final int MAX_BATCH_SIZE = 100; @Override public void syncExtractedAPIs( CodeAnalysisRepo codeAnalysisRepo, List codeAnalysisApisList, boolean isLastBatch) { - return; + + if (codeAnalysisApisList == null) { + return; + } + + if (codeAnalysisRepo == null) { + return; + } + String apiCollectionName = codeAnalysisRepo.getProjectName() + "/" + codeAnalysisRepo.getRepoName(); + + // Ensure batch size is not exceeded + if (codeAnalysisApisList.size() > MAX_BATCH_SIZE) { + return; + } + + // populate code analysis api map + Map codeAnalysisApisMap = new HashMap<>(); + for (CodeAnalysisApi codeAnalysisApi: codeAnalysisApisList) { + codeAnalysisApisMap.put(codeAnalysisApi.generateCodeAnalysisApisMapKey(), codeAnalysisApi); + } + + ApiCollection apiCollection = ApiCollectionsDao.instance.findByName(apiCollectionName); + if (apiCollection == null) { + apiCollection = new ApiCollection(Context.now(), apiCollectionName, Context.now(), new HashSet<>(), null, 0, false, false); + ApiCollectionsDao.instance.insertOne(apiCollection); + } + + /* + * In some cases it is not possible to determine the type of template url from source code + * In such cases, we can use the information from traffic endpoints to match the traffic and source code endpoints + * + * Eg: + * Source code endpoints: + * GET /books/STRING -> GET /books/AKTO_TEMPLATE_STR -> GET /books/INTEGER + * POST /city/STRING/district/STRING -> POST /city/AKTO_TEMPLATE_STR/district/AKTO_TEMPLATE_STR -> POST /city/STRING/district/INTEGER + * Traffic endpoints: + * GET /books/INTEGER -> GET /books/AKTO_TEMPLATE_STR + * POST /city/STRING/district/INTEGER -> POST /city/AKTO_TEMPLATE_STR/district/AKTO_TEMPLATE_STR + */ + + List trafficApis = ApiCollectionsDao.fetchEndpointsInCollectionUsingHost(apiCollection.getId(), 0, -1, 60 * 24 * 60 * 60); + Map trafficApiEndpointAktoTemplateStrToOriginalMap = new HashMap<>(); + List trafficApiKeys = new ArrayList<>(); + for (BasicDBObject trafficApi: trafficApis) { + BasicDBObject trafficApiApiInfoKey = (BasicDBObject) trafficApi.get("_id"); + String trafficApiMethod = trafficApiApiInfoKey.getString("method"); + String trafficApiUrl = trafficApiApiInfoKey.getString("url"); + String trafficApiEndpoint = ""; + + // extract path name from url + try { + // Directly parse the trafficApiUrl as a URI + URI uri = new URI(trafficApiUrl); + trafficApiEndpoint = uri.getPath(); + + // Decode any percent-encoded characters in the path + trafficApiEndpoint = java.net.URLDecoder.decode(trafficApiEndpoint, "UTF-8"); + + } catch (Exception e) { + continue; + } + + + // Ensure endpoint doesn't end with a slash + if (trafficApiEndpoint.length() > 1 && trafficApiEndpoint.endsWith("/")) { + trafficApiEndpoint = trafficApiEndpoint.substring(0, trafficApiEndpoint.length() - 1); + } + + String trafficApiKey = trafficApiMethod + " " + trafficApiEndpoint; + trafficApiKeys.add(trafficApiKey); + + String trafficApiEndpointAktoTemplateStr = trafficApiEndpoint; + + for (SingleTypeInfo.SuperType type : SingleTypeInfo.SuperType.values()) { + // Replace each occurrence of Akto template url format with"AKTO_TEMPLATE_STRING" + trafficApiEndpointAktoTemplateStr = trafficApiEndpointAktoTemplateStr.replace(type.name(), "AKTO_TEMPLATE_STR"); + } + + trafficApiEndpointAktoTemplateStrToOriginalMap.put(trafficApiEndpointAktoTemplateStr, trafficApiEndpoint); + } + + Map tempCodeAnalysisApisMap = new HashMap<>(codeAnalysisApisMap); + for (Map.Entry codeAnalysisApiEntry: codeAnalysisApisMap.entrySet()) { + String codeAnalysisApiKey = codeAnalysisApiEntry.getKey(); + CodeAnalysisApi codeAnalysisApi = codeAnalysisApiEntry.getValue(); + + String codeAnalysisApiEndpointAktoTemplateStr = codeAnalysisApi.getEndpoint(); + + for (SingleTypeInfo.SuperType type : SingleTypeInfo.SuperType.values()) { + // Replace each occurrence of Akto template url format with "AKTO_TEMPLATE_STRING" + codeAnalysisApiEndpointAktoTemplateStr = codeAnalysisApiEndpointAktoTemplateStr.replace(type.name(), "AKTO_TEMPLATE_STR"); + } + + if(codeAnalysisApiEndpointAktoTemplateStr.contains("AKTO_TEMPLATE_STR") && trafficApiEndpointAktoTemplateStrToOriginalMap.containsKey(codeAnalysisApiEndpointAktoTemplateStr)) { + CodeAnalysisApi newCodeAnalysisApi = new CodeAnalysisApi( + codeAnalysisApi.getMethod(), + trafficApiEndpointAktoTemplateStrToOriginalMap.get(codeAnalysisApiEndpointAktoTemplateStr), + codeAnalysisApi.getLocation(), codeAnalysisApi.getRequestBody(), codeAnalysisApi.getResponseBody()); + + tempCodeAnalysisApisMap.remove(codeAnalysisApiKey); + tempCodeAnalysisApisMap.put(newCodeAnalysisApi.generateCodeAnalysisApisMapKey(), newCodeAnalysisApi); + } + } + + + /* + * Match endpoints between traffic and source code endpoints, when only method is different + * Eg: + * Source code endpoints: + * POST /books + * Traffic endpoints: + * PUT /books + * Add PUT /books to source code endpoints + */ + for(String trafficApiKey: trafficApiKeys) { + if (!codeAnalysisApisMap.containsKey(trafficApiKey)) { + for(Map.Entry codeAnalysisApiEntry: tempCodeAnalysisApisMap.entrySet()) { + CodeAnalysisApi codeAnalysisApi = codeAnalysisApiEntry.getValue(); + String codeAnalysisApiEndpoint = codeAnalysisApi.getEndpoint(); + + String trafficApiMethod = "", trafficApiEndpoint = ""; + try { + String[] trafficApiKeyParts = trafficApiKey.split(" "); + trafficApiMethod = trafficApiKeyParts[0]; + trafficApiEndpoint = trafficApiKeyParts[1]; + } catch (Exception e) { + continue; + } + + if (codeAnalysisApiEndpoint.equals(trafficApiEndpoint)) { + CodeAnalysisApi newCodeAnalysisApi = new CodeAnalysisApi( + trafficApiMethod, + trafficApiEndpoint, + codeAnalysisApi.getLocation(), codeAnalysisApi.getRequestBody(), codeAnalysisApi.getResponseBody()); + + tempCodeAnalysisApisMap.put(newCodeAnalysisApi.generateCodeAnalysisApisMapKey(), newCodeAnalysisApi); + break; + } + } + } + } + + codeAnalysisApisMap = tempCodeAnalysisApisMap; + + ObjectId codeAnalysisCollectionId = null; + try { + // ObjectId for new code analysis collection + codeAnalysisCollectionId = new ObjectId(); + + String projectDir = codeAnalysisRepo.getProjectName() + "/" + codeAnalysisRepo.getRepoName(); //todo: + + CodeAnalysisCollection codeAnalysisCollection = CodeAnalysisCollectionDao.instance.updateOne( + Filters.eq("codeAnalysisCollectionName", apiCollectionName), + Updates.combine( + Updates.setOnInsert(CodeAnalysisCollection.ID, codeAnalysisCollectionId), + Updates.setOnInsert(CodeAnalysisCollection.NAME, apiCollectionName), + Updates.set(CodeAnalysisCollection.PROJECT_DIR, projectDir), + Updates.setOnInsert(CodeAnalysisCollection.API_COLLECTION_ID, apiCollection.getId()) + ) + ); + + // Set code analysis collection id if existing collection is updated + if (codeAnalysisCollection != null) { + codeAnalysisCollectionId = codeAnalysisCollection.getId(); + } + } catch (Exception e) { + return; + } + + int now = Context.now(); + + if (codeAnalysisCollectionId != null) { + List> bulkUpdates = new ArrayList<>(); + List> bulkUpdatesSTI = new ArrayList<>(); + + for(Map.Entry codeAnalysisApiEntry: codeAnalysisApisMap.entrySet()) { + CodeAnalysisApi codeAnalysisApi = codeAnalysisApiEntry.getValue(); + CodeAnalysisApiInfo.CodeAnalysisApiInfoKey codeAnalysisApiInfoKey = new CodeAnalysisApiInfo.CodeAnalysisApiInfoKey(codeAnalysisCollectionId, codeAnalysisApi.getMethod(), codeAnalysisApi.getEndpoint()); + + bulkUpdates.add( + new UpdateOneModel<>( + Filters.eq(CodeAnalysisApiInfo.ID, codeAnalysisApiInfoKey), + Updates.combine( + Updates.setOnInsert(CodeAnalysisApiInfo.ID, codeAnalysisApiInfoKey), + Updates.set(CodeAnalysisApiInfo.LOCATION, codeAnalysisApi.getLocation()), + Updates.setOnInsert(CodeAnalysisApiInfo.DISCOVERED_TS, now), + Updates.set(CodeAnalysisApiInfo.LAST_SEEN_TS, now) + ), + new UpdateOptions().upsert(true) + ) + ); + + String requestBody = codeAnalysisApi.getRequestBody(); + String responseBody = codeAnalysisApi.getResponseBody(); + + List singleTypeInfos = new ArrayList<>(); + singleTypeInfos.addAll(generateSTIsFromPayload(apiCollection.getId(), codeAnalysisApi.getEndpoint(), codeAnalysisApi.getMethod(), requestBody, -1)); + singleTypeInfos.addAll(generateSTIsFromPayload(apiCollection.getId(), codeAnalysisApi.getEndpoint(), codeAnalysisApi.getMethod(), responseBody, 200)); + + Bson update = Updates.combine(Updates.max(SingleTypeInfo.LAST_SEEN, now), Updates.setOnInsert("timestamp", now)); + + for (SingleTypeInfo singleTypeInfo: singleTypeInfos) { + bulkUpdatesSTI.add( + new UpdateOneModel<>( + SingleTypeInfoDao.createFilters(singleTypeInfo), + update, + new UpdateOptions().upsert(true) + ) + ); + } + + } + + if (!bulkUpdatesSTI.isEmpty()) { + CodeAnalysisSingleTypeInfoDao.instance.getMCollection().bulkWrite(bulkUpdatesSTI); + } + + if (bulkUpdates.size() > 0) { + try { + CodeAnalysisApiInfoDao.instance.getMCollection().bulkWrite(bulkUpdates); + } catch (Exception e) { + return; + } + } + } + + if (isLastBatch) {//Remove scheduled state from codeAnalysisRepo + Bson sourceCodeFilter; + if (codeAnalysisRepo.getSourceCodeType() == CodeAnalysisRepo.SourceCodeType.BITBUCKET) { + sourceCodeFilter = Filters.or( + Filters.eq(CodeAnalysisRepo.SOURCE_CODE_TYPE, codeAnalysisRepo.getSourceCodeType()), + Filters.exists(CodeAnalysisRepo.SOURCE_CODE_TYPE, false) + + ); + } else { + sourceCodeFilter = Filters.eq(CodeAnalysisRepo.SOURCE_CODE_TYPE, codeAnalysisRepo.getSourceCodeType()); + } + + Bson filters = Filters.and( + Filters.eq(CodeAnalysisRepo.REPO_NAME, codeAnalysisRepo.getRepoName()), + Filters.eq(CodeAnalysisRepo.PROJECT_NAME, codeAnalysisRepo.getProjectName()), + sourceCodeFilter + ); + + CodeAnalysisRepoDao.instance.updateOneNoUpsert(filters, Updates.set(CodeAnalysisRepo.LAST_RUN, Context.now())); + } } @Override public void updateRepoLastRun(CodeAnalysisRepo codeAnalysisRepo) { - return; + Bson sourceCodeFilter; + if (codeAnalysisRepo == null) { + return; + } + + if (codeAnalysisRepo.getSourceCodeType() == CodeAnalysisRepo.SourceCodeType.BITBUCKET) { + sourceCodeFilter = Filters.or( + Filters.eq(CodeAnalysisRepo.SOURCE_CODE_TYPE, codeAnalysisRepo.getSourceCodeType()), + Filters.exists(CodeAnalysisRepo.SOURCE_CODE_TYPE, false) + + ); + } else { + sourceCodeFilter = Filters.eq(CodeAnalysisRepo.SOURCE_CODE_TYPE, codeAnalysisRepo.getSourceCodeType()); + } + + Bson filters = Filters.and( + Filters.eq(CodeAnalysisRepo.REPO_NAME, codeAnalysisRepo.getRepoName()), + Filters.eq(CodeAnalysisRepo.PROJECT_NAME, codeAnalysisRepo.getProjectName()), + sourceCodeFilter + ); + + CodeAnalysisRepoDao.instance.updateOneNoUpsert(filters, Updates.set(CodeAnalysisRepo.LAST_RUN, Context.now())); } public Set fetchMergedUrls() { From fc6eefa0da5299144f35c12fc4143a636ed0ff1f Mon Sep 17 00:00:00 2001 From: SHIVAM RAWAT Date: Wed, 13 Nov 2024 07:00:15 +0530 Subject: [PATCH 6/6] mongo based code-analysis --- .../src/main/java/com/akto/BitbucketRepo.java | 1 + .../src/main/java/com/akto/GithubRepo.java | 3 ++- .../src/main/java/com/akto/Main.java | 10 +++++----- .../java/com/akto/SourceCodeAnalyserRepo.java | 15 +++++++++++++-- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/apps/source-code-analyser/src/main/java/com/akto/BitbucketRepo.java b/apps/source-code-analyser/src/main/java/com/akto/BitbucketRepo.java index cf2cced3fe..d2aa581cbd 100644 --- a/apps/source-code-analyser/src/main/java/com/akto/BitbucketRepo.java +++ b/apps/source-code-analyser/src/main/java/com/akto/BitbucketRepo.java @@ -96,6 +96,7 @@ public BasicDBObject getCodeAnalysisBody(String path) { requestBody.put("repoName",this.getRepoToBeAnalysed().getRepoName()); requestBody.put("bitbucketHost",BITBUCKET_URL); requestBody.put("is_bitbucket",true); + requestBody.put("is_aktogpt",false); return requestBody; } diff --git a/apps/source-code-analyser/src/main/java/com/akto/GithubRepo.java b/apps/source-code-analyser/src/main/java/com/akto/GithubRepo.java index 18eadc446a..ec84102729 100644 --- a/apps/source-code-analyser/src/main/java/com/akto/GithubRepo.java +++ b/apps/source-code-analyser/src/main/java/com/akto/GithubRepo.java @@ -34,7 +34,7 @@ public String getRepoUrl() { @Override public BasicDBObject getCodeAnalysisBody(String path) { - if (path == null || StringUtils.isEmpty(GITHUB_ACCESS_TOKEN)) { + if (path == null) { return null; } @@ -43,6 +43,7 @@ public BasicDBObject getCodeAnalysisBody(String path) { requestBody.put("orgName",this.getRepoToBeAnalysed().getProjectName()); requestBody.put("repoName",this.getRepoToBeAnalysed().getRepoName()); requestBody.put("is_github",true); + requestBody.put("is_aktogpt",false); return requestBody; } diff --git a/apps/source-code-analyser/src/main/java/com/akto/Main.java b/apps/source-code-analyser/src/main/java/com/akto/Main.java index 6543aebf0c..c1ac35f0a7 100644 --- a/apps/source-code-analyser/src/main/java/com/akto/Main.java +++ b/apps/source-code-analyser/src/main/java/com/akto/Main.java @@ -88,11 +88,11 @@ public static void main(String[] args) throws InterruptedException { boolean isConnectedToMongo = connectToMongo();//When mongo connection, fetch for all accounts while (true) { - if (!BitbucketRepo.doesEnvVariablesExists() && !GithubRepo.doesEnvVariablesExists()) { - loggerMaker.infoAndAddToDb("No tokens found"); - Thread.sleep(SLEEP_TIME); - continue; - } +// if (!BitbucketRepo.doesEnvVariablesExists() && !GithubRepo.doesEnvVariablesExists()) { +// loggerMaker.infoAndAddToDb("No tokens found"); +// Thread.sleep(SLEEP_TIME); +// continue; +// } if (isConnectedToMongo) { AccountTask.instance.executeTask(t -> { List repos = fetchReposToSync(); diff --git a/apps/source-code-analyser/src/main/java/com/akto/SourceCodeAnalyserRepo.java b/apps/source-code-analyser/src/main/java/com/akto/SourceCodeAnalyserRepo.java index e517361982..a6ee6be5b2 100644 --- a/apps/source-code-analyser/src/main/java/com/akto/SourceCodeAnalyserRepo.java +++ b/apps/source-code-analyser/src/main/java/com/akto/SourceCodeAnalyserRepo.java @@ -29,6 +29,7 @@ abstract public class SourceCodeAnalyserRepo { private CodeAnalysisRepo repoToBeAnalysed; + private boolean isAktoGPTEnabled; private static final LoggerMaker loggerMaker = new LoggerMaker(SourceCodeAnalyserRepo.class, LoggerMaker.LogDb.RUNTIME); abstract public String getToken(); abstract public String getRepoUrl(); @@ -58,7 +59,7 @@ abstract public class SourceCodeAnalyserRepo { public String downloadRepository () { String finalUrl = this.getRepoUrl(); String token = this.getToken(); - if (finalUrl == null || token == null) { + if (finalUrl == null) { return null; } String outputFilePath = System.getenv("DOCKER_VOLUME") + repoToBeAnalysed.getRepoName()+ ".zip"; // The local file where the repository will be saved @@ -67,7 +68,9 @@ public String downloadRepository () { Request.Builder builder = new Request.Builder(); builder.url(finalUrl); builder.get(); -// builder.addHeader("Authorization", "Bearer " + token); + if (token != null) { + builder.addHeader("Authorization", "Bearer " + token); + } Request request = builder.build(); try { @@ -179,4 +182,12 @@ public CodeAnalysisRepo getRepoToBeAnalysed() { public void setRepoToBeAnalysed(CodeAnalysisRepo repoToBeAnalysed) { this.repoToBeAnalysed = repoToBeAnalysed; } + + public boolean isAktoGPTEnabled() { + return isAktoGPTEnabled; + } + + public void setAktoGPTEnabled(boolean aktoGPTEnabled) { + isAktoGPTEnabled = aktoGPTEnabled; + } }