From 8af6f5c84a59f1512e4f4783498bf333a782c84f Mon Sep 17 00:00:00 2001 From: Pranav Gaikwad Date: Mon, 20 May 2024 19:20:04 -0400 Subject: [PATCH] :sparkles: Path inclusion (#606) Bundle PR: 94 --------- Signed-off-by: Pranav Gaikwad --- .github/workflows/demo-testing.yml | 40 ++++-- demo-output.yaml | 125 +++++++++++++++--- .../inclusion_tests/dir-0/inclusion-test.json | 5 + .../inclusion_tests/dir-0/inclusion-test.xml | 5 + .../inclusion_tests/inclusion-test.json | 5 + .../inclusion_tests/inclusion-test.xml | 5 + .../src/main/java/com/example/apps/App.java | 7 + .../src/main/java/com/example/apps/Bean.java | 1 + .../com/example/apps/util/FileReader.java | 13 ++ .../pkg/java_external_provider/provider.go | 1 + .../java_external_provider/service_client.go | 8 +- provider/internal/builtin/provider.go | 1 + provider/internal/builtin/service_client.go | 105 ++++++++++----- .../internal/builtin/service_client_test.go | 62 +++++++++ provider/lib.go | 27 ++++ provider/provider.go | 1 + provider_container_settings.json | 21 ++- provider_local_external_images.json | 21 ++- provider_pod_local_settings.json | 21 ++- rule-example.yaml | 39 ++++++ 20 files changed, 452 insertions(+), 61 deletions(-) create mode 100644 examples/builtin/inclusion_tests/dir-0/inclusion-test.json create mode 100644 examples/builtin/inclusion_tests/dir-0/inclusion-test.xml create mode 100644 examples/builtin/inclusion_tests/inclusion-test.json create mode 100644 examples/builtin/inclusion_tests/inclusion-test.xml create mode 100644 external-providers/java-external-provider/examples/java/example/src/main/java/com/example/apps/util/FileReader.java diff --git a/.github/workflows/demo-testing.yml b/.github/workflows/demo-testing.yml index e70330ca..95f176c8 100644 --- a/.github/workflows/demo-testing.yml +++ b/.github/workflows/demo-testing.yml @@ -13,25 +13,49 @@ jobs: env: PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} run: | + echo "BUILD_BUNDLE=false" >> $GITHUB_OUTPUT # if this is a PR, we should use the base branch # else, use the branch on which this is running if [ ! -z ${GITHUB_BASE_REF} ]; then echo "Using ${GITHUB_BASE_REF}" echo "API_TESTS_REF=${GITHUB_BASE_REF}" >> $GITHUB_OUTPUT - echo "ADDON_REF=${GITHUB_BASE_REF}" >>$GITHUB_ENV + echo "ADDON_REF=${GITHUB_BASE_REF}" >>$GITHUB_OUTPUT + echo "JAVA_BUNDLE_REF=${GITHUB_BASE_REF}" >>$GITHUB_OUTPUT else echo "Using ${GITHUB_REF_NAME}" echo "API_TESTS_REF=${GITHUB_REF_NAME}" >> $GITHUB_OUTPUT - echo "ADDON_REF=${GITHUB_REF_NAME}" >>$GITHUB_ENV + echo "ADDON_REF=${GITHUB_REF_NAME}" >>$GITHUB_OUTPUT + echo "JAVA_BUNDLE_REF=${GITHUB_REF_NAME}" >>$GITHUB_OUTPUT fi # override with explicitely set value in PR description echo "${PULL_REQUEST_BODY}" - PULL_REQUEST_NUMBER=$(echo "${PULL_REQUEST_BODY}" | grep -oP '[A|a]ddon [P|p][R|r]: \K\d+' || true) - if [ ! -z "$PULL_REQUEST_NUMBER" ]; then - echo "Using pull/${PULL_REQUEST_NUMBER} for addon" - echo "ADDON_REF=refs/pull/$PULL_REQUEST_NUMBER/merge" >>$GITHUB_ENV - fi + ADDON_PULL_REQUEST_NUMBER=$(echo "${PULL_REQUEST_BODY}" | grep -oP '[A|a]ddon [P|p][R|r]: \K\d+' || true) + if [ ! -z "$ADDON_PULL_REQUEST_NUMBER" ]; then + echo "Using pull/${ADDON_PULL_REQUEST_NUMBER} for addon" + echo "ADDON_REF=refs/pull/$ADDON_PULL_REQUEST_NUMBER/merge" >>$GITHUB_OUTPUT + fi + + JAVA_BUNDLE_PR_NUMBER=$(echo "${PULL_REQUEST_BODY}" | grep -oP '[B|b]undle [P|p][R|r]: \K\d+' || true) + if [ ! -z "$JAVA_BUNDLE_PR_NUMBER" ]; then + echo "Using bundle PR pull/${JAVA_BUNDLE_PR_NUMBER}" + echo "JAVA_BUNDLE_REF=refs/pull/$JAVA_BUNDLE_PR_NUMBER/merge" >> $GITHUB_OUTPUT + echo "BUILD_BUNDLE=true" >>$GITHUB_OUTPUT + fi + + - uses: actions/checkout@v3 + if: steps.extract-info.outputs.BUILD_BUNDLE + with: + fetch-depth: 0 + repository: konveyor/java-analyzer-bundle + ref: "${{ steps.extract-info.outputs.JAVA_BUNDLE_REF }}" + path: java-analyzer-bundle + + - name: build java analyzer bundle image + if: steps.extract-info.outputs.BUILD_BUNDLE + working-directory: java-analyzer-bundle + run: | + podman build -t quay.io/konveyor/jdtls-server-base:latest -f Dockerfile . - uses: actions/checkout@v3 @@ -64,7 +88,7 @@ jobs: with: fetch-depth: 0 repository: konveyor/tackle2-addon-analyzer - ref: "${{ env.ADDON_REF}}" + ref: ${{ steps.extract-info.outputs.ADDON_REF }} path: tackle2-addon-analyzer - name: Build addon and save image diff --git a/demo-output.yaml b/demo-output.yaml index 5a1b441f..c4c0f803 100644 --- a/demo-output.yaml +++ b/demo-output.yaml @@ -6,6 +6,58 @@ - Language=Golang - License=Apache violations: + builtin-inclusion-test-json: + description: | + This is same as java-io-file-usage but for the builtin providers. There are multiple instances of the same incidents in different directories. + We are filtering some out using includedPaths setting. + category: optional + incidents: + - uri: file:///examples/builtin/inclusion_tests/dir-0/inclusion-test.json + message: Only incidents in dir-0/test.json should be found + - uri: file:///examples/builtin/inclusion_tests/dir-0/inclusion-test.json + message: Only incidents in dir-0/test.json should be found + codeSnip: |2 + 1 { + 2 "description": "Does your JSON search work?", + 3 "name": "test-your-json-search", + 4 "inclusionTestNode": "Test this node" + 5 } + lineNumber: 4 + variables: + data: inclusionTestNode + matchingJSON: Test this node + - uri: file:///examples/builtin/inclusion_tests/dir-0/inclusion-test.xml + message: Only incidents in dir-0/test.json should be found + codeSnip: |2 + 1 + 2 Does your XML search work? + 3 xml-search + 4 Test this node + 5 + lineNumber: 4 + variables: + matchingText: inclusionTestNode + builtin-inclusion-test-xml: + description: | + This is same as java-io-file-usage but for the builtin providers. There are multiple instances of the same incidents in different directories. + We are filtering some out using includedPaths setting. + category: optional + incidents: + - uri: file:///examples/builtin/inclusion_tests/dir-0/inclusion-test.xml + message: Only incidents in dir-0/test.xml should be found + - uri: file:///examples/builtin/inclusion_tests/dir-0/inclusion-test.xml + message: Only incidents in dir-0/test.xml should be found + codeSnip: |2 + 1 + 2 Does your XML search work? + 3 xml-search + 4 Test this node + 5 + lineNumber: 4 + variables: + data: inclusionTestNode + innerText: Test this node + matchingXML: Test this node chain-pom-001: description: "" category: potential @@ -302,6 +354,47 @@ variables: name: sigs.k8s.io/structured-merge-diff/v4 version: v4.2.1 + java-inclusion-test: + description: "This rule tests includedPaths config of the java provider. There should be two instances of this issue in the example app. \nWe are filtering one of them using includedPaths in provider config.\n" + category: mandatory + incidents: + - uri: file:///examples/java/example/src/main/java/com/example/apps/App.java + message: Only incidents in util/FileReader.java should be found + codeSnip: " 1 package com.example.apps;\n 2 \n 3 import java.io.File;\n 4 import io.fabric8.kubernetes.api.model.apiextensions.v1beta1.CustomResourceDefinition;\n 5 \n 6 public class App \n 7 {\n 8 \n 9 /**\n10 * {@link CustomResourceDefinition}\n11 * @param args\n12 */\n13 public static void main( String[] args )" + lineNumber: 3 + variables: + file: file:///examples/java/example/src/main/java/com/example/apps/App.java + kind: Module + name: java.io.File + package: com.example.apps + - uri: file:///examples/java/example/src/main/java/com/example/apps/App.java + message: Only incidents in util/FileReader.java should be found + codeSnip: "12 */\n13 public static void main( String[] args )\n14 {\n15 CustomResourceDefinition crd = new CustomResourceDefinition();\n16 System.out.println( crd );\n17 \n18 GenericClass element = new GenericClass(\"Hello world!\");\n19 element.get();\n20 \n21 // test file usage\n22 File file = new File(\"test\");\n23 if (file.exists()) {\n24 System.out.println(\"file exists\");\n25 }\n26 }\n27 }\n" + lineNumber: 22 + variables: + file: file:///examples/java/example/src/main/java/com/example/apps/App.java + kind: Method + name: main + package: com.example.apps + - uri: file:///examples/java/example/src/main/java/com/example/apps/util/FileReader.java + message: Only incidents in util/FileReader.java should be found + codeSnip: " 1 package com.example.apps.util;\n 2 \n 3 import java.io.File;\n 4 \n 5 public class FileReader {\n 6 public static void main(String[] args) {\n 7 File file = new File(\"test\");\n 8 if (file.exists()) {\n 9 System.out.println(\"file exists\");\n10 }\n11 }\n12 \n13 }" + lineNumber: 3 + variables: + file: file:///examples/java/example/src/main/java/com/example/apps/util/FileReader.java + kind: Module + name: java.io.File + package: com.example.apps.util + - uri: file:///examples/java/example/src/main/java/com/example/apps/util/FileReader.java + message: Only incidents in util/FileReader.java should be found + codeSnip: " 1 package com.example.apps.util;\n 2 \n 3 import java.io.File;\n 4 \n 5 public class FileReader {\n 6 public static void main(String[] args) {\n 7 File file = new File(\"test\");\n 8 if (file.exists()) {\n 9 System.out.println(\"file exists\");\n10 }\n11 }\n12 \n13 }\n" + lineNumber: 7 + variables: + file: file:///examples/java/example/src/main/java/com/example/apps/util/FileReader.java + kind: Method + name: main + package: com.example.apps.util + effort: 3 java-pomxml-dependencies: description: "" category: potential @@ -370,8 +463,8 @@ file: file:///examples/golang/main.go - uri: file:///examples/java/example/src/main/java/com/example/apps/App.java message: apiextensions/v1beta1/customresourcedefinitions is deprecated, apiextensions/v1/customresourcedefinitions should be used instead - codeSnip: " 1 package com.example.apps;\n 2 \n 3 import io.fabric8.kubernetes.api.model.apiextensions.v1beta1.CustomResourceDefinition;\n 4 \n 5 public class App \n 6 {\n 7 \n 8 /**\n 9 * {@link CustomResourceDefinition}\n10 * @param args\n11 */\n12 public static void main( String[] args )\n13 {" - lineNumber: 3 + codeSnip: " 1 package com.example.apps;\n 2 \n 3 import java.io.File;\n 4 import io.fabric8.kubernetes.api.model.apiextensions.v1beta1.CustomResourceDefinition;\n 5 \n 6 public class App \n 7 {\n 8 \n 9 /**\n10 * {@link CustomResourceDefinition}\n11 * @param args\n12 */\n13 public static void main( String[] args )\n14 {" + lineNumber: 4 variables: file: file:///examples/java/example/src/main/java/com/example/apps/App.java kind: Module @@ -379,8 +472,8 @@ package: com.example.apps - uri: file:///examples/java/example/src/main/java/com/example/apps/App.java message: apiextensions/v1beta1/customresourcedefinitions is deprecated, apiextensions/v1/customresourcedefinitions should be used instead - codeSnip: " 4 \n 5 public class App \n 6 {\n 7 \n 8 /**\n 9 * {@link CustomResourceDefinition}\n10 * @param args\n11 */\n12 public static void main( String[] args )\n13 {\n14 CustomResourceDefinition crd = new CustomResourceDefinition();\n15 System.out.println( crd );\n16 \n17 GenericClass element = new GenericClass(\"Hello world!\");\n18 element.get();\n19 }\n20 }\n" - lineNumber: 14 + codeSnip: " 5 \n 6 public class App \n 7 {\n 8 \n 9 /**\n10 * {@link CustomResourceDefinition}\n11 * @param args\n12 */\n13 public static void main( String[] args )\n14 {\n15 CustomResourceDefinition crd = new CustomResourceDefinition();\n16 System.out.println( crd );\n17 \n18 GenericClass element = new GenericClass(\"Hello world!\");\n19 element.get();\n20 \n21 // test file usage\n22 File file = new File(\"test\");\n23 if (file.exists()) {\n24 System.out.println(\"file exists\");\n25 }" + lineNumber: 15 variables: file: file:///examples/java/example/src/main/java/com/example/apps/App.java kind: Method @@ -391,18 +484,18 @@ category: potential incidents: - uri: file:///examples/java/example/src/main/java/com/example/apps/App.java - message: java found apiextensions/v1/customresourcedefinitions found file:///examples/java/example/src/main/java/com/example/apps/App.java:14 - codeSnip: " 4 \n 5 public class App \n 6 {\n 7 \n 8 /**\n 9 * {@link CustomResourceDefinition}\n10 * @param args\n11 */\n12 public static void main( String[] args )\n13 {\n14 CustomResourceDefinition crd = new CustomResourceDefinition();\n15 System.out.println( crd );\n16 \n17 GenericClass element = new GenericClass(\"Hello world!\");\n18 element.get();\n19 }\n20 }\n" - lineNumber: 14 + message: java found apiextensions/v1/customresourcedefinitions found file:///examples/java/example/src/main/java/com/example/apps/App.java:15 + codeSnip: " 5 \n 6 public class App \n 7 {\n 8 \n 9 /**\n10 * {@link CustomResourceDefinition}\n11 * @param args\n12 */\n13 public static void main( String[] args )\n14 {\n15 CustomResourceDefinition crd = new CustomResourceDefinition();\n16 System.out.println( crd );\n17 \n18 GenericClass element = new GenericClass(\"Hello world!\");\n19 element.get();\n20 \n21 // test file usage\n22 File file = new File(\"test\");\n23 if (file.exists()) {\n24 System.out.println(\"file exists\");\n25 }" + lineNumber: 15 variables: file: file:///examples/java/example/src/main/java/com/example/apps/App.java kind: Method name: main package: com.example.apps - uri: file:///examples/java/example/src/main/java/com/example/apps/App.java - message: java found apiextensions/v1/customresourcedefinitions found file:///examples/java/example/src/main/java/com/example/apps/App.java:3 - codeSnip: " 1 package com.example.apps;\n 2 \n 3 import io.fabric8.kubernetes.api.model.apiextensions.v1beta1.CustomResourceDefinition;\n 4 \n 5 public class App \n 6 {\n 7 \n 8 /**\n 9 * {@link CustomResourceDefinition}\n10 * @param args\n11 */\n12 public static void main( String[] args )\n13 {" - lineNumber: 3 + message: java found apiextensions/v1/customresourcedefinitions found file:///examples/java/example/src/main/java/com/example/apps/App.java:4 + codeSnip: " 1 package com.example.apps;\n 2 \n 3 import java.io.File;\n 4 import io.fabric8.kubernetes.api.model.apiextensions.v1beta1.CustomResourceDefinition;\n 5 \n 6 public class App \n 7 {\n 8 \n 9 /**\n10 * {@link CustomResourceDefinition}\n11 * @param args\n12 */\n13 public static void main( String[] args )\n14 {" + lineNumber: 4 variables: file: file:///examples/java/example/src/main/java/com/example/apps/App.java kind: Module @@ -414,8 +507,8 @@ incidents: - uri: file:///examples/java/example/src/main/java/com/example/apps/App.java message: found generic call - codeSnip: " 8 /**\n 9 * {@link CustomResourceDefinition}\n10 * @param args\n11 */\n12 public static void main( String[] args )\n13 {\n14 CustomResourceDefinition crd = new CustomResourceDefinition();\n15 System.out.println( crd );\n16 \n17 GenericClass element = new GenericClass(\"Hello world!\");\n18 element.get();\n19 }\n20 }\n" - lineNumber: 18 + codeSnip: " 9 /**\n10 * {@link CustomResourceDefinition}\n11 * @param args\n12 */\n13 public static void main( String[] args )\n14 {\n15 CustomResourceDefinition crd = new CustomResourceDefinition();\n16 System.out.println( crd );\n17 \n18 GenericClass element = new GenericClass(\"Hello world!\");\n19 element.get();\n20 \n21 // test file usage\n22 File file = new File(\"test\");\n23 if (file.exists()) {\n24 System.out.println(\"file exists\");\n25 }\n26 }\n27 }\n" + lineNumber: 19 variables: VariableName: element file: file:///examples/java/example/src/main/java/com/example/apps/App.java @@ -490,7 +583,7 @@ incidents: - uri: file:///examples/java/example/src/main/java/com/example/apps/Bean.java message: condition entries should evaluate out of order - codeSnip: " 1 package com.example.apps;\n 2 \n 3 import javax.ejb.SessionBean;\n 4 import javax.ejb.Singleton;\n 5 \n 6 @Singleton\n 7 public abstract class Bean implements SessionBean {\n 8 }\n" + codeSnip: " 1 package com.example.apps;\n 2 \n 3 import javax.ejb.SessionBean;\n 4 import javax.ejb.Singleton;\n 5 \n 6 @Singleton\n 7 public abstract class Bean implements SessionBean {\n 8 \n 9 }\n" lineNumber: 6 variables: file: file:///examples/java/example/src/main/java/com/example/apps/Bean.java @@ -499,7 +592,7 @@ package: com.example.apps - uri: file:///examples/java/example/src/main/java/com/example/apps/Bean.java message: condition entries should evaluate out of order - codeSnip: " 1 package com.example.apps;\n 2 \n 3 import javax.ejb.SessionBean;\n 4 import javax.ejb.Singleton;\n 5 \n 6 @Singleton\n 7 public abstract class Bean implements SessionBean {\n 8 }\n" + codeSnip: " 1 package com.example.apps;\n 2 \n 3 import javax.ejb.SessionBean;\n 4 import javax.ejb.Singleton;\n 5 \n 6 @Singleton\n 7 public abstract class Bean implements SessionBean {\n 8 \n 9 }\n" lineNumber: 7 variables: file: file:///examples/java/example/src/main/java/com/example/apps/Bean.java @@ -512,7 +605,7 @@ incidents: - uri: file:///examples/java/example/src/main/java/com/example/apps/Bean.java message: condition entries should evaluate in order - codeSnip: " 1 package com.example.apps;\n 2 \n 3 import javax.ejb.SessionBean;\n 4 import javax.ejb.Singleton;\n 5 \n 6 @Singleton\n 7 public abstract class Bean implements SessionBean {\n 8 }\n" + codeSnip: " 1 package com.example.apps;\n 2 \n 3 import javax.ejb.SessionBean;\n 4 import javax.ejb.Singleton;\n 5 \n 6 @Singleton\n 7 public abstract class Bean implements SessionBean {\n 8 \n 9 }\n" lineNumber: 6 variables: file: file:///examples/java/example/src/main/java/com/example/apps/Bean.java @@ -521,7 +614,7 @@ package: com.example.apps - uri: file:///examples/java/example/src/main/java/com/example/apps/Bean.java message: condition entries should evaluate in order - codeSnip: " 1 package com.example.apps;\n 2 \n 3 import javax.ejb.SessionBean;\n 4 import javax.ejb.Singleton;\n 5 \n 6 @Singleton\n 7 public abstract class Bean implements SessionBean {\n 8 }\n" + codeSnip: " 1 package com.example.apps;\n 2 \n 3 import javax.ejb.SessionBean;\n 4 import javax.ejb.Singleton;\n 5 \n 6 @Singleton\n 7 public abstract class Bean implements SessionBean {\n 8 \n 9 }\n" lineNumber: 7 variables: file: file:///examples/java/example/src/main/java/com/example/apps/Bean.java diff --git a/examples/builtin/inclusion_tests/dir-0/inclusion-test.json b/examples/builtin/inclusion_tests/dir-0/inclusion-test.json new file mode 100644 index 00000000..a3a5bc90 --- /dev/null +++ b/examples/builtin/inclusion_tests/dir-0/inclusion-test.json @@ -0,0 +1,5 @@ +{ + "description": "Does your JSON search work?", + "name": "test-your-json-search", + "inclusionTestNode": "Test this node" +} \ No newline at end of file diff --git a/examples/builtin/inclusion_tests/dir-0/inclusion-test.xml b/examples/builtin/inclusion_tests/dir-0/inclusion-test.xml new file mode 100644 index 00000000..0cdffeee --- /dev/null +++ b/examples/builtin/inclusion_tests/dir-0/inclusion-test.xml @@ -0,0 +1,5 @@ + + Does your XML search work? + xml-search + Test this node + \ No newline at end of file diff --git a/examples/builtin/inclusion_tests/inclusion-test.json b/examples/builtin/inclusion_tests/inclusion-test.json new file mode 100644 index 00000000..a3a5bc90 --- /dev/null +++ b/examples/builtin/inclusion_tests/inclusion-test.json @@ -0,0 +1,5 @@ +{ + "description": "Does your JSON search work?", + "name": "test-your-json-search", + "inclusionTestNode": "Test this node" +} \ No newline at end of file diff --git a/examples/builtin/inclusion_tests/inclusion-test.xml b/examples/builtin/inclusion_tests/inclusion-test.xml new file mode 100644 index 00000000..0cdffeee --- /dev/null +++ b/examples/builtin/inclusion_tests/inclusion-test.xml @@ -0,0 +1,5 @@ + + Does your XML search work? + xml-search + Test this node + \ No newline at end of file diff --git a/external-providers/java-external-provider/examples/java/example/src/main/java/com/example/apps/App.java b/external-providers/java-external-provider/examples/java/example/src/main/java/com/example/apps/App.java index 1bf6ecd6..549e1a3c 100644 --- a/external-providers/java-external-provider/examples/java/example/src/main/java/com/example/apps/App.java +++ b/external-providers/java-external-provider/examples/java/example/src/main/java/com/example/apps/App.java @@ -1,5 +1,6 @@ package com.example.apps; +import java.io.File; import io.fabric8.kubernetes.api.model.apiextensions.v1beta1.CustomResourceDefinition; public class App @@ -16,5 +17,11 @@ public static void main( String[] args ) GenericClass element = new GenericClass("Hello world!"); element.get(); + + // test file usage + File file = new File("test"); + if (file.exists()) { + System.out.println("file exists"); + } } } diff --git a/external-providers/java-external-provider/examples/java/example/src/main/java/com/example/apps/Bean.java b/external-providers/java-external-provider/examples/java/example/src/main/java/com/example/apps/Bean.java index 21fc114b..8a1f9fba 100644 --- a/external-providers/java-external-provider/examples/java/example/src/main/java/com/example/apps/Bean.java +++ b/external-providers/java-external-provider/examples/java/example/src/main/java/com/example/apps/Bean.java @@ -5,4 +5,5 @@ @Singleton public abstract class Bean implements SessionBean { + } diff --git a/external-providers/java-external-provider/examples/java/example/src/main/java/com/example/apps/util/FileReader.java b/external-providers/java-external-provider/examples/java/example/src/main/java/com/example/apps/util/FileReader.java new file mode 100644 index 00000000..245676ea --- /dev/null +++ b/external-providers/java-external-provider/examples/java/example/src/main/java/com/example/apps/util/FileReader.java @@ -0,0 +1,13 @@ +package com.example.apps.util; + +import java.io.File; + +public class FileReader { + public static void main(String[] args) { + File file = new File("test"); + if (file.exists()) { + System.out.println("file exists"); + } + } + +} diff --git a/external-providers/java-external-provider/pkg/java_external_provider/provider.go b/external-providers/java-external-provider/pkg/java_external_provider/provider.go index e57d2d37..1bedb85b 100644 --- a/external-providers/java-external-provider/pkg/java_external_provider/provider.go +++ b/external-providers/java-external-provider/pkg/java_external_provider/provider.go @@ -308,6 +308,7 @@ func (p *javaProvider) Init(ctx context.Context, log logr.Logger, config provide isLocationBinary: isBinary, mvnSettingsFile: mavenSettingsFile, depsLocationCache: make(map[string]int), + includedPaths: provider.GetIncludedPathsFromConfig(config, false), } svcClient.initialization(ctx) diff --git a/external-providers/java-external-provider/pkg/java_external_provider/service_client.go b/external-providers/java-external-provider/pkg/java_external_provider/service_client.go index c40f7d54..1749b7cb 100644 --- a/external-providers/java-external-provider/pkg/java_external_provider/service_client.go +++ b/external-providers/java-external-provider/pkg/java_external_provider/service_client.go @@ -32,6 +32,7 @@ type javaServiceClient struct { depsMutex sync.RWMutex depsCache map[uri.URI][]*provider.Dep depsLocationCache map[string]int + includedPaths []string } type depLabelItem struct { @@ -101,13 +102,18 @@ func (p *javaServiceClient) GetAllSymbols(ctx context.Context, query, location s // This command will run the added bundle to the language server. The command over the wire needs too look like this. // in this case the project is hardcoded in the init of the Langauge Server above // workspace/executeCommand '{"command": "io.konveyor.tackle.ruleEntry", "arguments": {"query":"*customresourcedefinition","project": "java"}}' - argumentsMap := map[string]string{ + argumentsMap := map[string]interface{}{ "query": query, "project": "java", "location": fmt.Sprintf("%v", locationToCode[strings.ToLower(location)]), "analysisMode": string(p.config.AnalysisMode), } + if p.includedPaths != nil && len(p.includedPaths) > 0 { + argumentsMap[provider.IncludedPathsConfigKey] = p.includedPaths + p.log.V(5).Info("setting search scope by filepaths", "paths", p.includedPaths) + } + argumentsBytes, _ := json.Marshal(argumentsMap) arguments := []json.RawMessage{argumentsBytes} diff --git a/provider/internal/builtin/provider.go b/provider/internal/builtin/provider.go index 279bcb34..7e821a1b 100644 --- a/provider/internal/builtin/provider.go +++ b/provider/internal/builtin/provider.go @@ -150,6 +150,7 @@ func (p *builtinProvider) Init(ctx context.Context, log logr.Logger, config prov UnimplementedDependenciesComponent: provider.UnimplementedDependenciesComponent{}, locationCache: make(map[string]float64), log: log, + includedPaths: provider.GetIncludedPathsFromConfig(config, true), }, nil } diff --git a/provider/internal/builtin/service_client.go b/provider/internal/builtin/service_client.go index 1019bb8c..18c3c56f 100644 --- a/provider/internal/builtin/service_client.go +++ b/provider/internal/builtin/service_client.go @@ -30,6 +30,7 @@ type builtinServiceClient struct { cacheMutex sync.RWMutex locationCache map[string]float64 + includedPaths []string } type fileTemplateContext struct { @@ -58,30 +59,24 @@ func (p *builtinServiceClient) Evaluate(ctx context.Context, cap string, conditi return response, fmt.Errorf("unable to find files using pattern `%s`: %v", c.Pattern, err) } - if len(matchingFiles) != 0 { - response.Matched = true - } - response.TemplateContext = map[string]interface{}{"filepaths": matchingFiles} for _, match := range matchingFiles { - if filepath.IsAbs(match) { - response.Incidents = append(response.Incidents, provider.IncidentContext{ - FileURI: uri.File(match), - }) - continue - + absPath := match + if !filepath.IsAbs(match) { + absPath, err = filepath.Abs(match) + if err != nil { + p.log.V(5).Error(err, "failed to get absolute path to file", "path", match) + absPath = match + } } - ab, err := filepath.Abs(match) - if err != nil { - //TODO: Probably want to log or something to let us know we can't get absolute path here. - fmt.Printf("\n%v\n\n", err) - ab = match + if !p.isFileIncluded(absPath) { + continue } response.Incidents = append(response.Incidents, provider.IncidentContext{ - FileURI: uri.File(ab), + FileURI: uri.File(absPath), }) - } + response.Matched = len(response.Incidents) > 0 return response, nil case "filecontent": c := cond.Filecontent @@ -122,16 +117,21 @@ func (p *builtinServiceClient) Evaluate(ctx context.Context, cap string, conditi continue } - ab, err := filepath.Abs(pieces[0]) + absPath, err := filepath.Abs(pieces[0]) if err != nil { - ab = pieces[0] + absPath = pieces[0] } + + if !p.isFileIncluded(absPath) { + continue + } + lineNumber, err := strconv.Atoi(pieces[1]) if err != nil { return response, fmt.Errorf("cannot convert line number string to integer") } response.Incidents = append(response.Incidents, provider.IncidentContext{ - FileURI: uri.File(ab), + FileURI: uri.File(absPath), LineNumber: &lineNumber, Variables: map[string]interface{}{ "matchingText": pieces[2], @@ -155,7 +155,6 @@ func (p *builtinServiceClient) Evaluate(ctx context.Context, cap string, conditi if err != nil { return response, fmt.Errorf("unable to find XML files: %v", err) } - for _, file := range xmlFiles { nodes, err := queryXMLFile(file, query) if err != nil { @@ -165,12 +164,15 @@ func (p *builtinServiceClient) Evaluate(ctx context.Context, cap string, conditi if len(nodes) != 0 { response.Matched = true for _, node := range nodes { - ab, err := filepath.Abs(file) + absPath, err := filepath.Abs(file) if err != nil { - ab = file + absPath = file + } + if !p.isFileIncluded(absPath) { + continue } incident := provider.IncidentContext{ - FileURI: uri.File(ab), + FileURI: uri.File(absPath), Variables: map[string]interface{}{ "matchingXML": node.OutputXML(false), "innerText": node.InnerText(), @@ -181,7 +183,7 @@ func (p *builtinServiceClient) Evaluate(ctx context.Context, cap string, conditi if content == "" { content = node.Data } - location, err := p.getLocation(ctx, ab, content) + location, err := p.getLocation(ctx, absPath, content) if err == nil { incident.CodeLocation = &location lineNo := int(location.StartPosition.Line) @@ -206,7 +208,6 @@ func (p *builtinServiceClient) Evaluate(ctx context.Context, cap string, conditi if err != nil { return response, fmt.Errorf("unable to find XML files: %v", err) } - for _, file := range xmlFiles { nodes, err := queryXMLFile(file, query) if err != nil { @@ -220,12 +221,15 @@ func (p *builtinServiceClient) Evaluate(ctx context.Context, cap string, conditi if attr.Name.Local == "public-id" { if regex.MatchString(attr.Value) { response.Matched = true - ab, err := filepath.Abs(file) + absPath, err := filepath.Abs(file) if err != nil { - ab = file + absPath = file + } + if !p.isFileIncluded(absPath) { + continue } response.Incidents = append(response.Incidents, provider.IncidentContext{ - FileURI: uri.File(ab), + FileURI: uri.File(absPath), Variables: map[string]interface{}{ "matchingXML": node.OutputXML(false), "innerText": node.InnerText(), @@ -268,18 +272,21 @@ func (p *builtinServiceClient) Evaluate(ctx context.Context, cap string, conditi if len(list) != 0 { response.Matched = true for _, node := range list { - ab, err := filepath.Abs(file) + absPath, err := filepath.Abs(file) if err != nil { - ab = file + absPath = file + } + if !p.isFileIncluded(absPath) { + continue } incident := provider.IncidentContext{ - FileURI: uri.File(ab), + FileURI: uri.File(absPath), Variables: map[string]interface{}{ "matchingJSON": node.InnerText(), "data": node.Data, }, } - location, err := p.getLocation(ctx, ab, node.InnerText()) + location, err := p.getLocation(ctx, absPath, node.InnerText()) if err == nil { incident.CodeLocation = &location lineNo := int(location.StartPosition.Line) @@ -451,3 +458,35 @@ func queryXMLFile(filePath string, query *xpath.Expr) (nodes []*xmlquery.Node, e nodes = xmlquery.QuerySelectorAll(doc, query) return nodes, err } + +// filterByIncludedPaths given a list of file paths, +// filters-out the ones not present in includedPaths +func (b *builtinServiceClient) isFileIncluded(absolutePath string) bool { + if b.includedPaths == nil || len(b.includedPaths) == 0 { + return true + } + + getSegments := func(path string) []string { + segments := []string{} + path = filepath.Clean(path) + for _, segment := range strings.Split( + path, string(os.PathSeparator)) { + if segment != "" { + segments = append(segments, segment) + } + } + return segments + } + + pathSegments := getSegments(absolutePath) + for _, includedPath := range b.includedPaths { + includedPathSegments := getSegments(includedPath) + if len(pathSegments) >= len(includedPathSegments) && + strings.HasPrefix(strings.Join(pathSegments, ""), + strings.Join(includedPathSegments, "")) { + return true + } + } + b.log.V(5).Info("excluding file from search", "file", absolutePath) + return false +} diff --git a/provider/internal/builtin/service_client_test.go b/provider/internal/builtin/service_client_test.go index 41825980..8051ef54 100644 --- a/provider/internal/builtin/service_client_test.go +++ b/provider/internal/builtin/service_client_test.go @@ -60,3 +60,65 @@ func Test_builtinServiceClient_getLocation(t *testing.T) { }) } } + +func Test_builtinServiceClient_filterByIncludedPaths(t *testing.T) { + tests := []struct { + name string + inputPath string + includedPaths []string + want bool + }{ + { + name: "no included paths given, match all", + inputPath: "/test/a/b", + includedPaths: []string{}, + want: true, + }, + { + name: "included file path doesn't match", + inputPath: "/test/a/b/file.py", + includedPaths: []string{"/test/a/c/file.py"}, + want: false, + }, + { + name: "included file path matches", + inputPath: "/test/a/b/file.py", + includedPaths: []string{"/test/a/b/file.py"}, + want: true, + }, + { + name: "input dir path is equivalent to included paths", + inputPath: "/test/a/b/", + includedPaths: []string{"////test/a/b//"}, + want: true, + }, + { + name: "input dir path is a sub-tree of included path", + inputPath: "///test/a/b/c/e/", + includedPaths: []string{"////test/a/b//"}, + want: true, + }, + { + name: "input dir path is not equal to included path and is not a sub-tree", + inputPath: "///test/a/b/c/e/file.java", + includedPaths: []string{"////test/a/d//"}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &builtinServiceClient{ + config: provider.InitConfig{ + ProviderSpecificConfig: map[string]interface{}{ + "includedPaths": tt.includedPaths, + }, + }, + includedPaths: tt.includedPaths, + log: testr.New(t), + } + if got := b.isFileIncluded(tt.inputPath); !reflect.DeepEqual(got, tt.want) { + t.Errorf("builtinServiceClient.filterByIncludedPaths() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/provider/lib.go b/provider/lib.go index e908ba39..b26a5c34 100644 --- a/provider/lib.go +++ b/provider/lib.go @@ -142,3 +142,30 @@ func MultilineGrep(ctx context.Context, window int, path, pattern string) (int, return -1, scanner.Err() } + +// GetIncludedPathsFromConfig returns validated includedPaths from provider settings +// if allowFilePaths is not set, path to a file is converted into a path to its base dir +func GetIncludedPathsFromConfig(i InitConfig, allowFilePaths bool) []string { + validatedPaths := []string{} + if includedPaths, ok := i.ProviderSpecificConfig[IncludedPathsConfigKey].([]interface{}); ok { + for _, ipathRaw := range includedPaths { + if ipath, ok := ipathRaw.(string); ok { + absPath := ipath + if !filepath.IsAbs(ipath) { + if ab, err := filepath.Abs( + filepath.Join(i.Location, ipath)); err == nil { + absPath = ab + } + } + if stat, err := os.Stat(absPath); err == nil { + if allowFilePaths || stat.IsDir() { + validatedPaths = append(validatedPaths, absPath) + } else { + validatedPaths = append(validatedPaths, filepath.Dir(absPath)) + } + } + } + } + } + return validatedPaths +} diff --git a/provider/provider.go b/provider/provider.go index 495e6ca6..42a721e7 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -33,6 +33,7 @@ const ( DepExcludeLabel = "konveyor.io/exclude" // LspServerPath is a provider specific config used to specify path to a LSP server LspServerPathConfigKey = "lspServerPath" + IncludedPathsConfigKey = "includedPaths" ) // We need to make these Vars, because you can not take a pointer of the constant. diff --git a/provider_container_settings.json b/provider_container_settings.json index 25599497..3f140f8e 100644 --- a/provider_container_settings.json +++ b/provider_container_settings.json @@ -79,6 +79,19 @@ }, "analysisMode": "source-only" }, + { + "location": "/analyzer-lsp/examples/java", + "providerSpecificConfig": { + "lspServerName": "java", + "bundles": "/jdtls/java-analyzer-bundle/java-analyzer-bundle.core/target/java-analyzer-bundle.core-1.0.0-SNAPSHOT.jar", + "depOpenSourceLabelsFile": "/usr/local/etc/maven.default.index", + "lspServerPath": "/jdtls/bin/jdtls", + "includedPaths": [ + "example/src/main/java/com/example/apps/util/FileReader.java" + ] + }, + "analysisMode": "source-only" + }, { "location": "/analyzer-lsp/examples/customers-tomcat-legacy", "providerSpecificConfig": { @@ -97,7 +110,13 @@ {"location": "examples/java/"}, {"location": "examples/python/"}, {"location": "examples/golang/"}, - {"location": "examples/customers-tomcat-legacy/"} + {"location": "examples/customers-tomcat-legacy/"}, + { + "location": "examples/builtin/", + "providerSpecificConfig": { + "includedPaths": ["inclusion_tests/dir-0"] + } + } ] } ] diff --git a/provider_local_external_images.json b/provider_local_external_images.json index 55f55757..6da3d6d4 100644 --- a/provider_local_external_images.json +++ b/provider_local_external_images.json @@ -79,6 +79,19 @@ }, "analysisMode": "source-only" }, + { + "location": "/analyzer-lsp/examples/java", + "providerSpecificConfig": { + "lspServerName": "java", + "bundles": "/jdtls/java-analyzer-bundle/java-analyzer-bundle.core/target/java-analyzer-bundle.core-1.0.0-SNAPSHOT.jar", + "depOpenSourceLabelsFile": "/usr/local/etc/maven.default.index", + "lspServerPath": "/jdtls/bin/jdtls", + "includedPaths": [ + "example/src/main/java/com/example/apps/util/FileReader.java" + ] + }, + "analysisMode": "source-only" + }, { "location": "examples/customers-tomcat-legacy", "providerSpecificConfig": { @@ -97,7 +110,13 @@ {"location": "external-providers/java-external-provider/examples/java"}, {"location": "external-providers/java-external-provider/examples/customers-tomcat-legacy"}, {"location": "examples/python/"}, - {"location": "examples/golang/"} + {"location": "examples/golang/"}, + { + "location": "examples/builtin/", + "providerSpecificConfig": { + "includedPaths": ["inclusion_tests/dir-0"] + } + } ] } ] diff --git a/provider_pod_local_settings.json b/provider_pod_local_settings.json index 6f83f1a6..1fe9f01e 100644 --- a/provider_pod_local_settings.json +++ b/provider_pod_local_settings.json @@ -79,6 +79,19 @@ }, "analysisMode": "source-only" }, + { + "location": "/analyzer-lsp/examples/java", + "providerSpecificConfig": { + "lspServerName": "java", + "bundles": "/jdtls/java-analyzer-bundle/java-analyzer-bundle.core/target/java-analyzer-bundle.core-1.0.0-SNAPSHOT.jar", + "depOpenSourceLabelsFile": "/usr/local/etc/maven.default.index", + "lspServerPath": "/jdtls/bin/jdtls", + "includedPaths": [ + "example/src/main/java/com/example/apps/util/FileReader.java" + ] + }, + "analysisMode": "source-only" + }, { "location": "/analyzer-lsp/examples/customers-tomcat-legacy", "providerSpecificConfig": { @@ -97,7 +110,13 @@ {"location": "examples/java/"}, {"location": "examples/python/"}, {"location": "examples/golang/"}, - {"location": "examples/customers-tomcat-legacy/"} + {"location": "examples/customers-tomcat-legacy/"}, + { + "location": "examples/builtin/", + "providerSpecificConfig": { + "includedPaths": ["inclusion_tests/dir-0"] + } + } ] } ] diff --git a/rule-example.yaml b/rule-example.yaml index 9708fe02..4d614909 100644 --- a/rule-example.yaml +++ b/rule-example.yaml @@ -253,3 +253,42 @@ java.dependency: lowerbound: 0.0.0 name: javax.activation.activation +- category: mandatory + description: | + This rule tests includedPaths config of the java provider. There should be two instances of this issue in the example app. + We are filtering one of them using includedPaths in provider config. + effort: 3 + message: Only incidents in util/FileReader.java should be found + ruleID: java-inclusion-test + when: + java.referenced: + pattern: java.io.File +- category: optional + description: | + This is same as java-io-file-usage but for the builtin providers. There are multiple instances of the same incidents in different directories. + We are filtering some out using includedPaths setting. + message: Only incidents in dir-0/test.json should be found + ruleID: builtin-inclusion-test-json + when: + and: + - builtin.json: + xpath: //inclusionTestNode + - builtin.file: + pattern: inclusion-test.json + - builtin.filecontent: + pattern: "inclusionTestNode" +- category: optional + description: | + This is same as java-io-file-usage but for the builtin providers. There are multiple instances of the same incidents in different directories. + We are filtering some out using includedPaths setting. + message: Only incidents in dir-0/test.xml should be found + ruleID: builtin-inclusion-test-xml + when: + and: + - builtin.xml: + xpath: //inclusionTestNode + - builtin.file: + pattern: inclusion-test.xml + - builtin.filecontent: + pattern: "" + \ No newline at end of file