Merge pull request #1151 from keshav-space/advisory_compare

Support Advisory Comparison in VulnTotal

requirements.txt

@@ -106,7 +106,7 @@ toml==0.10.2
 tomli==2.0.1
 traitlets==5.1.1
 typing_extensions==4.1.1
-univers==30.11.0
+univers==30.12.0
 urllib3==1.26.19
 wcwidth==0.2.5
 websocket-client==0.59.0

setup.cfg

@@ -71,7 +71,7 @@ install_requires =
 
     #essentials
     packageurl-python>=0.10.5rc1
-    univers>=30.11.0
+    univers>=30.12.0
     license-expression>=21.6.14
 
     # file and data formats

vulnerabilities/tests/test_data/apache_kafka/apache-kafka-improver-expected.json

@@ -1,4 +1,206 @@
 [
+  {
+    "vulnerability_id": null,
+    "aliases": [
+      "CVE-2021-38153"
+    ],
+    "confidence": 100,
+    "summary": "Some components in Apache Kafka use Arrays.equals to validate a password or key, which is vulnerable to timing attacks that make brute force attacks for such credentials more likely to be successful. Users should upgrade to 2.8.1 or higher, or 3.0.0 or higher where this vulnerability has been fixed.",
+    "affected_purls": [
+      {
+        "type": "apache",
+        "namespace": "",
+        "name": "kafka",
+        "version": "1.1.0",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      {
+        "type": "apache",
+        "namespace": "",
+        "name": "kafka",
+        "version": "1.1.1",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      {
+        "type": "apache",
+        "namespace": "",
+        "name": "kafka",
+        "version": "1.1.2",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      {
+        "type": "apache",
+        "namespace": "",
+        "name": "kafka",
+        "version": "1.1.3",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      {
+        "type": "apache",
+        "namespace": "",
+        "name": "kafka",
+        "version": "1.1.4",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      {
+        "type": "apache",
+        "namespace": "",
+        "name": "kafka",
+        "version": "1.1.5",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      {
+        "type": "apache",
+        "namespace": "",
+        "name": "kafka",
+        "version": "1.1.6",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      {
+        "type": "apache",
+        "namespace": "",
+        "name": "kafka",
+        "version": "1.1.7",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      {
+        "type": "apache",
+        "namespace": "",
+        "name": "kafka",
+        "version": "1.1.8",
+        "qualifiers": "",
+        "subpath": ""
+      }
+    ],
+    "fixed_purl": null,
+    "references": [
+      {
+        "reference_id": "CVE-2021-38153",
+        "url": "https://kafka.apache.org/cve-list",
+        "severities": []
+      },
+      {
+        "reference_id": "CVE-2021-38153",
+        "url": "https://kafka.apache.org/cve-list#CVE-2021-38153",
+        "severities": []
+      },
+      {
+        "reference_id": "CVE-2021-38153",
+        "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-38153",
+        "severities": []
+      }
+    ],
+    "weaknesses": []
+  },
+  {
+    "vulnerability_id": null,
+    "aliases": [
+      "CVE-2019-12399"
+    ],
+    "confidence": 100,
+    "summary": "When Connect workers in Apache Kafka 2.0.0, 2.0.1, 2.1.0, 2.1.1, 2.2.0, 2.2.1, or 2.3.0 are configured with one or more config providers, and a connector is created/updated on that Connect cluster to use an externalized secret variable in a substring of a connector configuration property value (the externalized secret variable is not the whole configuration property value), then any client can issue a request to the same Connect cluster to obtain the connector's task configurations and the response will contain the plaintext secret rather than the externalized secrets variable. Users should upgrade to 2.2.2 or higher, or 2.3.1 or higher where this vulnerability has been fixed.",
+    "affected_purls": [
+      {
+        "type": "apache",
+        "namespace": "",
+        "name": "kafka",
+        "version": "1.1.0",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      {
+        "type": "apache",
+        "namespace": "",
+        "name": "kafka",
+        "version": "1.1.1",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      {
+        "type": "apache",
+        "namespace": "",
+        "name": "kafka",
+        "version": "1.1.2",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      {
+        "type": "apache",
+        "namespace": "",
+        "name": "kafka",
+        "version": "1.1.3",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      {
+        "type": "apache",
+        "namespace": "",
+        "name": "kafka",
+        "version": "1.1.4",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      {
+        "type": "apache",
+        "namespace": "",
+        "name": "kafka",
+        "version": "1.1.5",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      {
+        "type": "apache",
+        "namespace": "",
+        "name": "kafka",
+        "version": "1.1.6",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      {
+        "type": "apache",
+        "namespace": "",
+        "name": "kafka",
+        "version": "1.1.7",
+        "qualifiers": "",
+        "subpath": ""
+      },
+      {
+        "type": "apache",
+        "namespace": "",
+        "name": "kafka",
+        "version": "1.1.8",
+        "qualifiers": "",
+        "subpath": ""
+      }
+    ],
+    "fixed_purl": null,
+    "references": [
+      {
+        "reference_id": "CVE-2019-12399",
+        "url": "https://kafka.apache.org/cve-list",
+        "severities": []
+      },
+      {
+        "reference_id": "CVE-2019-12399",
+        "url": "https://kafka.apache.org/cve-list#CVE-2019-12399",
+        "severities": []
+      },
+      {
+        "reference_id": "CVE-2019-12399",
+        "url": "https://nvd.nist.gov/vuln/detail/CVE-2019-12399",
+        "severities": []
+      }
+    ],
+    "weaknesses": []
+  },
   {
     "vulnerability_id": null,
     "aliases": [

vulntotal/datasources/deps.py

@@ -42,6 +42,8 @@ def datasource_advisory(self, purl) -> Iterable[VendorData]:
             A list of VendorData objects containing the advisory information.
         """
         payload = generate_meta_payload(purl)
+        if not payload:
+            return
         response = self.fetch_json_response(payload)
         if response:
             advisories = parse_advisories_from_meta(response)

vulntotal/datasources/github.py

@@ -101,8 +101,8 @@ def parse_advisory(interesting_edges, purl) -> Iterable[VendorData]:
     """
     for edge in interesting_edges:
         node = edge["node"]
-        aliases = [alias["value"] for alias in get_item(node, "advisory", "identifiers")]
-        affected_versions = node["vulnerableVersionRange"].strip().replace(" ", "").split(",")
+        aliases = [aliase["value"] for aliase in get_item(node, "advisory", "identifiers")]
+        affected_versions = [node["vulnerableVersionRange"].strip()]
         parsed_fixed_versions = get_item(node, "firstPatchedVersion", "identifier")
         fixed_versions = [parsed_fixed_versions] if parsed_fixed_versions else []
         yield VendorData(

vulntotal/datasources/gitlab.py

@@ -180,7 +180,7 @@ def get_casesensitive_slug(path, package_slug):
 
             # If the namespace/subfolder contains multiple packages, then progressive transverse through folders tree
             if package_slug.lower().startswith(slug_flatpath.lower()):
-                return get_gitlab_style_slug(slug_flatpath, package_slug)
+                return get_casesensitive_slug(slug_flatpath, package_slug)
 
         payload[0]["variables"]["nextPageCursor"] = paginated_tree["pageInfo"]["endCursor"]
         has_next = paginated_tree["pageInfo"]["hasNextPage"]

vulntotal/datasources/snyk.py

@@ -8,6 +8,7 @@
 #
 
 import logging
+import re
 from typing import Iterable
 from urllib.parse import quote
 from urllib.parse import unquote_plus
@@ -23,6 +24,8 @@
 
 logger = logging.getLogger(__name__)
 
+fixed_version_pattern = re.compile(r"\b\d[\w.-]*\b")
+
 
 class SnykDataSource(DataSource):
     spdx_license_expression = "TODO"
@@ -107,6 +110,7 @@ def datasource_advisory_from_cve(self, cve: str) -> Iterable[VendorData]:
     @classmethod
     def supported_ecosystem(cls):
         return {
+            "cargo": "cargo",
             "cocoapods": "cocoapods",
             "composer": "composer",
             "golang": "golang",
@@ -115,6 +119,7 @@ def supported_ecosystem(cls):
             "maven": "maven",
             "npm": "npm",
             "nuget": "nuget",
+            "pub": "pub",
             "pypi": "pip",
             "gem": "rubygems",
             # any purl.type not in supported_ecosystem shall implicitly be treated as unmanaged type
@@ -272,18 +277,16 @@ def parse_html_advisory(advisory_html, snyk_id, affected, purl) -> VendorData:
     advisory_soup = BeautifulSoup(advisory_html, "html.parser")
     cve_span = advisory_soup.find("span", class_="cve")
     if cve_span:
-        cve_anchor = cve_span.find("a", class_="vue--anchor")
-        aliases.append(cve_anchor["id"])
+        if cve_anchor := cve_span.find("a", class_="vue--anchor"):
+            aliases.append(cve_anchor.get("id"))
 
     how_to_fix = advisory_soup.find(
         "div", class_="vue--block vuln-page__instruction-block vue--block--instruction"
     )
-    if how_to_fix:
-        fixed = how_to_fix.find("p").text.split(" ")
-        if "Upgrade" in fixed:
-            lower = fixed.index("version") if "version" in fixed else fixed.index("versions")
-            upper = fixed.index("or")
-            fixed_versions = "".join(fixed[lower + 1 : upper]).split(",")
+
+    if how_to_fix and (fixed := how_to_fix.find("p").text):
+        fixed_versions = fixed_version_pattern.findall(fixed)
+
     aliases.append(snyk_id)
     return VendorData(
         purl=PackageURL(purl.type, purl.namespace, purl.name),

vulntotal/tests/test_data/github/parse_advisory-expected.json

@@ -2,7 +2,7 @@
   {
     "purl": "pkg:generic/namespace/test",
     "affected_versions": [
-      "<2.7.2"
+      "< 2.7.2"
     ],
     "fixed_versions": [
       "2.7.2"
@@ -15,7 +15,7 @@
   {
     "purl": "pkg:generic/namespace/test",
     "affected_versions": [
-      "<2.11.3"
+      "< 2.11.3"
     ],
     "fixed_versions": [
       "2.11.3"
@@ -28,7 +28,7 @@
   {
     "purl": "pkg:generic/namespace/test",
     "affected_versions": [
-      "<2.8.1"
+      "< 2.8.1"
     ],
     "fixed_versions": [
       "2.8.1"
@@ -41,7 +41,7 @@
   {
     "purl": "pkg:generic/namespace/test",
     "affected_versions": [
-      "<2.10.1"
+      "< 2.10.1"
     ],
     "fixed_versions": [
       "2.10.1"