Remove GitLabBasicImprover (#1137)

Add get_cwes_from_github_advisory function Add CWE support for github importer Add CWE support for osv Add CWE support for gitlab and redhat Signed-off-by: ziadhany <[email protected]>
aboutcode-org · Nov 15, 2023 · a114deb · a114deb
1 parent dc94005
commit a114deb
Show file tree

Hide file tree

Showing 22 changed files with 311 additions and 22 deletions.
diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/importers/github.py
@@ -11,6 +11,7 @@
 from typing import Iterable
 from typing import Optional
 
+from cwe2.database import Database
 from dateutil import parser as dateparser
 from packageurl import PackageURL
 from univers.version_range import RANGE_CLASS_BY_SCHEMES
@@ -24,11 +25,11 @@
 from vulnerabilities.importer import Reference
 from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.utils import dedupe
+from vulnerabilities.utils import get_cwe_id
 from vulnerabilities.utils import get_item
 
 logger = logging.getLogger(__name__)
 
-
 PACKAGE_TYPE_BY_GITHUB_ECOSYSTEM = {
     "MAVEN": "maven",
     "NUGET": "nuget",
@@ -63,6 +64,11 @@
                         url
                     }
                     severity
+                    cwes(first: 10){ 
+                        nodes {
+                            cweId       
+                        }
+                    }
                     publishedAt
                 }
                 firstPatchedVersion{
@@ -227,10 +233,34 @@ def process_response(resp: dict, package_type: str) -> Iterable[AdvisoryData]:
             else:
                 logger.error(f"Unknown identifier type {identifier_type!r} and value {value!r}")
 
+        weaknesses = get_cwes_from_github_advisory(advisory)
+
         yield AdvisoryData(
             aliases=sorted(dedupe(aliases)),
             summary=summary,
             references=references,
             affected_packages=affected_packages,
             date_published=date_published,
+            weaknesses=weaknesses,
         )
+
+
+def get_cwes_from_github_advisory(advisory) -> [int]:
+    """
+    Return the cwe-id list from advisory ex: [ 522 ]
+    by extracting the cwe_list from advisory ex: [{'cweId': 'CWE-522'}]
+    then remove the CWE- from string and convert it to integer 522 and Check if the CWE in CWE-Database
+    """
+    weaknesses = []
+    db = Database()
+    cwe_list = get_item(advisory, "cwes", "nodes") or []
+    for cwe_item in cwe_list:
+        cwe_string = get_item(cwe_item, "cweId")
+        if cwe_string:
+            cwe_id = get_cwe_id(cwe_string)
+            try:
+                db.get(cwe_id)
+                weaknesses.append(cwe_id)
+            except Exception:
+                logger.error("Invalid CWE id")
+    return weaknesses
diff --git a/vulnerabilities/importers/gitlab.py b/vulnerabilities/importers/gitlab.py
@@ -28,10 +28,10 @@
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
 from vulnerabilities.utils import build_description
+from vulnerabilities.utils import get_cwe_id
 
 logger = logging.getLogger(__name__)
 
-
 PURL_TYPE_BY_GITLAB_SCHEME = {
     "conan": "conan",
     "gem": "gem",
@@ -44,7 +44,6 @@
     "pypi": "pypi",
 }
 
-
 GITLAB_SCHEME_BY_PURL_TYPE = {v: k for k, v in PURL_TYPE_BY_GITLAB_SCHEME.items()}
 
 
@@ -186,6 +185,10 @@ def parse_gitlab_advisory(file):
     summary = build_description(gitlab_advisory.get("title"), gitlab_advisory.get("description"))
     urls = gitlab_advisory.get("urls")
     references = [Reference.from_url(u) for u in urls]
+
+    cwe_ids = gitlab_advisory.get("cwe_ids") or []
+    cwe_list = list(map(get_cwe_id, cwe_ids))
+
     date_published = dateparser.parse(gitlab_advisory.get("pubdate"))
     date_published = date_published.replace(tzinfo=pytz.UTC)
     package_slug = gitlab_advisory.get("package_slug")
@@ -251,4 +254,5 @@ def parse_gitlab_advisory(file):
         references=references,
         date_published=date_published,
         affected_packages=affected_packages,
+        weaknesses=cwe_list,
     )
diff --git a/vulnerabilities/importers/osv.py b/vulnerabilities/importers/osv.py
@@ -27,6 +27,7 @@
 from vulnerabilities.severity_systems import SCORING_SYSTEMS
 from vulnerabilities.utils import build_description
 from vulnerabilities.utils import dedupe
+from vulnerabilities.utils import get_cwe_id
 
 logger = logging.getLogger(__name__)
 
@@ -74,13 +75,17 @@ def parse_advisory_data(raw_data: dict, supported_ecosystem) -> Optional[Advisor
                         fixed_version=version,
                     )
                 )
+    database_specific = raw_data.get("database_specific") or {}
+    cwe_ids = database_specific.get("cwe_ids") or []
+    weaknesses = list(map(get_cwe_id, cwe_ids))
 
     return AdvisoryData(
         aliases=aliases,
         summary=summary,
         references=references,
         affected_packages=affected_packages,
         date_published=date_published,
+        weaknesses=weaknesses,
     )
 
 

diff --git a/vulnerabilities/importers/redhat.py b/vulnerabilities/importers/redhat.py
@@ -8,6 +8,7 @@
 #
 
 import logging
+import re
 from typing import Dict
 from typing import Iterable
 from typing import List
@@ -23,6 +24,7 @@
 from vulnerabilities.importer import Reference
 from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.rpm_utils import rpm_to_purl
+from vulnerabilities.utils import get_cwe_id
 from vulnerabilities.utils import get_item
 from vulnerabilities.utils import requests_with_5xx_retry
 
@@ -61,7 +63,6 @@ def get_data_from_url(url):
 
 
 class RedhatImporter(Importer):
-
     spdx_license_expression = "CC-BY-4.0"
     license_url = "https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0/html/red_hat_security_data_api/legal-notice"
 
@@ -135,6 +136,11 @@ def to_advisory(advisory_data):
                 scoring_elements=cvssv3_vector,
             )
         )
+    cwe_list = []
+    # cwe_string : CWE-409","CWE-121->CWE-787","(CWE-401|CWE-404)","(CWE-190|CWE-911)->CWE-416"
+    cwe_string = advisory_data.get("CWE")
+    if cwe_string:
+        cwe_list = list(map(get_cwe_id, re.findall("CWE-[0-9]+", cwe_string)))
 
     aliases = []
     alias = advisory_data.get("CVE")
@@ -148,4 +154,5 @@ def to_advisory(advisory_data):
         summary=advisory_data.get("bugzilla_description") or "",
         affected_packages=affected_packages,
         references=references,
+        weaknesses=cwe_list,
     )
diff --git a/vulnerabilities/improvers/default.py b/vulnerabilities/improvers/default.py
@@ -64,6 +64,7 @@ def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]:
                         affected_purls=affected_purls,
                         fixed_purl=None,
                         references=advisory_data.references,
+                        weaknesses=advisory_data.weaknesses,
                     )
                 else:
                     for fixed_purl in fixed_purls or []:
@@ -74,6 +75,7 @@ def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]:
                             affected_purls=affected_purls,
                             fixed_purl=fixed_purl,
                             references=advisory_data.references,
+                            weaknesses=advisory_data.weaknesses,
                         )
 
         else:

diff --git a/vulnerabilities/tests/test_data/gitlab/composer-expected.json b/vulnerabilities/tests/test_data/gitlab/composer-expected.json
@@ -25,5 +25,5 @@
     }
   ],
   "date_published": "2018-03-15T00:00:00+00:00",
-  "weaknesses": []
+  "weaknesses": [1035,937]
 }
diff --git a/vulnerabilities/tests/test_data/gitlab/composer-improver-expected.json b/vulnerabilities/tests/test_data/gitlab/composer-improver-expected.json
@@ -22,6 +22,6 @@
         "severities": []
       }
     ],
-    "weaknesses": []
+    "weaknesses": [1035,937]
   }
 ]
diff --git a/vulnerabilities/tests/test_data/gitlab/golang-expected.json b/vulnerabilities/tests/test_data/gitlab/golang-expected.json
@@ -31,5 +31,5 @@
     }
   ],
   "date_published": "2021-05-20T00:00:00+00:00",
-  "weaknesses": []
+  "weaknesses": [1035,937]
 }
diff --git a/vulnerabilities/tests/test_data/gitlab/golang-improver-expected.json b/vulnerabilities/tests/test_data/gitlab/golang-improver-expected.json
@@ -37,7 +37,7 @@
         "severities": []
       }
     ],
-    "weaknesses": []
+    "weaknesses": [1035,937]
   },
   {
     "vulnerability_id": null,
@@ -68,6 +68,6 @@
         "severities": []
       }
     ],
-    "weaknesses": []
+    "weaknesses": [1035,937]
   }
 ]
diff --git a/vulnerabilities/tests/test_data/gitlab/maven-expected.json b/vulnerabilities/tests/test_data/gitlab/maven-expected.json
@@ -46,5 +46,5 @@
     }
   ],
   "date_published": "2021-11-15T00:00:00+00:00",
-  "weaknesses": []
+  "weaknesses": [1035,937,94]
 }
diff --git a/vulnerabilities/tests/test_data/gitlab/maven-improver-expected.json b/vulnerabilities/tests/test_data/gitlab/maven-improver-expected.json
@@ -100,7 +100,7 @@
         "severities": []
       }
     ],
-    "weaknesses": []
+    "weaknesses": [1035,937,94]
   },
   {
     "vulnerability_id": null,
@@ -146,6 +146,6 @@
         "severities": []
       }
     ],
-    "weaknesses": []
+    "weaknesses": [1035,937,94]
   }
 ]
diff --git a/vulnerabilities/tests/test_data/gitlab/npm-expected.json b/vulnerabilities/tests/test_data/gitlab/npm-expected.json
@@ -36,5 +36,5 @@
     }
   ],
   "date_published": "2020-06-05T00:00:00+00:00",
-  "weaknesses": []
+  "weaknesses": [1035,937]
 }
diff --git a/vulnerabilities/tests/test_data/gitlab/npm-improver-expected.json b/vulnerabilities/tests/test_data/gitlab/npm-improver-expected.json
@@ -50,7 +50,7 @@
         "severities": []
       }
     ],
-    "weaknesses": []
+    "weaknesses": [1035,937]
   },
   {
     "vulnerability_id": null,
@@ -86,6 +86,6 @@
         "severities": []
       }
     ],
-    "weaknesses": []
+    "weaknesses": [1035,937]
   }
 ]
diff --git a/vulnerabilities/tests/test_data/gitlab/nuget-expected.json b/vulnerabilities/tests/test_data/gitlab/nuget-expected.json
@@ -31,5 +31,5 @@
     }
   ],
   "date_published": "2022-01-08T00:00:00+00:00",
-  "weaknesses": []
+  "weaknesses": [1035,770,937]
 }
diff --git a/vulnerabilities/tests/test_data/gitlab/nuget-improver-expected.json b/vulnerabilities/tests/test_data/gitlab/nuget-improver-expected.json
@@ -37,7 +37,7 @@
         "severities": []
       }
     ],
-    "weaknesses": []
+    "weaknesses": [1035,770,937]
   },
   {
     "vulnerability_id": null,
@@ -68,6 +68,6 @@
         "severities": []
       }
     ],
-    "weaknesses": []
+    "weaknesses": [1035,770,937]
   }
 ]
diff --git a/vulnerabilities/tests/test_data/gitlab/pypi-expected.json b/vulnerabilities/tests/test_data/gitlab/pypi-expected.json
@@ -30,5 +30,5 @@
     }
   ],
   "date_published": "2019-07-17T00:00:00+00:00",
-  "weaknesses": []
+  "weaknesses": [1035, 937]
 }
diff --git a/vulnerabilities/tests/test_data/gitlab/pypi-improver-expected.json b/vulnerabilities/tests/test_data/gitlab/pypi-improver-expected.json
@@ -44,7 +44,7 @@
         "severities": []
       }
     ],
-    "weaknesses": []
+    "weaknesses": [1035, 937]
   },
   {
     "vulnerability_id": null,
@@ -74,6 +74,6 @@
         "severities": []
       }
     ],
-    "weaknesses": []
+    "weaknesses": [1035, 937]
   }
 ]