Improve license scanning check

- The check has been renamed from FOSSA badge to License scanning - It's now able to detect Snyk badges in README files - A link to the license scanning report is stored and exposed in the UI - In addition to FOSSA and Snyk, it's now possible to provide a custom license scanning url in the `.clomonitor.yml` metadata file Closes #50 Signed-off-by: Sergio Castaño Arteaga <[email protected]> Signed-off-by: Cintia Sanchez Garcia <[email protected]> Co-authored-by: Sergio Castaño Arteaga <[email protected]> Co-authored-by: Cintia Sanchez Garcia <[email protected]>
cncf · Feb 28, 2022 · 83615b6 · 83615b6
1 parent 2308d3c
commit 83615b6
Show file tree

Hide file tree

Showing 18 changed files with 272 additions and 137 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/clomonitor-core/Cargo.toml b/clomonitor-core/Cargo.toml
@@ -13,5 +13,6 @@ lazy_static = "1.4.0"
 octocrab = "0.15.4"
 regex = "1.5.4"
 serde = { version = "1.0", features = ["derive"] }
+serde_yaml = "0.8.23"
 tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
 tracing = "0.1.29"
diff --git a/clomonitor-core/src/linter/check.rs b/clomonitor-core/src/linter/check.rs
@@ -37,6 +37,33 @@ where
     pub case_sensitive: bool,
 }
 
+/// Check if the content of any of the files that match the globs provided
+/// matches any of the regular expressions given, returning the captured value
+/// when there is a match. This function expects that the regular expressions
+/// provided contain one capture group.
+pub(crate) fn content_find<P, R>(globs: Globs<P>, regexps: R) -> Result<Option<String>, Error>
+where
+    P: IntoIterator,
+    P::Item: AsRef<str>,
+    R: IntoIterator,
+    R::Item: AsRef<str>,
+{
+    let mut res = Vec::<Regex>::new();
+    for regexp in regexps {
+        res.push(Regex::new(regexp.as_ref())?);
+    }
+    for path in matching_paths(globs)?.iter() {
+        if let Ok(content) = fs::read_to_string(path) {
+            for re in res.iter() {
+                if let Some(c) = re.captures(&content) {
+                    return Ok(Some(c[1].to_string()));
+                }
+            }
+        }
+    }
+    Ok(None)
+}
+
 /// Check if the content of any of the files that match the globs provided
 /// matches any of the regular expressions given.
 pub(crate) fn content_matches<P, R>(globs: Globs<P>, regexps: R) -> Result<bool, Error>
@@ -179,6 +206,85 @@ mod tests {
 
     const TESTDATA_PATH: &str = "src/linter/testdata";
 
+    #[test]
+    fn content_find_found() {
+        assert_eq!(
+            content_find(
+                Globs {
+                    root: Path::new(TESTDATA_PATH),
+                    patterns: README_FILE,
+                    case_sensitive: true,
+                },
+                LICENSE_SCANNING_URL
+            )
+            .unwrap()
+            .unwrap(),
+            "https://snyk.io/test/github/username/repo".to_string()
+        );
+    }
+
+    #[test]
+    fn content_find_not_found() {
+        assert_eq!(
+            content_find(
+                Globs {
+                    root: Path::new(TESTDATA_PATH),
+                    patterns: README_FILE,
+                    case_sensitive: true,
+                },
+                [r"non-existing pattern"]
+            )
+            .unwrap(),
+            None
+        );
+    }
+
+    #[test]
+    fn content_find_file_not_found() {
+        assert_eq!(
+            content_find(
+                Globs {
+                    root: Path::new(TESTDATA_PATH),
+                    patterns: vec!["nonexisting"],
+                    case_sensitive: true,
+                },
+                [r"pattern"]
+            )
+            .unwrap(),
+            None
+        );
+    }
+
+    #[test]
+    fn content_find_invalid_glob_pattern() {
+        assert!(matches!(
+            content_find(
+                Globs {
+                    root: Path::new(TESTDATA_PATH),
+                    patterns: vec!["invalid***"],
+                    case_sensitive: true,
+                },
+                [r"pattern"]
+            ),
+            Err(_)
+        ));
+    }
+
+    #[test]
+    fn content_find_invalid_regexp() {
+        assert!(matches!(
+            content_find(
+                Globs {
+                    root: Path::new(TESTDATA_PATH),
+                    patterns: README_FILE,
+                    case_sensitive: true,
+                },
+                [r"***"]
+            ),
+            Err(_)
+        ));
+    }
+
     #[test]
     fn content_matches_match() {
         assert!(content_matches(

diff --git a/clomonitor-core/src/linter/metadata.rs b/clomonitor-core/src/linter/metadata.rs
@@ -0,0 +1,69 @@
+use anyhow::Error;
+use serde::Deserialize;
+use std::ffi::OsStr;
+use std::fs;
+use std::path::Path;
+
+/// Metadata file name.
+pub const METADATA_FILE: &str = ".clomonitor.yml";
+
+/// CloMonitor metadata.
+#[derive(Debug, Deserialize)]
+pub struct Metadata {
+    pub license_scanning: Option<LicenseScanning>,
+}
+
+impl Metadata {
+    /// Create a new metadata instance from the contents of the file located at
+    /// the path provided.
+    pub fn from<P: AsRef<OsStr>>(path: P) -> Result<Option<Self>, Error> {
+        if !Path::new(&path).exists() {
+            return Ok(None);
+        }
+        let content = fs::read_to_string(path.as_ref())?;
+        Ok(serde_yaml::from_str(&content)?)
+    }
+}
+
+/// License scanning section of the metadata.
+#[derive(Debug, Deserialize)]
+pub struct LicenseScanning {
+    pub url: Option<String>,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const TESTDATA_PATH: &str = "src/linter/testdata";
+
+    #[test]
+    fn metadata_from_path_success() {
+        assert_eq!(
+            Metadata::from(Path::new(TESTDATA_PATH).join(METADATA_FILE))
+                .unwrap()
+                .unwrap()
+                .license_scanning
+                .unwrap()
+                .url
+                .unwrap(),
+            "https://license-scanning.url"
+        );
+    }
+
+    #[test]
+    fn metadata_from_path_not_found() {
+        assert!(matches!(
+            Metadata::from(Path::new(TESTDATA_PATH).join("not-found")),
+            Ok(None)
+        ));
+    }
+
+    #[test]
+    fn metadata_from_path_invalid_metadata_file() {
+        assert!(matches!(
+            Metadata::from(Path::new(TESTDATA_PATH).join("LICENSE")),
+            Err(_)
+        ));
+    }
+}
diff --git a/clomonitor-core/src/linter/mod.rs b/clomonitor-core/src/linter/mod.rs
@@ -5,6 +5,7 @@ use std::path::Path;
 use std::str::FromStr;
 
 mod check;
+mod metadata;
 mod patterns;
 pub mod primary;
 pub mod secondary;

diff --git a/clomonitor-core/src/linter/patterns.rs b/clomonitor-core/src/linter/patterns.rs
@@ -24,7 +24,10 @@ pub(crate) static ROADMAP_HEADER: [&str; 1] = [r"(?im)^#+.*roadmap.*$"];
 
 // License
 pub(crate) static LICENSE_FILE: [&str; 2] = ["LICENSE*", "COPYING*"];
-pub(crate) static FOSSA_BADGE_URL: [&str; 1] = [r"https://app.fossa.*/api/projects/.*"];
+pub(crate) static LICENSE_SCANNING_URL: [&str; 2] = [
+    r"\[!\[.*\]\(https://app.fossa.*/api/projects/.*\)\]\((.*)\)",
+    r"\[!\[.*\]\(https://snyk.io/test/github/[^/]+/[^/]+/badge.svg\)\]\((.*)\)",
+];
 
 // Best practices
 pub(crate) static ARTIFACTHUB_BADGE_URL: [&str; 1] =

diff --git a/clomonitor-core/src/linter/primary.rs b/clomonitor-core/src/linter/primary.rs
@@ -1,5 +1,6 @@
 use super::{
     check::{self, Globs},
+    metadata::*,
     patterns::*,
     LintOptions,
 };
@@ -37,7 +38,7 @@ pub struct Documentation {
 #[non_exhaustive]
 pub struct License {
     pub approved: Option<bool>,
-    pub fossa_badge: bool,
+    pub scanning: Option<String>,
     pub spdx_id: Option<String>,
 }
 
@@ -60,6 +61,9 @@ pub struct Security {
 
 /// Lint the path provided and return a report.
 pub async fn lint(options: LintOptions<'_>) -> Result<Report, Error> {
+    // Read and parse metadata
+    let md = Metadata::from(options.root.join(METADATA_FILE))?;
+
     // Async checks: documentation, best_practices
     let (documentation, best_practices) = tokio::try_join!(
         lint_documentation(options.root, options.url),
@@ -68,7 +72,7 @@ pub async fn lint(options: LintOptions<'_>) -> Result<Report, Error> {
 
     Ok(Report {
         documentation,
-        license: lint_license(options.root)?,
+        license: lint_license(options.root, &md)?,
         best_practices,
         security: lint_security(options.root)?,
     })
@@ -184,7 +188,7 @@ async fn lint_documentation(root: &Path, repo_url: &str) -> Result<Documentation
 }
 
 /// Run license checks and prepare the report's license section.
-fn lint_license(root: &Path) -> Result<License, Error> {
+fn lint_license(root: &Path, md: &Option<Metadata>) -> Result<License, Error> {
     // SPDX id
     let spdx_id = check::license(Globs {
         root,
@@ -198,19 +202,29 @@ fn lint_license(root: &Path) -> Result<License, Error> {
         approved = Some(check::is_approved_license(spdx_id))
     }
 
-    // FOSSA badge
-    let fossa_badge = check::content_matches(
-        Globs {
-            root,
-            patterns: README_FILE,
-            case_sensitive: true,
-        },
-        FOSSA_BADGE_URL,
-    )?;
+    // Scanning url
+    let mut scanning_url: Option<String> = None;
+    if let Some(md) = md {
+        if let Some(license_scanning) = &md.license_scanning {
+            if let Some(url) = &license_scanning.url {
+                scanning_url = Some(url.to_owned())
+            }
+        }
+    }
+    if scanning_url.is_none() {
+        scanning_url = check::content_find(
+            Globs {
+                root,
+                patterns: README_FILE,
+                case_sensitive: true,
+            },
+            LICENSE_SCANNING_URL,
+        )?;
+    }
 
     Ok(License {
         approved,
-        fossa_badge,
+        scanning: scanning_url,
         spdx_id,
     })
 }

diff --git a/clomonitor-core/src/linter/testdata/.clomonitor.yml b/clomonitor-core/src/linter/testdata/.clomonitor.yml
@@ -0,0 +1,2 @@
+license_scanning:
+  url: https://license-scanning.url
diff --git a/clomonitor-core/src/linter/testdata/README.md b/clomonitor-core/src/linter/testdata/README.md
@@ -1,5 +1,7 @@
 # Sample README file
 
+[![Known Vulnerabilities](https://snyk.io/test/github/username/repo/badge.svg)](https://snyk.io/test/github/username/repo)
+
 Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi id ex nunc.
 
 ## Adopters

diff --git a/clomonitor-core/src/score/primary.rs b/clomonitor-core/src/score/primary.rs
@@ -66,7 +66,7 @@ pub(crate) fn calculate_score(report: &Report) -> Score {
             score.license += 60;
         }
     }
-    if report.license.fossa_badge {
+    if report.license.scanning.is_some() {
         score.license += 20;
     }
     if report.license.spdx_id.is_some() {
@@ -138,7 +138,7 @@ mod tests {
                 },
                 license: License {
                     approved: Some(true),
-                    fossa_badge: true,
+                    scanning: Some("https://license-scanning.url".to_string()),
                     spdx_id: Some("Apache-2.0".to_string()),
                 },
                 best_practices: BestPractices {
@@ -178,7 +178,7 @@ mod tests {
                 },
                 license: License {
                     approved: None,
-                    fossa_badge: false,
+                    scanning: None,
                     spdx_id: None,
                 },
                 best_practices: BestPractices {

diff --git a/clomonitor-linter/src/display.rs b/clomonitor-linter/src/display.rs
@@ -105,8 +105,8 @@ pub(crate) fn display_primary(report: &linter::primary::Report, score: &score::p
             cell_check(report.license.approved.unwrap_or(false)),
         ])
         .add_row(vec![
-            cell_entry("License / FOSSA badge"),
-            cell_check(report.license.fossa_badge),
+            cell_entry("License / Scanning"),
+            cell_check(report.license.scanning.is_some()),
         ])
         .add_row(vec![
             cell_entry("Best practices / Artifact Hub badge"),

diff --git a/database/tests/functions/projects/get_project.sql b/database/tests/functions/projects/get_project.sql
@@ -75,7 +75,7 @@ insert into report (
         "license": {
             "spdx_id": "Apache-2.0",
             "approved": true,
-            "fossa_badge": true
+            "scanning": "https://license-scanning.url"
         },
         "security": {
             "security_policy": true
@@ -145,7 +145,7 @@ select is(
                             },
                             "license": {
                                 "approved": true,
-                                "fossa_badge": true,
+                                "scanning": "https://license-scanning.url",
                                 "spdx_id": "Apache-2.0"
                             },
                             "report_kind": "Primary",
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		license_scanning:
		url: https://license-scanning.url