feature/ adapt tests for warning (#66)

* changed each test to also check for warnings and specified which warning must be displayed

---------

Signed-off-by: Anton Utz <[email protected]>

test/systemtest/test_separate_pkgs.py

@@ -20,8 +20,13 @@
 import subprocess
 import unittest
 from test.systemtest._test_helpers import make_repo, remove_repo
+from typing import Optional
 
-from ros_license_toolkit.main import main
+from ros_license_toolkit.checks import Status
+
+SUCCESS = Status.SUCCESS
+WARNING = Status.WARNING
+FAILURE = Status.FAILURE
 
 
 class TestPkgs(unittest.TestCase):
@@ -54,16 +59,19 @@ def test_pkg_both_tags_not_spdx(self):
         """ Test on a package that has two different Licenses that both
         have a not SPDX conform Tag. License files and source files
         are SPDX conform"""
-        self.assertEqual(os.EX_OK, main([
-            "test/_test_data/test_pkg_both_tags_not_spdx"
-        ]))
+        process, stdout = open_subprocess("test_pkg_both_tags_not_spdx")
+        self.assertEqual(os.EX_OK, process.returncode)
+        self.assertTrue(check_output_status(
+            stdout, SUCCESS, WARNING, WARNING, WARNING, WARNING))
 
     def test_pkg_both_tags_not_spdx_one_file_own(self):
         """Test on a package that has two licenses. One is self-defined, other
         one with not SPDX tag but therefore code and license file in SPDX"""
-        self.assertEqual(os.EX_DATAERR, main([
-            "test/_test_data/test_pkg_both_tags_not_spdx_one_file_own"
-        ]))
+        process, stdout = open_subprocess(
+            "test_pkg_both_tags_not_spdx_one_file_own")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(check_output_status(
+            stdout, SUCCESS, WARNING, FAILURE, WARNING, WARNING))
 
     def test_pkg_code_has_no_license(self):
         """Test on a package that has a correct package.xml with a license
@@ -76,72 +84,89 @@ def test_pkg_code_has_no_license(self):
     def test_pkg_has_code_disjoint(self):
         """Test on a package with two disjoint sets of source files under
         a license different from the package main license."""
-        self.assertEqual(os.EX_OK, main(
-            ["test/_test_data/test_pkg_has_code_disjoint"]))
+        process, stdout = open_subprocess("test_pkg_has_code_disjoint")
+        self.assertEqual(os.EX_OK, process.returncode)
+        self.assertTrue(check_output_status(stdout))
 
     def test_pkg_has_code_of_different_license(self):
         """Test on a package with source files under a license different
         from the package main license (here LGPL). It should fail, because
         the additional license is not declared in the package.xml."""
-        self.assertEqual(os.EX_DATAERR, main(
-            ["test/_test_data/test_pkg_has_code_of_different_license"]))
+        process, stdout = open_subprocess(
+            "test_pkg_has_code_of_different_license")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(check_output_status(stdout, exp_lic_in_code=FAILURE))
 
     def test_pkg_has_code_of_different_license_and_tag(self):
         """Test on a package with source files under a license different
         from the package main license, but the additional license is declared
         in the package.xml."""
-        self.assertEqual(os.EX_OK, main(
-            ["test/_test_data/"
-                "test_pkg_has_code_of_different_license_and_tag"]))
+        process, stdout = open_subprocess(
+            "test_pkg_has_code_of_different_license_and_tag")
+        self.assertEqual(os.EX_OK, process.returncode)
+        self.assertTrue(check_output_status(stdout))
 
     def test_pkg_has_code_of_different_license_and_wrong_tag(self):
         """Test on a package with source files under a license different
         from the package main license, but the additional license is declared
         in the package.xml, but with the wrong name."""
-        self.assertEqual(os.EX_DATAERR, main(
-            ["test/_test_data/"
-             "test_pkg_has_code_of_different_license_and_wrong_tag"]))
+        process, stdout = open_subprocess(
+            "test_pkg_has_code_of_different_license_and_wrong_tag")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(check_output_status(
+            stdout, exp_lic_text_exits=FAILURE, exp_lic_in_code=FAILURE))
 
     def test_pkg_ignore_readme_contents(self):
         """Test on a package with readme files. READMEs mention licenses
         that are not in package and shall therefore be ignored."""
-        test_result = main(["test/_test_data/test_pkg_ignore_readme_contents"])
-        self.assertEqual(os.EX_OK, test_result)
+        process, stdout = open_subprocess("test_pkg_ignore_readme_contents")
+        self.assertEqual(os.EX_OK, process.returncode)
+        self.assertTrue(check_output_status(stdout))
 
     def test_pkg_name_not_in_spdx(self):
         """Test on a package that has valid License file with BSD-3-Clause
         but its license tag BSD is not in SPDX format"""
         process, stdout = open_subprocess("test_pkg_name_not_in_spdx")
         self.assertEqual(os.EX_OK, process.returncode)
-        self.assertIn(b"WARNING", stdout)
+        self.assertTrue(check_output_status(
+            stdout, SUCCESS, WARNING, WARNING, SUCCESS, WARNING))
 
     def test_pkg_no_file_attribute(self):
         """Test on a package with License file that is not referenced in
         package.xml"""
-        self.assertEqual(os.EX_OK, main(
-            ["test/_test_data/test_pkg_no_file_attribute"]))
+        process, stdout = open_subprocess("test_pkg_no_file_attribute")
+        self.assertEqual(os.EX_OK, process.returncode)
+        self.assertTrue(check_output_status(stdout))
 
     def test_pkg_no_license(self):
         """Test on a package with no license declared in the package.xml."""
-        self.assertEqual(os.EX_DATAERR, main(
-            ["test/_test_data/test_pkg_no_license"]))
+        process, stdout = open_subprocess("test_pkg_no_license")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(check_output_status(
+            stdout, FAILURE, SUCCESS, FAILURE, FAILURE, SUCCESS))
 
     def test_pkg_no_license_file(self):
         """Test on a package with no license text file."""
-        self.assertEqual(os.EX_DATAERR, main(
-            ["test/_test_data/test_pkg_no_license_file"]))
+        process, stdout = open_subprocess("test_pkg_no_license_file")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(check_output_status(
+            stdout, exp_lic_text_exits=FAILURE))
 
     def test_pkg_one_correct_one_license_file_missing(self):
         """Test on a package that has one correct license with file
         and code, but also one not known license tag without file"""
-        self.assertEqual(os.EX_DATAERR, main(
-            ["test/_test_data/test_pkg_one_correct_one_license_file_missing"]))
+        process, stdout = open_subprocess(
+            "test_pkg_one_correct_one_license_file_missing")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(check_output_status(
+            stdout, SUCCESS, WARNING, FAILURE, FAILURE, SUCCESS))
 
     def test_pkg_spdx_tag(self):
         """Test on a package with a license declared in the package.xml
         with the SPDX tag."""
-        self.assertEqual(os.EX_OK, main(
-            ["test/_test_data/test_pkg_spdx_tag"]))
+        process, stdout = open_subprocess("test_pkg_spdx_tag")
+        self.assertEqual(os.EX_OK, process.returncode)
+        self.assertTrue(check_output_status(stdout))
 
     def test_pkg_too_many_license_files(self):
         """"Test on a package with multiple License files that are not
@@ -151,64 +176,69 @@ def test_pkg_too_many_license_files(self):
         self.assertIn(b"bsd.LICENSE", stdout)
         self.assertIn(b"apl.LICENSE", stdout)
         self.assertNotIn(b"../../../LICENSE", stdout)
+        self.assertTrue(check_output_status(
+            stdout, exp_lic_files_referenced=FAILURE))
 
     def test_pkg_tag_not_spdx(self):
         """Test on a package that has one linked declaration, one code file
         but not in SPDX tag. Tag must be gotten from declaration."""
         process, stdout = open_subprocess("test_pkg_tag_not_spdx")
         self.assertEqual(os.EX_OK, process.returncode)
-        self.assertIn(b"WARNING", stdout)
-        self.assertIn(b"'code_with_afl.py' is of AFL-2.0 but its Tag is AFL.",
-                      stdout)
+        self.assertTrue(check_output_status(
+            stdout, SUCCESS, WARNING, WARNING, WARNING, WARNING))
 
     def test_pkg_unknown_license(self):
         """Test on a package with an unknown license declared in the
         package.xml."""
-        # using subprocess.Popen instead of main() to capture stdout
-        with subprocess.Popen(
-            ["ros_license_toolkit",
-             "test/_test_data/test_pkg_unknown_license"],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE
-        ) as process:
-            stdout, _ = process.communicate()
-        self.assertNotEqual(os.EX_OK, process.returncode)
-        self.assertIn(b'not in SPDX list of licenses', stdout)
-        self.assertIn(b'my own fancy license 1.0', stdout)
+        process, stdout = open_subprocess("test_pkg_unknown_license")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(check_output_status(
+            stdout, exp_lic_tag_spdx=WARNING, exp_lic_text_exits=FAILURE))
 
     def test_pkg_unknown_license_missing_file(self):
         """Test on a package that has an unknown license
         without a license file"""
-        self.assertEqual(os.EX_DATAERR, main(
-            ["test/_test_data/test_pkg_unknown_license_missing_file"]))
+        process, stdout = open_subprocess(
+            "test_pkg_unknown_license_missing_file")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(check_output_status(
+            stdout, exp_lic_tag_spdx=WARNING, exp_lic_text_exits=FAILURE))
 
     def test_pkg_with_license_and_file(self):
         """Test on a package with a license declared in the package.xml
         and a matching license text file."""
-        self.assertEqual(os.EX_OK, main(
-            ["test/_test_data/test_pkg_with_license_and_file"]))
+        process, stdout = open_subprocess("test_pkg_with_license_and_file")
+        self.assertEqual(os.EX_OK, process.returncode)
+        self.assertTrue(check_output_status(stdout))
 
     def test_pkg_with_multiple_licenses_no_source_files_tag(self):
         """Test on a package with multiple licenses declared in the
         package.xml, none of which have source file tags."""
-        self.assertEqual(os.EX_DATAERR, main(
-            ["test/_test_data/"
-             "test_pkg_with_multiple_licenses_no_source_files_tag"]))
+        process, stdout = open_subprocess(
+            "test_pkg_with_multiple_licenses_no_source_files_tag")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(check_output_status(
+            stdout, exp_lic_tag_exists=FAILURE))
 
     def test_pkg_with_multiple_licenses_one_referenced_incorrect(self):
         """Test on a package with multiple licenses declared in the
-        package.xml. First has tag not in SPDX list with correct source file,
-        second is in SPDX."""
+        package.xml. First has tag not in SPDX list with correct
+        source file, second is in SPDX."""
         process, stdout = open_subprocess(
             "test_pkg_with_multiple_licenses_one_referenced_incorrect")
         self.assertEqual(os.EX_OK, process.returncode)
         self.assertIn(b"WARNING Licenses ['BSD'] are not in SPDX list", stdout)
+        self.assertTrue(check_output_status(
+            stdout, SUCCESS, WARNING, WARNING, WARNING, WARNING))
 
     def test_pkg_wrong_license_file(self):
         """Test on a package with a license text file that does not match
         the license declared in the package.xml, both tag and file in spdx"""
-        self.assertEqual(os.EX_DATAERR, main(
-            ["test/_test_data/test_pkg_wrong_license_file"]))
+        process, stdout = open_subprocess("test_pkg_wrong_license_file")
+        self.assertEqual(os.EX_DATAERR, process.returncode)
+        self.assertTrue(check_output_status(
+            stdout, exp_lic_text_exits=FAILURE,
+            exp_lic_files_referenced=WARNING))
 
 
 def open_subprocess(test_data_name: str):
@@ -223,5 +253,50 @@ def open_subprocess(test_data_name: str):
     return process, stdout
 
 
+def check_output_status(output: str,
+                        exp_lic_tag_exists: Status = Status.SUCCESS,
+                        exp_lic_tag_spdx: Status = Status.SUCCESS,
+                        exp_lic_text_exits: Status = Status.SUCCESS,
+                        exp_lic_in_code: Status = Status.SUCCESS,
+                        exp_lic_files_referenced: Status = Status.SUCCESS
+                        ) -> bool:
+    """Check output of each check for expected status.
+    each argument except for output tells the expected status of a
+    certain check. The default is always SUCCESS."""
+    # pylint: disable=too-many-arguments
+
+    real_lic_tag_exists = get_test_result(output, "LicenseTagExistsCheck")
+    real_lic_tag_spdx = get_test_result(output, "LicenseTagIsInSpdxListCheck")
+    real_lic_text_exits = get_test_result(output, "LicenseTextExistsCheck")
+    real_lic_in_code = get_test_result(output, "LicensesInCodeCheck")
+    real_lic_files_referenced = get_test_result(
+        output, "LicenseFilesReferencedCheck")
+
+    return exp_lic_tag_exists == real_lic_tag_exists \
+        and exp_lic_tag_spdx == real_lic_tag_spdx \
+        and exp_lic_text_exits == real_lic_text_exits \
+        and exp_lic_in_code == real_lic_in_code \
+        and exp_lic_files_referenced == real_lic_files_referenced
+
+
+def get_test_result(output: str, test_name: str) -> Optional[Status]:
+    """Get result status for specific test.
+    Return None if no status could be found"""
+    lines = output.splitlines()
+    for i, line in enumerate(lines):
+        if test_name not in str(line):
+            continue
+        if i + 1 > len(lines):
+            continue
+        result_line = str(lines[i + 1])
+    if "FAILURE" in result_line:
+        return FAILURE
+    if "WARNING" in result_line:
+        return WARNING
+    if "SUCCESS" in result_line:
+        return SUCCESS
+    return None
+
+
 if __name__ == '__main__':
     unittest.main()