Merge branch 'develop' into restructure-api-docs

.pre-commit-config.yaml

@@ -1,6 +1,7 @@
+
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0  # Use the ref you want to point at
+    rev: v5.0.0  # Use the ref you want to point at
     hooks:
       - id: check-merge-conflict
       - id: check-yaml
@@ -13,13 +14,13 @@ repos:
       - id: yapf
         args: [--in-place, --parallel, --recursive, --style, .yapf-config]
         files: "^(trestle|tests|scripts)"
-        stages: [commit]
+        stages: [pre-commit]
         additional_dependencies: [toml]
   - repo: https://github.com/PyCQA/flake8
-    rev: 5.0.4
+    rev: 7.1.1
     hooks:
       - id: flake8
-        args: [--extend-ignore, "P1,C812,C813,C814,C815,C816,W503,W605,B017,B028", "--illegal-import-packages=filecmp"]
+        args: [--extend-ignore, "P1,C812,C813,C814,C815,C816,W503,W605,B017,B028"]
         additional_dependencies:
           [
             flake8-2020,
@@ -37,14 +38,13 @@ repos:
             flake8-quotes,
             flake8-string-format,
             flake8-use-fstring,
-            flake8-illegal-import,
             pep8-naming,
           ]
         files: "^(tests|scripts)"
         exclude: "(oscal/|third_party)"
-        stages: [commit]
+        stages: [pre-commit]
       - id: flake8
-        args: [--extend-ignore, "P1,C812,C813,C814,C815,C816,W503,W605,B017,B028", "--illegal-import-packages=filecmp"]
+        args: [--extend-ignore, "P1,C812,C813,C814,C815,C816,W503,W605,B017,B028"]
         additional_dependencies:
           [
             flake8-2020,
@@ -62,16 +62,15 @@ repos:
             flake8-quotes,
             flake8-string-format,
             flake8-use-fstring,
-            flake8-illegal-import,
             pep8-naming,
             flake8-bandit,
             dlint  
           ]
         files: "^(trestle)"
         exclude: "(oscal/)"
-        stages: [commit]
+        stages: [pre-commit]
   - repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.16
+    rev: 0.7.17
     hooks:
     - id: mdformat
       exclude: "CHANGELOG.md|docs/contributing/maintainers.md|docs/reference/API|tests/data/author|docs/contributing/index.md|tests/data/jinja_markdown_include|tests/data/jinja_cmd/number_captions_data.md|tests/data/jinja_cmd/number_captions_expected_output.md"
@@ -80,3 +79,4 @@ repos:
       - mdformat-config
       - mdformat-frontmatter
       - mdformat-gfm
+

scripts/flatten_schema.py

@@ -56,7 +56,7 @@ def _replace_ref(self, d, ref_set):
 
     def _replace_refs(self, obj, ref_set):
         """Given an object recurse into it replacing any found ref: defs with what is in def_list."""
-        if type(obj) == dict:
+        if isinstance(obj, dict):
             # first check if it is a simple $ref line and replace it directly
             if len(obj.items()) == 1 and obj.get(self._ref_str, None) is not None:
                 return self._replace_ref(obj, ref_set)
@@ -71,9 +71,9 @@ def _replace_refs(self, obj, ref_set):
                     if changed:
                         dirty = True
             return new_dict, ref_set, dirty
-        elif type(obj) == str:
+        elif isinstance(obj, str):
             return obj, ref_set, False
-        elif type(obj) == list:
+        elif isinstance(obj, list):
             n_list = len(obj)
             changed = False
             dirty = False
@@ -82,7 +82,7 @@ def _replace_refs(self, obj, ref_set):
                 if changed:
                     dirty = True
             return obj, ref_set, dirty
-        elif type(obj) == tuple:
+        elif isinstance(obj, tuple):
             new_val, ref_set, changed = self._replace_refs(obj[1], ref_set)
             return (obj[0], new_val), ref_set, changed
         if hasattr(obj, '__iter__'):

scripts/oscal_normalize.py

@@ -235,7 +235,7 @@ def add_ref_pattern(self, p, line):
         new_refs = p.findall(line)
         if new_refs:
             for r in new_refs:
-                if type(r) == tuple:
+                if isinstance(r, tuple):
                     for s in r:
                         self.add_ref_if_good(s)
                 else:

scripts/schema_integrity.py

@@ -77,11 +77,11 @@ def recursive_ref(self, ref_key: str, dict_of_interest: Dict[str, Any]) -> List[
         for key, value in dict_of_interest.items():
             if key == ref_key:
                 returner.append(value)
-            elif type(value) == dict:
+            elif isinstance(value, dict):
                 returner = returner + self.recursive_ref(ref_key, value)
-            elif type(value) == list:
+            elif isinstance(value, list):
                 for item in value:
-                    if type(item) == dict:
+                    if isinstance(item, dict):
                         returner = returner + self.recursive_ref(ref_key, item)
                     elif key == ref_key:
                         returner.append(value)

tests/data/csv/rule-name-overlap.csv

@@ -0,0 +1,7 @@
+$$Component_Title,$$Component_Description,$$Component_Type,$$Rule_Id,$$Rule_Description,Check_Id,Check_Description,$$Namespace,Target_Component,$$Control_Id_List,$$Profile_Source,$$Profile_Description
+OSCO,OSCO,validation,RULE-1.1,RULE1.1,Check.1,Check.1,http://oscal-compass/compliance-trestle/schemas/oscal/cd,Target-A,,,
+OSCO,OSCO,validation,RULE-1.1,RULE1.1,Check.2,Check.2,http://oscal-compass/compliance-trestle/schemas/oscal/cd,Target-A,,,
+OSCO,OSCO,validation,RULE-1.1,RULE1.1,Check.3,Check.3,http://oscal-compass/compliance-trestle/schemas/oscal/cd,Target-A,,,
+OSCO,OSCO,validation,RULE-1.1,RULE1.1,Check.1,Check.1,http://oscal-compass/compliance-trestle/schemas/oscal/cd,Target-B,,,
+OSCO,OSCO,validation,RULE-1.1,RULE1.1,Check.3,Check.2,http://oscal-compass/compliance-trestle/schemas/oscal/cd,Target-B,,,
+OSCO,OSCO,validation,RULE-1.1,RULE1.1,Check.2,Check.3,http://oscal-compass/compliance-trestle/schemas/oscal/cd,Target-B,,,

tests/data/tasks/csv-to-oscal-cd/test-csv-to-oscal-cd-rule-name-overlap.config

@@ -0,0 +1,7 @@
+[task.csv-to-oscal-cd]
+
+title = Component definition for validation with rule overlaps
+version = V1.1
+csv-file = tests/data/csv/rule-name-overlap.csv
+output-dir = tmp
+output-overwrite = true

tests/trestle/core/base_model_test.py

@@ -86,7 +86,7 @@ def test_is_oscal_base() -> None:
     """Test that the typing information is as expected."""
     catalog = simple_catalog()
 
-    assert (isinstance(catalog, ospydantic.OscalBaseModel))
+    assert isinstance(catalog, ospydantic.OscalBaseModel)
 
 
 def test_no_timezone_exception() -> None:
@@ -104,7 +104,7 @@ def test_with_timezone() -> None:
 
     popo_json = json.loads(jsoned_catalog)
     time = popo_json['metadata']['last-modified']
-    assert (type(time) == str)
+    assert isinstance(time, str)
     assert ('Z' in time or '+' in time or '-' in time)
 
 

tests/trestle/core/generator_test.py

@@ -154,7 +154,7 @@ def test_gen_control() -> None:
 def test_ensure_optional_exists() -> None:
     """Explicit test to ensure that optional variables are populated."""
     my_catalog = gens.generate_sample_model(catalog.Catalog, include_optional=True, depth=-1)
-    assert type(my_catalog.controls[0]) == catalog.Control
+    assert isinstance(my_catalog.controls[0], catalog.Control)
 
 
 def test_gen_party() -> None:

tests/trestle/core/remote/cache_test.py

@@ -276,13 +276,13 @@ def test_fetcher_factory(tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch
                 as_file_uri('user/oscal_file.json'),
                 as_file_uri('../user/oscal_file.json')]:
         fetcher = cache.FetcherFactory.get_fetcher(tmp_trestle_dir, uri)
-        assert type(fetcher) == cache.LocalFetcher
+        assert isinstance(fetcher, cache.LocalFetcher)
 
     # paths with drive letter
     for uri in ['C:\\Users\\user\\this.json', 'C:/Users/user/this.json', 'C:file.json']:
         if file_utils.is_windows():
             fetcher = cache.FetcherFactory.get_fetcher(tmp_trestle_dir, uri)
-            assert type(fetcher) == cache.LocalFetcher
+            assert isinstance(fetcher, cache.LocalFetcher)
         else:
             with pytest.raises(TrestleError):
                 cache.FetcherFactory.get_fetcher(tmp_trestle_dir, uri)
@@ -291,15 +291,15 @@ def test_fetcher_factory(tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch
     monkeypatch.setenv('myusername', 'user123')
     monkeypatch.setenv('mypassword', 'somep4ss')
     fetcher = cache.FetcherFactory.get_fetcher(tmp_trestle_dir, https_uri)
-    assert type(fetcher) == cache.HTTPSFetcher
+    assert isinstance(fetcher, cache.HTTPSFetcher)
 
     sftp_uri = 'sftp://user@hostname:/path/to/file.json'
     fetcher = cache.FetcherFactory.get_fetcher(tmp_trestle_dir, sftp_uri)
-    assert type(fetcher) == cache.SFTPFetcher
+    assert isinstance(fetcher, cache.SFTPFetcher)
 
     sftp_uri = 'sftp://user@hostname:2000/path/to/file.json'
     fetcher = cache.FetcherFactory.get_fetcher(tmp_trestle_dir, sftp_uri)
-    assert type(fetcher) == cache.SFTPFetcher
+    assert isinstance(fetcher, cache.SFTPFetcher)
 
 
 def test_fetcher_expiration(tmp_trestle_dir: pathlib.Path) -> None:

tests/trestle/tasks/csv_to_oscal_cd_test.py

@@ -908,10 +908,10 @@ def test_execute_correct_rule_key(tmp_path: pathlib.Path) -> None:
         retval = tgt.execute()
         assert retval == TaskOutcome.SUCCESS
         # insure expected key exists
-        expected_key = (component_title, component_type, rule_id)
+        expected_key = (component_title, component_type, rule_id, None, None)
         assert expected_key in tgt._csv_mgr.get_rule_keys()
         # insure unexpected key does not exist
-        unexpected_key = (component_description, component_type, rule_id)
+        unexpected_key = (component_description, component_type, rule_id, None, None)
         assert unexpected_key not in tgt._csv_mgr.get_rule_keys()
 
 
@@ -1345,6 +1345,32 @@ def test_execute_with_ignored_risk_properties(tmp_path: pathlib.Path) -> None:
         assert prop.name != 'Risk_Adjustment'
 
 
+def test_execute_rule_name_overlap(tmp_path: pathlib.Path) -> None:
+    """Test execute rule name overlap."""
+    _, section = _get_config_section_init(tmp_path, 'test-csv-to-oscal-cd-rule-name-overlap.config')
+    rows = _get_rows(section['csv-file'])
+    with mock.patch('trestle.tasks.csv_to_oscal_cd.csv.reader') as mock_csv_reader:
+        mock_csv_reader.return_value = rows
+        tgt = csv_to_oscal_cd.CsvToOscalComponentDefinition(section)
+        retval = tgt.execute()
+        assert retval == TaskOutcome.SUCCESS
+    # read component-definition
+    fp = pathlib.Path(tmp_path) / 'component-definition.json'
+    cd = ComponentDefinition.oscal_read(fp)
+    # spot check
+    component = cd.components[0]
+    assert component.type.lower() == 'validation'
+    assert len(component.props) == 20
+    assert component.props[0].name == 'Rule_Id'
+    assert component.props[0].value == 'RULE-1.1'
+    assert component.props[3].name == 'Target_Component'
+    assert component.props[3].value == 'Target-A'
+    assert component.props[12].name == 'Rule_Id'
+    assert component.props[12].value == 'RULE-1.1'
+    assert component.props[15].name == 'Target_Component'
+    assert component.props[15].value == 'Target-B'
+
+
 def test_execute_add_user_property(tmp_path: pathlib.Path) -> None:
     """Test execute add user property."""
     _, section = _get_config_section_init(tmp_path, 'test-csv-to-oscal-cd-bp.config')

trestle/core/markdown/markdown_validator.py

@@ -181,8 +181,8 @@ def compare_keys(
             return False
         for key in template.keys():
             if key in candidate.keys():
-                if type(template[key]) == dict:
-                    if type(candidate[key]) == dict:
+                if isinstance(template[key], dict):
+                    if isinstance(candidate[key], dict):
                         status = cls.compare_keys(template[key], candidate[key], ignore_fields)
                         if not status:
                             return status

trestle/tasks/cis_xlsx_to_oscal_catalog.py

@@ -41,7 +41,7 @@
 
 logger = logging.getLogger(__name__)
 
-timestamp = datetime.datetime.utcnow().replace(microsecond=0).replace(tzinfo=datetime.timezone.utc).isoformat()
+timestamp = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat()
 
 
 class XlsxHelper: