From ae3bae7357ded9e0d2b8ab9c657f745716dd710c Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Fri, 22 Dec 2023 09:50:48 +0000 Subject: [PATCH 01/20] feat(versioned_resource): Add line number and options attribute. --- infrapatch/core/models/versioned_resource.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/infrapatch/core/models/versioned_resource.py b/infrapatch/core/models/versioned_resource.py index c708789..dd356a6 100644 --- a/infrapatch/core/models/versioned_resource.py +++ b/infrapatch/core/models/versioned_resource.py @@ -16,16 +16,23 @@ class ResourceStatus: PATCHED = "patched" PATCH_ERROR = "patch_error" NO_VERSION_FOUND = "no_version_found" + + +@dataclass +class VersionedResourceOptions: + ignore_resource: bool @dataclass class VersionedResource: name: str current_version: str + start_line_number: int _source_file: str _newest_version: Optional[str] = None _status: str = ResourceStatus.UNPATCHED _github_repo: Optional[str] = None + options: Optional[VersionedResourceOptions] = None @property def source_file(self) -> Path: From ecc5f9821e2e88942226eff1a606c8589dd7b8b1 Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Fri, 22 Dec 2023 09:51:29 +0000 Subject: [PATCH 02/20] test(versioned_resources): Add new attributes to tests. --- .../models/tests/test_versioned_resource.py | 40 ++++++++++--------- .../test_versioned_terraform_resource.py | 37 +++++++++-------- 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/infrapatch/core/models/tests/test_versioned_resource.py b/infrapatch/core/models/tests/test_versioned_resource.py index c1b66b6..8abee77 100644 --- a/infrapatch/core/models/tests/test_versioned_resource.py +++ b/infrapatch/core/models/tests/test_versioned_resource.py @@ -7,7 +7,7 @@ def test_version_management(): # Create new resource with newer version - resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py") + resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py", start_line_number=1) resource.newest_version = "2.0.0" assert resource.status == ResourceStatus.UNPATCHED @@ -17,14 +17,14 @@ def test_version_management(): assert resource.status == ResourceStatus.PATCHED # Check new_version the same as current_version - resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py") + resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py", start_line_number=1) resource.newest_version = "1.0.0" assert resource.status == ResourceStatus.UP_TO_DATE assert resource.installed_version_equal_or_newer_than_new_version() is True # Check new_version older than current_version - resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py") + resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py", start_line_number=1) resource.newest_version = "0.1.0" assert resource.status == ResourceStatus.UP_TO_DATE @@ -32,7 +32,7 @@ def test_version_management(): def test_tile_constraint(): - resource = VersionedResource(name="test_resource", current_version="~>1.0.0", _source_file="test_file.py") + resource = VersionedResource(name="test_resource", current_version="~>1.0.0", _source_file="test_file.py", start_line_number=1) resource.newest_version = "~>1.0.1" assert resource.has_tile_constraint() is True assert resource.installed_version_equal_or_newer_than_new_version() is True @@ -40,16 +40,16 @@ def test_tile_constraint(): resource.newest_version = "~>1.1.0" assert resource.installed_version_equal_or_newer_than_new_version() is False - resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py") + resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py", start_line_number=1) assert resource.has_tile_constraint() is False - resource = VersionedResource(name="test_resource", current_version="~>1.0.0", _source_file="test_file.py") + resource = VersionedResource(name="test_resource", current_version="~>1.0.0", _source_file="test_file.py", start_line_number=1) resource.newest_version = "1.1.0" assert resource.newest_version == "~>1.1.0" def test_git_repo(): - resource = VersionedResource(name="test_resource", current_version="~>1.0.0", _source_file="test_file.py") + resource = VersionedResource(name="test_resource", current_version="~>1.0.0", _source_file="test_file.py", start_line_number=1) assert resource.github_repo is None @@ -67,39 +67,39 @@ def test_git_repo(): def test_patch_error(): - resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py") + resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py", start_line_number=1) resource.set_patch_error() assert resource.status == ResourceStatus.PATCH_ERROR def test_version_not_found(): # Test manual setting - resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py") + resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py", start_line_number=1) resource.set_no_version_found() assert resource.status == ResourceStatus.NO_VERSION_FOUND assert resource.installed_version_equal_or_newer_than_new_version() is True # Test by setting None as new version - resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py") + resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py", start_line_number=1) resource.newest_version = None assert resource.status == ResourceStatus.NO_VERSION_FOUND assert resource.installed_version_equal_or_newer_than_new_version() is True def test_path(): - resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="/var/testdir/test_file.py") + resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="/var/testdir/test_file.py", start_line_number=1) assert resource.source_file == Path("/var/testdir/test_file.py") def test_find(): - findably_resource = VersionedResource(name="test_resource3", current_version="1.0.0", _source_file="test_file3.py") - unfindably_resource = VersionedResource(name="test_resource6", current_version="1.0.0", _source_file="test_file8.py") + findably_resource = VersionedResource(name="test_resource3", current_version="1.0.0", _source_file="test_file3.py", start_line_number=1) + unfindably_resource = VersionedResource(name="test_resource6", current_version="1.0.0", _source_file="test_file8.py", start_line_number=1) resources = [ - VersionedResource(name="test_resource1", current_version="1.0.0", _source_file="test_file1.py"), - VersionedResource(name="test_resource2", current_version="1.0.0", _source_file="test_file2.py"), - VersionedResource(name="test_resource3", current_version="1.0.0", _source_file="test_file3.py"), - VersionedResource(name="test_resource4", current_version="1.0.0", _source_file="test_file4.py"), - VersionedResource(name="test_resource5", current_version="1.0.0", _source_file="test_file5.py"), + VersionedResource(name="test_resource1", current_version="1.0.0", _source_file="test_file1.py", start_line_number=1), + VersionedResource(name="test_resource2", current_version="1.0.0", _source_file="test_file2.py", start_line_number=1), + VersionedResource(name="test_resource3", current_version="1.0.0", _source_file="test_file3.py", start_line_number=1), + VersionedResource(name="test_resource4", current_version="1.0.0", _source_file="test_file4.py", start_line_number=1), + VersionedResource(name="test_resource5", current_version="1.0.0", _source_file="test_file5.py", start_line_number=1), ] assert len(findably_resource.find(resources)) == 1 assert findably_resource.find(resources) == [resources[2]] @@ -107,7 +107,7 @@ def test_find(): def test_versioned_resource_to_dict(): - resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py") + resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py", start_line_number=1) expected_dict = { "name": "test_resource", "current_version": "1.0.0", @@ -115,5 +115,7 @@ def test_versioned_resource_to_dict(): "_newest_version": None, "_status": ResourceStatus.UNPATCHED, "_github_repo": None, + "start_line_number": 1, + "options": None, } assert resource.to_dict() == expected_dict diff --git a/infrapatch/core/models/tests/test_versioned_terraform_resource.py b/infrapatch/core/models/tests/test_versioned_terraform_resource.py index bc40976..dad1e14 100644 --- a/infrapatch/core/models/tests/test_versioned_terraform_resource.py +++ b/infrapatch/core/models/tests/test_versioned_terraform_resource.py @@ -5,8 +5,8 @@ def test_attributes(): # test with default registry - module = TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test/test_module/test_provider") - provider = TerraformProvider(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test_provider/test_provider") + module = TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test/test_module/test_provider", start_line_number=1) + provider = TerraformProvider(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test_provider/test_provider", start_line_number=1) assert module.source == "test/test_module/test_provider" assert module.base_domain is None @@ -17,8 +17,8 @@ def test_attributes(): assert provider.identifier == "test_provider/test_provider" # test with custom registry - module = TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="testregistry.ch/test/test_module/test_provider") - provider = TerraformProvider(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="testregistry.ch/test_provider/test_provider") + module = TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="testregistry.ch/test/test_module/test_provider", start_line_number=1) + provider = TerraformProvider(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="testregistry.ch/test_provider/test_provider", start_line_number=1) assert module.source == "testregistry.ch/test/test_module/test_provider" assert module.base_domain == "testregistry.ch" @@ -30,23 +30,23 @@ def test_attributes(): # test invalid sources with pytest.raises(Exception): - TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test/test_module/test_provider/test") - TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="/test_module") + TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test/test_module/test_provider/test", start_line_number=1) + TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="/test_module", start_line_number=1) with pytest.raises(Exception): - TerraformProvider(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="/test_module") - TerraformProvider(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="kfdsjflksdj/kldfsjflsdkj/dkljflsk/test_module") + TerraformProvider(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="/test_module", start_line_number=1) + TerraformProvider(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="kfdsjflksdj/kldfsjflsdkj/dkljflsk/test_module", start_line_number=1) def test_find(): - findably_resource = TerraformModule(name="test_resource3", current_version="1.0.0", _source_file="test_file3.py", _source="test/test_module3/test_provider") - unfindably_resource = TerraformModule(name="test_resource6", current_version="1.0.0", _source_file="test_file8.py", _source="test/test_module3/test_provider") + findably_resource = TerraformModule(name="test_resource3", current_version="1.0.0", _source_file="test_file3.py", _source="test/test_module3/test_provider", start_line_number=1) + unfindably_resource = TerraformModule(name="test_resource6", current_version="1.0.0", _source_file="test_file8.py", _source="test/test_module3/test_provider", start_line_number=1) resources = [ - TerraformModule(name="test_resource1", current_version="1.0.0", _source_file="test_file1.py", _source="test/test_module1/test_provider"), - TerraformModule(name="test_resource2", current_version="1.0.0", _source_file="test_file2.py", _source="test/test_module2/test_provider"), - TerraformModule(name="test_resource3", current_version="1.0.0", _source_file="test_file3.py", _source="test/test_module3/test_provider"), - TerraformModule(name="test_resource4", current_version="1.0.0", _source_file="test_file4.py", _source="test/test_module4/test_provider"), - TerraformModule(name="test_resource5", current_version="1.0.0", _source_file="test_file5.py", _source="test/test_module5/test_provider"), + TerraformModule(name="test_resource1", current_version="1.0.0", _source_file="test_file1.py", _source="test/test_module1/test_provider", start_line_number=1), + TerraformModule(name="test_resource2", current_version="1.0.0", _source_file="test_file2.py", _source="test/test_module2/test_provider", start_line_number=1), + TerraformModule(name="test_resource3", current_version="1.0.0", _source_file="test_file3.py", _source="test/test_module3/test_provider", start_line_number=1), + TerraformModule(name="test_resource4", current_version="1.0.0", _source_file="test_file4.py", _source="test/test_module4/test_provider", start_line_number=1), + TerraformModule(name="test_resource5", current_version="1.0.0", _source_file="test_file5.py", _source="test/test_module5/test_provider", start_line_number=1), ] assert len(findably_resource.find(resources)) == 1 assert findably_resource.find(resources) == [resources[2]] @@ -54,12 +54,13 @@ def test_find(): def test_to_dict(): - module = TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test/test_module/test_provider") + module = TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test/test_module/test_provider", start_line_number=1) provider = TerraformProvider( name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test_provider/test_provider", + start_line_number=1 ) module_dict = module.to_dict() @@ -75,6 +76,8 @@ def test_to_dict(): "_base_domain": None, "_identifier": "test/test_module/test_provider", "_github_repo": None, + "start_line_number": 1, + "options": None, } assert provider_dict == { "name": "test_resource", @@ -86,4 +89,6 @@ def test_to_dict(): "_base_domain": None, "_identifier": "test_provider/test_provider", "_github_repo": None, + "start_line_number": 1, + "options": None, } From ac0f4501b5a18f2d57c246046d4c7eae1921cf23 Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Fri, 22 Dec 2023 09:52:29 +0000 Subject: [PATCH 03/20] refac(tf_test_projects): Move resources to get different line numbers. --- tf_test_files/project1/main.tf | 2 ++ tf_test_files/project2/main.tf | 1 + tf_test_files/project2/meta.tf | 2 ++ 3 files changed, 5 insertions(+) diff --git a/tf_test_files/project1/main.tf b/tf_test_files/project1/main.tf index 601f093..f2be59f 100644 --- a/tf_test_files/project1/main.tf +++ b/tf_test_files/project1/main.tf @@ -1,3 +1,5 @@ + + module "test_module" { source = "hashicorp/consul/aws" version = "0.2.0" diff --git a/tf_test_files/project2/main.tf b/tf_test_files/project2/main.tf index b689679..fd3fa7a 100644 --- a/tf_test_files/project2/main.tf +++ b/tf_test_files/project2/main.tf @@ -1,3 +1,4 @@ + module "test_module" { source = "hashicorp/consul/aws" version = "0.2.0" diff --git a/tf_test_files/project2/meta.tf b/tf_test_files/project2/meta.tf index 931a76f..7c194c4 100644 --- a/tf_test_files/project2/meta.tf +++ b/tf_test_files/project2/meta.tf @@ -1,3 +1,5 @@ + + terraform { required_providers { azurerm = { From 3fa64e000a5f5495a649c1987dc1aaed9d211867 Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Fri, 22 Dec 2023 09:54:21 +0000 Subject: [PATCH 04/20] feat(hcl_handler): Extract start line number for terraform providers and modules. --- .../core/utils/terraform/hcl_handler.py | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/infrapatch/core/utils/terraform/hcl_handler.py b/infrapatch/core/utils/terraform/hcl_handler.py index a0ffaae..97750b7 100644 --- a/infrapatch/core/utils/terraform/hcl_handler.py +++ b/infrapatch/core/utils/terraform/hcl_handler.py @@ -2,6 +2,7 @@ import logging as log import platform from pathlib import Path +import re from typing import Protocol, Sequence import pygohcl @@ -64,30 +65,37 @@ def get_terraform_resources_from_file(self, tf_file: Path, get_modules: bool = T with open(tf_file.absolute(), "r") as file: try: - terraform_file_dict = pygohcl.loads(file.read()) + content = file.read() + terraform_file_dict = pygohcl.loads(content) except Exception as e: raise HclParserException(f"Could not parse file '{tf_file}': {e}") found_resources = [] if get_modules: - found_resources.extend(self._get_terraform_modules_from_dict(terraform_file_dict, tf_file)) + found_resources.extend(self._get_terraform_modules_from_dict(terraform_file_dict, tf_file, content)) if get_providers: - found_resources.extend(self._get_terraform_providers_from_dict(terraform_file_dict, tf_file)) + found_resources.extend(self._get_terraform_providers_from_dict(terraform_file_dict, tf_file, content)) return found_resources - def _get_terraform_providers_from_dict(self, terraform_file_dict: dict, tf_file: Path) -> Sequence[TerraformProvider]: + def _get_terraform_providers_from_dict(self, terraform_file_dict: dict, tf_file: Path, content: str) -> Sequence[TerraformProvider]: found_resources = [] if "terraform" in terraform_file_dict: if "required_providers" in terraform_file_dict["terraform"]: providers = terraform_file_dict["terraform"]["required_providers"] for provider_name, provider_config in providers.items(): + source = provider_config["source"] + start_line_number = self._get_start_line_number(content, file=tf_file, search_regex=rf'{provider_name}\s*=\s*\{{[^}}]*source\s*=\s*"{source}"[^}}]*\}}') found_resources.append( TerraformProvider( - name=provider_name, _source=provider_config["source"], current_version=provider_config["version"], _source_file=tf_file.absolute().as_posix() + name=provider_name, + _source=source, + current_version=provider_config["version"], + _source_file=tf_file.absolute().as_posix(), + start_line_number=start_line_number, ) ) return found_resources - def _get_terraform_modules_from_dict(self, terraform_file_dict: dict, tf_file: Path) -> Sequence[TerraformProvider]: + def _get_terraform_modules_from_dict(self, terraform_file_dict: dict, tf_file: Path, content: str) -> Sequence[TerraformProvider]: found_resources = [] if "module" in terraform_file_dict: modules = terraform_file_dict["module"] @@ -98,9 +106,22 @@ def _get_terraform_modules_from_dict(self, terraform_file_dict: dict, tf_file: P if "version" not in value: log.debug(f"Skipping module '{module_name}' because it has no version attribute.") continue - found_resources.append(TerraformModule(name=module_name, _source=value["source"], current_version=value["version"], _source_file=tf_file.absolute().as_posix())) + start_line_number = self._get_start_line_number(content, file=tf_file, search_regex=f'module\s+"{module_name}"\s+\{{') + found_resources.append( + TerraformModule( + name=module_name, _source=value["source"], current_version=value["version"], _source_file=tf_file.absolute().as_posix(), start_line_number=start_line_number + ) + ) return found_resources + def _get_start_line_number(self, content: str, file: Path, search_regex: str) -> int: + result = re.search(search_regex, content, re.DOTALL) + if result is None: + raise Exception(f"Could not find line matching regex '{search_regex}' in file '{file.name}'") + line_number = content[: result.start()].count("\n") + 1 + log.debug(f"Found line number {line_number} for regex '{search_regex}' in file '{file.name}'") + return line_number + def get_all_terraform_files(self, root: Path) -> Sequence[Path]: if not root.is_dir(): raise Exception(f"Path '{root}' is not a directory.") From 751b7e8c4d9deb031f45b56fffbad13277de6061 Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Fri, 22 Dec 2023 09:54:50 +0000 Subject: [PATCH 05/20] test(hcl_handler): Change hcl_handler tests to check line numbers. --- infrapatch/core/utils/terraform/tests/test_hcl_handler.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infrapatch/core/utils/terraform/tests/test_hcl_handler.py b/infrapatch/core/utils/terraform/tests/test_hcl_handler.py index 566864f..8b5efda 100644 --- a/infrapatch/core/utils/terraform/tests/test_hcl_handler.py +++ b/infrapatch/core/utils/terraform/tests/test_hcl_handler.py @@ -99,6 +99,7 @@ def test_get_terraform_resources_from_file(hcl_handler: HclHandler, valid_terraf assert resource.current_version == "2.0.0" assert resource.source == "test/test_module/test_provider" assert resource.identifier == "test/test_module/test_provider" + assert resource.start_line_number == 14 assert resource.base_domain is None elif resource.name == "test_module2": assert isinstance(resource, TerraformModule) @@ -106,17 +107,20 @@ def test_get_terraform_resources_from_file(hcl_handler: HclHandler, valid_terraf assert resource.source == "spacelift.io/test/test_module/test_provider" assert resource.identifier == "test/test_module/test_provider" assert resource.base_domain == "spacelift.io" + assert resource.start_line_number == 19 elif resource.name == "test_provider": assert isinstance(resource, TerraformProvider) assert resource.current_version == ">1.0.0" assert resource.source == "test_provider/test_provider" assert resource.identifier == "test_provider/test_provider" + assert resource.start_line_number == 4 assert resource.base_domain is None elif resource.name == "test_provider2": assert isinstance(resource, TerraformProvider) assert resource.current_version == "1.0.5" assert resource.source == "spacelift.io/test_provider/test_provider2" assert resource.identifier == "test_provider/test_provider2" + assert resource.start_line_number == 8 assert resource.base_domain == "spacelift.io" else: raise Exception(f"Unknown resource '{resource.name}'.") From 78a205d95b7677d7f85287c1661ccdc535a403d4 Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Fri, 22 Dec 2023 09:55:11 +0000 Subject: [PATCH 06/20] fix(ruff): Format code. --- .../test_versioned_terraform_resource.py | 24 ++++++++++--------- infrapatch/core/models/versioned_resource.py | 4 ++-- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/infrapatch/core/models/tests/test_versioned_terraform_resource.py b/infrapatch/core/models/tests/test_versioned_terraform_resource.py index dad1e14..b76134a 100644 --- a/infrapatch/core/models/tests/test_versioned_terraform_resource.py +++ b/infrapatch/core/models/tests/test_versioned_terraform_resource.py @@ -17,8 +17,12 @@ def test_attributes(): assert provider.identifier == "test_provider/test_provider" # test with custom registry - module = TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="testregistry.ch/test/test_module/test_provider", start_line_number=1) - provider = TerraformProvider(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="testregistry.ch/test_provider/test_provider", start_line_number=1) + module = TerraformModule( + name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="testregistry.ch/test/test_module/test_provider", start_line_number=1 + ) + provider = TerraformProvider( + name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="testregistry.ch/test_provider/test_provider", start_line_number=1 + ) assert module.source == "testregistry.ch/test/test_module/test_provider" assert module.base_domain == "testregistry.ch" @@ -39,8 +43,12 @@ def test_attributes(): def test_find(): - findably_resource = TerraformModule(name="test_resource3", current_version="1.0.0", _source_file="test_file3.py", _source="test/test_module3/test_provider", start_line_number=1) - unfindably_resource = TerraformModule(name="test_resource6", current_version="1.0.0", _source_file="test_file8.py", _source="test/test_module3/test_provider", start_line_number=1) + findably_resource = TerraformModule( + name="test_resource3", current_version="1.0.0", _source_file="test_file3.py", _source="test/test_module3/test_provider", start_line_number=1 + ) + unfindably_resource = TerraformModule( + name="test_resource6", current_version="1.0.0", _source_file="test_file8.py", _source="test/test_module3/test_provider", start_line_number=1 + ) resources = [ TerraformModule(name="test_resource1", current_version="1.0.0", _source_file="test_file1.py", _source="test/test_module1/test_provider", start_line_number=1), TerraformModule(name="test_resource2", current_version="1.0.0", _source_file="test_file2.py", _source="test/test_module2/test_provider", start_line_number=1), @@ -55,13 +63,7 @@ def test_find(): def test_to_dict(): module = TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test/test_module/test_provider", start_line_number=1) - provider = TerraformProvider( - name="test_resource", - current_version="1.0.0", - _source_file="test_file.py", - _source="test_provider/test_provider", - start_line_number=1 - ) + provider = TerraformProvider(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test_provider/test_provider", start_line_number=1) module_dict = module.to_dict() provider_dict = provider.to_dict() diff --git a/infrapatch/core/models/versioned_resource.py b/infrapatch/core/models/versioned_resource.py index dd356a6..828967c 100644 --- a/infrapatch/core/models/versioned_resource.py +++ b/infrapatch/core/models/versioned_resource.py @@ -16,8 +16,8 @@ class ResourceStatus: PATCHED = "patched" PATCH_ERROR = "patch_error" NO_VERSION_FOUND = "no_version_found" - - + + @dataclass class VersionedResourceOptions: ignore_resource: bool From 1e6dc419181e57b500b9ab56d744be09bd9c6f1a Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Fri, 22 Dec 2023 10:31:24 +0000 Subject: [PATCH 07/20] feat(packages): Add pydantic as requirement. --- requirements.txt | 3 ++- setup.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 16c9359..b769cfd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ setuptools~=65.5.1 pygit2~=1.13.1 semantic-version~=2.10.0 PyGithub~=2.1.1 -pytablewriter~=1.2.0 \ No newline at end of file +pytablewriter~=1.2.0 +pydantic~=2.5.2 \ No newline at end of file diff --git a/setup.py b/setup.py index ececde9..fdbf7f3 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ "semantic_version~=2.10.0", "pytablewriter~=1.2.0", "PyGithub~=2.1.1", + "pydantic~=2.5.2", ], python_requires=">=3.11", entry_points=""" From 9f66fb37aad89226fdc8e381eb4124aa7520de41 Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Fri, 22 Dec 2023 12:37:42 +0000 Subject: [PATCH 08/20] feat(cli): Bump version. --- infrapatch/cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrapatch/cli/__init__.py b/infrapatch/cli/__init__.py index 3d18726..906d362 100644 --- a/infrapatch/cli/__init__.py +++ b/infrapatch/cli/__init__.py @@ -1 +1 @@ -__version__ = "0.5.0" +__version__ = "0.6.0" From a9593ce57c951140c26b9405366de64ea42e1b89 Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Fri, 22 Dec 2023 13:12:40 +0000 Subject: [PATCH 09/20] feat(options_processor): Add helper to process resource options. --- infrapatch/core/utils/options_processor.py | 48 ++++++ .../utils/tests/test_options_processor.py | 140 ++++++++++++++++++ 2 files changed, 188 insertions(+) create mode 100644 infrapatch/core/utils/options_processor.py create mode 100644 infrapatch/core/utils/tests/test_options_processor.py diff --git a/infrapatch/core/utils/options_processor.py b/infrapatch/core/utils/options_processor.py new file mode 100644 index 0000000..e71ddd2 --- /dev/null +++ b/infrapatch/core/utils/options_processor.py @@ -0,0 +1,48 @@ +import logging as log +from typing import Any, Protocol, Union +import infrapatch.core.constants as cs + +from infrapatch.core.models.versioned_resource import VersionedResource, VersionedResourceOptions + + +class OptionsProcessorInterface(Protocol): + def process_options_for_resource(self, resource: VersionedResource) -> VersionedResource: + ... + + +class OptionsProcessor(OptionsProcessorInterface): + def _get_upper_line_content(self, resource: VersionedResource) -> Union[str, None]: + if resource.start_line_number == 0: + raise Exception(f"Resource '{resource.name}' has invalid start line number 0.") + if resource.start_line_number == 1: + return None + with open(resource.source_file, "r") as f: + lines = f.readlines() + return lines[resource.start_line_number - 2].strip() + + def _process_options_string(self, options: str) -> dict[str, Any]: + options_dict = {} + for option in options.split(","): + key, value = option.split("=") + options_dict[key.strip()] = value.strip() + return options_dict + + def _get_options_object(self, line: str) -> VersionedResourceOptions: + # Get the rigth part of the options line + options_string = line.split(cs.infrapatch_options_prefix)[1].strip() + optioons_dict = self._process_options_string(options_string) + return VersionedResourceOptions(**optioons_dict) + + def process_options_for_resource(self, resource: VersionedResource) -> VersionedResource: + upper_line_content = self._get_upper_line_content(resource) + + if upper_line_content is None: + log.debug(f"Resource '{resource.name}' has no options.") + return resource + + if cs.infrapatch_options_prefix not in upper_line_content: + log.debug(f"Resource '{resource.name}' has no options.") + return resource + + resource.options = self._get_options_object(upper_line_content) + return resource diff --git a/infrapatch/core/utils/tests/test_options_processor.py b/infrapatch/core/utils/tests/test_options_processor.py new file mode 100644 index 0000000..ce2a250 --- /dev/null +++ b/infrapatch/core/utils/tests/test_options_processor.py @@ -0,0 +1,140 @@ +from pathlib import Path +from unittest import mock + +import pytest + +import infrapatch.core.constants as cs +from infrapatch.core.models.versioned_terraform_resources import TerraformModule +from infrapatch.core.utils.options_processor import OptionsProcessor +from infrapatch.core.utils.terraform.hcl_edit_cli import HclEditCli +from infrapatch.core.utils.terraform.hcl_handler import HclHandler + + +@pytest.fixture +def options_processor(): + return OptionsProcessor() + + +@pytest.fixture +def tmp_user_home(tmp_path: Path): + return tmp_path + + +@pytest.fixture +def hcl_handler(): + return HclHandler(hcl_edit_cli=HclEditCli()) + + +@pytest.fixture +def terraform_code_with_options(): + return """ + terraform { + required_providers { + # infrapatch_options: ignore_resource=true + test_provider = { + source = "test_provider/test_provider" + version = ">1.0.0" + } + test_provider2 = { + source = "spacelift.io/test_provider/test_provider2" + version = "1.0.5" + } + } + } + module "test_module" { + source = "test/test_module/test_provider" + version = "2.0.0" + name = "Test_module" + } + # infrapatch_options: ignore_resource=true, test=test + module "test_module2" { + source = "spacelift.io/test/test_module/test_provider" + version = "1.0.2" + name = "Test_module2" + } + # This module should be ignored since it has no version + module "test_module3" { + source = "C:/test/test_module/test_provider" + name = "Test_module3" + } + """ + + +def test_options_processing_from_file(options_processor: OptionsProcessor, hcl_handler: HclHandler, terraform_code_with_options: str, tmp_path: Path): + # Create a temporary Terraform file for testing + tf_file = tmp_path.joinpath("test_file.tf") + tf_file.write_text(terraform_code_with_options) + resouces = hcl_handler.get_terraform_resources_from_file(tf_file, get_modules=True, get_providers=True) + + # Check all resources parsed from the file for correct options + for resource in resouces: + resource = options_processor.process_options_for_resource(resource) + if resource.name == "test_module": + assert resource.options is None + elif resource.name == "test_module2": + assert resource.options is not None + assert resource.options.ignore_resource is True + elif resource.name == "test_provider": + assert resource.options is not None + assert resource.options.ignore_resource is True + elif resource.name == "test_provider2": + assert resource.options is None + else: + raise Exception(f"Unknown resource '{resource.name}'.") + + +def test_get_upper_line_with_non_valid_line_numbers(options_processor: OptionsProcessor): + # Should return none since start line number is 1 + resource = TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test/test_module/test_provider", start_line_number=1) + upper_line_content = options_processor._get_upper_line_content(resource) + assert upper_line_content is None + + # Should raise an exception since start line number is 0 + with pytest.raises(Exception): + resource = TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test/test_module/test_provider", start_line_number=0) + upper_line_content = options_processor._get_upper_line_content(resource) + + +def test_get_upper_line_with_valid_line_numbers(options_processor: OptionsProcessor): + resource1 = TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test/test_module/test_provider", start_line_number=2) + resource2 = TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test/test_module/test_provider", start_line_number=5) + with mock.patch("builtins.open", mock.mock_open(read_data="line1\nline2\nline3\nline4\nline5\nline6\nline7\nline8\nline9\nline10\n")): + resource1_line = options_processor._get_upper_line_content(resource1) + resource2_line = options_processor._get_upper_line_content(resource2) + assert resource1_line == "line1" + assert resource2_line == "line4" + + +options_strings = ["ignore_resource=true", "ignore_resource = true, test_option=2", "ignore_resource =false,test_option2=test4, test_option3 = test5"] + + +@pytest.mark.parametrize("options_string", options_strings) +def test_options_format_processing(options_processor: OptionsProcessor, options_string: str): + options_dict = options_processor._process_options_string(options_string) + assert options_dict is not None + assert options_dict is not {} + + if options_string == "ignore_resource=true": + assert options_dict["ignore_resource"] == "true" + elif options_string == "ignore_resource = true, test_option=2": + assert options_dict["ignore_resource"] == "true" + assert options_dict["test_option"] == "2" + elif options_string == "ignore_resource =false,test_option2=test4, test_option3 = test5": + assert options_dict["ignore_resource"] == "false" + assert options_dict["test_option2"] == "test4" + assert options_dict["test_option3"] == "test5" + else: + raise Exception(f"Unknown options string '{options_string}'.") + + +@pytest.mark.parametrize("options_string", options_strings) +def test_get_options_object(options_processor: OptionsProcessor, options_string: str): + options = options_processor._get_options_object(f"{cs.infrapatch_options_prefix} {options_string}") + assert options is not None + + if options_string == "ignore_resource=true": + assert options.ignore_resource is True + elif options_string == "ignore_resource =false,test_option2=test4, test_option3 = test5": + assert options.ignore_resource is False + with pytest.raises(AttributeError): + options.test_option2 # type: ignore From 4142451259592bbe48266dce3a8e716d35184b84 Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Fri, 22 Dec 2023 13:13:00 +0000 Subject: [PATCH 10/20] feat(constants): Add new constant for option prefix --- infrapatch/core/constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infrapatch/core/constants.py b/infrapatch/core/constants.py index e9f46b0..03a5766 100644 --- a/infrapatch/core/constants.py +++ b/infrapatch/core/constants.py @@ -5,3 +5,5 @@ APP_NAME = "InfraPatch" DEFAULT_CREDENTIALS_FILE_NAME = "infrapatch_credentials.json" + +infrapatch_options_prefix = "# infrapatch_options:" From 4caed865781205aecebc2620aacaf4baaeae76bb Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Fri, 22 Dec 2023 13:13:40 +0000 Subject: [PATCH 11/20] refac(hcl_handler_tests): Edit tests to include check for line numbers. --- .../utils/terraform/tests/test_hcl_handler.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/infrapatch/core/utils/terraform/tests/test_hcl_handler.py b/infrapatch/core/utils/terraform/tests/test_hcl_handler.py index 8b5efda..dc0a259 100644 --- a/infrapatch/core/utils/terraform/tests/test_hcl_handler.py +++ b/infrapatch/core/utils/terraform/tests/test_hcl_handler.py @@ -23,11 +23,12 @@ def valid_terraform_code(): return """ terraform { required_providers { - test_provider = { + test_provider = { + test = "test" source = "test_provider/test_provider" version = ">1.0.0" } - test_provider2 = { + test_provider2 = { source = "spacelift.io/test_provider/test_provider2" version = "1.0.5" } @@ -38,7 +39,7 @@ def valid_terraform_code(): version = "2.0.0" name = "Test_module" } - module "test_module2" { + module "test_module2" { source = "spacelift.io/test/test_module/test_provider" version = "1.0.2" name = "Test_module2" @@ -56,7 +57,7 @@ def invalid_terraform_code(): return """ terraform { required_providers { - test_provider = { + test_provider = { source = "test_provider/test_provider" version = ">1.0.0" } @@ -66,7 +67,7 @@ def invalid_terraform_code(): } } } - module "test_module" { + module "test_module" { source = "test/test_module/test_provider" version = "2.0.0" name = Test_module" @@ -99,7 +100,7 @@ def test_get_terraform_resources_from_file(hcl_handler: HclHandler, valid_terraf assert resource.current_version == "2.0.0" assert resource.source == "test/test_module/test_provider" assert resource.identifier == "test/test_module/test_provider" - assert resource.start_line_number == 14 + assert resource.start_line_number == 15 assert resource.base_domain is None elif resource.name == "test_module2": assert isinstance(resource, TerraformModule) @@ -107,7 +108,7 @@ def test_get_terraform_resources_from_file(hcl_handler: HclHandler, valid_terraf assert resource.source == "spacelift.io/test/test_module/test_provider" assert resource.identifier == "test/test_module/test_provider" assert resource.base_domain == "spacelift.io" - assert resource.start_line_number == 19 + assert resource.start_line_number == 20 elif resource.name == "test_provider": assert isinstance(resource, TerraformProvider) assert resource.current_version == ">1.0.0" @@ -120,7 +121,7 @@ def test_get_terraform_resources_from_file(hcl_handler: HclHandler, valid_terraf assert resource.current_version == "1.0.5" assert resource.source == "spacelift.io/test_provider/test_provider2" assert resource.identifier == "test_provider/test_provider2" - assert resource.start_line_number == 8 + assert resource.start_line_number == 9 assert resource.base_domain == "spacelift.io" else: raise Exception(f"Unknown resource '{resource.name}'.") From f2116ca78b3112189265a321f350073f4c12038c Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Fri, 22 Dec 2023 13:14:14 +0000 Subject: [PATCH 12/20] refac(resource_options): Switch from dataclass to pydantic. --- infrapatch/core/models/versioned_resource.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infrapatch/core/models/versioned_resource.py b/infrapatch/core/models/versioned_resource.py index 828967c..6f0b961 100644 --- a/infrapatch/core/models/versioned_resource.py +++ b/infrapatch/core/models/versioned_resource.py @@ -6,6 +6,7 @@ from urllib.parse import urlparse import logging as log from git import Sequence +from pydantic import BaseModel import semantic_version @@ -18,9 +19,8 @@ class ResourceStatus: NO_VERSION_FOUND = "no_version_found" -@dataclass -class VersionedResourceOptions: - ignore_resource: bool +class VersionedResourceOptions(BaseModel): + ignore_resource: bool = False @dataclass From 28bf3dca492de97921dbf836a1e1b3b842f97eee Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Fri, 22 Dec 2023 13:14:30 +0000 Subject: [PATCH 13/20] feat(vscode): Update code actions on save. --- .vscode/settings.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 549a43a..42e3633 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,8 +11,8 @@ "[python]": { "editor.formatOnSave": true, "editor.codeActionsOnSave": { - "source.fixAll": true, - "source.organizeImports": true + "source.fixAll": "explicit", + "source.organizeImports": "explicit" }, "editor.defaultFormatter": "charliermarsh.ruff" }, From d77b44f61b52eab87897920f415dbccfe86d95f9 Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Fri, 22 Dec 2023 15:07:47 +0000 Subject: [PATCH 14/20] doc(README): Add documentation regarding resource options. --- README.md | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/README.md b/README.md index 7dd3995..31e238b 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,10 @@ The CLI works by scanning your .tf files for versioned providers and modules and - [infrapatch\_credentials.json file:](#infrapatch_credentialsjson-file) - [Setup Development Environment for InfraPatch](#setup-development-environment-for-infrapatch) - [Contributing](#contributing) + - [Global](#global) + - [Resource Options](#resource-options) + - [Available Options](#available-options) + - [Example](#example) ## GitHub Action @@ -197,3 +201,58 @@ During the first start, the devcontainer will build the container image and inst If you have any ideas for improvements or find any bugs, feel free to open an issue or create a pull request. +## Global + +The following section describes configurations and behaviors that are applicable to the Github Action and the CLI. + +### Resource Options + +InfraPatch supports individual resource options to change the behavior for a specific resource. +Resource options can be specified one line obove your resource definition with the following syntax: + +```hcl +# infrapatch_options: = +module "example" { + source = "terraform-aws-modules/example" + name: "demo" +} + +terraform { + required_providers { + # infrapatch_options: = + aws = { + source = "hashicorp/aws" + } + } +} +``` + +#### Available Options + +Currently, the following options are available: + +| Option Name | Description | Default Value | +| ----------- | ----------- | ------------- | +| `ignore_resource` | If set to `true`, the resource will be ignored by InfraPatch. | `false` | + +#### Example + +The following example shows how to ignore a terraform module and a terraform provider: + + ```hcl + # infrapatch_options: ignore_resource=true + module "example" { + source = "terraform-aws-modules/example" + name: "demo" + } + + terraform { + required_providers { + # infrapatch_options: ignore_resource=true + aws = { + source = "hashicorp/aws" + } + } + } + ``` + From 5ea81cd1842f6ac03d394deb7ae8a73176cfa0c1 Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Fri, 22 Dec 2023 15:08:15 +0000 Subject: [PATCH 15/20] feat(tf_test_files): Add module that should be ignored by infrapatch. --- tf_test_files/project2/main.tf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tf_test_files/project2/main.tf b/tf_test_files/project2/main.tf index fd3fa7a..e04fdf5 100644 --- a/tf_test_files/project2/main.tf +++ b/tf_test_files/project2/main.tf @@ -2,4 +2,11 @@ module "test_module" { source = "hashicorp/consul/aws" version = "0.2.0" +} + +# This resource should be ignored by infrapatch +# infrapatch_options: ignore_resource=true +module "test_module6" { + source = "hashicorp/consul/aws" + version = "0.2.0" } \ No newline at end of file From 2fe37a204aa57452db4fe31563dbec7efed53c39 Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Fri, 22 Dec 2023 15:09:36 +0000 Subject: [PATCH 16/20] refac(pydantic): Switch models from dataclass to pydantic BaseModel. --- infrapatch/core/models/statistics.py | 10 +-- infrapatch/core/models/versioned_resource.py | 82 ++++++++----------- .../models/versioned_terraform_resources.py | 58 ++++++------- 3 files changed, 62 insertions(+), 88 deletions(-) diff --git a/infrapatch/core/models/statistics.py b/infrapatch/core/models/statistics.py index a4f76bd..6ece766 100644 --- a/infrapatch/core/models/statistics.py +++ b/infrapatch/core/models/statistics.py @@ -1,28 +1,24 @@ -from dataclasses import dataclass -import dataclasses from typing import Any, Sequence +from pydantic import BaseModel from pytablewriter import MarkdownTableWriter from rich.table import Table from infrapatch.core.models.versioned_resource import VersionedResource -@dataclass -class BaseStatistics: +class BaseStatistics(BaseModel): errors: int resources_patched: int resources_pending_update: int total_resources: int def to_dict(self) -> dict[str, Any]: - return dataclasses.asdict(self) + return self.model_dump() -@dataclass class ProviderStatistics(BaseStatistics): resources: Sequence[VersionedResource] -@dataclass class Statistics(BaseStatistics): providers: dict[str, ProviderStatistics] diff --git a/infrapatch/core/models/versioned_resource.py b/infrapatch/core/models/versioned_resource.py index 6f0b961..a46bc7d 100644 --- a/infrapatch/core/models/versioned_resource.py +++ b/infrapatch/core/models/versioned_resource.py @@ -1,14 +1,12 @@ -import dataclasses +import logging as log import re -from dataclasses import dataclass from pathlib import Path -from typing import Any, Optional, Union +from typing import Any, Optional from urllib.parse import urlparse -import logging as log -from git import Sequence -from pydantic import BaseModel import semantic_version +from git import Sequence +from pydantic import BaseModel class ResourceStatus: @@ -23,59 +21,52 @@ class VersionedResourceOptions(BaseModel): ignore_resource: bool = False -@dataclass -class VersionedResource: +class VersionedResource(BaseModel): name: str current_version: str start_line_number: int - _source_file: str - _newest_version: Optional[str] = None - _status: str = ResourceStatus.UNPATCHED - _github_repo: Optional[str] = None - options: Optional[VersionedResourceOptions] = None - - @property - def source_file(self) -> Path: - return Path(self._source_file) - - @property - def status(self) -> str: - return self._status - - @property - def github_repo(self) -> Union[str, None]: - return self._github_repo + source_file: Path + newest_version_string: Optional[str] = None + status: str = ResourceStatus.UNPATCHED + github_repo_string: Optional[str] = None + options: VersionedResourceOptions = VersionedResourceOptions() @property def resource_name(self): raise NotImplementedError() - @property - def newest_version(self) -> Optional[str]: - return self._newest_version - @property def newest_version_base(self): if self.has_tile_constraint(): - if self.newest_version is None: + if self.newest_version_string is None: raise Exception(f"Newest version of resource '{self.name}' is not set.") - return self.newest_version.strip("~>") - return self.newest_version + return self.newest_version_string.strip("~>") + return self.newest_version_string + + @property + def newest_version(self): + return self.newest_version_string @newest_version.setter def newest_version(self, version: Optional[str]): if self.has_tile_constraint(): - self._newest_version = f"~>{version}" + self.newest_version_string = f"~>{version}" else: - self._newest_version = version + self.newest_version_string = version if version is None: self.set_no_version_found() return + if self.installed_version_equal_or_newer_than_new_version(): self.set_up_to_date() - def set_github_repo(self, github_repo_url: str): + @property + def github_repo(self): + return self.github_repo_string + + @github_repo.setter + def github_repo(self, github_repo_url: str): url = urlparse(github_repo_url) if url.path is None or url.path == "" or url.path == "/": raise Exception(f"Invalid github repo url '{github_repo_url}'.") @@ -84,16 +75,16 @@ def set_github_repo(self, github_repo_url: str): path = path[:-4] repo = "/".join(path.split("/")[1:3]) log.debug(f"Setting github repo for resource '{self.name}' to '{repo}'") - self._github_repo = repo + self.github_repo_string = repo def set_patched(self): - self._status = ResourceStatus.PATCHED + self.status = ResourceStatus.PATCHED def set_no_version_found(self): - self._status = ResourceStatus.NO_VERSION_FOUND + self.status = ResourceStatus.NO_VERSION_FOUND def set_up_to_date(self): - self._status = ResourceStatus.UP_TO_DATE + self.status = ResourceStatus.UP_TO_DATE def has_tile_constraint(self) -> bool: result = re.match(r"^~>[0-9]+\.[0-9]+\.[0-9]+$", self.current_version) @@ -102,16 +93,16 @@ def has_tile_constraint(self) -> bool: return True def set_patch_error(self): - self._status = ResourceStatus.PATCH_ERROR + self.status = ResourceStatus.PATCH_ERROR def find(self, resources): - result = [resource for resource in resources if resource.name == self.name and resource._source_file == self._source_file] + result = [resource for resource in resources if resource.name == self.name and resource.source_file == self.source_file] return result def installed_version_equal_or_newer_than_new_version(self): - if self._status == ResourceStatus.NO_VERSION_FOUND: + if self.status == ResourceStatus.NO_VERSION_FOUND: return True - if self.newest_version is None: + if self.newest_version_string is None: raise Exception(f"Newest version of resource '{self.name}' is not set.") newest = semantic_version.Version(self.newest_version_base) @@ -147,11 +138,10 @@ def check_if_up_to_date(self): return False def to_dict(self) -> dict[str, Any]: - return dataclasses.asdict(self) + return self.model_dump() -@dataclass -class VersionedResourceReleaseNotes: +class VersionedResourceReleaseNotes(BaseModel): resources: Sequence[VersionedResource] name: str body: str diff --git a/infrapatch/core/models/versioned_terraform_resources.py b/infrapatch/core/models/versioned_terraform_resources.py index a98b84f..9554dc2 100644 --- a/infrapatch/core/models/versioned_terraform_resources.py +++ b/infrapatch/core/models/versioned_terraform_resources.py @@ -1,48 +1,37 @@ import logging as log import re -from dataclasses import dataclass -from typing import Optional, Union +from typing import Optional from infrapatch.core.models.versioned_resource import VersionedResource -@dataclass(kw_only=True) class VersionedTerraformResource(VersionedResource): - _source: str - _base_domain: Optional[str] = None - _identifier: Optional[str] = None + source_string: str + base_domain: Optional[str] = None + identifier: Optional[str] = None @property def source(self) -> str: - if self._source is None: + if self.source_string is None: raise Exception("Source is None.") - return self._source - - @property - def base_domain(self) -> Optional[str]: - return self._base_domain + return self.source_string @property def resource_name(self): raise NotImplementedError() - @property - def identifier(self) -> Union[str, None]: - return self._identifier - def find(self, resources): filtered_resources = super().find(resources) - return [resource for resource in filtered_resources if resource._source == self._source] + return [resource for resource in filtered_resources if resource.source == self.source] -@dataclass class TerraformModule(VersionedTerraformResource): - def __post_init__(self): - self.source = self._source + def model_post_init(self, __context): + self.source = self.source_string @property def source(self) -> str: - return self._source + return self.source_string @property def resource_name(self): @@ -51,27 +40,26 @@ def resource_name(self): @source.setter def source(self, source: str): source_lower_case = source.lower() - self._source = source_lower_case - self._newest_version = None + self.source_string = source_lower_case + self.newest_version_string = None if re.match(r"^[a-zA-Z0-9-]+\.[a-zA-Z0-9-]+/[a-zA-Z0-9-_]+/[a-zA-Z0-9-_]+/[a-zA-Z0-9-_]+$", source_lower_case): log.debug(f"Source '{source_lower_case}' is from a generic registry.") - self._base_domain = source_lower_case.split("/")[0] - self._identifier = "/".join(source_lower_case.split("/")[1:]) + self.base_domain = source_lower_case.split("/")[0] + self.identifier = "/".join(source_lower_case.split("/")[1:]) elif re.match(r"^[a-zA-Z0-9-_]+/[a-zA-Z0-9-_]+/[a-zA-Z0-9-_]+$", source_lower_case): log.debug(f"Source '{source_lower_case}' is from the public registry.") - self._identifier = source_lower_case + self.identifier = source_lower_case else: raise Exception(f"Source '{source_lower_case}' is not a valid terraform resource source.") -@dataclass class TerraformProvider(VersionedTerraformResource): - def __post_init__(self): - self.source = self._source + def model_post_init(self, __context): + self.source = self.source_string @property def source(self) -> str: - return self._source + return self.source_string @property def resource_name(self): @@ -80,14 +68,14 @@ def resource_name(self): @source.setter def source(self, source: str) -> None: source_lower_case = source.lower() - self._source = source_lower_case - self._newest_version = None + self.source_string = source_lower_case + self.newest_version_string = None if re.match(r"^[a-zA-Z0-9-]+\.[a-zA-Z0-9-]+/[a-zA-Z0-9-_]+/[a-zA-Z0-9-_]+$", source_lower_case): log.debug(f"Source '{source_lower_case}' is from a generic registry.") - self._base_domain = source_lower_case.split("/")[0] - self._identifier = "/".join(source_lower_case.split("/")[1:]) + self.base_domain = source_lower_case.split("/")[0] + self.identifier = "/".join(source_lower_case.split("/")[1:]) elif re.match(r"^[a-zA-Z0-9-_]+/[a-zA-Z0-9-_]+$", source_lower_case): log.debug(f"Source '{source_lower_case}' is from the public registry.") - self._identifier = source_lower_case + self.identifier = source_lower_case else: raise Exception(f"Source '{source_lower_case}' is not a valid terraform resource source.") From bcdc06b03a0ffa9de258537aba960a1d953910a0 Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Fri, 22 Dec 2023 15:10:45 +0000 Subject: [PATCH 17/20] refac(Pydantic): Fix Code and tests to work with new pydantic models. --- .../models/tests/test_versioned_resource.py | 62 ++++++++------- .../test_versioned_terraform_resource.py | 78 ++++++++++--------- .../terraform/base_terraform_provider.py | 2 +- .../core/utils/terraform/hcl_handler.py | 8 +- .../utils/terraform/tests/test_hcl_handler.py | 2 +- .../utils/tests/test_options_processor.py | 21 +++-- 6 files changed, 93 insertions(+), 80 deletions(-) diff --git a/infrapatch/core/models/tests/test_versioned_resource.py b/infrapatch/core/models/tests/test_versioned_resource.py index 8abee77..ebe019f 100644 --- a/infrapatch/core/models/tests/test_versioned_resource.py +++ b/infrapatch/core/models/tests/test_versioned_resource.py @@ -1,4 +1,4 @@ -from pathlib import Path +from pathlib import Path, PosixPath import pytest @@ -7,7 +7,7 @@ def test_version_management(): # Create new resource with newer version - resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py", start_line_number=1) + resource = VersionedResource(name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), start_line_number=1) resource.newest_version = "2.0.0" assert resource.status == ResourceStatus.UNPATCHED @@ -17,14 +17,14 @@ def test_version_management(): assert resource.status == ResourceStatus.PATCHED # Check new_version the same as current_version - resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py", start_line_number=1) + resource = VersionedResource(name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), start_line_number=1) resource.newest_version = "1.0.0" assert resource.status == ResourceStatus.UP_TO_DATE assert resource.installed_version_equal_or_newer_than_new_version() is True # Check new_version older than current_version - resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py", start_line_number=1) + resource = VersionedResource(name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), start_line_number=1) resource.newest_version = "0.1.0" assert resource.status == ResourceStatus.UP_TO_DATE @@ -32,7 +32,7 @@ def test_version_management(): def test_tile_constraint(): - resource = VersionedResource(name="test_resource", current_version="~>1.0.0", _source_file="test_file.py", start_line_number=1) + resource = VersionedResource(name="test_resource", current_version="~>1.0.0", source_file=Path("test_file.py"), start_line_number=1) resource.newest_version = "~>1.0.1" assert resource.has_tile_constraint() is True assert resource.installed_version_equal_or_newer_than_new_version() is True @@ -40,66 +40,66 @@ def test_tile_constraint(): resource.newest_version = "~>1.1.0" assert resource.installed_version_equal_or_newer_than_new_version() is False - resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py", start_line_number=1) + resource = VersionedResource(name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), start_line_number=1) assert resource.has_tile_constraint() is False - resource = VersionedResource(name="test_resource", current_version="~>1.0.0", _source_file="test_file.py", start_line_number=1) + resource = VersionedResource(name="test_resource", current_version="~>1.0.0", source_file=Path("test_file.py"), start_line_number=1) resource.newest_version = "1.1.0" assert resource.newest_version == "~>1.1.0" def test_git_repo(): - resource = VersionedResource(name="test_resource", current_version="~>1.0.0", _source_file="test_file.py", start_line_number=1) + resource = VersionedResource(name="test_resource", current_version="~>1.0.0", source_file=Path("test_file.py"), start_line_number=1) assert resource.github_repo is None - resource.set_github_repo("https://github.com/noahnc/test_repo.git") + resource.github_repo = "https://github.com/noahnc/test_repo.git" assert resource.github_repo == "noahnc/test_repo" - resource.set_github_repo("https://github.com/noahnc/test_repo") + resource.github_repo = "https://github.com/noahnc/test_repo" assert resource.github_repo == "noahnc/test_repo" with pytest.raises(Exception): - resource.set_github_repo("https://github.com/") + resource.github_repo = "https://github.com/" with pytest.raises(Exception): - resource.set_github_repo("https://github.com") + resource.github_repo = "https://github.com" def test_patch_error(): - resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py", start_line_number=1) + resource = VersionedResource(name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), start_line_number=1) resource.set_patch_error() assert resource.status == ResourceStatus.PATCH_ERROR def test_version_not_found(): # Test manual setting - resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py", start_line_number=1) + resource = VersionedResource(name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), start_line_number=1) resource.set_no_version_found() assert resource.status == ResourceStatus.NO_VERSION_FOUND assert resource.installed_version_equal_or_newer_than_new_version() is True # Test by setting None as new version - resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py", start_line_number=1) + resource = VersionedResource(name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), start_line_number=1) resource.newest_version = None assert resource.status == ResourceStatus.NO_VERSION_FOUND assert resource.installed_version_equal_or_newer_than_new_version() is True def test_path(): - resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="/var/testdir/test_file.py", start_line_number=1) + resource = VersionedResource(name="test_resource", current_version="1.0.0", source_file=Path("/var/testdir/test_file.py"), start_line_number=1) assert resource.source_file == Path("/var/testdir/test_file.py") def test_find(): - findably_resource = VersionedResource(name="test_resource3", current_version="1.0.0", _source_file="test_file3.py", start_line_number=1) - unfindably_resource = VersionedResource(name="test_resource6", current_version="1.0.0", _source_file="test_file8.py", start_line_number=1) + findably_resource = VersionedResource(name="test_resource3", current_version="1.0.0", source_file=Path("test_file3.py"), start_line_number=1) + unfindably_resource = VersionedResource(name="test_resource6", current_version="1.0.0", source_file=Path("test_file8.py"), start_line_number=1) resources = [ - VersionedResource(name="test_resource1", current_version="1.0.0", _source_file="test_file1.py", start_line_number=1), - VersionedResource(name="test_resource2", current_version="1.0.0", _source_file="test_file2.py", start_line_number=1), - VersionedResource(name="test_resource3", current_version="1.0.0", _source_file="test_file3.py", start_line_number=1), - VersionedResource(name="test_resource4", current_version="1.0.0", _source_file="test_file4.py", start_line_number=1), - VersionedResource(name="test_resource5", current_version="1.0.0", _source_file="test_file5.py", start_line_number=1), + VersionedResource(name="test_resource1", current_version="1.0.0", source_file=Path("test_file1.py"), start_line_number=1), + VersionedResource(name="test_resource2", current_version="1.0.0", source_file=Path("test_file2.py"), start_line_number=1), + VersionedResource(name="test_resource3", current_version="1.0.0", source_file=Path("test_file3.py"), start_line_number=1), + VersionedResource(name="test_resource4", current_version="1.0.0", source_file=Path("test_file4.py"), start_line_number=1), + VersionedResource(name="test_resource5", current_version="1.0.0", source_file=Path("test_file5.py"), start_line_number=1), ] assert len(findably_resource.find(resources)) == 1 assert findably_resource.find(resources) == [resources[2]] @@ -107,15 +107,17 @@ def test_find(): def test_versioned_resource_to_dict(): - resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py", start_line_number=1) + resource = VersionedResource(name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), start_line_number=1) expected_dict = { "name": "test_resource", "current_version": "1.0.0", - "_source_file": "test_file.py", - "_newest_version": None, - "_status": ResourceStatus.UNPATCHED, - "_github_repo": None, + "source_file": PosixPath("test_file.py"), + "newest_version_string": None, + "status": ResourceStatus.UNPATCHED, + "github_repo_string": None, "start_line_number": 1, - "options": None, + "options": { + "ignore_resource": False, + }, } - assert resource.to_dict() == expected_dict + assert resource.model_dump() == expected_dict diff --git a/infrapatch/core/models/tests/test_versioned_terraform_resource.py b/infrapatch/core/models/tests/test_versioned_terraform_resource.py index b76134a..99aab9c 100644 --- a/infrapatch/core/models/tests/test_versioned_terraform_resource.py +++ b/infrapatch/core/models/tests/test_versioned_terraform_resource.py @@ -1,3 +1,5 @@ +from pathlib import Path, PosixPath + import pytest from infrapatch.core.models.versioned_terraform_resources import TerraformModule, TerraformProvider @@ -5,8 +7,8 @@ def test_attributes(): # test with default registry - module = TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test/test_module/test_provider", start_line_number=1) - provider = TerraformProvider(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test_provider/test_provider", start_line_number=1) + module = TerraformModule(name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), source_string="test/test_module/test_provider", start_line_number=1) + provider = TerraformProvider(name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), source_string="test_provider/test_provider", start_line_number=1) assert module.source == "test/test_module/test_provider" assert module.base_domain is None @@ -18,10 +20,10 @@ def test_attributes(): # test with custom registry module = TerraformModule( - name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="testregistry.ch/test/test_module/test_provider", start_line_number=1 + name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), source_string="testregistry.ch/test/test_module/test_provider", start_line_number=1 ) provider = TerraformProvider( - name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="testregistry.ch/test_provider/test_provider", start_line_number=1 + name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), source_string="testregistry.ch/test_provider/test_provider", start_line_number=1 ) assert module.source == "testregistry.ch/test/test_module/test_provider" @@ -34,27 +36,29 @@ def test_attributes(): # test invalid sources with pytest.raises(Exception): - TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test/test_module/test_provider/test", start_line_number=1) - TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="/test_module", start_line_number=1) + TerraformModule(name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), source_string="test/test_module/test_provider/test", start_line_number=1) + TerraformModule(name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), source_string="/test_module", start_line_number=1) with pytest.raises(Exception): - TerraformProvider(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="/test_module", start_line_number=1) - TerraformProvider(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="kfdsjflksdj/kldfsjflsdkj/dkljflsk/test_module", start_line_number=1) + TerraformProvider(name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), source_string="/test_module", start_line_number=1) + TerraformProvider( + name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), source_string="kfdsjflksdj/kldfsjflsdkj/dkljflsk/test_module", start_line_number=1 + ) def test_find(): findably_resource = TerraformModule( - name="test_resource3", current_version="1.0.0", _source_file="test_file3.py", _source="test/test_module3/test_provider", start_line_number=1 + name="test_resource3", current_version="1.0.0", source_file=Path("test_file3.py"), source_string="test/test_module3/test_provider", start_line_number=1 ) unfindably_resource = TerraformModule( - name="test_resource6", current_version="1.0.0", _source_file="test_file8.py", _source="test/test_module3/test_provider", start_line_number=1 + name="test_resource6", current_version="1.0.0", source_file=Path("test_file8.py"), source_string="test/test_module3/test_provider", start_line_number=1 ) resources = [ - TerraformModule(name="test_resource1", current_version="1.0.0", _source_file="test_file1.py", _source="test/test_module1/test_provider", start_line_number=1), - TerraformModule(name="test_resource2", current_version="1.0.0", _source_file="test_file2.py", _source="test/test_module2/test_provider", start_line_number=1), - TerraformModule(name="test_resource3", current_version="1.0.0", _source_file="test_file3.py", _source="test/test_module3/test_provider", start_line_number=1), - TerraformModule(name="test_resource4", current_version="1.0.0", _source_file="test_file4.py", _source="test/test_module4/test_provider", start_line_number=1), - TerraformModule(name="test_resource5", current_version="1.0.0", _source_file="test_file5.py", _source="test/test_module5/test_provider", start_line_number=1), + TerraformModule(name="test_resource1", current_version="1.0.0", source_file=Path("test_file1.py"), source_string="test/test_module1/test_provider", start_line_number=1), + TerraformModule(name="test_resource2", current_version="1.0.0", source_file=Path("test_file2.py"), source_string="test/test_module2/test_provider", start_line_number=1), + TerraformModule(name="test_resource3", current_version="1.0.0", source_file=Path("test_file3.py"), source_string="test/test_module3/test_provider", start_line_number=1), + TerraformModule(name="test_resource4", current_version="1.0.0", source_file=Path("test_file4.py"), source_string="test/test_module4/test_provider", start_line_number=1), + TerraformModule(name="test_resource5", current_version="1.0.0", source_file=Path("test_file5.py"), source_string="test/test_module5/test_provider", start_line_number=1), ] assert len(findably_resource.find(resources)) == 1 assert findably_resource.find(resources) == [resources[2]] @@ -62,35 +66,39 @@ def test_find(): def test_to_dict(): - module = TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test/test_module/test_provider", start_line_number=1) - provider = TerraformProvider(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test_provider/test_provider", start_line_number=1) + module = TerraformModule(name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), source_string="test/test_module/test_provider", start_line_number=1) + provider = TerraformProvider(name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), source_string="test_provider/test_provider", start_line_number=1) - module_dict = module.to_dict() - provider_dict = provider.to_dict() + module_dict = module.model_dump() + provider_dict = provider.model_dump() assert module_dict == { "name": "test_resource", "current_version": "1.0.0", - "_newest_version": None, - "_status": "unpatched", - "_source_file": "test_file.py", - "_source": "test/test_module/test_provider", - "_base_domain": None, - "_identifier": "test/test_module/test_provider", - "_github_repo": None, + "newest_version_string": None, + "status": "unpatched", + "source_file": PosixPath("test_file.py"), + "source_string": "test/test_module/test_provider", + "base_domain": None, + "identifier": "test/test_module/test_provider", + "github_repo_string": None, "start_line_number": 1, - "options": None, + "options": { + "ignore_resource": False, + }, } assert provider_dict == { "name": "test_resource", "current_version": "1.0.0", - "_newest_version": None, - "_status": "unpatched", - "_source_file": "test_file.py", - "_source": "test_provider/test_provider", - "_base_domain": None, - "_identifier": "test_provider/test_provider", - "_github_repo": None, + "newest_version_string": None, + "status": "unpatched", + "source_file": PosixPath("test_file.py"), + "source_string": "test_provider/test_provider", + "base_domain": None, + "identifier": "test_provider/test_provider", + "github_repo_string": None, "start_line_number": 1, - "options": None, + "options": { + "ignore_resource": False, + }, } diff --git a/infrapatch/core/providers/terraform/base_terraform_provider.py b/infrapatch/core/providers/terraform/base_terraform_provider.py index be917e5..0e6f3c9 100644 --- a/infrapatch/core/providers/terraform/base_terraform_provider.py +++ b/infrapatch/core/providers/terraform/base_terraform_provider.py @@ -55,7 +55,7 @@ def get_resources(self) -> Sequence[VersionedResource]: resource.newest_version = self.registry_handler.get_newest_version(resource) source = self.registry_handler.get_source(resource) if source is not None and "github.com" in source: - resource.set_github_repo(source) + resource.github_repo = source return resources def patch_resource(self, resource: VersionedTerraformResource) -> VersionedTerraformResource: diff --git a/infrapatch/core/utils/terraform/hcl_handler.py b/infrapatch/core/utils/terraform/hcl_handler.py index 97750b7..637875d 100644 --- a/infrapatch/core/utils/terraform/hcl_handler.py +++ b/infrapatch/core/utils/terraform/hcl_handler.py @@ -87,9 +87,9 @@ def _get_terraform_providers_from_dict(self, terraform_file_dict: dict, tf_file: found_resources.append( TerraformProvider( name=provider_name, - _source=source, + source_string=source, current_version=provider_config["version"], - _source_file=tf_file.absolute().as_posix(), + source_file=tf_file, start_line_number=start_line_number, ) ) @@ -108,9 +108,7 @@ def _get_terraform_modules_from_dict(self, terraform_file_dict: dict, tf_file: P continue start_line_number = self._get_start_line_number(content, file=tf_file, search_regex=f'module\s+"{module_name}"\s+\{{') found_resources.append( - TerraformModule( - name=module_name, _source=value["source"], current_version=value["version"], _source_file=tf_file.absolute().as_posix(), start_line_number=start_line_number - ) + TerraformModule(name=module_name, source_string=value["source"], current_version=value["version"], source_file=tf_file, start_line_number=start_line_number) ) return found_resources diff --git a/infrapatch/core/utils/terraform/tests/test_hcl_handler.py b/infrapatch/core/utils/terraform/tests/test_hcl_handler.py index dc0a259..787751c 100644 --- a/infrapatch/core/utils/terraform/tests/test_hcl_handler.py +++ b/infrapatch/core/utils/terraform/tests/test_hcl_handler.py @@ -94,7 +94,7 @@ def test_get_terraform_resources_from_file(hcl_handler: HclHandler, valid_terraf assert len(providers_filtered) == len(providers) for resource in resouces: - assert resource._source_file == tf_file.absolute().as_posix() + assert resource.source_file == tf_file if resource.name == "test_module": assert isinstance(resource, TerraformModule) assert resource.current_version == "2.0.0" diff --git a/infrapatch/core/utils/tests/test_options_processor.py b/infrapatch/core/utils/tests/test_options_processor.py index ce2a250..d0acaec 100644 --- a/infrapatch/core/utils/tests/test_options_processor.py +++ b/infrapatch/core/utils/tests/test_options_processor.py @@ -69,35 +69,40 @@ def test_options_processing_from_file(options_processor: OptionsProcessor, hcl_h # Check all resources parsed from the file for correct options for resource in resouces: resource = options_processor.process_options_for_resource(resource) + assert resource.options is not None if resource.name == "test_module": - assert resource.options is None + assert resource.options.ignore_resource is False elif resource.name == "test_module2": - assert resource.options is not None assert resource.options.ignore_resource is True elif resource.name == "test_provider": - assert resource.options is not None assert resource.options.ignore_resource is True elif resource.name == "test_provider2": - assert resource.options is None + assert resource.options.ignore_resource is False else: raise Exception(f"Unknown resource '{resource.name}'.") def test_get_upper_line_with_non_valid_line_numbers(options_processor: OptionsProcessor): # Should return none since start line number is 1 - resource = TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test/test_module/test_provider", start_line_number=1) + resource = TerraformModule(name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), source_string="test/test_module/test_provider", start_line_number=1) upper_line_content = options_processor._get_upper_line_content(resource) assert upper_line_content is None # Should raise an exception since start line number is 0 with pytest.raises(Exception): - resource = TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test/test_module/test_provider", start_line_number=0) + resource = TerraformModule( + name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), source_string="test/test_module/test_provider", start_line_number=0 + ) upper_line_content = options_processor._get_upper_line_content(resource) def test_get_upper_line_with_valid_line_numbers(options_processor: OptionsProcessor): - resource1 = TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test/test_module/test_provider", start_line_number=2) - resource2 = TerraformModule(name="test_resource", current_version="1.0.0", _source_file="test_file.py", _source="test/test_module/test_provider", start_line_number=5) + resource1 = TerraformModule( + name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), source_string="test/test_module/test_provider", start_line_number=2 + ) + resource2 = TerraformModule( + name="test_resource", current_version="1.0.0", source_file=Path("test_file.py"), source_string="test/test_module/test_provider", start_line_number=5 + ) with mock.patch("builtins.open", mock.mock_open(read_data="line1\nline2\nline3\nline4\nline5\nline6\nline7\nline8\nline9\nline10\n")): resource1_line = options_processor._get_upper_line_content(resource1) resource2_line = options_processor._get_upper_line_content(resource2) From 68806b0ee1187d578c6615b6d16efffebcd3c27e Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Fri, 22 Dec 2023 15:11:25 +0000 Subject: [PATCH 18/20] feat(resource_options): Implement ignore_resource in provider handler. --- infrapatch/core/provider_handler.py | 25 ++++++++++++++------- infrapatch/core/provider_handler_builder.py | 5 ++++- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/infrapatch/core/provider_handler.py b/infrapatch/core/provider_handler.py index 851c677..0b6ec0c 100644 --- a/infrapatch/core/provider_handler.py +++ b/infrapatch/core/provider_handler.py @@ -1,4 +1,3 @@ -import json import logging as log from pathlib import Path from typing import Sequence, Union @@ -11,10 +10,13 @@ from infrapatch.core.models.statistics import ProviderStatistics, Statistics from infrapatch.core.models.versioned_resource import ResourceStatus, VersionedResource, VersionedResourceReleaseNotes from infrapatch.core.providers.base_provider_interface import BaseProviderInterface +from infrapatch.core.utils.options_processor import OptionsProcessorInterface class ProviderHandler: - def __init__(self, providers: Sequence[BaseProviderInterface], console: Console, statistics_file: Path, repo: Union[Repo, None] = None) -> None: + def __init__( + self, providers: Sequence[BaseProviderInterface], console: Console, statistics_file: Path, options_processor: OptionsProcessorInterface, repo: Union[Repo, None] = None + ) -> None: self.providers: dict[str, BaseProviderInterface] = {} for provider in providers: self.providers[provider.get_provider_name()] = provider @@ -23,19 +25,26 @@ def __init__(self, providers: Sequence[BaseProviderInterface], console: Console, self.console = console self.statistics_file = statistics_file self.repo = repo + self.options_processor = options_processor def get_resources(self, disable_cache: bool = False) -> dict[str, Sequence[VersionedResource]]: for provider_name, provider in self.providers.items(): if provider_name not in self._resource_cache: log.debug(f"Fetching resources for provider {provider.get_provider_name()} since cache is empty.") - self._resource_cache[provider.get_provider_name()] = provider.get_resources() - continue elif disable_cache: log.debug(f"Fetching resources for provider {provider.get_provider_name()} since cache is disabled.") - self._resource_cache[provider.get_provider_name()] = provider.get_resources() - continue else: log.debug(f"Using cached resources for provider {provider.get_provider_name()}.") + continue + resources = provider.get_resources() + for resource in resources: + self.options_processor.process_options_for_resource(resource) + ignored_resources = [resource for resource in resources if resource.options.ignore_resource] + un_ignored_resources = [resource for resource in resources if not resource.options.ignore_resource] + for resource in ignored_resources: + log.debug(f"Ignoring resource '{resource.name}' from provider {provider.get_provider_display_name()}since its marked as ignored.") + + self._resource_cache[provider.get_provider_name()] = un_ignored_resources return self._resource_cache def get_patched_resources(self) -> dict[str, Sequence[VersionedResource]]: @@ -127,9 +136,9 @@ def dump_statistics(self, disable_cache: bool = False): log.debug(f"Deleting existing statistics file {self.statistics_file.absolute().as_posix()}.") self.statistics_file.unlink() log.debug(f"Writing statistics to {self.statistics_file.absolute().as_posix()}.") - statistics_dict = self._get_statistics(disable_cache).to_dict() + statistics = self._get_statistics(disable_cache) with open(self.statistics_file, "w") as f: - f.write(json.dumps(statistics_dict)) + f.write(statistics.model_dump_json()) def print_statistics_table(self, disable_cache: bool = False): table = self._get_statistics(disable_cache).get_rich_table() diff --git a/infrapatch/core/provider_handler_builder.py b/infrapatch/core/provider_handler_builder.py index 0db5175..ae87611 100644 --- a/infrapatch/core/provider_handler_builder.py +++ b/infrapatch/core/provider_handler_builder.py @@ -12,6 +12,7 @@ import infrapatch.core.constants as const import infrapatch.core.constants as cs from infrapatch.core.provider_handler import ProviderHandler +from infrapatch.core.utils.options_processor import OptionsProcessor from infrapatch.core.utils.terraform.hcl_edit_cli import HclEditCli from infrapatch.core.utils.terraform.hcl_handler import HclHandler from infrapatch.core.utils.terraform.registry_handler import RegistryHandler @@ -61,4 +62,6 @@ def build(self) -> ProviderHandler: if len(self.providers) == 0: raise Exception("No providers added to ProviderHandlerBuilder.") statistics_file = self.working_directory.joinpath(f"{cs.APP_NAME}_Statistics.json") - return ProviderHandler(providers=self.providers, console=Console(width=const.CLI_WIDTH), statistics_file=statistics_file, repo=self.git_repo) + return ProviderHandler( + providers=self.providers, console=Console(width=const.CLI_WIDTH), options_processor=OptionsProcessor(), statistics_file=statistics_file, repo=self.git_repo + ) From 535a3e2182ff2a7212220c3504198bf83ea86dca Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Wed, 3 Jan 2024 13:58:30 +0000 Subject: [PATCH 19/20] feat(vscode): Set markdown formatter. --- .vscode/settings.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index 42e3633..d2db7f7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -53,5 +53,8 @@ "python.analysis.autoSearchPaths": false, "[json]": { "editor.defaultFormatter": "vscode.json-language-features" + }, + "[markdown]": { + "editor.defaultFormatter": "yzhang.markdown-all-in-one" } } \ No newline at end of file From af803e64db260eee7b82f1db18344b473e7a3f9a Mon Sep 17 00:00:00 2001 From: Noah Canadea Date: Wed, 3 Jan 2024 13:59:14 +0000 Subject: [PATCH 20/20] doc(README): Move development chapter to bottom and add multiple options to example. --- README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 31e238b..2b56e34 100644 --- a/README.md +++ b/README.md @@ -18,12 +18,12 @@ The CLI works by scanning your .tf files for versioned providers and modules and - [Authentication](#authentication-1) - [.terraformrc file:](#terraformrc-file) - [infrapatch\_credentials.json file:](#infrapatch_credentialsjson-file) - - [Setup Development Environment for InfraPatch](#setup-development-environment-for-infrapatch) - - [Contributing](#contributing) - [Global](#global) - [Resource Options](#resource-options) - [Available Options](#available-options) - [Example](#example) + - [Setup Development Environment for InfraPatch](#setup-development-environment-for-infrapatch) + - [Contributing](#contributing) ## GitHub Action @@ -188,19 +188,6 @@ You can also specify the path to the credentials file with the `--credentials-fi infrapatch --credentials-file-path "path/to/credentials/file" update ``` -### Setup Development Environment for InfraPatch - -This repository contains a devcontainer configuration for VSCode. To use it, you need to install the following tools: -* ["Dev Containers VSCode Extension"](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) for VSCode. -* A local Docker installation like [Docker Desktop](https://www.docker.com/products/docker-desktop). - -After installation, you can open the repository in the devcontainer by clicking on the green "Open in Container" button in the bottom left corner of VSCode. -During the first start, the devcontainer will build the container image and install all dependencies. - -### Contributing - -If you have any ideas for improvements or find any bugs, feel free to open an issue or create a pull request. - ## Global The following section describes configurations and behaviors that are applicable to the Github Action and the CLI. @@ -211,7 +198,7 @@ InfraPatch supports individual resource options to change the behavior for a spe Resource options can be specified one line obove your resource definition with the following syntax: ```hcl -# infrapatch_options: = +# infrapatch_options: =, = module "example" { source = "terraform-aws-modules/example" name: "demo" @@ -219,7 +206,7 @@ module "example" { terraform { required_providers { - # infrapatch_options: = + # infrapatch_options: =,= aws = { source = "hashicorp/aws" } @@ -231,9 +218,9 @@ terraform { Currently, the following options are available: -| Option Name | Description | Default Value | -| ----------- | ----------- | ------------- | -| `ignore_resource` | If set to `true`, the resource will be ignored by InfraPatch. | `false` | +| Option Name | Description | Default Value | +| ----------------- | ------------------------------------------------------------- | ------------- | +| `ignore_resource` | If set to `true`, the resource will be ignored by InfraPatch. | `false` | #### Example @@ -256,3 +243,16 @@ The following example shows how to ignore a terraform module and a terraform pro } ``` + ## Setup Development Environment for InfraPatch + +This repository contains a devcontainer configuration for VSCode. To use it, you need to install the following tools: +* ["Dev Containers VSCode Extension"](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) for VSCode. +* A local Docker installation like [Docker Desktop](https://www.docker.com/products/docker-desktop). + +After installation, you can open the repository in the devcontainer by clicking on the green "Open in Container" button in the bottom left corner of VSCode. +During the first start, the devcontainer will build the container image and install all dependencies. + +## Contributing + +If you have any ideas for improvements or find any bugs, feel free to open an issue or create a pull request. +