diff --git a/.gitignore b/.gitignore index 29f4640..e14016b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ __pycache__/ # C extensions *.so +Pipfile + **/.terraform/* # Distribution / packaging diff --git a/asset/infrapatch_report.gif b/asset/infrapatch_report.gif index af31240..fbfcc56 100644 Binary files a/asset/infrapatch_report.gif and b/asset/infrapatch_report.gif differ diff --git a/asset/infrapatch_update.gif b/asset/infrapatch_update.gif index a914006..a306285 100644 Binary files a/asset/infrapatch_update.gif and b/asset/infrapatch_update.gif differ diff --git a/infrapatch/action/__main__.py b/infrapatch/action/__main__.py index 6c512ef..a021ee9 100644 --- a/infrapatch/action/__main__.py +++ b/infrapatch/action/__main__.py @@ -34,9 +34,9 @@ def main(debug: bool): if "terraform_modules" in config.enabled_providers or "terraform_providers" in config.enabled_providers: builder.add_terraform_registry_configuration(config.default_registry_domain, config.terraform_registry_secrets) if "terraform_modules" in config.enabled_providers: - builder.with_terraform_module_provider() + builder.with_terraform_module_provider(github) if "terraform_providers" in config.enabled_providers: - builder.with_terraform_provider_provider() + builder.with_terraform_provider_provider(github) provider_handler = builder.build() @@ -107,9 +107,20 @@ def update_pr_body(pr, provider_handler): def get_pr_body(provider_handler: ProviderHandler) -> str: body = "" markdown_tables = provider_handler.get_markdown_table_for_changed_resources() - for table in markdown_tables: - body += table.dumps() - body += "\n" + patched_resources = provider_handler.get_patched_resources() + release_notes = provider_handler.get_release_notes(patched_resources) + for provider_name in provider_handler.providers: + if provider_name in markdown_tables: + body += markdown_tables[provider_name].dumps() + body += "\n" + if provider_name in release_notes: + log.debug(f"Adding release notes for provider '{provider_name}' to pull request body.") + body += "## Changelog\n" + for release_note in release_notes[provider_name]: + body += "
\n" + body += f"{release_note.name} - {release_note.version}\n" + body += f"{release_note.body}\n" + body += "
\n\n" body += provider_handler._get_statistics().get_markdown_table().dumps() body += "\n" diff --git a/infrapatch/cli/__init__.py b/infrapatch/cli/__init__.py index 27fdca4..3d18726 100644 --- a/infrapatch/cli/__init__.py +++ b/infrapatch/cli/__init__.py @@ -1 +1 @@ -__version__ = "0.0.3" +__version__ = "0.5.0" diff --git a/infrapatch/core/models/tests/test_versioned_resource.py b/infrapatch/core/models/tests/test_versioned_resource.py index d6f3d77..c1b66b6 100644 --- a/infrapatch/core/models/tests/test_versioned_resource.py +++ b/infrapatch/core/models/tests/test_versioned_resource.py @@ -1,5 +1,7 @@ from pathlib import Path +import pytest + from infrapatch.core.models.versioned_resource import ResourceStatus, VersionedResource @@ -14,10 +16,18 @@ def test_version_management(): resource.set_patched() assert resource.status == ResourceStatus.PATCHED + # Check new_version the same as current_version resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py") resource.newest_version = "1.0.0" - assert resource.status == ResourceStatus.UNPATCHED + assert resource.status == ResourceStatus.UP_TO_DATE + assert resource.installed_version_equal_or_newer_than_new_version() is True + + # Check new_version older than current_version + resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py") + resource.newest_version = "0.1.0" + + assert resource.status == ResourceStatus.UP_TO_DATE assert resource.installed_version_equal_or_newer_than_new_version() is True @@ -38,12 +48,44 @@ def test_tile_constraint(): assert resource.newest_version == "~>1.1.0" +def test_git_repo(): + resource = VersionedResource(name="test_resource", current_version="~>1.0.0", _source_file="test_file.py") + + assert resource.github_repo is None + + resource.set_github_repo("https://github.com/noahnc/test_repo.git") + assert resource.github_repo == "noahnc/test_repo" + + resource.set_github_repo("https://github.com/noahnc/test_repo") + assert resource.github_repo == "noahnc/test_repo" + + with pytest.raises(Exception): + resource.set_github_repo("https://github.com/") + + with pytest.raises(Exception): + resource.set_github_repo("https://github.com") + + def test_patch_error(): resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py") resource.set_patch_error() assert resource.status == ResourceStatus.PATCH_ERROR +def test_version_not_found(): + # Test manual setting + resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py") + resource.set_no_version_found() + assert resource.status == ResourceStatus.NO_VERSION_FOUND + assert resource.installed_version_equal_or_newer_than_new_version() is True + + # Test by setting None as new version + resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py") + resource.newest_version = None + assert resource.status == ResourceStatus.NO_VERSION_FOUND + assert resource.installed_version_equal_or_newer_than_new_version() is True + + def test_path(): resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="/var/testdir/test_file.py") assert resource.source_file == Path("/var/testdir/test_file.py") @@ -66,5 +108,12 @@ def test_find(): def test_versioned_resource_to_dict(): resource = VersionedResource(name="test_resource", current_version="1.0.0", _source_file="test_file.py") - expected_dict = {"name": "test_resource", "current_version": "1.0.0", "_source_file": "test_file.py", "_newest_version": None, "_status": ResourceStatus.UNPATCHED} + expected_dict = { + "name": "test_resource", + "current_version": "1.0.0", + "_source_file": "test_file.py", + "_newest_version": None, + "_status": ResourceStatus.UNPATCHED, + "_github_repo": None, + } assert resource.to_dict() == expected_dict diff --git a/infrapatch/core/models/tests/test_versioned_terraform_resource.py b/infrapatch/core/models/tests/test_versioned_terraform_resource.py index a4b568b..efeb34e 100644 --- a/infrapatch/core/models/tests/test_versioned_terraform_resource.py +++ b/infrapatch/core/models/tests/test_versioned_terraform_resource.py @@ -69,6 +69,7 @@ def test_to_dict(): "_source": "test/test_module/test_provider", "_base_domain": None, "_identifier": "test/test_module/test_provider", + "_github_repo": None, } assert provider_dict == { "name": "test_resource", @@ -79,4 +80,5 @@ def test_to_dict(): "_source": "test_provider/test_provider", "_base_domain": None, "_identifier": "test_provider/test_provider", + "_github_repo": None, } diff --git a/infrapatch/core/models/versioned_resource.py b/infrapatch/core/models/versioned_resource.py index 0147f6d..c708789 100644 --- a/infrapatch/core/models/versioned_resource.py +++ b/infrapatch/core/models/versioned_resource.py @@ -3,14 +3,19 @@ from dataclasses import dataclass from pathlib import Path from typing import Any, Optional, Union +from urllib.parse import urlparse +import logging as log +from git import Sequence import semantic_version class ResourceStatus: UNPATCHED = "unpatched" + UP_TO_DATE = "up_to_date" PATCHED = "patched" PATCH_ERROR = "patch_error" + NO_VERSION_FOUND = "no_version_found" @dataclass @@ -18,8 +23,9 @@ class VersionedResource: name: str current_version: str _source_file: str - _newest_version: Union[str, None] = None + _newest_version: Optional[str] = None _status: str = ResourceStatus.UNPATCHED + _github_repo: Optional[str] = None @property def source_file(self) -> Path: @@ -29,6 +35,10 @@ def source_file(self) -> Path: def status(self) -> str: return self._status + @property + def github_repo(self) -> Union[str, None]: + return self._github_repo + @property def resource_name(self): raise NotImplementedError() @@ -46,15 +56,38 @@ def newest_version_base(self): return self.newest_version @newest_version.setter - def newest_version(self, version: str): + def newest_version(self, version: Optional[str]): if self.has_tile_constraint(): self._newest_version = f"~>{version}" + else: + self._newest_version = version + + if version is None: + self.set_no_version_found() return - self._newest_version = version + if self.installed_version_equal_or_newer_than_new_version(): + self.set_up_to_date() + + def set_github_repo(self, github_repo_url: str): + url = urlparse(github_repo_url) + if url.path is None or url.path == "" or url.path == "/": + raise Exception(f"Invalid github repo url '{github_repo_url}'.") + path = url.path + if path.endswith(".git"): + path = path[:-4] + repo = "/".join(path.split("/")[1:3]) + log.debug(f"Setting github repo for resource '{self.name}' to '{repo}'") + self._github_repo = repo def set_patched(self): self._status = ResourceStatus.PATCHED + def set_no_version_found(self): + self._status = ResourceStatus.NO_VERSION_FOUND + + def set_up_to_date(self): + self._status = ResourceStatus.UP_TO_DATE + def has_tile_constraint(self) -> bool: result = re.match(r"^~>[0-9]+\.[0-9]+\.[0-9]+$", self.current_version) if result is None: @@ -69,6 +102,8 @@ def find(self, resources): return result def installed_version_equal_or_newer_than_new_version(self): + if self._status == ResourceStatus.NO_VERSION_FOUND: + return True if self.newest_version is None: raise Exception(f"Newest version of resource '{self.name}' is not set.") @@ -106,3 +141,11 @@ def check_if_up_to_date(self): def to_dict(self) -> dict[str, Any]: return dataclasses.asdict(self) + + +@dataclass +class VersionedResourceReleaseNotes: + resources: Sequence[VersionedResource] + name: str + body: str + version: str diff --git a/infrapatch/core/models/versioned_terraform_resources.py b/infrapatch/core/models/versioned_terraform_resources.py index 1882db0..cc3b5c7 100644 --- a/infrapatch/core/models/versioned_terraform_resources.py +++ b/infrapatch/core/models/versioned_terraform_resources.py @@ -6,14 +6,16 @@ from infrapatch.core.models.versioned_resource import VersionedResource -@dataclass +@dataclass(kw_only=True) class VersionedTerraformResource(VersionedResource): - _base_domain: Union[str, None] = None - _identifier: Union[str, None] = None - _source: Union[str, None] = None + _source: str + _base_domain: Optional[str] = None + _identifier: Optional[str] = None @property - def source(self) -> Union[str, None]: + def source(self) -> str: + if self._source is None: + raise Exception("Source is None.") return self._source @property @@ -36,12 +38,10 @@ def find(self, resources): @dataclass class TerraformModule(VersionedTerraformResource): def __post_init__(self): - if self._source is None: - raise Exception("Source is None.") self.source = self._source @property - def source(self) -> Union[str, None]: + def source(self) -> str: return self._source @property @@ -67,12 +67,10 @@ def source(self, source: str): @dataclass class TerraformProvider(VersionedTerraformResource): def __post_init__(self): - if self._source is None: - raise Exception("Source is None.") self.source = self._source @property - def source(self) -> Union[str, None]: + def source(self) -> str: return self._source @property diff --git a/infrapatch/core/provider_handler.py b/infrapatch/core/provider_handler.py index dcaea56..851c677 100644 --- a/infrapatch/core/provider_handler.py +++ b/infrapatch/core/provider_handler.py @@ -9,7 +9,7 @@ from rich.console import Console from infrapatch.core.models.statistics import ProviderStatistics, Statistics -from infrapatch.core.models.versioned_resource import ResourceStatus, VersionedResource +from infrapatch.core.models.versioned_resource import ResourceStatus, VersionedResource, VersionedResourceReleaseNotes from infrapatch.core.providers.base_provider_interface import BaseProviderInterface @@ -38,6 +38,13 @@ def get_resources(self, disable_cache: bool = False) -> dict[str, Sequence[Versi log.debug(f"Using cached resources for provider {provider.get_provider_name()}.") return self._resource_cache + def get_patched_resources(self) -> dict[str, Sequence[VersionedResource]]: + resources = self.get_resources() + patched_resources: dict[str, Sequence[VersionedResource]] = {} + for provider_name, provider in self.providers.items(): + patched_resources[provider.get_provider_name()] = [resource for resource in resources[provider_name] if resource.status == ResourceStatus.PATCHED] + return patched_resources + def get_upgradable_resources(self, disable_cache: bool = False) -> dict[str, Sequence[VersionedResource]]: upgradable_resources: dict[str, Sequence[VersionedResource]] = {} resources = self.get_resources(disable_cache) @@ -128,11 +135,11 @@ def print_statistics_table(self, disable_cache: bool = False): table = self._get_statistics(disable_cache).get_rich_table() self.console.print(table) - def get_markdown_table_for_changed_resources(self) -> list[MarkdownTableWriter]: + def get_markdown_table_for_changed_resources(self) -> dict[str, MarkdownTableWriter]: if self._resource_cache is None: raise Exception("No resources found. Run get_resources() first.") - markdown_tables = [] + markdown_tables = {} for provider_name, provider in self.providers.items(): changed_resources = [ resource for resource in self._resource_cache[provider_name] if resource.status == ResourceStatus.PATCHED or resource.status == ResourceStatus.PATCH_ERROR @@ -140,7 +147,7 @@ def get_markdown_table_for_changed_resources(self) -> list[MarkdownTableWriter]: if len(changed_resources) == 0: log.debug(f"No changed resources found for provider {provider_name}. Skipping.") continue - markdown_tables.append(provider.get_markdown_table(changed_resources)) + markdown_tables[provider_name] = provider.get_markdown_table(changed_resources) return markdown_tables def set_resources_patched_based_on_existing_resources(self, original_resources: dict[str, Sequence[VersionedResource]]) -> None: @@ -157,3 +164,21 @@ def set_resources_patched_based_on_existing_resources(self, original_resources: found_resource = found_resources[0] found_resource.set_patched() self._resource_cache[provider_name][i] = found_resource # type: ignore + + def get_release_notes(self, resources: dict[str, Sequence[VersionedResource]]) -> dict[str, Sequence[VersionedResourceReleaseNotes]]: + release_notes: dict[str, Sequence[VersionedResourceReleaseNotes]] = {} + for provider_name, provider in self.providers.items(): + provider_release_notes: list[VersionedResourceReleaseNotes] = [] + patched_resources = [resource for resource in resources[provider_name] if resource.status == ResourceStatus.PATCHED] + grouped_resources = provider.get_grouped_by_identifier(patched_resources) + for identifier in progress.track(grouped_resources, description=f"Getting release notes for resources of Provider {provider.get_provider_display_name()}..."): + identifier_resources = grouped_resources[identifier] + if identifier_resources[0].status == ResourceStatus.NO_VERSION_FOUND: + log.debug(f"Skipping resource '{identifier_resources[0].name}' since no version was found.") + continue + resource_release_note = provider.get_resource_release_notes(grouped_resources[identifier][0]) + if resource_release_note is not None: + resource_release_note.resources = grouped_resources[identifier] + provider_release_notes.append(resource_release_note) + release_notes[provider_name] = provider_release_notes + return release_notes diff --git a/infrapatch/core/provider_handler_builder.py b/infrapatch/core/provider_handler_builder.py index 1bf76d4..0db5175 100644 --- a/infrapatch/core/provider_handler_builder.py +++ b/infrapatch/core/provider_handler_builder.py @@ -1,6 +1,8 @@ import logging as log from pathlib import Path -from typing import Self +from typing import Self, Union + +from github import Github from infrapatch.core.providers.terraform.terraform_provider_provider import TerraformProviderProvider from infrapatch.core.providers.terraform.terraform_module_provider import TerraformModuleProvider @@ -29,19 +31,23 @@ def add_terraform_registry_configuration(self, default_registry_domain: str, cre self.registry_handler = RegistryHandler(default_registry_domain, credentials) return self - def with_terraform_module_provider(self) -> Self: + def with_terraform_module_provider(self, github: Union[Github, None] = None) -> Self: if self.registry_handler is None: raise Exception("No registry configuration added to ProviderHandlerBuilder.") log.debug("Adding TerraformModuleProvider to ProviderHandlerBuilder.") - tf_module_provider = TerraformModuleProvider(HclEditCli(), self.registry_handler, HclHandler(HclEditCli()), self.working_directory) + if github is None: + github = Github() + tf_module_provider = TerraformModuleProvider(HclEditCli(), self.registry_handler, HclHandler(HclEditCli()), self.working_directory, github) self.providers.append(tf_module_provider) return self - def with_terraform_provider_provider(self) -> Self: + def with_terraform_provider_provider(self, github: Union[Github, None] = None) -> Self: if self.registry_handler is None: raise Exception("No registry configuration added to ProviderHandlerBuilder.") log.debug("Adding TerraformModuleProvider to ProviderHandlerBuilder.") - tf_module_provider = TerraformProviderProvider(HclEditCli(), self.registry_handler, HclHandler(HclEditCli()), self.working_directory) + if github is None: + github = Github() + tf_module_provider = TerraformProviderProvider(HclEditCli(), self.registry_handler, HclHandler(HclEditCli()), self.working_directory, github) self.providers.append(tf_module_provider) return self diff --git a/infrapatch/core/providers/base_provider_interface.py b/infrapatch/core/providers/base_provider_interface.py index 24e8335..ccfd7cd 100644 --- a/infrapatch/core/providers/base_provider_interface.py +++ b/infrapatch/core/providers/base_provider_interface.py @@ -1,7 +1,9 @@ -from typing import Protocol, Sequence +from typing import Protocol, Sequence, Union + from pytablewriter import MarkdownTableWriter from rich.table import Table -from infrapatch.core.models.versioned_resource import VersionedResource + +from infrapatch.core.models.versioned_resource import VersionedResource, VersionedResourceReleaseNotes class BaseProviderInterface(Protocol): @@ -25,3 +27,9 @@ def get_markdown_table(self, resources: Sequence[VersionedResource]) -> Markdown def get_resources_as_dict_list(self, resources: Sequence[VersionedResource]): ... + + def get_resource_release_notes(self, resource: VersionedResource) -> Union[VersionedResourceReleaseNotes, None]: + ... + + def get_grouped_by_identifier(self, resources: Sequence[VersionedResource]) -> dict[str, Sequence[VersionedResource]]: + ... diff --git a/infrapatch/core/providers/terraform/base_terraform_provider.py b/infrapatch/core/providers/terraform/base_terraform_provider.py index b0776f7..97a853e 100644 --- a/infrapatch/core/providers/terraform/base_terraform_provider.py +++ b/infrapatch/core/providers/terraform/base_terraform_provider.py @@ -1,26 +1,30 @@ +import logging as log +from abc import abstractmethod from pathlib import Path -from typing import Any, Sequence +from typing import Any, Sequence, Union -from abc import abstractmethod +from github import Github from pytablewriter import MarkdownTableWriter - from rich import progress from rich.table import Table -from infrapatch.core.models.versioned_resource import VersionedResource + +from infrapatch.core.models.versioned_resource import VersionedResource, VersionedResourceReleaseNotes from infrapatch.core.models.versioned_terraform_resources import VersionedTerraformResource from infrapatch.core.providers.base_provider_interface import BaseProviderInterface from infrapatch.core.utils.terraform.hcl_edit_cli import HclEditCliInterface from infrapatch.core.utils.terraform.hcl_handler import HclHandlerInterface from infrapatch.core.utils.terraform.registry_handler import RegistryHandlerInterface -import logging as log class TerraformProvider(BaseProviderInterface): - def __init__(self, hcledit: HclEditCliInterface, registry_handler: RegistryHandlerInterface, hcl_handler: HclHandlerInterface, project_root: Path) -> None: + def __init__( + self, hcledit: HclEditCliInterface, registry_handler: RegistryHandlerInterface, hcl_handler: HclHandlerInterface, project_root: Path, github: Union[Github, None] + ) -> None: self.hcledit = hcledit self.registry_handler = registry_handler self.hcl_handler = hcl_handler self.project_root = project_root + self._github = github @abstractmethod def get_provider_name(self) -> str: @@ -49,6 +53,9 @@ def get_resources(self) -> Sequence[VersionedResource]: for resource in progress.track(resources, description=f"Getting newest resource versions for Provider {self.get_provider_display_name()}..."): resource.newest_version = self.registry_handler.get_newest_version(resource) + source = self.registry_handler.get_source(resource) + if source is not None and "github.com" in source: + resource.set_github_repo(source) return resources def patch_resource(self, resource: VersionedTerraformResource) -> VersionedTerraformResource: @@ -64,9 +71,9 @@ def get_rich_table(self, resources: Sequence[VersionedTerraformResource]) -> Tab table.add_column("Source", overflow="fold") table.add_column("Current") table.add_column("Newest") - table.add_column("Upgradeable") + table.add_column("Status") for resource in resources: - table.add_row(resource.name, resource.source, resource.current_version, resource.newest_version, str(not resource.installed_version_equal_or_newer_than_new_version())) + table.add_row(resource.name, resource.source, resource.current_version, resource.newest_version, resource.status) return table def get_markdown_table(self, resources: Sequence[VersionedTerraformResource]) -> MarkdownTableWriter: @@ -77,7 +84,7 @@ def get_markdown_table(self, resources: Sequence[VersionedTerraformResource]) -> "Source": resource.source, "Current": resource.current_version, "Newest": resource.newest_version, - "Upgradeable": str(not resource.installed_version_equal_or_newer_than_new_version()), + "Status": resource.status, } dict_list.append(dict_element) return MarkdownTableWriter( @@ -88,3 +95,28 @@ def get_markdown_table(self, resources: Sequence[VersionedTerraformResource]) -> def get_resources_as_dict_list(self, resources: Sequence[VersionedTerraformResource]) -> list[dict[str, Any]]: return [resource.to_dict() for resource in resources] + + def get_resource_release_notes(self, resource: VersionedTerraformResource) -> Union[VersionedResourceReleaseNotes, None]: + if resource.newest_version is None: + raise Exception(f"Newest version of resource '{resource.name}' is not set.") + if self._github is None: + raise Exception("Github integration is not enabled.") + if resource.github_repo is None: + log.debug(f"Resource '{resource.name}' has no github repo set, skipping release notes.") + return None + try: + repo = self._github.get_repo(resource.github_repo) + release_notes = repo.get_release(f"v{resource.newest_version}").body + except Exception as e: + log.warning(f"Could not get release notes from repo '{resource.github_repo}' for version '{resource.newest_version}': {e}") + return None + return VersionedResourceReleaseNotes(resources=[resource], body=release_notes, name=resource.source, version=resource.newest_version) + + def get_grouped_by_identifier(self, resources: Sequence[VersionedTerraformResource]) -> dict[str, Sequence[VersionedTerraformResource]]: + identifiers: dict[str, Sequence[VersionedTerraformResource]] = {} + for resource in resources: + if resource.source not in identifiers: + identifiers[resource.source] = [resource] + continue + list(identifiers[resource.source]).append(resource) + return identifiers diff --git a/infrapatch/core/utils/terraform/registry_handler.py b/infrapatch/core/utils/terraform/registry_handler.py index 7b41f2d..eaf8f69 100644 --- a/infrapatch/core/utils/terraform/registry_handler.py +++ b/infrapatch/core/utils/terraform/registry_handler.py @@ -1,3 +1,5 @@ +from dataclasses import dataclass +from typing import Union import json import logging as log from distutils.version import StrictVersion @@ -5,18 +7,11 @@ from urllib import request from urllib.parse import urlparse -from infrapatch.core.models.versioned_terraform_resources import VersionedTerraformResource, TerraformModule, TerraformProvider - - -class RegistryNotFoundException(Exception): - pass - -class RegistryMetadataException(Exception): - pass +from infrapatch.core.models.versioned_terraform_resources import VersionedTerraformResource, TerraformModule, TerraformProvider -class ResourceNotFoundException(Exception): +class TerraformRegistryException(Exception): pass @@ -24,29 +19,72 @@ class RegistryHandlerInterface(Protocol): def get_newest_version(self, resource: VersionedTerraformResource): ... + def get_source(self, resource: VersionedTerraformResource): + ... + + +@dataclass +class TerraformRegistryResourceCache: + newest_version: Union[str, None] = None + source: Union[str, None] = None + class RegistryHandler(RegistryHandlerInterface): def __init__(self, default_registry_domain: str, credentials: dict): self.default_registry_domain = default_registry_domain self.cached_registry_metadata = {} - self.cached_module_version = {} - self.cached_provider_version = {} + self.module_cache: dict[str, TerraformRegistryResourceCache] = {} + self.provider_cache: dict[str, TerraformRegistryResourceCache] = {} self.credentials = credentials - def get_newest_version(self, resource: VersionedTerraformResource): + def get_newest_version(self, resource: VersionedTerraformResource) -> Union[str, None]: if not isinstance(resource, TerraformModule) and not isinstance(resource, TerraformProvider): raise Exception(f"Resource type '{type(resource)}' is not supported.") + cache = self._get_from_cache(resource) + if cache.newest_version is not None: + return cache.newest_version + + registry_api_base_endpoint, registry_base_domain = self._compose_base_url(resource) + version_endpoint = f"{registry_api_base_endpoint}/versions" + log.debug(f"Getting versions from {version_endpoint}") + + response = self._send_request(version_endpoint, registry_base_domain) + response_data = json.loads(response.read()) if isinstance(resource, TerraformModule): - if resource.source in self.cached_module_version: - log.debug(f"Module versions for '{resource.source}' already cached.") - return self.cached_module_version[resource.source] + versions = response_data["modules"][0]["versions"] + elif isinstance(resource, TerraformProvider): + versions = response_data["versions"] + else: + raise Exception(f"Resource type '{type(resource)}' is not supported.") + if len(versions) == 0: + log.debug(f"No versions found for resource '{resource.source}'.") + return None + sorted_versions = sorted(versions, key=lambda k: StrictVersion(k["version"]), reverse=True) + newest_version = sorted_versions[0]["version"] + cache.newest_version = newest_version + + return newest_version + + def _get_from_cache(self, resource: VersionedTerraformResource) -> TerraformRegistryResourceCache: + if isinstance(resource, TerraformModule): + cache = self.module_cache elif isinstance(resource, TerraformProvider): - if resource.source in self.cached_provider_version: - log.debug(f"Provider versions for '{resource.source}' already cached.") - return self.cached_provider_version[resource.source] + cache = self.provider_cache + else: + raise Exception(f"Resource type '{type(resource)}' is not supported.") + + if resource.source in cache: + log.debug(f"Cache found for resource {resource.source}.") + return cache[resource.source] + log.debug(f"No cache found for resource {resource.source}.") + new_cache = TerraformRegistryResourceCache() + cache[resource.source] = new_cache + return new_cache + + def _compose_base_url(self, resource) -> tuple[str, str]: registry_base_domain = self.default_registry_domain if resource.base_domain is not None: registry_base_domain = resource.base_domain @@ -59,56 +97,65 @@ def get_newest_version(self, resource: VersionedTerraformResource): else: raise Exception(f"Resource type '{type(resource)}' is not supported.") - version_url = urlparse(registry_metadata[metadata_key]) + url_from_meta = urlparse(registry_metadata[metadata_key]) - if version_url.hostname is not None: - version_endpoint = f"https://{version_url.hostname}" + if url_from_meta.hostname is not None: + endpoint = f"https://{url_from_meta.hostname}" else: - version_endpoint = f"https://{registry_base_domain}" + endpoint = f"https://{registry_base_domain}" - version_endpoint = f"{version_endpoint}{version_url.path}{resource.identifier}/versions" - log.debug(f"Getting versions from {version_endpoint}") + endpoint = f"{endpoint}{url_from_meta.path}{resource.identifier}" + return endpoint, registry_base_domain + + def get_source(self, resource: VersionedTerraformResource) -> Union[str, None]: + if not isinstance(resource, TerraformModule) and not isinstance(resource, TerraformProvider): + raise Exception(f"Resource type '{type(resource)}' is not supported.") - request_object = request.Request(version_endpoint) + cache = self._get_from_cache(resource) + if cache.source is not None: + return cache.source + + base_endpoint, registry_base_domain = self._compose_base_url(resource) + version_info_endpoint = f"{base_endpoint}/{resource.newest_version}" try: + response = self._send_request(version_info_endpoint, registry_base_domain) + except TerraformRegistryException as e: + log.debug(f"Could not get source for resource '{resource.source}': {e}") + return None + response_data = json.loads(response.read()) + if "source" not in response_data: + log.debug(f"Source not found in response data: {response_data}") + return None + source = response_data["source"] + log.debug(f"Source for '{resource.source}' is '{source}'") + cache.source = source + return source + + def _send_request(self, url: str, registry_base_domain: str): + request_object = request.Request(url) + + if registry_base_domain in self.credentials: token = self.credentials[registry_base_domain] log.debug(f"Found credentials for registry '{registry_base_domain}', using token: {token[0:5]}...") request_object.add_header("Authorization", f"Bearer {token}") - except KeyError: + else: log.debug(f"No credentials found for registry '{registry_base_domain}', using unauthenticated request.") - response = request.urlopen(request_object) + try: + response = request.urlopen(request_object) + except Exception as e: + raise TerraformRegistryException(f"Registry request returned an error '{url}': {e}") if response.status == 404: - raise ResourceNotFoundException(f"Resource '{resource.name}' not found in registry '{registry_base_domain}'.") + raise TerraformRegistryException(f"Registry resource '{url}' not found.") elif response.status >= 400: - raise RegistryMetadataException(f"Could not get versions from '{version_endpoint}'.") - response_data = json.loads(response.read()) - if isinstance(resource, TerraformModule): - versions = response_data["modules"][0]["versions"] - elif isinstance(resource, TerraformProvider): - versions = response_data["versions"] - else: - raise Exception(f"Resource type '{type(resource)}' is not supported.") - sorted_versions = sorted(versions, key=lambda k: StrictVersion(k["version"]), reverse=True) - newest_version = sorted_versions[0]["version"] + raise TerraformRegistryException(f"Registry request '{url}' returned error code '{response.status}'.") + return response - if isinstance(resource, TerraformModule): - self.cached_module_version[resource.source] = newest_version - elif isinstance(resource, TerraformProvider): - self.cached_provider_version[resource.source] = newest_version - - return newest_version - - def get_registry_metadata(self, registry_base_domain: str): + def get_registry_metadata(self, registry_base_domain: str) -> dict: if registry_base_domain in self.cached_registry_metadata: log.debug(f"Registry metadata for '{registry_base_domain}' already cached.") return self.cached_registry_metadata[registry_base_domain] discovery_url = f"https://{registry_base_domain}/.well-known/terraform.json" - log.debug(f"Getting registry metadata from {discovery_url}") - response = request.urlopen(discovery_url) - if response.status == 404: - raise RegistryNotFoundException(f"Registry '{registry_base_domain}' not found.") - elif response.status >= 400: - raise RegistryMetadataException(f"Could not get registry metadata from '{discovery_url}'.") + response = self._send_request(discovery_url, registry_base_domain) metadata = json.loads(response.read()) self.cached_registry_metadata[registry_base_domain] = metadata return metadata diff --git a/setup.py b/setup.py index 381caeb..ececde9 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,16 @@ version=__version__, packages=find_packages(where=".", include=["infrapatch*"], exclude=["action*"]), package_data={"infrapatch": ["core/utils/terraform/bin/*"]}, - install_requires=["click~=8.1.7", "rich~=13.6.0", "pygohcl~=1.0.7", "GitPython~=3.1.40", "setuptools~=65.5.1", "semantic_version~=2.10.0", "pytablewriter~=1.2.0"], + install_requires=[ + "click~=8.1.7", + "rich~=13.6.0", + "pygohcl~=1.0.7", + "GitPython~=3.1.40", + "setuptools~=65.5.1", + "semantic_version~=2.10.0", + "pytablewriter~=1.2.0", + "PyGithub~=2.1.1", + ], python_requires=">=3.11", entry_points=""" [console_scripts]