Support Migration of Components to a New GitHub Organization (#46)

## what * Automatically migrate existing components to the new GitHub organization ## why * Makes migration to new component structure simple and stable ## Refs * cloudposse/terraform-aws-components#1177
cloudposse · Nov 27, 2024 · 97ba053 · 97ba053
1 parent c40923e
commit 97ba053
Show file tree

Hide file tree

Showing 10 changed files with 256 additions and 23 deletions.
diff --git a/Makefile b/Makefile
@@ -13,4 +13,4 @@ install-deps:
 	pip install -r src/requirements.txt
 
 test:
-	pytest -s -v --log-level DEBUG -rP --pyargs src/
+	pytest -s -v --log-level DEBUG -rP --pyargs src/
diff --git a/action.yml b/action.yml
@@ -9,6 +9,10 @@ inputs:
     description: "GitHub Token used to perform git and GitHub operations"
     required: false
     default: ${{ github.token }}
+  infra-repo-dir:
+    description: "Path to the infra repository. Default '/github/workspace/'"
+    required: false
+    default: '/github/workspace/'
   infra-terraform-dirs:
     description: "Comma or new line separated list of terraform directories in infra repo. For example 'components/terraform,components/terraform-old. Default 'components/terraform'"
     required: false
@@ -64,6 +68,7 @@ runs:
   env:
     GITHUB_ACCESS_TOKEN: ${{ inputs.github-access-token }}
     INFRA_TERRAFORM_DIRS: ${{ inputs.infra-terraform-dirs }}
+    INFRA_REPO_DIR: ${{ inputs.infra-repo-dir }}
     VENDORING_ENABLED: ${{ inputs.vendoring-enabled }}
     MAX_NUMBER_OF_PRS: ${{ inputs.max-number-of-prs }}
     INCLUDE: ${{ inputs.include }}

diff --git a/entrypoint.sh b/entrypoint.sh
@@ -10,7 +10,7 @@ python3 src/main.py \
     --github-api-token ${GITHUB_ACCESS_TOKEN} \
     --go-getter-tool ${GO_GETTER_TOOL} \
     --infra-repo-name ${GITHUB_REPOSITORY} \
-    --infra-repo-dir /github/workspace/ \
+    --infra-repo-dir ${INFRA_REPO_DIR} \
     --infra-terraform-dirs "${INFRA_TERRAFORM_DIRS}" \
     --vendoring-enabled ${VENDORING_ENABLED} \
     --max-number-of-prs ${MAX_NUMBER_OF_PRS} \

diff --git a/src/assets/config.yaml b/src/assets/config.yaml
@@ -0,0 +1,161 @@
+repo_settings:
+  prefix: aws
+component_map:
+  access-analyzer: access-analyzer
+  account: account
+  account-map: account-map
+  account-quotas: account-quotas
+  account-settings: account-settings
+  acm: acm
+  alb: alb
+  amplify: amplify
+  api-gateway-account-settings: api-gateway-account-settings
+  api-gateway-rest-api: api-gateway-rest-api
+  athena: athena
+  aurora-mysql: aurora-mysql
+  aurora-mysql-resources: aurora-mysql-resources
+  aurora-postgres: aurora-postgres
+  aurora-postgres-resources: aurora-postgres-resources
+  auth0/app: auth0-app
+  auth0/connection: auth0-connection
+  auth0/tenant: auth0-tenant
+  aws-backup: backup
+  aws-config: config
+  aws-inspector: inspector
+  aws-inspector2: inspector2
+  aws-saml: saml
+  aws-shield: shield
+  aws-sso: identity-center
+  aws-ssosync: ssosync
+  aws-teams: teams
+  aws-team-roles: team-roles
+  bastion: bastion
+  cloudmap-namespace: cloudmap-namespace
+  cloudtrail: cloudtrail
+  cloudtrail-bucket: cloudtrail-bucket
+  cloudwatch-logs: cloudwatch-logs
+  cognito: cognito
+  config-bucket: config-bucket
+  datadog-configuration: datadog-credentials
+  datadog-integration: datadog-integration
+  datadog-lambda-forwarder: datadog-lambda-forwarder
+  datadog-logs-archive: datadog-logs-archive
+  datadog-monitor: datadog-monitor
+  datadog-private-location-ecs: datadog-private-location-ecs
+  datadog-synthetics: datadog-synthetics
+  datadog-synthetics-private-location: datadog-synthetics-private-location
+  dms/endpoint: dms-endpoint
+  dms/iam: dms-iam
+  dms/replication-instance: dms-replication-instance
+  dms/replication-task: dms-replication-task
+  dns-delegated: dns-delegated
+  dns-primary: dns-primary
+  documentdb: documentdb
+  dynamodb: dynamodb
+  ec2-client-vpn: ec2-client-vpn
+  ec2-instance: ec2-instance
+  ecr: ecr
+  ecs: ecs
+  ecs-service: ecs-service
+  efs: efs
+  eks/actions-runner-controller: eks-actions-runner-controller
+  eks/alb-controller: eks-alb-controller
+  eks/alb-controller-ingress-class: eks-alb-controller-ingress-class
+  eks/alb-controller-ingress-group: eks-alb-controller-ingress-group
+  argocd-repo: argocd-github-repo
+  eks/argocd: eks-argocd
+  eks/aws-node-termination-handler: eks-node-termination-handler
+  eks/cert-manager: eks-cert-manager
+  eks/cluster: eks-cluster
+  eks/datadog-agent: eks-datadog-agent
+  eks/echo-server: eks-echo-server
+  eks/external-dns: eks-external-dns
+  eks/external-secrets-operator: eks-external-secrets-operator
+  eks/github-actions-runner: eks-github-actions-runner
+  eks/idp-roles: eks-idp-roles
+  eks/karpenter: eks-karpenter-controller
+  eks/karpenter-node-pool: eks-karpenter-node-pool
+  eks/keda: eks-keda
+  eks/loki: eks-loki
+  eks/metrics-server: eks-metrics-server
+  eks/prometheus-scraper: eks-prometheus-scraper
+  eks/promtail: eks-promtail
+  eks/redis: eks-redis
+  eks/redis-operator: eks-redis-operator
+  eks/reloader: eks-reloader
+  eks/storage-class: eks-storage-class
+  eks/tailscale: eks-tailscale
+  elasticache-redis: elasticache-redis
+  elasticsearch: elasticsearch
+  eventbridge: eventbridge
+  github-action-token-rotator: github-action-token-rotator
+  github-oidc-provider: github-oidc-provider
+  github-oidc-role: github-oidc-role
+  github-runners: github-runners
+  github-webhook: github-webhook
+  global-accelerator: global-accelerator
+  global-accelerator-endpoint-group: global-accelerator-endpoint-group
+  glue/catalog-database: glue-catalog-database
+  glue/catalog-table: glue-catalog-table
+  glue/connection: glue-connection
+  glue/crawler: glue-crawler
+  glue/iam: glue-iam
+  glue/job: glue-job
+  glue/registry: glue-registry
+  glue/schema: glue-schema
+  glue/trigger: glue-trigger
+  glue/workflow: glue-workflow
+  guardduty: guardduty
+  iam-role: iam-role
+  iam-service-linked-roles: iam-service-linked-roles
+  ipam: ipam
+  kinesis-stream: kinesis-stream
+  kms: kms
+  lakeformation: lakeformation
+  lambda: lambda
+  macie: macie
+  managed-grafana/api-key: managed-grafana-api-key
+  managed-grafana/dashboard: managed-grafana-dashboard
+  managed-grafana/data-source/loki: managed-grafana-data-source-loki
+  managed-grafana/data-source/managed-prometheus: managed-grafana-data-source-managed-prometheus
+  managed-grafana/workspace: managed-grafana-workspace
+  managed-prometheus/workspace: managed-prometheus-workspace
+  memorydb: memorydb
+  mq-broker: mq-broker
+  msk: msk
+  mwaa: mwaa
+  network-firewall: network-firewall
+  opsgenie-team: opsgenie-team
+  philips-labs-github-runners: philips-labs-github-runners
+  rds: rds
+  redshift: redshift
+  redshift-serverless: redshift-serverless
+  route53-resolver-dns-firewall: route53-resolver-dns-firewall
+  runs-on: runs-on
+  s3-bucket: s3-bucket
+  security-hub: security-hub
+  ses: ses
+  sftp: sftp
+  site-to-site-vpn: site-to-site-vpn
+  snowflake-account: snowflake-account
+  snowflake-database: snowflake-database
+  sns-topic: sns-topic
+  spa-s3-cloudfront: spa-s3-cloudfront
+  spacelift/admin-stack: spacelift-admin-stack
+  spacelift/spaces: spacelift-spaces
+  spacelift/worker-pool: spacelift-worker-pool-asg
+  eks/spacelift-worker-pool-controller: eks-spacelift-worker-pool-controller
+  eks/spacelift-worker-pool: eks-spacelift-worker-pool
+  sqs-queue: sqs-queue
+  ssm-parameters: ssm-parameters
+  sso-saml-provider: sso-saml-provider
+  strongdm: strongdm
+  tfstate-backend: tfstate-backend
+  tgw/cross-region-hub-connector: tgw-hub-connector
+  tgw/hub: tgw-hub
+  tgw/spoke: tgw-spoke
+  vpc: vpc
+  vpc-flow-logs-bucket: vpc-flow-logs-bucket
+  vpc-peering: vpc-peering
+  waf: waf
+  zscaler: zscaler
diff --git a/src/atmos_component.py b/src/atmos_component.py
@@ -1,12 +1,17 @@
+import logging
 import re
 import os
 from typing import Tuple
 import yaml
+import semver
 
 from utils import io
 
-VERSION_PATTERN = r"version:\s*\d+\.\d+\.\d+"
+VERSION_PATTERN = r"(?<=source:)(?<!mixins:)(.*?)(version:\s*['\"]?v?\d+\.\d+\.\d+['\"]?)"
+URI_PATTERN = r"(?<=source:)(?<!mixins:)(.*?)(uri:\s*[^\n]*)"
 COMPONENT_YAML = 'component.yaml'
+README_EXTENTION = '.md'
+MONOREPO_MAXIMUM_VERSION = '1.532.0'
 
 
 class AtmosComponent:
@@ -21,7 +26,7 @@ def __init__(self, infra_repo_dir: str, infra_terraform_dir: str, component_file
     @property
     def version(self):
         version = self.__yaml_content.get('spec', {}).get('source', {}).get('version')
-        return version.strip() if version else None
+        return version.strip().lstrip("v") if version else None
 
     @property
     def uri_repo(self) -> str:
@@ -74,6 +79,28 @@ def __initialize(self):
         self.__content: str = self.__load_file()
         self.__yaml_content = self.__load_yaml_content()
         (self.__uri_repo, self.__uri_path) = self.__parse_uri()
+        self.__migrate_new_org()
+
+    def __migrate_new_org(self):
+        if self.has_version() and semver.compare(self.version, MONOREPO_MAXIMUM_VERSION) != -1:
+            self.migrate()
+
+    def migrate(self):
+        if (self.has_version() and
+                self.has_valid_uri() and
+                self.__uri_repo == 'github.com/cloudposse/terraform-aws-components.git'):
+            component_name = '/'.join(self.__uri_path.split('/')[1:])
+            config_path = os.path.join(os.path.dirname(__file__), "assets", "config.yaml")
+            migration_config = yaml.load(io.read_file_to_string(config_path), Loader=yaml.FullLoader)
+            prefix = migration_config.get('repo_settings').get('prefix')
+            new_component_name = migration_config.get('component_map').get(component_name)
+            if new_component_name:
+                destination = new_component_name.replace('/', '-')
+                self.__uri_repo = f"github.com/cloudposse-terraform-components/{prefix}-{destination}.git"
+                self.__uri_path = "src"
+                template = f"\g<1>uri: {self.__uri_repo}//{self.__uri_path}?ref={{{{ .Version }}}}"
+                self.__content = re.sub(URI_PATTERN, template, self.__content, flags=re.DOTALL)
+                self.__yaml_content = self.__load_yaml_content()
 
     def __fetch_name(self) -> str:
         return os.path.dirname(os.path.relpath(self.__component_file, os.path.join(self.__infra_repo_dir, self.__infra_terraform_dir)))
@@ -101,13 +128,17 @@ def __load_yaml_content(self):
         return yaml.load(self.__content, Loader=yaml.FullLoader)
 
     def has_version(self) -> bool:
-        return bool(self.version)
+        try:
+            return bool(semver.parse(self.version))
+        except Exception:
+            return False
+
 
     def has_valid_uri(self) -> bool:
         return bool(self.uri_repo and self.uri_path)
 
     def update_version(self, new_version: str):
-        self.__content = re.sub(VERSION_PATTERN, f"version: {new_version}", self.__content)
+        self.__content = re.sub(VERSION_PATTERN, f"\g<1>version: {new_version}", self.__content, flags=re.DOTALL)
         self.__yaml_content = self.__load_yaml_content()
 
     def persist(self, output_file=None):

diff --git a/src/component_updater.py b/src/component_updater.py
@@ -1,3 +1,4 @@
+import copy
 import os
 import sys
 import logging
@@ -6,7 +7,7 @@
 from enum import Enum
 from tools_manager import ToolsManager, ToolExecutionError
 from utils import io
-from atmos_component import AtmosComponent, COMPONENT_YAML
+from atmos_component import AtmosComponent, COMPONENT_YAML, README_EXTENTION
 from github_provider import GitHubProvider, PullRequestCreationResponse
 from config import Config
 
@@ -142,14 +143,18 @@ def __update_component(self, infra_terraform_dir, component_file: str) -> Compon
             response.state = ComponentUpdaterResponseState.NOT_VALID_URI_FOUND_IN_SOURCE_YAML
             return response
 
-        repo_dir = self.__fetch_component_repo(original_component) if not self.__config.skip_component_repo_fetching else self.__config.components_download_dir
+        migrated_component = copy.deepcopy(original_component)
+        migrated_component.migrate()
+
+        repo_dir = self.__fetch_component_repo(migrated_component) if not self.__config.skip_component_repo_fetching else self.__config.components_download_dir
 
         if not self.__tools_manager.is_git_repo(repo_dir):
             logging.error(f"Component '{original_component.name}' uri is not git repo. Can't figure out latest version. Skipping")
             response.state = ComponentUpdaterResponseState.URI_IS_NOT_GIT_REPO
             return response
 
         latest_tag = self.__tools_manager.git_get_latest_tag(repo_dir)
+        logging.info(f"Latest tag for component '{original_component.name}' is '{latest_tag}'")
 
         if not latest_tag:
             logging.error(f"Unable to figure out latest tag for component '{original_component.name}' source uri. Skipping")
@@ -161,7 +166,8 @@ def __update_component(self, infra_terraform_dir, component_file: str) -> Compon
             response.state = ComponentUpdaterResponseState.ALREADY_UP_TO_DATE
             return response
 
-        updated_component = self.__clone_infra_for_component(infra_terraform_dir, original_component)
+        updated_component = self.__clone_infra_for_component(infra_terraform_dir, migrated_component)
+        updated_component.migrate()
 
         logging.debug(f"Updated component:\n{str(updated_component)}")
 
@@ -253,6 +259,9 @@ def __does_component_needs_to_be_updated(self, original_component: AtmosComponen
             if updated_file.endswith(COMPONENT_YAML):
                 continue
 
+            if updated_file.endswith(README_EXTENTION):
+                continue
+
             # skip folders
             if not os.path.isfile(updated_file):
                 continue

diff --git a/src/github_provider.py b/src/github_provider.py
@@ -131,16 +131,19 @@ def open_pr(self,
         original_component_release_link = self.__build_component_release_tag_link(original_component)
         updated_component_release_link = self.__build_component_release_tag_link(updated_component)
 
-        source_name, source_link = self.__build_get_source(original_component)
+        old_source_name, old_source_link = self.__build_get_source(original_component)
+        new_source_name, new_source_link = self.__build_get_source(updated_component)
 
         title = self.__pr_title_template.render(component_name=original_component.name,
-                                                source_name=source_name,
+                                                source_name=old_source_name,
                                                 old_version=original_component.version,
                                                 new_version=updated_component.version)
 
         body = self.__pr_body_template.render(component_name=original_component.name,
-                                              source_name=source_name,
-                                              source_link=source_link,
+                                              old_source_name=old_source_name,
+                                              old_source_link=old_source_link,
+                                              new_source_name=new_source_name,
+                                              new_source_link=old_source_link,
                                               old_version=original_component.version,
                                               new_version=updated_component.version,
                                               old_version_link=original_component_version_link,
@@ -190,24 +193,31 @@ def close_pr(self, pull_request: PullRequest, message: str):
     def __build_component_version_link(self, component: AtmosComponent):
         component_version_link = None
 
+        version = component.version
+        if "cloudposse-terraform-components" not in component.uri_repo:
+            version = f"v{version.lstrip('v')}"
+
         if component.uri_repo.startswith('github.com'):
             normalized_repo_uri = self.__remove_git_suffix(component.uri_repo)
-            component_version_link = f'https://{normalized_repo_uri}/tree/{component.version}/{component.uri_path}'
+            component_version_link = f'https://{normalized_repo_uri}/tree/{version}/{component.uri_path}'
         elif component.uri_repo.startswith('https://github.com'):
             normalized_repo_uri = self.__remove_git_suffix(component.uri_repo)
-            component_version_link = f'{normalized_repo_uri}/tree/{component.version}/{component.uri_path}'
+            component_version_link = f'{normalized_repo_uri}/tree/{version}/{component.uri_path}'
 
         return component_version_link
 
     def __build_component_release_tag_link(self, component: AtmosComponent):
         component_release_tag_link = None
+        version = component.version
+        if "cloudposse-terraform-components" not in component.uri_repo:
+            version = f"v{version.lstrip('v')}"
 
         if component.uri_repo.startswith('github.com'):
             normalized_repo_uri = self.__remove_git_suffix(component.uri_repo)
-            component_release_tag_link = f'https://{normalized_repo_uri}/releases/tag/{component.version}'
+            component_release_tag_link = f'https://{normalized_repo_uri}/releases/tag/{version}'
         elif component.uri_repo.startswith('https://github.com'):
             normalized_repo_uri = self.__remove_git_suffix(component.uri_repo)
-            component_release_tag_link = f'{normalized_repo_uri}/releases/tag/{component.version}'
+            component_release_tag_link = f'{normalized_repo_uri}/releases/tag/{version}'
 
         return component_release_tag_link
 

diff --git a/src/requirements.txt b/src/requirements.txt
@@ -37,3 +37,4 @@ smmap==5.0.0
 tomlkit==0.11.7
 urllib3==1.26.15
 wrapt==1.15.0
+semver==3.0.2