feat: allow-for-empty-targets-in-deployment-maps (awslabs#634)

* feat: allow-for-empty-targets-in-deployment-maps

* fix: consider comments from review

* Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/generate_pipeline_inputs.py

Changed

Co-authored-by: Simon Kok

* fix: consider comments from review

* fix: consider comments from review

* Fix lint issues and few minor review changes

* Fix indentation issue

* feat: add unit tests

* feat: add unit tests

* fix: rename variables for consistency

* Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/target.py

* Allow additional return statement

* Extract final pipeline reporting to method

* Fix return statements for target.py / fetch_accounts_for_target

---------

Co-authored-by: Andreas Falkenberg
Co-authored-by: Simon Kok

docs/admin-guide.md

@@ -214,6 +214,17 @@ Config has five components in `main-notification-endpoint`, `scp`, `scm`,
     `master` instead. We recommend configuring the main scm branch name to
     `main`. As new repositories will most likely use this branch name as their
     default branch.
+- `deployment-maps` tracks all source code management configuration.
+  - **allow-empty-target** allows you to configure deployment maps with empty
+    targets.
+    If all targets get evaluated to empty, the ADF pipeline is still created
+    based on the remaining providers (e.g. source and build). It just does not
+    have a deploy stage.
+    This is useful when you need to:
+    - target an OU that does not have any AWS Accounts (initially or
+      temporarily).
+    - target AWS Accounts by tag with no AWS Accounts having that tag assigned
+      (yet).
 - `org` configures settings in case of staged multi-organization ADF deployments.
   - `stage` defines the AWS Organization stage in case of staged multi-
     organization ADF deployments. This is an optional setting. In enterprise-

src/lambda_codebase/initial_commit/adfconfig.yml.j2

@@ -27,6 +27,9 @@ config:
   scm:
     auto-create-repositories: enabled
     default-scm-branch: main
+  deployment-maps:
+    allow-empty-target: "False"
+    # ^ Needs to be "True" or "False". Defaults to "False" when not set.
   #org:
     # Optional: Use this variable to define the AWS Organization in case of staged multi-organization ADF deployments
     #stage: dev

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/generate_pipeline_inputs.py

@@ -86,6 +86,20 @@ def fetch_required_ssm_params(pipeline_input, regions):
     return output
 
 
+def report_final_pipeline_targets(pipeline_object):
+    number_of_targets = 0
+    LOGGER.info(
+        "Targets found: %s",
+        pipeline_object.template_dictionary["targets"],
+    )
+    for target in pipeline_object.template_dictionary["targets"]:
+        for target_accounts in target:
+            number_of_targets = number_of_targets + len(target_accounts)
+    LOGGER.info("Number of targets found: %d", number_of_targets)
+    if number_of_targets == 0:
+        LOGGER.info("Attempting to create an empty pipeline as there were no targets found")
+
+
 def generate_pipeline_inputs(
     pipeline,
     deployment_map_source,
@@ -144,6 +158,8 @@ def generate_pipeline_inputs(
             ),
         )
 
+    report_final_pipeline_targets(pipeline_object)
+
     if DEPLOYMENT_ACCOUNT_REGION not in regions:
         pipeline_object.stage_regions.append(DEPLOYMENT_ACCOUNT_REGION)
 

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py

@@ -56,6 +56,7 @@
     "ACCOUNT_BOOTSTRAPPING_STATE_MACHINE_ARN"
 )
 ADF_DEFAULT_SCM_FALLBACK_BRANCH = 'master'
+ADF_DEFAULT_DEPLOYMENT_MAPS_ALLOW_EMPTY_TARGET = False
 ADF_DEFAULT_ORG_STAGE = "none"
 LOGGER = configure_logger(__name__)
 
@@ -152,6 +153,13 @@ def prepare_deployment_account(sts, deployment_account_id, config):
             ADF_DEFAULT_SCM_FALLBACK_BRANCH,
         )
     )
+    deployment_account_parameter_store.put_parameter(
+        '/adf/deployment-maps/allow-empty-target',
+        config.config.get('deployment-maps', {}).get(
+            'allow-empty-target',
+            ADF_DEFAULT_DEPLOYMENT_MAPS_ALLOW_EMPTY_TARGET,
+        )
+    )
     deployment_account_parameter_store.put_parameter(
         '/adf/org/stage',
         config.config.get('org', {}).get(

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/target.py

@@ -16,10 +16,14 @@
 )
 from logger import configure_logger
 from schema_validation import AWS_ACCOUNT_ID_REGEX_STR
-
+from botocore.exceptions import ClientError
+from parameter_store import ParameterStore
+import boto3
 
 LOGGER = configure_logger(__name__)
 ADF_DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"]
+DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"]
+
 AWS_ACCOUNT_ID_REGEX = re.compile(AWS_ACCOUNT_ID_REGEX_STR)
 CLOUDFORMATION_PROVIDER_NAME = "cloudformation"
 RECURSIVE_SUFFIX = "/**/*"
@@ -141,6 +145,13 @@ def __init__(
         )
         self.target_structure = target_structure
         self.organizations = organizations
+        # Set adf_deployment_maps_allow_empty_target as bool
+        parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3)
+        adf_deployment_maps_allow_empty_target_bool = parameter_store.fetch_parameter(
+            "/adf/deployment-maps/allow-empty-target"
+        ).lower().capitalize() == "True"
+        self.adf_deployment_maps_allow_empty_target = adf_deployment_maps_allow_empty_target_bool
+
 
     @staticmethod
     def _account_is_active(account):
@@ -176,14 +187,32 @@ def _create_response_object(self, responses):
                         response.get("Name"), str(response.get("Id"))
                     )
                 )
+
         if accounts_found == 0:
-            raise NoAccountsFoundError(f"No accounts found in {self.path}")
+            if self.adf_deployment_maps_allow_empty_target is False:
+                raise NoAccountsFoundError(f"No accounts found in {self.path}.")
+            LOGGER.info(
+                "Create_response_object: 0 AWS accounts found for path %s. "
+                "Continue with empty response.",
+                self.path,
+            )
 
     def _target_is_account_id(self):
-        responses = self.organizations.client.describe_account(
-            AccountId=str(self.path)
-        ).get("Account")
-        self._create_response_object([responses])
+        try:
+            responses = self.organizations.client.describe_account(
+                AccountId=str(self.path)
+            ).get('Account')
+            responses_list = [responses]
+        except ClientError as client_err:
+            if (
+                client_err.response["Error"]["Code"] == "AccountNotFoundException" and
+                self.adf_deployment_maps_allow_empty_target is True
+            ):
+                LOGGER.info("IGNORE - Account was not found in AWS Org for id %s", self.path)
+                responses_list = []
+            else:
+                raise
+        self._create_response_object(responses_list)
 
     def _target_is_tags(self):
         responses = self.organizations.get_account_ids_for_tags(self.path)
@@ -202,16 +231,46 @@ def _target_is_tags(self):
         self._create_response_object(accounts)
 
     def _target_is_ou_id(self):
-        responses = self.organizations.get_accounts_for_parent(str(self.path))
+        try:
+            # Check if ou exists - otherwise throw clean exception here
+            self.organizations.client.list_children(ParentId=self.path, ChildType="ACCOUNT")
+            responses = self.organizations.get_accounts_for_parent(
+                str(self.path)
+            )
+        except ClientError as client_err:
+            no_target_found = (
+                client_err.response["Error"]["Code"] == "ParentNotFoundException"
+            )
+            if no_target_found and self.adf_deployment_maps_allow_empty_target is True:
+                LOGGER.info(
+                    "Note: Target OU was not found in the AWS Org for id %s",
+                    self.path,
+                )
+                responses = []
+            else:
+                raise
         self._create_response_object(responses)
 
     def _target_is_ou_path(self, resolve_children=False):
-        responses = self.organizations.get_accounts_in_path(
-            self.path,
-            resolve_children=resolve_children,
-            ou_id=None,
-            excluded_paths=[],
-        )
+        try:
+            responses = self.organizations.get_accounts_in_path(
+                self.path,
+                resolve_children=resolve_children,
+                ou_id=None,
+                excluded_paths=[],
+            )
+        except ClientError as client_err:
+            no_target_found = (
+                client_err.response["Error"]["Code"] == "ParentNotFoundException"
+            )
+            if no_target_found and self.adf_deployment_maps_allow_empty_target is True:
+                LOGGER.info(
+                    "Note: Target OU was not found in AWS Org for path %s",
+                    self.path,
+                )
+                responses = []
+            else:
+                raise
         self._create_response_object(responses)
 
     def _target_is_null_path(self):
@@ -220,15 +279,20 @@ def _target_is_null_path(self):
         responses = self.organizations.dir_to_ou(self.path)
         self._create_response_object(responses)
 
+    # pylint: disable=R0911
     def fetch_accounts_for_target(self):
         if self.path == "approval":
-            return self._target_is_approval()
+            self._target_is_approval()
+            return
         if isinstance(self.path, dict):
-            return self._target_is_tags()
+            self._target_is_tags()
+            return
         if str(self.path).startswith("ou-"):
-            return self._target_is_ou_id()
+            self._target_is_ou_id()
+            return
         if AWS_ACCOUNT_ID_REGEX.match(str(self.path)):
-            return self._target_is_account_id()
+            self._target_is_account_id()
+            return
         if str(self.path).isnumeric():
             LOGGER.warning(
                 "The specified path is numeric, but is not 12 chars long. "
@@ -248,10 +312,16 @@ def fetch_accounts_for_target(self):
                 str(oct(int(self.path))).replace("o", ""),
             )
         if str(self.path).startswith("/"):
-            return self._target_is_ou_path(
+            self._target_is_ou_path(
                 resolve_children=str(self.path).endswith(RECURSIVE_SUFFIX)
             )
+            return
+
+        if self.adf_deployment_maps_allow_empty_target is True:
+            return
+
         if self.path is None:
             # No path/target has been passed, path will default to /deployment
-            return self._target_is_null_path()
+            self._target_is_null_path()
+            return
         raise InvalidDeploymentMapError(f"Unknown definition for target: {self.path}")