Add support for CodeStar CodeBuild clone ref and update CodeStar setup

**Why?**

The CodeStar provider did not support the `CODEBUILD_CLONE_REF` artifact mode
yet. This is useful if you would like to access the git repository, for example
to analyze the history or checkout a different branch.

For CodeStar Connections, it was trying to resolve the CodeStar Connection Arn
from SSM Parameter Store at the pipeline stack generation. However, resolving
parameters should be performed in the input generation step.

**What?**

* Added support to define the `output_artifact_format` in the CodeStar source
  provider. Allowing `CODEBUILD_CLONE_REF` to be defined as the output format.
* Moved resolving the `codestar_connection_path` in SSM Parameter Store to the
  pipeline input generation. As this is where it should be performed.
  The Lambda function resolving these parameters has the required permissions
  already.
* Added support for our CloudFormation role to perform
  `codestar-connections:PassConnection`, such that CodeStar connections can be
  configured in the CodePipeline stack that it generates.

docs/providers-guide.md

@@ -211,6 +211,11 @@ Provider type: `codestar`.
     web hook as part of the pipeline. Read the CodeStar Connections
     documentation for more
     [information](https://docs.aws.amazon.com/dtconsole/latest/userguide/connections.html).
+- *output_artifact_format* - *(String)* default: `CODE_ZIP`
+  - The output artifact format. Values can be either `CODEBUILD_CLONE_REF` or
+    `CODE_ZIP`. If unspecified, the default is `CODE_ZIP`.
+  - NB: The `CODEBUILD_CLONE_REF` value can only be used by CodeBuild
+    downstream actions.
 
 ## Build
 

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/generate_pipeline_inputs.py

@@ -21,7 +21,11 @@
 ROOT_ACCOUNT_ID = os.environ["ROOT_ACCOUNT_ID"]
 
 
-def store_regional_parameter_config(pipeline, parameter_store, deployment_map_source):
+def store_regional_parameter_config(
+    pipeline,
+    parameter_store,
+    deployment_map_source,
+):
     """
     Responsible for storing the region information for specific
     pipelines. These regions are defined in the deployment_map
@@ -41,21 +45,36 @@ def store_regional_parameter_config(pipeline, parameter_store, deployment_map_so
     )
 
 
-def fetch_required_ssm_params(regions):
+def fetch_required_ssm_params(pipeline_input, regions):
     output = {}
     for region in regions:
         parameter_store = ParameterStore(region, boto3)
         output[region] = {
             "s3": parameter_store.fetch_parameter(
-                f"/cross_region/s3_regional_bucket/{region}"
+                f"/cross_region/s3_regional_bucket/{region}",
+            ),
+            "kms": parameter_store.fetch_parameter(
+                f"/cross_region/kms_arn/{region}",
             ),
-            "kms": parameter_store.fetch_parameter(f"/cross_region/kms_arn/{region}"),
         }
         if region == DEPLOYMENT_ACCOUNT_REGION:
             output[region]["modules"] = parameter_store.fetch_parameter(
                 "deployment_account_bucket"
             )
-            output['default_scm_branch'] = parameter_store.fetch_parameter('default_scm_branch')
+            output['default_scm_branch'] = parameter_store.fetch_parameter(
+                'default_scm_branch',
+            )
+            codestar_connection_path = (
+                pipeline_input
+                .get('default_providers', {})
+                .get('source')
+                .get('properties', {})
+                .get('codestar_connection_path', {})
+            )
+            if codestar_connection_path:
+                output['codestar_connection_arn'] = (
+                    parameter_store.fetch_parameter(codestar_connection_path)
+                )
     return output
 
 
@@ -84,9 +103,12 @@ def generate_pipeline_inputs(pipeline, organizations, parameter_store):
         # Targets should be a list of lists.
 
         # Note: This is a big shift away from how ADF handles targets natively.
-        # Previously this would be a list of [accountId(s)] it now returns a list of [[account_ids], [account_ids]]
-        # for the sake of consistency we should probably think of a target consisting of multiple "waves". So if you see
-        # any reference to a wave going forward it will be the individual batch of account ids
+        # Previously this would be a list of [accountId(s)] it now returns a
+        # list of [[account_ids], [account_ids]].
+        #
+        # For the sake of consistency we should probably think of a target
+        # consisting of multiple "waves". So if you see any reference to
+        # a wave going forward it will be the individual batch of account ids.
         pipeline_object.template_dictionary["targets"].append(
             list(target_structure.generate_waves()),
         )
@@ -96,20 +118,37 @@ def generate_pipeline_inputs(pipeline, organizations, parameter_store):
 
     pipeline_object.generate_input()
     data["ssm_params"] = fetch_required_ssm_params(
+        pipeline_object.input,
         pipeline_object.input["regions"] or [DEPLOYMENT_ACCOUNT_REGION]
     )
     data["input"] = pipeline_object.input
-    data['input']['default_scm_branch'] = data["ssm_params"].get('default_scm_branch')
-    store_regional_parameter_config(pipeline_object, parameter_store, pipeline.get("deployment_map_source"))
+    if 'codestar_connection_arn' in data["ssm_params"]:
+        data['input']['default_providers']['source']['properties'][
+            'codestar_connection_arn'
+        ] = data["ssm_params"]['codestar_connection_arn']
+    data['input']['default_scm_branch'] = data["ssm_params"].get(
+        'default_scm_branch',
+    )
+    store_regional_parameter_config(
+        pipeline_object,
+        parameter_store,
+        pipeline.get("deployment_map_source"),
+    )
     return data
 
 
 def lambda_handler(pipeline, _):
     """Main Lambda Entry point"""
     parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3)
     sts = STS()
+    cross_account_role_name = parameter_store.fetch_parameter(
+        "cross_account_access_role",
+    )
     role = sts.assume_cross_account_role(
-        f'arn:{get_partition(DEPLOYMENT_ACCOUNT_REGION)}:iam::{ROOT_ACCOUNT_ID}:role/{parameter_store.fetch_parameter("cross_account_access_role")}-readonly',
+        (
+            f'arn:{get_partition(DEPLOYMENT_ACCOUNT_REGION)}:iam::'
+            f'{ROOT_ACCOUNT_ID}:role/{cross_account_role_name}-readonly'
+        ),
         "pipeline",
     )
     organizations = Organizations(role)

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/identify_out_of_date_pipelines.py

@@ -73,9 +73,9 @@ def lambda_handler(event, _):
         )
     parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3)
     current_pipelines = {
-            parameter.get("Name").split("/")[-2]
-            for parameter in get_current_pipelines(parameter_store)
-        }
+        parameter.get("Name").split("/")[-2]
+        for parameter in get_current_pipelines(parameter_store)
+    }
 
     pipeline_names = {
         p.get("name") for p in deployment_map.map_contents["pipelines"]

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml

@@ -818,6 +818,15 @@ Resources:
                   - "iam:PassRole"
                 Resource:
                   - !Sub arn:${AWS::Partition}:iam::*:role/*
+              - Effect: Allow
+                Sid: "AllowCodeStarConnection"
+                Action:
+                  - "codestar-connections:PassConnection"
+                Resource:
+                  - !Sub arn:${AWS::Partition}:codestar-connections:${AWS::Region}:${AWS::AccountId}:connection/*
+                Condition:
+                  StringEquals:
+                    'codestar-connections:PassedToService': 'codepipeline.amazonaws.com'
               - Effect: Allow
                 Action:
                   - "events:PutRule"

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py

@@ -6,7 +6,6 @@
 
 import os
 import json
-import boto3
 
 from aws_cdk import (
     aws_codepipeline as _codepipeline,
@@ -61,7 +60,11 @@ def __init__(self, **kwargs):
         self.action_name = kwargs.get('action_name')
         self.action_mode = kwargs.get('action_mode', '').upper()
         self.region = kwargs.get('region') or ADF_DEPLOYMENT_REGION
-        self.account_id = self.map_params["default_providers"]["source"].get('properties', {}).get("account_id")
+        self.account_id = (
+            self.map_params["default_providers"]["source"]
+            .get('properties', {})
+            .get("account_id")
+        )
         self.role_arn = self._generate_role_arn()
         self.notification_endpoint = self.map_params.get("topic_arn")
         self.default_scm_branch = self.map_params.get(
@@ -122,25 +125,41 @@ def _generate_configuration(self): #pylint: disable=R0912, R0911, R0915
                                         'object_key')
             }
         if self.provider == "CodeStarSourceConnection":
-            owner = self.map_params.get('default_providers', {}).get('source').get('properties', {}).get('owner', {})
-            repo = self.map_params.get('default_providers', {}).get('source', {}).get('properties', {}).get('repository', {}) or self.map_params['name']
-            codestar_connection_path = self.map_params.get('default_providers', {}).get('source').get('properties', {}).get('codestar_connection_path', {})
-            ssm_client = boto3.client('ssm')
-            try:
-                response = ssm_client.get_parameter(Name=codestar_connection_path)
-            except Exception as e:
-                LOGGER.error(f"No parameter found at {codestar_connection_path}. Check the path/value.")
-                raise e
-            connection_arn = response['Parameter']['Value']
-            return {
-                "ConnectionArn": connection_arn,
+            default_source_props = (
+                self.map_params
+                .get('default_providers', {})
+                .get('source', {})
+                .get('properties', {})
+            )
+            owner = default_source_props.get('owner')
+            repo = (
+                default_source_props.get('repository')
+                or self.map_params['name']
+            )
+            if not default_source_props.get('codestar_connection_arn'):
+                raise Exception(
+                    "The CodeStar Connection Arn could not be resolved for "
+                    f"the {self.map_params['name']} pipeline. Please check "
+                    "whether the codestar_connection_path is setup correctly "
+                    "and validate that the Parameter it points to is properly "
+                    "configured in SSM Parameter Store."
+                )
+            props = {
+                "ConnectionArn": default_source_props.get(
+                    'codestar_connection_arn',
+                ),
                 "FullRepositoryId": f"{owner}/{repo}",
-                "BranchName": self.map_params.get('default_providers', {}).get(
-                    'source', {}).get('properties', {}).get(
-                        'branch',
-                        self.default_scm_branch
-                    )
+                "BranchName": default_source_props.get(
+                    'branch',
+                    self.default_scm_branch,
+                )
             }
+            output_artifact_format = default_source_props.get(
+                'output_artifact_format',
+            )
+            if output_artifact_format:
+                props["OutputArtifactFormat"] = output_artifact_format
+            return props
         if self.provider == "GitHub":
             return {
                 "Owner": self.map_params.get('default_providers', {}).get('source').get('properties', {}).get('owner', {}),
@@ -273,7 +292,11 @@ def _generate_configuration(self): #pylint: disable=R0912, R0911, R0915
                     and self.map_params['default_providers']['source'].get('properties', {}).get('poll_for_changes', False)
                 )
             }
-            output_artifact_format = self.map_params['default_providers']['source'].get('properties', {}).get('output_artifact_format', None)
+            output_artifact_format = (
+                self.map_params['default_providers']['source']
+                .get('properties', {})
+                .get('output_artifact_format')
+            )
             if output_artifact_format:
                 props["OutputArtifactFormat"] = output_artifact_format
             return props

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/schema_validation.py

@@ -12,7 +12,7 @@
 
 NOTIFICATION_PROPS = {
     Optional("target"): str,
-    Optional("type") : Or("lambda", "chat_bot")
+    Optional("type"): Or("lambda", "chat_bot")
 }
 
 # Pipeline Params
@@ -47,6 +47,12 @@
     )
 )
 
+SOURCE_OUTPUT_ARTIFACT_FORMAT = Or(
+    "CODEBUILD_CLONE_REF",
+    "CODE_ZIP",
+    None,
+)
+
 # CodeCommit Source
 CODECOMMIT_SOURCE_PROPS = {
     "account_id": AWS_ACCOUNT_ID_SCHEMA,
@@ -56,7 +62,9 @@
     Optional("owner"): str,
     Optional("role"): str,
     Optional("trigger_on_changes"): bool,
-    Optional("output_artifact_format", default=None): Or("CODEBUILD_CLONE_REF", "CODE_ZIP", None)
+    Optional("output_artifact_format", default=None): (
+        SOURCE_OUTPUT_ARTIFACT_FORMAT
+    ),
 }
 CODECOMMIT_SOURCE = {
     "provider": 'codecommit',
@@ -82,7 +90,10 @@
     Optional("repository"): str,
     Optional("branch"): str,
     "owner": str,
-    "codestar_connection_path": str
+    "codestar_connection_path": str,
+    Optional("output_artifact_format", default=None): (
+        SOURCE_OUTPUT_ARTIFACT_FORMAT
+    ),
 }
 
 CODESTAR_SOURCE = {
@@ -114,7 +125,9 @@
     Optional("image"): Or(str, CODEBUILD_IMAGE_PROPS),
     Optional("size"): Or('small', 'medium', 'large'),
     Optional("spec_filename"): str,
-    Optional("environment_variables"): {Optional(str): Or(str, bool, int, object)},
+    Optional("environment_variables"): {
+        Optional(str): Or(str, bool, int, object)
+    },
     Optional("role"): str,
     Optional("timeout"): int,
     Optional("privileged"): bool,
@@ -273,15 +286,19 @@
             'provider': Or('codecommit', 'github', 's3', 'codestar'),
             'properties': dict,
         },
-        lambda x: PROVIDER_SOURCE_SCHEMAS[x['provider']].validate(x),  #pylint: disable=W0108
+        # pylint: disable=W0108
+        lambda x: PROVIDER_SOURCE_SCHEMAS[x['provider']].validate(x),
     ),
     Optional('build'): And(
         {
             Optional('provider'): Or('codebuild', 'jenkins'),
             Optional('enabled'): bool,
             Optional('properties'): dict,
         },
-        lambda x: PROVIDER_BUILD_SCHEMAS[x.get('provider', 'codebuild')].validate(x),  #pylint: disable=W0108
+        # pylint: disable=W0108
+        lambda x: PROVIDER_BUILD_SCHEMAS[
+            x.get('provider', 'codebuild')
+        ].validate(x),
     ),
     Optional('deploy'): And(
         {
@@ -292,7 +309,8 @@
             Optional('enabled'): bool,
             Optional('properties'): dict,
         },
-        lambda x: PROVIDER_DEPLOY_SCHEMAS[x['provider']].validate(x),  #pylint: disable=W0108
+        # pylint: disable=W0108
+        lambda x: PROVIDER_DEPLOY_SCHEMAS[x['provider']].validate(x),
     ),
 }
 REGION_SCHEMA = Or(
@@ -313,11 +331,31 @@
 
 TARGET_SCHEMA = {
     Optional("path"): Or(str, int, TARGET_LIST_SCHEMA),
-    Optional("tags"): {And(str, Regex(r"\A.{1,128}\Z")): And(str, Regex(r"\A.{0,256}\Z"))},
+    Optional("tags"): {
+        And(str, Regex(r"\A.{1,128}\Z")): And(str, Regex(r"\A.{0,256}\Z"))
+    },
     Optional("target"): Or(str, int, TARGET_LIST_SCHEMA),
     Optional("name"): str,
-    Optional("provider"): Or('lambda', 's3', 'codedeploy', 'cloudformation', 'service_catalog', 'approval', 'codebuild', 'jenkins'),
-    Optional("properties"): Or(CODEBUILD_PROPS, JENKINS_PROPS, CLOUDFORMATION_PROPS, CODEDEPLOY_PROPS, S3_DEPLOY_PROPS, SERVICECATALOG_PROPS, LAMBDA_PROPS, APPROVAL_PROPS),
+    Optional("provider"): Or(
+        'lambda',
+        's3',
+        'codedeploy',
+        'cloudformation',
+        'service_catalog',
+        'approval',
+        'codebuild',
+        'jenkins',
+    ),
+    Optional("properties"): Or(
+        CODEBUILD_PROPS,
+        JENKINS_PROPS,
+        CLOUDFORMATION_PROPS,
+        CODEDEPLOY_PROPS,
+        S3_DEPLOY_PROPS,
+        SERVICECATALOG_PROPS,
+        LAMBDA_PROPS,
+        APPROVAL_PROPS,
+    ),
     Optional("regions"): REGION_SCHEMA,
     Optional("exclude", default=[]): [str],
     Optional("wave", default={"size": 50}): TARGET_WAVE_SCHEME
@@ -348,8 +386,9 @@
 }
 TOP_LEVEL_SCHEMA = {
     "pipelines": [PIPELINE_SCHEMA],
-    # Allow any toplevel key starting with "x-" or "x_".
-    # ADF will ignore these, but users can use them to define anchors in one place.
+    # Allow any top level key starting with "x-" or "x_".
+    # ADF will ignore these, but users can use them to define anchors
+    # in one place.
     Optional(Regex('^[x][-_].*')): object
 }