From 68c8e7b9d2a06754f9602d1885581c6b3291920d Mon Sep 17 00:00:00 2001 From: Kaixiang Zhao Date: Fri, 7 Jun 2019 16:31:07 -0700 Subject: [PATCH] feat(codebuild): add functionality to allow using private registry and cross-account ECR repository as build image Fixes #2175 --- packages/@aws-cdk/aws-codebuild/README.md | 7 +- .../@aws-cdk/aws-codebuild/lib/project.ts | 98 +++++++----- packages/@aws-cdk/aws-codebuild/package.json | 2 + .../test/integ.docker-asset.lit.expected.json | 28 +--- .../integ.docker-registry.lit.expected.json | 148 ++++++++++++++++++ .../test/integ.docker-registry.lit.ts | 34 ++++ .../test/integ.ecr.lit.expected.json | 32 +--- 7 files changed, 254 insertions(+), 95 deletions(-) create mode 100644 packages/@aws-cdk/aws-codebuild/test/integ.docker-registry.lit.expected.json create mode 100644 packages/@aws-cdk/aws-codebuild/test/integ.docker-registry.lit.ts diff --git a/packages/@aws-cdk/aws-codebuild/README.md b/packages/@aws-cdk/aws-codebuild/README.md index d8e89469295db..91165822cfcdd 100644 --- a/packages/@aws-cdk/aws-codebuild/README.md +++ b/packages/@aws-cdk/aws-codebuild/README.md @@ -190,8 +190,7 @@ of the constants such as `WindowsBuildImage.WIN_SERVER_CORE_2016_BASE` or Alternatively, you can specify a custom image using one of the static methods on `XxxBuildImage`: -* Use `.fromDockerHub(image)` to reference an image publicly available in Docker - Hub. +* Use `.fromDockerRegistry(image[, secretsManagerCredential])` to reference an image in any public or private Docker registry. * Use `.fromEcrRepository(repo[, tag])` to reference an image available in an ECR repository. * Use `.fromAsset(directory)` to use an image created from a @@ -205,6 +204,10 @@ The following example shows how to define an image from an ECR repository: [ECR example](./test/integ.ecr.lit.ts) +The following example shows how to define an image from a private docker registry: + +[Docker Registry example](./test/integ.docker-registry.lit.ts) + ## Events CodeBuild projects can be used either as a source for events or be triggered diff --git a/packages/@aws-cdk/aws-codebuild/lib/project.ts b/packages/@aws-cdk/aws-codebuild/lib/project.ts index 02b0ed08152d8..6251492935ba6 100644 --- a/packages/@aws-cdk/aws-codebuild/lib/project.ts +++ b/packages/@aws-cdk/aws-codebuild/lib/project.ts @@ -5,7 +5,8 @@ import { DockerImageAsset, DockerImageAssetProps } from '@aws-cdk/aws-ecr-assets import events = require('@aws-cdk/aws-events'); import iam = require('@aws-cdk/aws-iam'); import kms = require('@aws-cdk/aws-kms'); -import { Aws, CfnResource, Construct, Duration, IResource, Lazy, PhysicalName, Resource, Stack } from '@aws-cdk/core'; +import secretsmanager = require('@aws-cdk/aws-secretsmanager'); +import { Aws, CfnResource, Construct, Duration, IResource, Lazy, PhysicalName, Resource, Stack, Token } from '@aws-cdk/core'; import { IArtifacts } from './artifacts'; import { BuildSpec } from './build-spec'; import { Cache } from './cache'; @@ -775,6 +776,18 @@ export class Project extends ProjectBase { }); } + private attachEcrPermission() { + this.addToRolePolicy(new iam.PolicyStatement({ + resources: ['*'], + actions: [ + 'ecr:GetAutheticationToken', + 'ecr:GetDownloadUrlForLayer', + 'ecr:BatchGetImage', + 'ecr:BatchCheckLayerAvailability' + ] + })); + } + private renderEnvironment(env: BuildEnvironment = {}, projectVars: { [name: string]: BuildEnvironmentVariable } = {}): CfnProject.EnvironmentProperty { const vars: { [name: string]: BuildEnvironmentVariable } = {}; @@ -792,6 +805,10 @@ export class Project extends ProjectBase { const hasEnvironmentVars = Object.keys(vars).length > 0; + if (isECRImage(this.buildImage.imageId)) { + this.attachEcrPermission(); + } + const errors = this.buildImage.validate(env); if (errors.length > 0) { throw new Error("Invalid CodeBuild environment: " + errors.join('\n')); @@ -800,6 +817,12 @@ export class Project extends ProjectBase { return { type: this.buildImage.type, image: this.buildImage.imageId, + imagePullCredentialsType: this.buildImage.imagePullCredentialsType, + registryCredential: this.buildImage.secretsManagerCredential ? + { + credentialProvider: 'SECRETS_MANAGER', + credential: this.buildImage.secretsManagerCredential.secretArn + } : undefined, privilegedMode: env.privileged || false, computeType: env.computeType || this.buildImage.defaultComputeType, environmentVariables: !hasEnvironmentVars ? undefined : Object.keys(vars).map(name => ({ @@ -924,6 +947,11 @@ export enum ComputeType { LARGE = 'BUILD_GENERAL1_LARGE' } +export enum ImagePullCredentialsType { + CODEBUILD = 'CODEBUILD', + SERVICE_ROLE = 'SERVICE_ROLE' +} + export interface BuildEnvironment { /** * The image used for the builds. @@ -982,6 +1010,16 @@ export interface IBuildImage { */ readonly defaultComputeType: ComputeType; + /** + * The type of credentials AWS CodeBuild uses to pull images in your build. + */ + readonly imagePullCredentialsType?: ImagePullCredentialsType; + + /** + * The credentials for access to a private registry. + */ + readonly secretsManagerCredential?: secretsmanager.ISecret; + /** * Allows the image a chance to validate whether the passed configuration is correct. * @@ -1002,7 +1040,7 @@ export interface IBuildImage { * * You can also specify a custom image using one of the static methods: * - * - LinuxBuildImage.fromDockerHub(image) + * - LinuxBuildImage.fromDockerRegistry(image[, secretsManagerCredential]) * - LinuxBuildImage.fromEcrRepository(repo[, tag]) * - LinuxBuildImage.fromAsset(parent, id, props) * @@ -1046,8 +1084,8 @@ export class LinuxBuildImage implements IBuildImage { /** * @returns a Linux build image from a Docker Hub image. */ - public static fromDockerHub(name: string): LinuxBuildImage { - return new LinuxBuildImage(name); + public static fromDockerRegistry(name: string, secretsManagerCredential?: secretsmanager.ISecret): LinuxBuildImage { + return new LinuxBuildImage(name, ImagePullCredentialsType.SERVICE_ROLE, secretsManagerCredential); } /** @@ -1062,9 +1100,7 @@ export class LinuxBuildImage implements IBuildImage { * @param tag Image tag (default "latest") */ public static fromEcrRepository(repository: ecr.IRepository, tag: string = 'latest'): LinuxBuildImage { - const image = new LinuxBuildImage(repository.repositoryUriForTag(tag)); - repository.addToResourcePolicy(ecrAccessForCodeBuildService()); - return image; + return new LinuxBuildImage(repository.repositoryUriForTag(tag), ImagePullCredentialsType.SERVICE_ROLE); } /** @@ -1072,19 +1108,16 @@ export class LinuxBuildImage implements IBuildImage { */ public static fromAsset(scope: Construct, id: string, props: DockerImageAssetProps): LinuxBuildImage { const asset = new DockerImageAsset(scope, id, props); - const image = new LinuxBuildImage(asset.imageUri); - - // allow this codebuild to pull this image (CodeBuild doesn't use a role, so - // we can't use `asset.grantUseImage()`. - asset.repository.addToResourcePolicy(ecrAccessForCodeBuildService()); - - return image; + return new LinuxBuildImage(asset.imageUri, ImagePullCredentialsType.SERVICE_ROLE); } public readonly type = 'LINUX_CONTAINER'; public readonly defaultComputeType = ComputeType.SMALL; - private constructor(public readonly imageId: string) { + private constructor( + public readonly imageId: string, + public readonly imagePullCredentialsType?: ImagePullCredentialsType, + public readonly secretsManagerCredential?: secretsmanager.ISecret) { } public validate(_: BuildEnvironment): string[] { @@ -1127,7 +1160,7 @@ export class LinuxBuildImage implements IBuildImage { * * You can also specify a custom image using one of the static methods: * - * - WindowsBuildImage.fromDockerHub(image) + * - WindowsBuildImage.fromDockerRegistry(image[, secretsManagerCredential]) * - WindowsBuildImage.fromEcrRepository(repo[, tag]) * - WindowsBuildImage.fromAsset(parent, id, props) * @@ -1139,8 +1172,8 @@ export class WindowsBuildImage implements IBuildImage { /** * @returns a Windows build image from a Docker Hub image. */ - public static fromDockerHub(name: string): WindowsBuildImage { - return new WindowsBuildImage(name); + public static fromDockerRegistry(name: string, secretsManagerCredential?: secretsmanager.ISecret): WindowsBuildImage { + return new WindowsBuildImage(name, ImagePullCredentialsType.SERVICE_ROLE, secretsManagerCredential); } /** @@ -1155,9 +1188,7 @@ export class WindowsBuildImage implements IBuildImage { * @param tag Image tag (default "latest") */ public static fromEcrRepository(repository: ecr.IRepository, tag: string = 'latest'): WindowsBuildImage { - const image = new WindowsBuildImage(repository.repositoryUriForTag(tag)); - repository.addToResourcePolicy(ecrAccessForCodeBuildService()); - return image; + return new WindowsBuildImage(repository.repositoryUriForTag(tag), ImagePullCredentialsType.SERVICE_ROLE); } /** @@ -1165,18 +1196,15 @@ export class WindowsBuildImage implements IBuildImage { */ public static fromAsset(scope: Construct, id: string, props: DockerImageAssetProps): WindowsBuildImage { const asset = new DockerImageAsset(scope, id, props); - const image = new WindowsBuildImage(asset.imageUri); - - // allow this codebuild to pull this image (CodeBuild doesn't use a role, so - // we can't use `asset.grantUseImage()`. - asset.repository.addToResourcePolicy(ecrAccessForCodeBuildService()); - - return image; + return new WindowsBuildImage(asset.imageUri, ImagePullCredentialsType.SERVICE_ROLE); } public readonly type = 'WINDOWS_CONTAINER'; public readonly defaultComputeType = ComputeType.MEDIUM; - private constructor(public readonly imageId: string) { + private constructor( + public readonly imageId: string, + public readonly imagePullCredentialsType?: ImagePullCredentialsType, + public readonly secretsManagerCredential?: secretsmanager.ISecret) { } public validate(buildEnvironment: BuildEnvironment): string[] { @@ -1239,11 +1267,9 @@ export enum BuildEnvironmentVariableType { PARAMETER_STORE = 'PARAMETER_STORE' } -function ecrAccessForCodeBuildService(): iam.PolicyStatement { - const s = new iam.PolicyStatement({ - principals: [new iam.ServicePrincipal('codebuild.amazonaws.com')], - actions: ['ecr:GetDownloadUrlForLayer', 'ecr:BatchGetImage', 'ecr:BatchCheckLayerAvailability'], - }); - s.sid = 'CodeBuild'; - return s; +function isECRImage(imageUri: string) { + if (!Token.isUnresolved(imageUri)) { + return /^(.+).dkr.ecr.(.+).amazonaws.com[.]{0,1}[a-z]{0,3}\/([^:]+):?.*$/.test(imageUri); + } + return false; } diff --git a/packages/@aws-cdk/aws-codebuild/package.json b/packages/@aws-cdk/aws-codebuild/package.json index d8554c0171991..8af0ed579b174 100644 --- a/packages/@aws-cdk/aws-codebuild/package.json +++ b/packages/@aws-cdk/aws-codebuild/package.json @@ -88,6 +88,7 @@ "@aws-cdk/aws-kms": "^0.36.0", "@aws-cdk/aws-s3": "^0.36.0", "@aws-cdk/aws-s3-assets": "^0.36.0", + "@aws-cdk/aws-secretsmanager": "^0.36.0", "@aws-cdk/core": "^0.36.0" }, "homepage": "https://github.com/awslabs/aws-cdk", @@ -103,6 +104,7 @@ "@aws-cdk/aws-kms": "^0.36.0", "@aws-cdk/aws-s3": "^0.36.0", "@aws-cdk/aws-s3-assets": "^0.36.0", + "@aws-cdk/aws-secretsmanager": "^0.36.0", "@aws-cdk/core": "^0.36.0" }, "engines": { diff --git a/packages/@aws-cdk/aws-codebuild/test/integ.docker-asset.lit.expected.json b/packages/@aws-cdk/aws-codebuild/test/integ.docker-asset.lit.expected.json index d4b6b0e10e84b..32ead4e776695 100644 --- a/packages/@aws-cdk/aws-codebuild/test/integ.docker-asset.lit.expected.json +++ b/packages/@aws-cdk/aws-codebuild/test/integ.docker-asset.lit.expected.json @@ -39,33 +39,6 @@ ] } ] - }, - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:BatchCheckLayerAvailability" - ], - "Effect": "Allow", - "Principal": { - "Service": { - "Fn::Join": [ - "", - [ - "codebuild.", - { - "Ref": "AWS::URLSuffix" - } - ] - ] - } - }, - "Sid": "CodeBuild" - } - ], - "Version": "2012-10-17" } }, "DependsOn": [ @@ -439,6 +412,7 @@ ] ] }, + "ImagePullCredentialsType": "SERVICE_ROLE", "PrivilegedMode": false, "Type": "LINUX_CONTAINER" }, diff --git a/packages/@aws-cdk/aws-codebuild/test/integ.docker-registry.lit.expected.json b/packages/@aws-cdk/aws-codebuild/test/integ.docker-registry.lit.expected.json new file mode 100644 index 0000000000000..b8cd00a66ffb2 --- /dev/null +++ b/packages/@aws-cdk/aws-codebuild/test/integ.docker-registry.lit.expected.json @@ -0,0 +1,148 @@ +{ + "Resources": { + "MyProjectRole9BBE5233": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": { + "Fn::Join": [ + "", + [ + "codebuild.", + { + "Ref": "AWS::URLSuffix" + } + ] + ] + } + } + } + ], + "Version": "2012-10-17" + } + } + }, + "MyProjectRoleDefaultPolicyB19B7C29": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":log-group:/aws/codebuild/", + { + "Ref": "MyProject39F7B0AE" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":log-group:/aws/codebuild/", + { + "Ref": "MyProject39F7B0AE" + }, + ":*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "MyProjectRoleDefaultPolicyB19B7C29", + "Roles": [ + { + "Ref": "MyProjectRole9BBE5233" + } + ] + } + }, + "MyProject39F7B0AE": { + "Type": "AWS::CodeBuild::Project", + "Properties": { + "Artifacts": { + "Type": "NO_ARTIFACTS" + }, + "Environment": { + "ComputeType": "BUILD_GENERAL1_SMALL", + "Image": "my-registry/my-repo", + "ImagePullCredentialsType": "SERVICE_ROLE", + "PrivilegedMode": false, + "RegistryCredential": { + "Credential": { + "Fn::Join": [ + "", + [ + "arn:aws:secretsmanager:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":secret:my-secrets-123456" + ] + ] + }, + "CredentialProvider": "SECRETS_MANAGER" + }, + "Type": "LINUX_CONTAINER" + }, + "ServiceRole": { + "Fn::GetAtt": [ + "MyProjectRole9BBE5233", + "Arn" + ] + }, + "Source": { + "BuildSpec": "{\n \"version\": \"0.2\",\n \"phases\": {\n \"build\": {\n \"commands\": [\n \"ls\"\n ]\n }\n }\n}", + "Type": "NO_SOURCE" + } + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-codebuild/test/integ.docker-registry.lit.ts b/packages/@aws-cdk/aws-codebuild/test/integ.docker-registry.lit.ts new file mode 100644 index 0000000000000..9e598127d317b --- /dev/null +++ b/packages/@aws-cdk/aws-codebuild/test/integ.docker-registry.lit.ts @@ -0,0 +1,34 @@ +import secretsmanager = require('@aws-cdk/aws-secretsmanager'); +import cdk = require('@aws-cdk/core'); +import codebuild = require('../lib'); + +class TestStack extends cdk.Stack { + constructor(scope: cdk.App, id: string) { + super(scope, id); + + const secrets = secretsmanager.Secret.fromSecretArn(this, "MySecrets", + `arn:aws:secretsmanager:${this.region}:${this.account}:secret:my-secrets-123456`); + + new codebuild.Project(this, 'MyProject', { + buildSpec: codebuild.BuildSpec.fromObject({ + version: "0.2", + phases: { + build: { + commands: [ 'ls' ] + } + } + }), + /// !show + environment: { + buildImage: codebuild.LinuxBuildImage.fromDockerRegistry("my-registry/my-repo", secrets) + } + /// !hide + }); + } +} + +const app = new cdk.App(); + +new TestStack(app, 'test-codebuild-docker-asset'); + +app.synth(); diff --git a/packages/@aws-cdk/aws-codebuild/test/integ.ecr.lit.expected.json b/packages/@aws-cdk/aws-codebuild/test/integ.ecr.lit.expected.json index 5bac318649a12..b7899d62a391b 100644 --- a/packages/@aws-cdk/aws-codebuild/test/integ.ecr.lit.expected.json +++ b/packages/@aws-cdk/aws-codebuild/test/integ.ecr.lit.expected.json @@ -1,37 +1,8 @@ { "Resources": { "MyRepoF4F48043": { - "DeletionPolicy": "Retain", "Type": "AWS::ECR::Repository", - "Properties": { - "RepositoryPolicyText": { - "Statement": [ - { - "Action": [ - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:BatchCheckLayerAvailability" - ], - "Effect": "Allow", - "Principal": { - "Service": { - "Fn::Join": [ - "", - [ - "codebuild.", - { - "Ref": "AWS::URLSuffix" - } - ] - ] - } - }, - "Sid": "CodeBuild" - } - ], - "Version": "2012-10-17" - } - } + "DeletionPolicy": "Retain" }, "MyProjectRole9BBE5233": { "Type": "AWS::IAM::Role", @@ -186,6 +157,7 @@ ] ] }, + "ImagePullCredentialsType": "SERVICE_ROLE", "PrivilegedMode": false, "Type": "LINUX_CONTAINER" },