From 335c9c87c3d7996c6fa428cae0287ac73aab6b6a Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Wed, 26 Jan 2022 11:54:24 +0200 Subject: [PATCH 01/14] feat(s3-deployment): deploy content with deploy-time values [WIP] Allow deploying test-based content that can potentially include deploy-time values such as attributes of cloud resources. Introduce a `Source.content(objectKey, text)` where `text` can naturally include tokens that resolve only at deploy time. This is implemented by replacing the deploy-time tokens with markers that are replaced inside the s3-deployment custome resource. Fixes #12903 TODO: - [ ] Implement + test the custom resource code - [ ] Documentation - [ ] Finalize integration test --- .../@aws-cdk/aws-s3-deployment/lib/content.ts | 59 +++ .../@aws-cdk/aws-s3-deployment/lib/source.ts | 35 ++ .../test/bucket-deployment.test.ts | 17 + .../aws-s3-deployment/test/content.test.ts | 72 +++ ...eg.bucket-deployment-content.expected.json | 428 ++++++++++++++++++ .../test/integ.bucket-deployment-content.ts | 20 + 6 files changed, 631 insertions(+) create mode 100644 packages/@aws-cdk/aws-s3-deployment/lib/content.ts create mode 100644 packages/@aws-cdk/aws-s3-deployment/test/content.test.ts create mode 100644 packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-content.expected.json create mode 100644 packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-content.ts diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/content.ts b/packages/@aws-cdk/aws-s3-deployment/lib/content.ts new file mode 100644 index 0000000000000..183a0ff7d5c43 --- /dev/null +++ b/packages/@aws-cdk/aws-s3-deployment/lib/content.ts @@ -0,0 +1,59 @@ +import { Stack } from '@aws-cdk/core'; + +// keep this import separate from other imports to reduce chance for merge conflicts with v2-main +// eslint-disable-next-line no-duplicate-imports, import/order +import { Construct } from '@aws-cdk/core'; + +export interface Content { + readonly text: string; + readonly markers: Record; +} + +export function renderContent(scope: Construct, content: string): Content { + const obj = Stack.of(scope).resolve(content); + if (typeof obj === 'string') { + return { text: obj, markers: {} }; + } + + if (!obj['Fn::Join']) { + throw new Error('Unexpected resolved value. Expecting Fn::Join'); + } + + const fnJoin: FnJoin = obj['Fn::Join']; + if (fnJoin[0] !== '') { + throw new Error('Unexpected join, expecting separator to be ""'); + } + + const markers: Record = {}; + const result = new Array(); + let markerIndex = 0; + + for (const part of fnJoin[1]) { + if (typeof(part) === 'string') { + result.push(part); + continue; + } + + if (typeof(part) === 'object') { + const keys: string[] = Object.keys(part); + if (keys.length !== 1) { + throw new Error('Invalid object'); + } + + if ('Ref' in part || 'Fn::GetAtt' in part) { + const marker = `<>`; + result.push(marker); + markers[marker] = part; + } else { + throw new Error('Invalid object'); + } + } + } + + return { text: result.join(''), markers }; +} + +type FnJoin = [string, FnJoinPart[]]; +type FnJoinPart = string | Ref | GetAtt; +type Ref = { Ref: string }; +type GetAtt = { 'Fn::GetAtt': [string, string] }; \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/source.ts b/packages/@aws-cdk/aws-s3-deployment/lib/source.ts index 8c22f49d791e3..34f6c93129ff5 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/source.ts +++ b/packages/@aws-cdk/aws-s3-deployment/lib/source.ts @@ -1,6 +1,10 @@ +import * as fs from 'fs'; +import { join, dirname } from 'path'; import * as iam from '@aws-cdk/aws-iam'; import * as s3 from '@aws-cdk/aws-s3'; import * as s3_assets from '@aws-cdk/aws-s3-assets'; +import { FileSystem } from '@aws-cdk/core'; +import { renderContent } from './content'; // keep this import separate from other imports to reduce chance for merge conflicts with v2-main // eslint-disable-next-line no-duplicate-imports, import/order @@ -19,6 +23,11 @@ export interface SourceConfig { * An S3 object key in the source bucket that points to a zip file. */ readonly zipObjectKey: string; + + /** + * A set of markers to substitute in the source content. + */ + readonly markers?: Record; } /** @@ -110,5 +119,31 @@ export class Source { }; } + /** + * Deploys a file with the specified textual contents into the bucket. The + * content can include deploy-time values that will get resolved only during + * deployment. + * + * @param objectKey The S3 object key to use for this file. + * @param content The contents + */ + public static content(objectKey: string, content: string): ISource { + return { + bind: (scope: Construct, context?: DeploymentSourceContext) => { + const workdir = FileSystem.mkdtemp('s3-deployment'); + const outputPath = join(workdir, objectKey); + const rendered = renderContent(scope, content); + fs.mkdirSync(dirname(outputPath), { recursive: true }); + fs.writeFileSync(outputPath, rendered.text); + const asset = this.asset(workdir).bind(scope, context); + return { + bucket: asset.bucket, + zipObjectKey: asset.zipObjectKey, + markers: rendered.markers, + }; + }, + }; + } + private constructor() { } } diff --git a/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts b/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts index 9507cdb5dace7..1d38816e2957b 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts +++ b/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts @@ -1056,3 +1056,20 @@ test('bucket has multiple deployments', () => { ], }); }); + +test('Source.content() can be used to create a file with contents', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Test'); + const bucket = new s3.Bucket(stack, 'Bucket'); + + const source = s3deploy.Source.content('my/path.txt', 'hello, world'); + + new s3deploy.BucketDeployment(stack, 'DeployWithVpc3', { + sources: [source], + destinationBucket: bucket, + destinationKeyPrefix: '/x/z', + }); + + const result = app.synth(); + expect(result.stacks[0].assets).toStrictEqual([]); +}); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-deployment/test/content.test.ts b/packages/@aws-cdk/aws-s3-deployment/test/content.test.ts new file mode 100644 index 0000000000000..d9120e1486c76 --- /dev/null +++ b/packages/@aws-cdk/aws-s3-deployment/test/content.test.ts @@ -0,0 +1,72 @@ +import * as lambda from '@aws-cdk/aws-lambda'; +import * as s3 from '@aws-cdk/aws-s3'; +import { Stack } from '@aws-cdk/core'; +import { Source } from '../lib'; +import { renderContent } from '../lib/content'; + +test('simple string', () => { + const stack = new Stack(); + expect(renderContent(stack, 'foo')).toStrictEqual({ + markers: {}, + text: 'foo', + }); +}); + +test('string with a "Ref" token', () => { + const stack = new Stack(); + const bucket = new s3.Bucket(stack, 'Bucket'); + + expect(renderContent(stack, `foo-${bucket.bucketName}`)).toStrictEqual({ + text: 'foo-<>', + markers: { '<>': { Ref: 'Bucket83908E77' } }, + }); +}); + +test('string with a "Fn::GetAtt" token', () => { + const stack = new Stack(); + const bucket = new s3.Bucket(stack, 'Bucket'); + + expect(renderContent(stack, `foo-${bucket.bucketArn}`)).toStrictEqual({ + text: 'foo-<>', + markers: { '<>': { 'Fn::GetAtt': ['Bucket83908E77', 'Arn'] } }, + }); +}); + +test('multiple markers', () => { + const stack = new Stack(); + const bucket = new s3.Bucket(stack, 'Bucket'); + + expect(renderContent(stack, `boom-${bucket.bucketName}-bam-${bucket.bucketArn}`)).toStrictEqual({ + text: 'boom-<>-bam-<>', + markers: { + '<>': { Ref: 'Bucket83908E77' }, + '<>': { 'Fn::GetAtt': ['Bucket83908E77', 'Arn'] }, + }, + }); +}); + +test('json-encoded string', () => { + const stack = new Stack(); + const bucket = new s3.Bucket(stack, 'Bucket'); + const json = { + BucketArn: bucket.bucketArn, + BucketName: bucket.bucketName, + }; + + expect(renderContent(stack, JSON.stringify(json))).toStrictEqual({ + text: JSON.stringify({ BucketArn: '<>', BucketName: '<>' }), + markers: { + '<>': { 'Fn::GetAtt': ['Bucket83908E77', 'Arn'] }, + '<>': { Ref: 'Bucket83908E77' }, + }, + }); +}); + +test('markers are returned in the source config', () => { + const stack = new Stack(); + const handler = new lambda.Function(stack, 'Handler', { runtime: lambda.Runtime.NODEJS_14_X, code: lambda.Code.fromInline('foo'), handler: 'index.handler' }); + const actual = Source.content('file1.txt', `boom-${stack.account}`).bind(stack, { handlerRole: handler.role! }); + expect(actual.markers).toStrictEqual({ + '<>': { Ref: 'AWS::AccountId' }, + }); +}); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-content.expected.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-content.expected.json new file mode 100644 index 0000000000000..8fccff7a6f382 --- /dev/null +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-content.expected.json @@ -0,0 +1,428 @@ +{ + "Resources": { + "Bucket83908E77": { + "Type": "AWS::S3::Bucket", + "Properties": { + "Tags": [ + { + "Key": "aws-cdk:cr-owned:deploy/here/:588fbb1f", + "Value": "true" + } + ] + }, + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "DeployMeAwsCliLayer5F9219E9": { + "Type": "AWS::Lambda::LayerVersion", + "Properties": { + "Content": { + "S3Bucket": { + "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + } + ] + } + ] + } + ] + ] + } + }, + "Description": "/opt/awscli/aws" + } + }, + "DeployMeCustomResource4455EE35": { + "Type": "Custom::CDKBucketDeployment", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756C81C01536", + "Arn" + ] + }, + "SourceBucketNames": [ + { + "Ref": "AssetParametersd09271be89b6cb0398f793b40c1531fd9b076aa92ba80b5e436914b1808fe18dS3BucketBC52CF96" + }, + { + "Ref": "AssetParametersf58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229S3BucketC0B57179" + } + ], + "SourceObjectKeys": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParametersd09271be89b6cb0398f793b40c1531fd9b076aa92ba80b5e436914b1808fe18dS3VersionKeyED6BBB32" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParametersd09271be89b6cb0398f793b40c1531fd9b076aa92ba80b5e436914b1808fe18dS3VersionKeyED6BBB32" + } + ] + } + ] + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParametersf58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229S3VersionKeyA1412544" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParametersf58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229S3VersionKeyA1412544" + } + ] + } + ] + } + ] + ] + } + ], + "DestinationBucketName": { + "Ref": "Bucket83908E77" + }, + "DestinationBucketKeyPrefix": "deploy/here/", + "Prune": true + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756CServiceRole89A01265": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + } + }, + "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756CServiceRoleDefaultPolicy88902FDF": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Ref": "AssetParametersd09271be89b6cb0398f793b40c1531fd9b076aa92ba80b5e436914b1808fe18dS3BucketBC52CF96" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Ref": "AssetParametersd09271be89b6cb0398f793b40c1531fd9b076aa92ba80b5e436914b1808fe18dS3BucketBC52CF96" + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Ref": "AssetParametersf58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229S3BucketC0B57179" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Ref": "AssetParametersf58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229S3BucketC0B57179" + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:Abort*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756CServiceRoleDefaultPolicy88902FDF", + "Roles": [ + { + "Ref": "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756CServiceRole89A01265" + } + ] + } + }, + "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756C81C01536": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Ref": "AssetParameterse3d9996b6fafcc7da88312672e15e3cc925b02cffc6f01a615d81f22303e3ae0S3BucketB3DDCC13" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameterse3d9996b6fafcc7da88312672e15e3cc925b02cffc6f01a615d81f22303e3ae0S3VersionKey3418DF70" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameterse3d9996b6fafcc7da88312672e15e3cc925b02cffc6f01a615d81f22303e3ae0S3VersionKey3418DF70" + } + ] + } + ] + } + ] + ] + } + }, + "Role": { + "Fn::GetAtt": [ + "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756CServiceRole89A01265", + "Arn" + ] + }, + "Handler": "index.handler", + "Layers": [ + { + "Ref": "DeployMeAwsCliLayer5F9219E9" + } + ], + "Runtime": "python3.7", + "Timeout": 900 + }, + "DependsOn": [ + "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756CServiceRoleDefaultPolicy88902FDF", + "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756CServiceRole89A01265" + ] + } + }, + "Parameters": { + "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7": { + "Type": "String", + "Description": "S3 bucket for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" + }, + "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F": { + "Type": "String", + "Description": "S3 key for asset version \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" + }, + "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68ArtifactHashD9A515C3": { + "Type": "String", + "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" + }, + "AssetParameterse3d9996b6fafcc7da88312672e15e3cc925b02cffc6f01a615d81f22303e3ae0S3BucketB3DDCC13": { + "Type": "String", + "Description": "S3 bucket for asset \"e3d9996b6fafcc7da88312672e15e3cc925b02cffc6f01a615d81f22303e3ae0\"" + }, + "AssetParameterse3d9996b6fafcc7da88312672e15e3cc925b02cffc6f01a615d81f22303e3ae0S3VersionKey3418DF70": { + "Type": "String", + "Description": "S3 key for asset version \"e3d9996b6fafcc7da88312672e15e3cc925b02cffc6f01a615d81f22303e3ae0\"" + }, + "AssetParameterse3d9996b6fafcc7da88312672e15e3cc925b02cffc6f01a615d81f22303e3ae0ArtifactHash9D8F179A": { + "Type": "String", + "Description": "Artifact hash for asset \"e3d9996b6fafcc7da88312672e15e3cc925b02cffc6f01a615d81f22303e3ae0\"" + }, + "AssetParametersd09271be89b6cb0398f793b40c1531fd9b076aa92ba80b5e436914b1808fe18dS3BucketBC52CF96": { + "Type": "String", + "Description": "S3 bucket for asset \"d09271be89b6cb0398f793b40c1531fd9b076aa92ba80b5e436914b1808fe18d\"" + }, + "AssetParametersd09271be89b6cb0398f793b40c1531fd9b076aa92ba80b5e436914b1808fe18dS3VersionKeyED6BBB32": { + "Type": "String", + "Description": "S3 key for asset version \"d09271be89b6cb0398f793b40c1531fd9b076aa92ba80b5e436914b1808fe18d\"" + }, + "AssetParametersd09271be89b6cb0398f793b40c1531fd9b076aa92ba80b5e436914b1808fe18dArtifactHashF6480042": { + "Type": "String", + "Description": "Artifact hash for asset \"d09271be89b6cb0398f793b40c1531fd9b076aa92ba80b5e436914b1808fe18d\"" + }, + "AssetParametersf58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229S3BucketC0B57179": { + "Type": "String", + "Description": "S3 bucket for asset \"f58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229\"" + }, + "AssetParametersf58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229S3VersionKeyA1412544": { + "Type": "String", + "Description": "S3 key for asset version \"f58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229\"" + }, + "AssetParametersf58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229ArtifactHash24BF3233": { + "Type": "String", + "Description": "Artifact hash for asset \"f58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229\"" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-content.ts b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-content.ts new file mode 100644 index 0000000000000..939c6ffc786ce --- /dev/null +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-content.ts @@ -0,0 +1,20 @@ +import { Bucket } from '@aws-cdk/aws-s3'; +import { App, CfnOutput, Stack } from '@aws-cdk/core'; +import { BucketDeployment, Source } from '../lib'; + +const app = new App(); +const stack = new Stack(app, 'TestBucketDeploymentContent'); +const bucket = new Bucket(stack, 'Bucket'); + +const file1 = Source.content('file1.txt', 'boom'); +const file2 = Source.content('path/to/file2.txt', `bam! ${bucket.bucketName}`); + +new BucketDeployment(stack, 'DeployMe', { + destinationBucket: bucket, + sources: [file1, file2], + destinationKeyPrefix: 'deploy/here/', +}); + +new CfnOutput(stack, 'BucketName', { value: bucket.bucketName }); + +app.synth(); \ No newline at end of file From 3c61b9b635fd41564801cf8ab5b6f45715c371de Mon Sep 17 00:00:00 2001 From: Otavio Macedo Date: Fri, 28 Jan 2022 15:02:40 +0000 Subject: [PATCH 02/14] Addressing case where the content is a single token --- .../@aws-cdk/aws-s3-deployment/lib/content.ts | 51 ++++++++++--------- .../aws-s3-deployment/test/content.test.ts | 10 ++++ 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/content.ts b/packages/@aws-cdk/aws-s3-deployment/lib/content.ts index 183a0ff7d5c43..19d0bd998acfa 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/content.ts +++ b/packages/@aws-cdk/aws-s3-deployment/lib/content.ts @@ -15,39 +15,44 @@ export function renderContent(scope: Construct, content: string): Content { return { text: obj, markers: {} }; } - if (!obj['Fn::Join']) { - throw new Error('Unexpected resolved value. Expecting Fn::Join'); - } - - const fnJoin: FnJoin = obj['Fn::Join']; - if (fnJoin[0] !== '') { - throw new Error('Unexpected join, expecting separator to be ""'); - } - const markers: Record = {}; const result = new Array(); let markerIndex = 0; - for (const part of fnJoin[1]) { - if (typeof(part) === 'string') { - result.push(part); - continue; + if (obj['Fn::Join']) { + const fnJoin: FnJoin = obj['Fn::Join']; + if (fnJoin[0] !== '') { + throw new Error('Unexpected join, expecting separator to be ""'); } - if (typeof(part) === 'object') { - const keys: string[] = Object.keys(part); - if (keys.length !== 1) { - throw new Error('Invalid object'); + for (const part of fnJoin[1]) { + if (typeof (part) === 'string') { + result.push(part); + continue; } - if ('Ref' in part || 'Fn::GetAtt' in part) { - const marker = `<>`; - result.push(marker); - markers[marker] = part; - } else { - throw new Error('Invalid object'); + if (typeof (part) === 'object') { + const keys: string[] = Object.keys(part); + if (keys.length !== 1) { + throw new Error('Invalid object'); + } + createMarker(part); } } + } else if (obj.Ref) { + createMarker(obj); + } else { + throw new Error('Unexpected resolved value. Expecting Fn::Join or Ref'); + } + + function createMarker(part: Ref | GetAtt) { + if ('Ref' in part || 'Fn::GetAtt' in part) { + const marker = `<>`; + result.push(marker); + markers[marker] = part; + } else { + throw new Error('Invalid object'); + } } return { text: result.join(''), markers }; diff --git a/packages/@aws-cdk/aws-s3-deployment/test/content.test.ts b/packages/@aws-cdk/aws-s3-deployment/test/content.test.ts index d9120e1486c76..eae6c1d2ae706 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/content.test.ts +++ b/packages/@aws-cdk/aws-s3-deployment/test/content.test.ts @@ -22,6 +22,16 @@ test('string with a "Ref" token', () => { }); }); +test('string is a single "Ref" token', () => { + const stack = new Stack(); + const bucket = new s3.Bucket(stack, 'Bucket'); + + expect(renderContent(stack, bucket.bucketName)).toStrictEqual({ + text: '<>', + markers: { '<>': { Ref: 'Bucket83908E77' } }, + }); +}); + test('string with a "Fn::GetAtt" token', () => { const stack = new Stack(); const bucket = new s3.Bucket(stack, 'Bucket'); From 9a1def56332af5f728b0e423bd2dccb0eb6e3162 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Tue, 1 Feb 2022 16:22:43 +0200 Subject: [PATCH 03/14] add unit tests for python lambda code --- .../lib/bucket-deployment.ts | 12 ++++ .../aws-s3-deployment/lib/lambda/index.py | 53 ++++++++++++++++-- .../aws-s3-deployment/test/lambda/.gitignore | 4 ++ .../test/lambda/Dockerfile.debug | 6 ++ .../aws-s3-deployment/test/lambda/debug.sh | 10 ++++ .../aws-s3-deployment/test/lambda/test.py | 46 ++++++++++++++- .../aws-s3-deployment/test/lambda/test.zip | Bin 179 -> 560 bytes 7 files changed, 124 insertions(+), 7 deletions(-) create mode 100644 packages/@aws-cdk/aws-s3-deployment/test/lambda/.gitignore create mode 100644 packages/@aws-cdk/aws-s3-deployment/test/lambda/Dockerfile.debug create mode 100755 packages/@aws-cdk/aws-s3-deployment/test/lambda/debug.sh diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts index 76bbae6a4b290..3519a203dd737 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts +++ b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts @@ -12,6 +12,10 @@ import { kebab as toKebabCase } from 'case'; import { Construct } from 'constructs'; import { ISource, SourceConfig } from './source'; +// keep this import separate from other imports to reduce chance for merge conflicts with v2-main +// eslint-disable-next-line no-duplicate-imports, import/order +import { Stack } from '@aws-cdk/core'; + // keep this import separate from other imports to reduce chance for merge conflicts with v2-main // eslint-disable-next-line no-duplicate-imports, import/order import { Construct as CoreConstruct } from '@aws-cdk/core'; @@ -323,6 +327,13 @@ export class BucketDeployment extends CoreConstruct { })); } + const markers: Record = {}; + for (const source of sources) { + for (const [k, v] of Object.entries(source.markers ?? {})) { + markers[k] = v; + } + } + const crUniqueId = `CustomResource${this.renderUniqueId(props.memoryLimit, props.vpc)}`; const cr = new cdk.CustomResource(this, crUniqueId, { serviceToken: handler.functionArn, @@ -330,6 +341,7 @@ export class BucketDeployment extends CoreConstruct { properties: { SourceBucketNames: sources.map(source => source.bucket.bucketName), SourceObjectKeys: sources.map(source => source.zipObjectKey), + SourceMarkers: sources.map(source => Stack.of(this).toJsonString(source.markers ?? {})), DestinationBucketName: props.destinationBucket.bucketName, DestinationBucketKeyPrefix: props.destinationKeyPrefix, RetainOnDelete: props.retainOnDelete, diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py b/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py index 466218b0fa134..63425cd972bae 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py +++ b/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py @@ -20,6 +20,7 @@ CFN_SUCCESS = "SUCCESS" CFN_FAILED = "FAILED" ENV_KEY_MOUNT_PATH = "MOUNT_PATH" +ENV_KEY_SKIP_CLEANUP = "SKIP_CLEANUP" CUSTOM_RESOURCE_OWNER_TAG = "aws-cdk:cr-owned" @@ -45,6 +46,7 @@ def cfn_error(message=None): try: source_bucket_names = props['SourceBucketNames'] source_object_keys = props['SourceObjectKeys'] + source_markers = props.get('SourceMarkers', None) dest_bucket_name = props['DestinationBucketName'] dest_bucket_prefix = props.get('DestinationBucketKeyPrefix', '') retain_on_delete = props.get('RetainOnDelete', "true") == "true" @@ -55,6 +57,11 @@ def cfn_error(message=None): exclude = props.get('Exclude', []) include = props.get('Include', []) + # backwards compatibility - if "SourceMarkers" is not specified, + # assume all sources have an empty market map + if source_markers is None: + source_markers = [json.dumps({}) for i in range(len(source_bucket_names))] + default_distribution_path = dest_bucket_prefix if not default_distribution_path.endswith("/"): default_distribution_path += "/" @@ -106,7 +113,7 @@ def cfn_error(message=None): aws_command("s3", "rm", old_s3_dest, "--recursive") if request_type == "Update" or request_type == "Create": - s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include) + s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers) if distribution_id: cloudfront_invalidate(distribution_id, distribution_paths) @@ -120,7 +127,11 @@ def cfn_error(message=None): #--------------------------------------------------------------------------------------------------- # populate all files from s3_source_zips to a destination bucket -def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include): +def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers): + # list lengths are equal + if len(list(s3_source_zips)) != len(source_markers): + raise Exception("'source_markers' and 's3_source_zips' must be the same length") + # create a temporary working directory in /tmp or if enabled an attached efs volume if ENV_KEY_MOUNT_PATH in os.environ: workdir = os.getenv(ENV_KEY_MOUNT_PATH) + "/" + str(uuid4()) @@ -136,13 +147,16 @@ def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, ex try: # download the archive from the source and extract to "contents" - for s3_source_zip in s3_source_zips: + for i in range(len(s3_source_zips)): + s3_source_zip = s3_source_zips[i] + markers = json.loads(source_markers[i]) + archive=os.path.join(workdir, str(uuid4())) logger.info("archive: %s" % archive) aws_command("s3", "cp", s3_source_zip, archive) logger.info("| extracting archive to: %s\n" % contents_dir) - with ZipFile(archive, "r") as zip: - zip.extractall(contents_dir) + logger.info("| markers: %s" % markers) + extract_and_replace_markers(archive, contents_dir, markers) # sync from "contents" to destination @@ -163,7 +177,8 @@ def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, ex s3_command.extend(create_metadata_args(user_metadata, system_metadata)) aws_command(*s3_command) finally: - shutil.rmtree(workdir) + if not os.getenv(ENV_KEY_SKIP_CLEANUP): + shutil.rmtree(workdir) #--------------------------------------------------------------------------------------------------- # invalidate files in the CloudFront distribution edge caches @@ -257,3 +272,29 @@ def bucket_owned(bucketName, keyPrefix): logger.info("| error getting tags from bucket") logger.exception(e) return False + +# extract archive and replace markers in output files +def extract_and_replace_markers(archive, contents_dir, markers): + with ZipFile(archive, "r") as zip: + zip.extractall(contents_dir) + + # replace markers for this source + for file in zip.namelist(): + file_path = os.path.join(contents_dir, file) + if os.path.isdir(file_path): continue + replace_markers(file_path, markers) + +def replace_markers(filename, markers): + # convert the dict of string markers to binary markers + replace_tokens = dict([(k.encode('utf-8'), v.encode('utf-8')) for k, v in markers.items()]) + + outfile = filename + '.new' + with open(filename, 'rb') as fi, open(outfile, 'wb') as fo: + for line in fi: + for token in replace_tokens: + line = line.replace(token, replace_tokens[token]) + fo.write(line) + + # # delete the original file and rename the new one to the original + os.remove(filename) + os.rename(outfile, filename) \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-deployment/test/lambda/.gitignore b/packages/@aws-cdk/aws-s3-deployment/test/lambda/.gitignore new file mode 100644 index 0000000000000..3a5f4f826d408 --- /dev/null +++ b/packages/@aws-cdk/aws-s3-deployment/test/lambda/.gitignore @@ -0,0 +1,4 @@ +# symlinked by debug.sh +index.py +__pycache__ +aws.out \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-deployment/test/lambda/Dockerfile.debug b/packages/@aws-cdk/aws-s3-deployment/test/lambda/Dockerfile.debug new file mode 100644 index 0000000000000..791343ea39cf8 --- /dev/null +++ b/packages/@aws-cdk/aws-s3-deployment/test/lambda/Dockerfile.debug @@ -0,0 +1,6 @@ +FROM public.ecr.aws/lambda/python:latest + +# install boto3, which is available on Lambda +RUN pip3 install boto3 + +ENTRYPOINT [ "/bin/bash" ] \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-deployment/test/lambda/debug.sh b/packages/@aws-cdk/aws-s3-deployment/test/lambda/debug.sh new file mode 100755 index 0000000000000..459116e55ab84 --- /dev/null +++ b/packages/@aws-cdk/aws-s3-deployment/test/lambda/debug.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# starts a debugging container for the python lambda function and tests + +tag="s3-deployment-test-environment" +docker build -f Dockerfile.debug -t $tag . + +echo "To iterate, run python3 ./test.py inside the container (source code is mapped into the container)." + +ln -fs /opt/lambda/index.py index.py +docker run -v $PWD:/opt/awscli -v $PWD/../../lib/lambda:/opt/lambda --workdir /opt/awscli -it $tag \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py b/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py index 30b59c26374f3..05be9a49ff9f8 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py +++ b/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py @@ -7,6 +7,7 @@ import traceback import logging import botocore +import tempfile from botocore.vendored import requests from botocore.exceptions import ClientError from unittest.mock import MagicMock @@ -584,7 +585,45 @@ def mock_make_api_call(self, operation_name, kwarg): self.assertAwsCommands( ["s3", "rm", "s3:///", "--recursive"] ) + + def test_replace_markers(self): + index.extract_and_replace_markers("test.zip", "/tmp/out", { + "_marker2_": "boom-marker2-replaced", + "_marker1_": "<>", + }) + + # assert that markers were replaced in the output + with open("/tmp/out/subfolder/boom.txt", "r") as file: + self.assertEqual(file.read().rstrip(), "Another <> file with boom-marker2-replaced hey!\nLine 2 with <> again :-)") + + with open("/tmp/out/test.txt") as file: + self.assertEqual(file.read().rstrip(), "Hello, <> world") + def test_marker_substitution(self): + outdir = tempfile.mkdtemp() + + invoke_handler("Create", { + "SourceBucketNames": ["", ""], + "SourceObjectKeys": ["", ""], + "DestinationBucketName": "", + "Prune": "false", + "SourceMarkers": [ + json.dumps({"_marker1_": "value1-source1", "_marker2_": "value2-source1"}), + json.dumps({"_marker1_": "value1-source2"}), + ], + }, outdir=outdir) + + # outdir is expected to have a single directory that contains the workdir + files = os.listdir(outdir) + self.assertEqual(len(files), 1) # defensive + + workdir = os.path.join(outdir, files[0], "contents") + + with open(os.path.join(workdir, "test.txt"), "r") as file: + self.assertEqual(file.read().rstrip(), "Hello, value1-source2 world") + + with open(os.path.join(workdir, "subfolder", "boom.txt"), "r") as file: + self.assertEqual(file.read().rstrip(), "Another value1-source2 file with _marker2_ hey!\nLine 2 with value1-source2 again :-)") # asserts that a given list of "aws xxx" commands have been invoked (in order) @@ -609,7 +648,7 @@ def read_aws_out(): # requestType: CloudFormation request type ("Create", "Update", "Delete") # resourceProps: map to pass to "ResourceProperties" # expected_status: "SUCCESS" or "FAILED" -def invoke_handler(requestType, resourceProps, old_resource_props=None, physical_id=None, expected_status='SUCCESS'): +def invoke_handler(requestType, resourceProps, old_resource_props=None, physical_id=None, expected_status='SUCCESS', outdir=None): response_url = 'http://' event={ @@ -636,6 +675,11 @@ class ResponseMock: context = ContextMock() index.urlopen = MagicMock(return_value=ResponseMock()) + # control the output directory and skip cleanup so we can examine the output + if outdir: + os.environ[index.ENV_KEY_MOUNT_PATH] = outdir + os.environ[index.ENV_KEY_SKIP_CLEANUP] = "1" + #-------------------- # invoke the handler #-------------------- diff --git a/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.zip b/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.zip index 56829f65a2681a83665ce8539b84c0dd081e97bd..784486f7d04cde8c6489a902d8f636e288eab392 100644 GIT binary patch literal 560 zcmWIWW@h1H00Dgu`w%b#N^mjAFcg<2rRC?Oq!#IihHx@4dv$(K^akS63T_5Q7VS;F z8Z06})d4^=Kr{!~i~^5R18X498;FHa%t*@5&($lbD8Xk|@tM>5Cp~n7I?ns*obuB( z>IwHaea16Xg)6|D zkx7mjS13sUEdT)lhPRF&8ad2ZAz_9QULf;ug&4#$ d<{|c3;=PefC2yj literal 179 zcmWIWW@h1H00AzGVBcl-QSQ7zHVAVt$S{d2?{eZMTh{FH?)6OLz From 2fa4acf124ea1f7ce932bef9da2fce50e5b288db Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Tue, 1 Feb 2022 20:09:33 +0200 Subject: [PATCH 04/14] tests, jsonData, and more --- packages/@aws-cdk/aws-s3-deployment/README.md | 29 ++++++++++- .../@aws-cdk/aws-s3-deployment/lib/content.ts | 2 +- .../@aws-cdk/aws-s3-deployment/lib/source.ts | 39 ++++++++++++--- .../test/bucket-deployment.test.ts | 50 +++++++++++++++++-- .../aws-s3-deployment/test/content.test.ts | 44 +++++++++++++++- .../test/integ.bucket-deployment-content.ts | 4 +- .../aws-s3-deployment/test/lambda/test.sh | 3 ++ 7 files changed, 153 insertions(+), 18 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-deployment/README.md b/packages/@aws-cdk/aws-s3-deployment/README.md index 1b86bb4fa92e4..9ba6686c71f9c 100644 --- a/packages/@aws-cdk/aws-s3-deployment/README.md +++ b/packages/@aws-cdk/aws-s3-deployment/README.md @@ -49,6 +49,7 @@ The following source types are supported for bucket deployments: - Local .zip file: `s3deploy.Source.asset('/path/to/local/file.zip')` - Local directory: `s3deploy.Source.asset('/path/to/local/directory')` - Another bucket: `s3deploy.Source.bucket(bucket, zipObjectKey)` +- Textual data: `s3deploy.Source.data('object-key.txt', 'hello, world!')` To create a source from a single file, you can pass `AssetOptions` to exclude all but a single file: @@ -268,6 +269,32 @@ new s3deploy.BucketDeployment(this, 'DeployMeWithEfsStorage', { }); ``` +## Data with deploy-time values + +The content passed to `Source.data()` or `Source.jsonData()` can include +references that will get resolved only during deployment. + +For example: + +```ts +declare const destinationBucket: s3.Bucket; +declare const topic: sns.Topic; + +const appConfig = { + topic_arn: topic.topicArn, + base_url: 'https://my-endpoint', +}; + +new s3deploy.BucketDeployment(this, 'BucketDeployment', { + sources: [s3deploy.Source.jsonData('config.json', config)], + destinationBucket, +}); +``` + +The value in `topic.topicArn` is a deploy-time value. It only gets resolved +during deployment by placing a marker in the generated source file and +substituting it when its deployed to the destination with the actual value. + ## Notes - This library uses an AWS CloudFormation custom resource which about 10MiB in @@ -282,7 +309,7 @@ new s3deploy.BucketDeployment(this, 'DeployMeWithEfsStorage', { be good enough: the custom resource will simply not run if the properties don't change. - If you use assets (`s3deploy.Source.asset()`) you don't need to worry - about this: the asset system will make sure that if the files have changed, + about this: the asset system will make sure that if the files have changed, the file name is unique and the deployment will run. ## Development diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/content.ts b/packages/@aws-cdk/aws-s3-deployment/lib/content.ts index 19d0bd998acfa..0fbc57e69f983 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/content.ts +++ b/packages/@aws-cdk/aws-s3-deployment/lib/content.ts @@ -39,7 +39,7 @@ export function renderContent(scope: Construct, content: string): Content { createMarker(part); } } - } else if (obj.Ref) { + } else if (obj.Ref || obj['Fn::GetAtt']) { createMarker(obj); } else { throw new Error('Unexpected resolved value. Expecting Fn::Join or Ref'); diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/source.ts b/packages/@aws-cdk/aws-s3-deployment/lib/source.ts index 34f6c93129ff5..bf86c398846d3 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/source.ts +++ b/packages/@aws-cdk/aws-s3-deployment/lib/source.ts @@ -3,7 +3,7 @@ import { join, dirname } from 'path'; import * as iam from '@aws-cdk/aws-iam'; import * as s3 from '@aws-cdk/aws-s3'; import * as s3_assets from '@aws-cdk/aws-s3-assets'; -import { FileSystem } from '@aws-cdk/core'; +import { FileSystem, Stack } from '@aws-cdk/core'; import { renderContent } from './content'; // keep this import separate from other imports to reduce chance for merge conflicts with v2-main @@ -26,6 +26,7 @@ export interface SourceConfig { /** * A set of markers to substitute in the source content. + * @default - no markers */ readonly markers?: Record; } @@ -59,6 +60,8 @@ export interface ISource { * Source.bucket(bucket, key) * Source.asset('/local/path/to/directory') * Source.asset('/local/path/to/a/file.zip') + * Source.data('hello/world/file.txt', 'Hello, world!') + * Source.data('config.json', { baz: topic.topicArn }) * */ export class Source { @@ -120,19 +123,22 @@ export class Source { } /** - * Deploys a file with the specified textual contents into the bucket. The - * content can include deploy-time values that will get resolved only during - * deployment. + * Deploys an object with the specified string contents into the bucket. The + * content can include deploy-time values (such as `snsTopic.topicArn`) that + * will get resolved only during deployment. * - * @param objectKey The S3 object key to use for this file. - * @param content The contents + * To store a JSON object use `Source.jsonData()`. + * + * @param objectKey The destination S3 object key (relative to the root of the + * S3 deployment). + * @param data The data to be stored in the object. */ - public static content(objectKey: string, content: string): ISource { + public static data(objectKey: string, data: string): ISource { return { bind: (scope: Construct, context?: DeploymentSourceContext) => { const workdir = FileSystem.mkdtemp('s3-deployment'); const outputPath = join(workdir, objectKey); - const rendered = renderContent(scope, content); + const rendered = renderContent(scope, data); fs.mkdirSync(dirname(outputPath), { recursive: true }); fs.writeFileSync(outputPath, rendered.text); const asset = this.asset(workdir).bind(scope, context); @@ -145,5 +151,22 @@ export class Source { }; } + /** + * Deploys an object with the specified JSON object into the bucket. The + * object can include deploy-time values (such as `snsTopic.topicArn`) that + * will get resolved only during deployment. + * + * @param objectKey The destination S3 object key (relative to the root of the + * S3 deployment). + * @param obj A JSON object. + */ + public static jsonData(objectKey: string, obj: any): ISource { + return { + bind: (scope: Construct, context?: DeploymentSourceContext) => { + return Source.data(objectKey, Stack.of(scope).toJsonString(obj)).bind(scope, context); + }, + }; + } + private constructor() { } } diff --git a/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts b/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts index 1d38816e2957b..a8372289dc0be 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts +++ b/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts @@ -1,3 +1,4 @@ +import { readFileSync } from 'fs'; import * as path from 'path'; import { Match, Template } from '@aws-cdk/assertions'; import * as cloudfront from '@aws-cdk/aws-cloudfront'; @@ -1057,12 +1058,12 @@ test('bucket has multiple deployments', () => { }); }); -test('Source.content() can be used to create a file with contents', () => { +test('Source.data() can be used to create a file with string contents', () => { const app = new cdk.App(); const stack = new cdk.Stack(app, 'Test'); const bucket = new s3.Bucket(stack, 'Bucket'); - const source = s3deploy.Source.content('my/path.txt', 'hello, world'); + const source = s3deploy.Source.data('my/path.txt', 'hello, world'); new s3deploy.BucketDeployment(stack, 'DeployWithVpc3', { sources: [source], @@ -1071,5 +1072,46 @@ test('Source.content() can be used to create a file with contents', () => { }); const result = app.synth(); - expect(result.stacks[0].assets).toStrictEqual([]); -}); \ No newline at end of file + const content = readDataFile(result, 'c5b1c01fc092abf1da35f6772e7c507e566aaa69404025c080ba074c69741755', 'my/path.txt'); + expect(content).toStrictEqual('hello, world'); +}); + +test('Source.jsonData() can be used to create a file with a JSON object', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Test'); + const bucket = new s3.Bucket(stack, 'Bucket'); + + const config = { + foo: 'bar', + sub: { + hello: bucket.bucketArn, + }, + }; + + new s3deploy.BucketDeployment(stack, 'DeployWithVpc3', { + sources: [s3deploy.Source.jsonData('app-config.json', config)], + destinationBucket: bucket, + }); + + const result = app.synth(); + const obj = JSON.parse(readDataFile(result, '6a9e1763f42401799363d87d16b238c89bf75a56f2a3f67498a3224573062b0c', 'app-config.json')); + expect(obj).toStrictEqual({ + foo: 'bar', + sub: { + hello: '<>', + }, + }); + + // verify marker is mapped to the bucket ARN in the resource props + Template.fromJSON(result.stacks[0].template).hasResourceProperties('Custom::CDKBucketDeployment', { + SourceMarkers: [ + '{"<>":{"Fn::GetAtt":["Bucket83908E77","Arn"]}}', + ], + }); +}); + +function readDataFile(casm: cxapi.CloudAssembly, assetId: string, filePath: string): string { + const asset = casm.stacks[0].assets.find(a => a.id === assetId); + if (!asset) { throw new Error('Asset not found'); } + return readFileSync(path.join(casm.directory, asset.path, filePath), 'utf-8'); +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-deployment/test/content.test.ts b/packages/@aws-cdk/aws-s3-deployment/test/content.test.ts index eae6c1d2ae706..a4fdb8af6a635 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/content.test.ts +++ b/packages/@aws-cdk/aws-s3-deployment/test/content.test.ts @@ -1,6 +1,6 @@ import * as lambda from '@aws-cdk/aws-lambda'; import * as s3 from '@aws-cdk/aws-s3'; -import { Stack } from '@aws-cdk/core'; +import { Lazy, Stack } from '@aws-cdk/core'; import { Source } from '../lib'; import { renderContent } from '../lib/content'; @@ -32,6 +32,16 @@ test('string is a single "Ref" token', () => { }); }); +test('string is a single "Fn::GetAtt" token', () => { + const stack = new Stack(); + const bucket = new s3.Bucket(stack, 'Bucket'); + + expect(renderContent(stack, bucket.bucketRegionalDomainName)).toStrictEqual({ + text: '<>', + markers: { '<>': { 'Fn::GetAtt': ['Bucket83908E77', 'RegionalDomainName'] } }, + }); +}); + test('string with a "Fn::GetAtt" token', () => { const stack = new Stack(); const bucket = new s3.Bucket(stack, 'Bucket'); @@ -75,8 +85,38 @@ test('json-encoded string', () => { test('markers are returned in the source config', () => { const stack = new Stack(); const handler = new lambda.Function(stack, 'Handler', { runtime: lambda.Runtime.NODEJS_14_X, code: lambda.Code.fromInline('foo'), handler: 'index.handler' }); - const actual = Source.content('file1.txt', `boom-${stack.account}`).bind(stack, { handlerRole: handler.role! }); + const actual = Source.data('file1.txt', `boom-${stack.account}`).bind(stack, { handlerRole: handler.role! }); expect(actual.markers).toStrictEqual({ '<>': { Ref: 'AWS::AccountId' }, }); +}); + +test('lazy string which can be fully resolved', () => { + const stack = new Stack(); + + expect(renderContent(stack, Lazy.string({ produce: () => 'resolved!' }))).toStrictEqual({ + text: 'resolved!', + markers: { }, + }); +}); + +test('lazy within a string which can be fully resolved', () => { + const stack = new Stack(); + const token = Lazy.string({ produce: () => 'resolved!' }); + + expect(renderContent(stack, `hello, ${token}`)).toStrictEqual({ + text: 'hello, resolved!', + markers: { }, + }); +}); + +test('lazy string which resolves to something with a deploy-time value', () => { + const stack = new Stack(); + const token = Lazy.string({ produce: () => 'resolved!' }); + + expect(renderContent(stack, `hello, ${token}`)).toStrictEqual({ + text: 'hello, resolved!', + markers: { }, + }); + }); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-content.ts b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-content.ts index 939c6ffc786ce..5679a83cd2da4 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-content.ts +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-content.ts @@ -6,8 +6,8 @@ const app = new App(); const stack = new Stack(app, 'TestBucketDeploymentContent'); const bucket = new Bucket(stack, 'Bucket'); -const file1 = Source.content('file1.txt', 'boom'); -const file2 = Source.content('path/to/file2.txt', `bam! ${bucket.bucketName}`); +const file1 = Source.data('file1.txt', 'boom'); +const file2 = Source.data('path/to/file2.txt', `bam! ${bucket.bucketName}`); new BucketDeployment(stack, 'DeployMe', { destinationBucket: bucket, diff --git a/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.sh b/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.sh index a094c8ae16cfa..8f0c0d2b473b7 100755 --- a/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.sh +++ b/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.sh @@ -6,6 +6,9 @@ set -e scriptdir=$(cd $(dirname $0) && pwd) +rm -f ${scriptdir}/index.py +rm -fr ${scriptdir}/__pycache__ + # prepare staging directory staging=$(mktemp -d) mkdir -p ${staging} From 044003ff14b7df1e425291dbc2fbda752e000587 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Tue, 1 Feb 2022 20:12:21 +0200 Subject: [PATCH 05/14] update readme --- packages/@aws-cdk/aws-s3-deployment/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-s3-deployment/README.md b/packages/@aws-cdk/aws-s3-deployment/README.md index 9ba6686c71f9c..eb0002593dbbe 100644 --- a/packages/@aws-cdk/aws-s3-deployment/README.md +++ b/packages/@aws-cdk/aws-s3-deployment/README.md @@ -49,7 +49,10 @@ The following source types are supported for bucket deployments: - Local .zip file: `s3deploy.Source.asset('/path/to/local/file.zip')` - Local directory: `s3deploy.Source.asset('/path/to/local/directory')` - Another bucket: `s3deploy.Source.bucket(bucket, zipObjectKey)` -- Textual data: `s3deploy.Source.data('object-key.txt', 'hello, world!')` +- String data: `s3deploy.Source.data('object-key.txt', 'hello, world!')` + (supports [deploy-time values](#data-with-deploy-time-values)) +- JSON data: `s3deploy.Source.jsonData('object-key.json', { json: 'object' })` + (supports [deploy-time values](#data-with-deploy-time-values)) To create a source from a single file, you can pass `AssetOptions` to exclude all but a single file: From 1c8e5ca8a211d57b6d5bb8ec6cb275c831b979ad Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Tue, 1 Feb 2022 21:17:26 +0200 Subject: [PATCH 06/14] integration tests pass --- .../lib/bucket-deployment.ts | 2 +- .../aws-s3-deployment/lib/lambda/index.py | 6 +- .../test/bucket-deployment.test.ts | 2 +- ...nteg.bucket-deployment-data.expected.json} | 152 +++++++++++++++--- ...ent.ts => integ.bucket-deployment-data.ts} | 3 +- .../aws-s3-deployment/test/lambda/test.py | 4 +- 6 files changed, 141 insertions(+), 28 deletions(-) rename packages/@aws-cdk/aws-s3-deployment/test/{integ.bucket-deployment-content.expected.json => integ.bucket-deployment-data.expected.json} (66%) rename packages/@aws-cdk/aws-s3-deployment/test/{integ.bucket-deployment-content.ts => integ.bucket-deployment-data.ts} (82%) diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts index 3519a203dd737..357a3ee04b617 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts +++ b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts @@ -341,7 +341,7 @@ export class BucketDeployment extends CoreConstruct { properties: { SourceBucketNames: sources.map(source => source.bucket.bucketName), SourceObjectKeys: sources.map(source => source.zipObjectKey), - SourceMarkers: sources.map(source => Stack.of(this).toJsonString(source.markers ?? {})), + SourceMarkers: sources.map(source => source.markers ?? {}), DestinationBucketName: props.destinationBucket.bucketName, DestinationBucketKeyPrefix: props.destinationKeyPrefix, RetainOnDelete: props.retainOnDelete, diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py b/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py index 63425cd972bae..4eae5f401a890 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py +++ b/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py @@ -78,7 +78,7 @@ def cfn_error(message=None): if dest_bucket_prefix == "/": dest_bucket_prefix = "" - s3_source_zips = map(lambda name, key: "s3://%s/%s" % (name, key), source_bucket_names, source_object_keys) + s3_source_zips = list(map(lambda name, key: "s3://%s/%s" % (name, key), source_bucket_names, source_object_keys)) s3_dest = "s3://%s/%s" % (dest_bucket_name, dest_bucket_prefix) old_s3_dest = "s3://%s/%s" % (old_props.get("DestinationBucketName", ""), old_props.get("DestinationBucketKeyPrefix", "")) @@ -129,7 +129,7 @@ def cfn_error(message=None): # populate all files from s3_source_zips to a destination bucket def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers): # list lengths are equal - if len(list(s3_source_zips)) != len(source_markers): + if len(s3_source_zips) != len(source_markers): raise Exception("'source_markers' and 's3_source_zips' must be the same length") # create a temporary working directory in /tmp or if enabled an attached efs volume @@ -149,7 +149,7 @@ def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, ex # download the archive from the source and extract to "contents" for i in range(len(s3_source_zips)): s3_source_zip = s3_source_zips[i] - markers = json.loads(source_markers[i]) + markers = source_markers[i] archive=os.path.join(workdir, str(uuid4())) logger.info("archive: %s" % archive) diff --git a/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts b/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts index a8372289dc0be..67c007f87c46f 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts +++ b/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts @@ -1105,7 +1105,7 @@ test('Source.jsonData() can be used to create a file with a JSON object', () => // verify marker is mapped to the bucket ARN in the resource props Template.fromJSON(result.stacks[0].template).hasResourceProperties('Custom::CDKBucketDeployment', { SourceMarkers: [ - '{"<>":{"Fn::GetAtt":["Bucket83908E77","Arn"]}}', + { '<>': { 'Fn::GetAtt': ['Bucket83908E77', 'Arn'] } }, ], }); }); diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-content.expected.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.expected.json similarity index 66% rename from packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-content.expected.json rename to packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.expected.json index 8fccff7a6f382..df1906a27c275 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-content.expected.json +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.expected.json @@ -71,7 +71,10 @@ "Ref": "AssetParametersd09271be89b6cb0398f793b40c1531fd9b076aa92ba80b5e436914b1808fe18dS3BucketBC52CF96" }, { - "Ref": "AssetParametersf58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229S3BucketC0B57179" + "Ref": "AssetParameters0f14dedeaf4386031c978375cbda0f65d7b52b29452cabb8873eb8f0d0fa936bS3BucketE46D7C76" + }, + { + "Ref": "AssetParameters0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5aS3Bucket485B8F98" } ], "SourceObjectKeys": [ @@ -119,7 +122,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersf58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229S3VersionKeyA1412544" + "Ref": "AssetParameters0f14dedeaf4386031c978375cbda0f65d7b52b29452cabb8873eb8f0d0fa936bS3VersionKey9FBF1498" } ] } @@ -132,7 +135,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersf58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229S3VersionKeyA1412544" + "Ref": "AssetParameters0f14dedeaf4386031c978375cbda0f65d7b52b29452cabb8873eb8f0d0fa936bS3VersionKey9FBF1498" } ] } @@ -140,6 +143,55 @@ } ] ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5aS3VersionKeyC8D63B0C" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5aS3VersionKeyC8D63B0C" + } + ] + } + ] + } + ] + ] + } + ], + "SourceMarkers": [ + {}, + { + "<>": { + "Ref": "Bucket83908E77" + } + }, + { + "<>": { + "Fn::GetAtt": [ + "Bucket83908E77", + "WebsiteURL" + ] + } } ], "DestinationBucketName": { @@ -246,7 +298,7 @@ }, ":s3:::", { - "Ref": "AssetParametersf58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229S3BucketC0B57179" + "Ref": "AssetParameters0f14dedeaf4386031c978375cbda0f65d7b52b29452cabb8873eb8f0d0fa936bS3BucketE46D7C76" } ] ] @@ -261,7 +313,48 @@ }, ":s3:::", { - "Ref": "AssetParametersf58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229S3BucketC0B57179" + "Ref": "AssetParameters0f14dedeaf4386031c978375cbda0f65d7b52b29452cabb8873eb8f0d0fa936bS3BucketE46D7C76" + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Ref": "AssetParameters0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5aS3Bucket485B8F98" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Ref": "AssetParameters0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5aS3Bucket485B8F98" }, "/*" ] @@ -318,7 +411,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameterse3d9996b6fafcc7da88312672e15e3cc925b02cffc6f01a615d81f22303e3ae0S3BucketB3DDCC13" + "Ref": "AssetParametersf26b845e072039dace324c286e111ec18c1f25ee43ee37ecb18353c990008db0S3BucketFB8090D0" }, "S3Key": { "Fn::Join": [ @@ -331,7 +424,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameterse3d9996b6fafcc7da88312672e15e3cc925b02cffc6f01a615d81f22303e3ae0S3VersionKey3418DF70" + "Ref": "AssetParametersf26b845e072039dace324c286e111ec18c1f25ee43ee37ecb18353c990008db0S3VersionKey4E8B9991" } ] } @@ -344,7 +437,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameterse3d9996b6fafcc7da88312672e15e3cc925b02cffc6f01a615d81f22303e3ae0S3VersionKey3418DF70" + "Ref": "AssetParametersf26b845e072039dace324c286e111ec18c1f25ee43ee37ecb18353c990008db0S3VersionKey4E8B9991" } ] } @@ -388,17 +481,17 @@ "Type": "String", "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" }, - "AssetParameterse3d9996b6fafcc7da88312672e15e3cc925b02cffc6f01a615d81f22303e3ae0S3BucketB3DDCC13": { + "AssetParametersf26b845e072039dace324c286e111ec18c1f25ee43ee37ecb18353c990008db0S3BucketFB8090D0": { "Type": "String", - "Description": "S3 bucket for asset \"e3d9996b6fafcc7da88312672e15e3cc925b02cffc6f01a615d81f22303e3ae0\"" + "Description": "S3 bucket for asset \"f26b845e072039dace324c286e111ec18c1f25ee43ee37ecb18353c990008db0\"" }, - "AssetParameterse3d9996b6fafcc7da88312672e15e3cc925b02cffc6f01a615d81f22303e3ae0S3VersionKey3418DF70": { + "AssetParametersf26b845e072039dace324c286e111ec18c1f25ee43ee37ecb18353c990008db0S3VersionKey4E8B9991": { "Type": "String", - "Description": "S3 key for asset version \"e3d9996b6fafcc7da88312672e15e3cc925b02cffc6f01a615d81f22303e3ae0\"" + "Description": "S3 key for asset version \"f26b845e072039dace324c286e111ec18c1f25ee43ee37ecb18353c990008db0\"" }, - "AssetParameterse3d9996b6fafcc7da88312672e15e3cc925b02cffc6f01a615d81f22303e3ae0ArtifactHash9D8F179A": { + "AssetParametersf26b845e072039dace324c286e111ec18c1f25ee43ee37ecb18353c990008db0ArtifactHash47188325": { "Type": "String", - "Description": "Artifact hash for asset \"e3d9996b6fafcc7da88312672e15e3cc925b02cffc6f01a615d81f22303e3ae0\"" + "Description": "Artifact hash for asset \"f26b845e072039dace324c286e111ec18c1f25ee43ee37ecb18353c990008db0\"" }, "AssetParametersd09271be89b6cb0398f793b40c1531fd9b076aa92ba80b5e436914b1808fe18dS3BucketBC52CF96": { "Type": "String", @@ -412,17 +505,36 @@ "Type": "String", "Description": "Artifact hash for asset \"d09271be89b6cb0398f793b40c1531fd9b076aa92ba80b5e436914b1808fe18d\"" }, - "AssetParametersf58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229S3BucketC0B57179": { + "AssetParameters0f14dedeaf4386031c978375cbda0f65d7b52b29452cabb8873eb8f0d0fa936bS3BucketE46D7C76": { + "Type": "String", + "Description": "S3 bucket for asset \"0f14dedeaf4386031c978375cbda0f65d7b52b29452cabb8873eb8f0d0fa936b\"" + }, + "AssetParameters0f14dedeaf4386031c978375cbda0f65d7b52b29452cabb8873eb8f0d0fa936bS3VersionKey9FBF1498": { + "Type": "String", + "Description": "S3 key for asset version \"0f14dedeaf4386031c978375cbda0f65d7b52b29452cabb8873eb8f0d0fa936b\"" + }, + "AssetParameters0f14dedeaf4386031c978375cbda0f65d7b52b29452cabb8873eb8f0d0fa936bArtifactHash1774126C": { "Type": "String", - "Description": "S3 bucket for asset \"f58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229\"" + "Description": "Artifact hash for asset \"0f14dedeaf4386031c978375cbda0f65d7b52b29452cabb8873eb8f0d0fa936b\"" }, - "AssetParametersf58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229S3VersionKeyA1412544": { + "AssetParameters0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5aS3Bucket485B8F98": { "Type": "String", - "Description": "S3 key for asset version \"f58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229\"" + "Description": "S3 bucket for asset \"0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5a\"" }, - "AssetParametersf58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229ArtifactHash24BF3233": { + "AssetParameters0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5aS3VersionKeyC8D63B0C": { "Type": "String", - "Description": "Artifact hash for asset \"f58a7925a8ef0a032be4511ea00bfb88121c6627b3703f16867496256c9c8229\"" + "Description": "S3 key for asset version \"0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5a\"" + }, + "AssetParameters0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5aArtifactHashA3962596": { + "Type": "String", + "Description": "Artifact hash for asset \"0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5a\"" + } + }, + "Outputs": { + "BucketName": { + "Value": { + "Ref": "Bucket83908E77" + } } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-content.ts b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.ts similarity index 82% rename from packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-content.ts rename to packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.ts index 5679a83cd2da4..45db0762527fd 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-content.ts +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.ts @@ -8,10 +8,11 @@ const bucket = new Bucket(stack, 'Bucket'); const file1 = Source.data('file1.txt', 'boom'); const file2 = Source.data('path/to/file2.txt', `bam! ${bucket.bucketName}`); +const file3 = Source.jsonData('my/config.json', { website_url: bucket.bucketWebsiteUrl }); new BucketDeployment(stack, 'DeployMe', { destinationBucket: bucket, - sources: [file1, file2], + sources: [file1, file2, file3], destinationKeyPrefix: 'deploy/here/', }); diff --git a/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py b/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py index 05be9a49ff9f8..b999aa5a4e797 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py +++ b/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py @@ -608,8 +608,8 @@ def test_marker_substitution(self): "DestinationBucketName": "", "Prune": "false", "SourceMarkers": [ - json.dumps({"_marker1_": "value1-source1", "_marker2_": "value2-source1"}), - json.dumps({"_marker1_": "value1-source2"}), + { "_marker1_": "value1-source1", "_marker2_": "value2-source1" }, + { "_marker1_": "value1-source2" }, ], }, outdir=outdir) From 22e8a6caa0d829912ec135ad4ea9a1e56a3aecf4 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Tue, 1 Feb 2022 21:33:34 +0200 Subject: [PATCH 07/14] cleanups, improved errors and commens --- .../lib/bucket-deployment.ts | 4 - .../@aws-cdk/aws-s3-deployment/lib/content.ts | 64 --------------- .../aws-s3-deployment/lib/render-data.ts | 81 +++++++++++++++++++ .../@aws-cdk/aws-s3-deployment/lib/source.ts | 4 +- .../aws-s3-deployment/test/content.test.ts | 22 ++--- 5 files changed, 94 insertions(+), 81 deletions(-) delete mode 100644 packages/@aws-cdk/aws-s3-deployment/lib/content.ts create mode 100644 packages/@aws-cdk/aws-s3-deployment/lib/render-data.ts diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts index 357a3ee04b617..db2d883718cda 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts +++ b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts @@ -12,10 +12,6 @@ import { kebab as toKebabCase } from 'case'; import { Construct } from 'constructs'; import { ISource, SourceConfig } from './source'; -// keep this import separate from other imports to reduce chance for merge conflicts with v2-main -// eslint-disable-next-line no-duplicate-imports, import/order -import { Stack } from '@aws-cdk/core'; - // keep this import separate from other imports to reduce chance for merge conflicts with v2-main // eslint-disable-next-line no-duplicate-imports, import/order import { Construct as CoreConstruct } from '@aws-cdk/core'; diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/content.ts b/packages/@aws-cdk/aws-s3-deployment/lib/content.ts deleted file mode 100644 index 0fbc57e69f983..0000000000000 --- a/packages/@aws-cdk/aws-s3-deployment/lib/content.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { Stack } from '@aws-cdk/core'; - -// keep this import separate from other imports to reduce chance for merge conflicts with v2-main -// eslint-disable-next-line no-duplicate-imports, import/order -import { Construct } from '@aws-cdk/core'; - -export interface Content { - readonly text: string; - readonly markers: Record; -} - -export function renderContent(scope: Construct, content: string): Content { - const obj = Stack.of(scope).resolve(content); - if (typeof obj === 'string') { - return { text: obj, markers: {} }; - } - - const markers: Record = {}; - const result = new Array(); - let markerIndex = 0; - - if (obj['Fn::Join']) { - const fnJoin: FnJoin = obj['Fn::Join']; - if (fnJoin[0] !== '') { - throw new Error('Unexpected join, expecting separator to be ""'); - } - - for (const part of fnJoin[1]) { - if (typeof (part) === 'string') { - result.push(part); - continue; - } - - if (typeof (part) === 'object') { - const keys: string[] = Object.keys(part); - if (keys.length !== 1) { - throw new Error('Invalid object'); - } - createMarker(part); - } - } - } else if (obj.Ref || obj['Fn::GetAtt']) { - createMarker(obj); - } else { - throw new Error('Unexpected resolved value. Expecting Fn::Join or Ref'); - } - - function createMarker(part: Ref | GetAtt) { - if ('Ref' in part || 'Fn::GetAtt' in part) { - const marker = `<>`; - result.push(marker); - markers[marker] = part; - } else { - throw new Error('Invalid object'); - } - } - - return { text: result.join(''), markers }; -} - -type FnJoin = [string, FnJoinPart[]]; -type FnJoinPart = string | Ref | GetAtt; -type Ref = { Ref: string }; -type GetAtt = { 'Fn::GetAtt': [string, string] }; \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/render-data.ts b/packages/@aws-cdk/aws-s3-deployment/lib/render-data.ts new file mode 100644 index 0000000000000..fa73c55dc2fba --- /dev/null +++ b/packages/@aws-cdk/aws-s3-deployment/lib/render-data.ts @@ -0,0 +1,81 @@ +import { Stack } from '@aws-cdk/core'; + +// keep this import separate from other imports to reduce chance for merge conflicts with v2-main +// eslint-disable-next-line no-duplicate-imports, import/order +import { Construct } from '@aws-cdk/core'; + +export interface Content { + readonly text: string; + readonly markers: Record; +} + +/** + * Renders the given string data as deployable content with markers substituted + * for all "Ref" and "Fn::GetAtt" objects. + * + * @param scope Construct scope + * @param data The input data + * @returns The markered text (`text`) and a map that maps marker names to their + * values (`markers`). + */ +export function renderData(scope: Construct, data: string): Content { + const obj = Stack.of(scope).resolve(data); + if (typeof(obj) === 'string') { + return { text: obj, markers: {} }; + } + + if (typeof(obj) !== 'object') { + throw new Error(`Unexpected: after resolve() data must either be a string or a CloudFormation intrinsic. Got: ${JSON.stringify(obj)}`); + } + + let markerIndex = 0; + const markers: Record = {}; + const result = new Array(); + const fnJoin: FnJoin | undefined = obj['Fn::Join']; + + if (fnJoin) { + const sep = fnJoin[0]; + const parts = fnJoin[1]; + + if (sep !== '') { + throw new Error(`Unexpected "Fn::Join", expecting separator to be an empty string but got "${sep}"`); + } + + for (const part of parts) { + if (typeof (part) === 'string') { + result.push(part); + continue; + } + + if (typeof (part) === 'object') { + addMarker(part); + continue; + } + + throw new Error(`Unexpected "Fn::Join" part, expecting string or object but got ${typeof (part)}`); + } + + } else if (obj.Ref || obj['Fn::GetAtt']) { + addMarker(obj); + } else { + throw new Error('Unexpected: Expecting `resolve()` to return "Fn::Join", "Ref" or "Fn::GetAtt"'); + } + + function addMarker(part: Ref | GetAtt) { + const keys = Object.keys(part); + if (keys.length !== 1 || (keys[0] != 'Ref' && keys[0] != 'Fn::GetAtt')) { + throw new Error(`Invalid CloudFormation reference. "Ref" or "Fn::GetAtt". Got ${JSON.stringify(part)}`); + } + + const marker = `<>`; + result.push(marker); + markers[marker] = part; + } + + return { text: result.join(''), markers }; +} + +type FnJoin = [string, FnJoinPart[]]; +type FnJoinPart = string | Ref | GetAtt; +type Ref = { Ref: string }; +type GetAtt = { 'Fn::GetAtt': [string, string] }; \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/source.ts b/packages/@aws-cdk/aws-s3-deployment/lib/source.ts index bf86c398846d3..0a02dd4503b01 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/source.ts +++ b/packages/@aws-cdk/aws-s3-deployment/lib/source.ts @@ -4,7 +4,7 @@ import * as iam from '@aws-cdk/aws-iam'; import * as s3 from '@aws-cdk/aws-s3'; import * as s3_assets from '@aws-cdk/aws-s3-assets'; import { FileSystem, Stack } from '@aws-cdk/core'; -import { renderContent } from './content'; +import { renderData } from './render-data'; // keep this import separate from other imports to reduce chance for merge conflicts with v2-main // eslint-disable-next-line no-duplicate-imports, import/order @@ -138,7 +138,7 @@ export class Source { bind: (scope: Construct, context?: DeploymentSourceContext) => { const workdir = FileSystem.mkdtemp('s3-deployment'); const outputPath = join(workdir, objectKey); - const rendered = renderContent(scope, data); + const rendered = renderData(scope, data); fs.mkdirSync(dirname(outputPath), { recursive: true }); fs.writeFileSync(outputPath, rendered.text); const asset = this.asset(workdir).bind(scope, context); diff --git a/packages/@aws-cdk/aws-s3-deployment/test/content.test.ts b/packages/@aws-cdk/aws-s3-deployment/test/content.test.ts index a4fdb8af6a635..3cc27a2dc7262 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/content.test.ts +++ b/packages/@aws-cdk/aws-s3-deployment/test/content.test.ts @@ -2,11 +2,11 @@ import * as lambda from '@aws-cdk/aws-lambda'; import * as s3 from '@aws-cdk/aws-s3'; import { Lazy, Stack } from '@aws-cdk/core'; import { Source } from '../lib'; -import { renderContent } from '../lib/content'; +import { renderData } from '../lib/render-data'; test('simple string', () => { const stack = new Stack(); - expect(renderContent(stack, 'foo')).toStrictEqual({ + expect(renderData(stack, 'foo')).toStrictEqual({ markers: {}, text: 'foo', }); @@ -16,7 +16,7 @@ test('string with a "Ref" token', () => { const stack = new Stack(); const bucket = new s3.Bucket(stack, 'Bucket'); - expect(renderContent(stack, `foo-${bucket.bucketName}`)).toStrictEqual({ + expect(renderData(stack, `foo-${bucket.bucketName}`)).toStrictEqual({ text: 'foo-<>', markers: { '<>': { Ref: 'Bucket83908E77' } }, }); @@ -26,7 +26,7 @@ test('string is a single "Ref" token', () => { const stack = new Stack(); const bucket = new s3.Bucket(stack, 'Bucket'); - expect(renderContent(stack, bucket.bucketName)).toStrictEqual({ + expect(renderData(stack, bucket.bucketName)).toStrictEqual({ text: '<>', markers: { '<>': { Ref: 'Bucket83908E77' } }, }); @@ -36,7 +36,7 @@ test('string is a single "Fn::GetAtt" token', () => { const stack = new Stack(); const bucket = new s3.Bucket(stack, 'Bucket'); - expect(renderContent(stack, bucket.bucketRegionalDomainName)).toStrictEqual({ + expect(renderData(stack, bucket.bucketRegionalDomainName)).toStrictEqual({ text: '<>', markers: { '<>': { 'Fn::GetAtt': ['Bucket83908E77', 'RegionalDomainName'] } }, }); @@ -46,7 +46,7 @@ test('string with a "Fn::GetAtt" token', () => { const stack = new Stack(); const bucket = new s3.Bucket(stack, 'Bucket'); - expect(renderContent(stack, `foo-${bucket.bucketArn}`)).toStrictEqual({ + expect(renderData(stack, `foo-${bucket.bucketArn}`)).toStrictEqual({ text: 'foo-<>', markers: { '<>': { 'Fn::GetAtt': ['Bucket83908E77', 'Arn'] } }, }); @@ -56,7 +56,7 @@ test('multiple markers', () => { const stack = new Stack(); const bucket = new s3.Bucket(stack, 'Bucket'); - expect(renderContent(stack, `boom-${bucket.bucketName}-bam-${bucket.bucketArn}`)).toStrictEqual({ + expect(renderData(stack, `boom-${bucket.bucketName}-bam-${bucket.bucketArn}`)).toStrictEqual({ text: 'boom-<>-bam-<>', markers: { '<>': { Ref: 'Bucket83908E77' }, @@ -73,7 +73,7 @@ test('json-encoded string', () => { BucketName: bucket.bucketName, }; - expect(renderContent(stack, JSON.stringify(json))).toStrictEqual({ + expect(renderData(stack, JSON.stringify(json))).toStrictEqual({ text: JSON.stringify({ BucketArn: '<>', BucketName: '<>' }), markers: { '<>': { 'Fn::GetAtt': ['Bucket83908E77', 'Arn'] }, @@ -94,7 +94,7 @@ test('markers are returned in the source config', () => { test('lazy string which can be fully resolved', () => { const stack = new Stack(); - expect(renderContent(stack, Lazy.string({ produce: () => 'resolved!' }))).toStrictEqual({ + expect(renderData(stack, Lazy.string({ produce: () => 'resolved!' }))).toStrictEqual({ text: 'resolved!', markers: { }, }); @@ -104,7 +104,7 @@ test('lazy within a string which can be fully resolved', () => { const stack = new Stack(); const token = Lazy.string({ produce: () => 'resolved!' }); - expect(renderContent(stack, `hello, ${token}`)).toStrictEqual({ + expect(renderData(stack, `hello, ${token}`)).toStrictEqual({ text: 'hello, resolved!', markers: { }, }); @@ -114,7 +114,7 @@ test('lazy string which resolves to something with a deploy-time value', () => { const stack = new Stack(); const token = Lazy.string({ produce: () => 'resolved!' }); - expect(renderContent(stack, `hello, ${token}`)).toStrictEqual({ + expect(renderData(stack, `hello, ${token}`)).toStrictEqual({ text: 'hello, resolved!', markers: { }, }); From efce5a9e32a32b2be1ccb4bf833b1af2bd564fee Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Wed, 2 Feb 2022 07:53:25 +0200 Subject: [PATCH 08/14] fix backwards compatibility --- packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py b/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py index 4eae5f401a890..14b1cc479e9d8 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py +++ b/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py @@ -60,7 +60,7 @@ def cfn_error(message=None): # backwards compatibility - if "SourceMarkers" is not specified, # assume all sources have an empty market map if source_markers is None: - source_markers = [json.dumps({}) for i in range(len(source_bucket_names))] + source_markers = [{} for i in range(len(source_bucket_names))] default_distribution_path = dest_bucket_prefix if not default_distribution_path.endswith("/"): From 9b0283dc0cc57060806d2dfe65972740dafdcdf2 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Wed, 2 Feb 2022 10:24:39 +0200 Subject: [PATCH 09/14] update integ --- ...integ.bucket-deployment-data.expected.json | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.expected.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.expected.json index df1906a27c275..f85e5f0821aca 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.expected.json +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.expected.json @@ -18,7 +18,7 @@ "Properties": { "Content": { "S3Bucket": { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3Bucket59E5CFEF" }, "S3Key": { "Fn::Join": [ @@ -31,7 +31,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C" } ] } @@ -44,7 +44,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C" } ] } @@ -411,7 +411,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParametersf26b845e072039dace324c286e111ec18c1f25ee43ee37ecb18353c990008db0S3BucketFB8090D0" + "Ref": "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eS3BucketC3F9EAA2" }, "S3Key": { "Fn::Join": [ @@ -424,7 +424,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersf26b845e072039dace324c286e111ec18c1f25ee43ee37ecb18353c990008db0S3VersionKey4E8B9991" + "Ref": "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eS3VersionKey030ACBFF" } ] } @@ -437,7 +437,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersf26b845e072039dace324c286e111ec18c1f25ee43ee37ecb18353c990008db0S3VersionKey4E8B9991" + "Ref": "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eS3VersionKey030ACBFF" } ] } @@ -469,29 +469,29 @@ } }, "Parameters": { - "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7": { + "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3Bucket59E5CFEF": { "Type": "String", - "Description": "S3 bucket for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" + "Description": "S3 bucket for asset \"187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0\"" }, - "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F": { + "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C": { "Type": "String", - "Description": "S3 key for asset version \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" + "Description": "S3 key for asset version \"187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0\"" }, - "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68ArtifactHashD9A515C3": { + "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0ArtifactHash8F73A2B0": { "Type": "String", - "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" + "Description": "Artifact hash for asset \"187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0\"" }, - "AssetParametersf26b845e072039dace324c286e111ec18c1f25ee43ee37ecb18353c990008db0S3BucketFB8090D0": { + "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eS3BucketC3F9EAA2": { "Type": "String", - "Description": "S3 bucket for asset \"f26b845e072039dace324c286e111ec18c1f25ee43ee37ecb18353c990008db0\"" + "Description": "S3 bucket for asset \"4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282e\"" }, - "AssetParametersf26b845e072039dace324c286e111ec18c1f25ee43ee37ecb18353c990008db0S3VersionKey4E8B9991": { + "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eS3VersionKey030ACBFF": { "Type": "String", - "Description": "S3 key for asset version \"f26b845e072039dace324c286e111ec18c1f25ee43ee37ecb18353c990008db0\"" + "Description": "S3 key for asset version \"4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282e\"" }, - "AssetParametersf26b845e072039dace324c286e111ec18c1f25ee43ee37ecb18353c990008db0ArtifactHash47188325": { + "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eArtifactHashE8052809": { "Type": "String", - "Description": "Artifact hash for asset \"f26b845e072039dace324c286e111ec18c1f25ee43ee37ecb18353c990008db0\"" + "Description": "Artifact hash for asset \"4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282e\"" }, "AssetParametersd09271be89b6cb0398f793b40c1531fd9b076aa92ba80b5e436914b1808fe18dS3BucketBC52CF96": { "Type": "String", From 956a7037c7c6f649bb6d900a7f22c6556c6d6675 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Wed, 2 Feb 2022 10:55:25 +0200 Subject: [PATCH 10/14] re-run integ tests --- ...bucket-deployment-cloudfront.expected.json | 61 +++++---- .../integ.bucket-deployment.expected.json | 124 +++++++++--------- 2 files changed, 93 insertions(+), 92 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json index e50759499b6cc..9684e61489f8e 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json @@ -114,7 +114,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters84e9b89449fe2573e51d08cc143e21116ed4608c6db56afffcb4ad85c8130709S3Bucket2C6C817C" + "Ref": "AssetParametersbe270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92824S3Bucket09A62232" }, "S3Key": { "Fn::Join": [ @@ -127,7 +127,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters84e9b89449fe2573e51d08cc143e21116ed4608c6db56afffcb4ad85c8130709S3VersionKeyFA215BD6" + "Ref": "AssetParametersbe270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92824S3VersionKeyA28118BE" } ] } @@ -140,7 +140,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters84e9b89449fe2573e51d08cc143e21116ed4608c6db56afffcb4ad85c8130709S3VersionKeyFA215BD6" + "Ref": "AssetParametersbe270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92824S3VersionKeyA28118BE" } ] } @@ -230,7 +230,7 @@ "Properties": { "Content": { "S3Bucket": { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3Bucket59E5CFEF" }, "S3Key": { "Fn::Join": [ @@ -243,7 +243,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C" } ] } @@ -256,7 +256,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C" } ] } @@ -318,6 +318,9 @@ ] } ], + "SourceMarkers": [ + {} + ], "DestinationBucketName": { "Ref": "Destination3E3DC043D" }, @@ -417,10 +420,6 @@ "s3:List*", "s3:DeleteObject*", "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", "s3:Abort*" ], "Effect": "Allow", @@ -471,7 +470,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4cS3Bucket1BE31DB0" + "Ref": "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eS3BucketC3F9EAA2" }, "S3Key": { "Fn::Join": [ @@ -484,7 +483,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4cS3VersionKeyDC38E49C" + "Ref": "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eS3VersionKey030ACBFF" } ] } @@ -497,7 +496,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4cS3VersionKeyDC38E49C" + "Ref": "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eS3VersionKey030ACBFF" } ] } @@ -529,41 +528,41 @@ } }, "Parameters": { - "AssetParameters84e9b89449fe2573e51d08cc143e21116ed4608c6db56afffcb4ad85c8130709S3Bucket2C6C817C": { + "AssetParametersbe270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92824S3Bucket09A62232": { "Type": "String", - "Description": "S3 bucket for asset \"84e9b89449fe2573e51d08cc143e21116ed4608c6db56afffcb4ad85c8130709\"" + "Description": "S3 bucket for asset \"be270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92824\"" }, - "AssetParameters84e9b89449fe2573e51d08cc143e21116ed4608c6db56afffcb4ad85c8130709S3VersionKeyFA215BD6": { + "AssetParametersbe270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92824S3VersionKeyA28118BE": { "Type": "String", - "Description": "S3 key for asset version \"84e9b89449fe2573e51d08cc143e21116ed4608c6db56afffcb4ad85c8130709\"" + "Description": "S3 key for asset version \"be270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92824\"" }, - "AssetParameters84e9b89449fe2573e51d08cc143e21116ed4608c6db56afffcb4ad85c8130709ArtifactHash17D48178": { + "AssetParametersbe270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92824ArtifactHash76F8FCF2": { "Type": "String", - "Description": "Artifact hash for asset \"84e9b89449fe2573e51d08cc143e21116ed4608c6db56afffcb4ad85c8130709\"" + "Description": "Artifact hash for asset \"be270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92824\"" }, - "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7": { + "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3Bucket59E5CFEF": { "Type": "String", - "Description": "S3 bucket for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" + "Description": "S3 bucket for asset \"187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0\"" }, - "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F": { + "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C": { "Type": "String", - "Description": "S3 key for asset version \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" + "Description": "S3 key for asset version \"187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0\"" }, - "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68ArtifactHashD9A515C3": { + "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0ArtifactHash8F73A2B0": { "Type": "String", - "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" + "Description": "Artifact hash for asset \"187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0\"" }, - "AssetParameters983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4cS3Bucket1BE31DB0": { + "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eS3BucketC3F9EAA2": { "Type": "String", - "Description": "S3 bucket for asset \"983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4c\"" + "Description": "S3 bucket for asset \"4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282e\"" }, - "AssetParameters983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4cS3VersionKeyDC38E49C": { + "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eS3VersionKey030ACBFF": { "Type": "String", - "Description": "S3 key for asset version \"983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4c\"" + "Description": "S3 key for asset version \"4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282e\"" }, - "AssetParameters983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4cArtifactHashBA6352EA": { + "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eArtifactHashE8052809": { "Type": "String", - "Description": "Artifact hash for asset \"983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4c\"" + "Description": "Artifact hash for asset \"4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282e\"" }, "AssetParametersfc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222eS3Bucket9CD8B20A": { "Type": "String", diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json index 51473300ac82b..df07caa2ad1f7 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json @@ -129,7 +129,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters84e9b89449fe2573e51d08cc143e21116ed4608c6db56afffcb4ad85c8130709S3Bucket2C6C817C" + "Ref": "AssetParametersbe270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92824S3Bucket09A62232" }, "S3Key": { "Fn::Join": [ @@ -142,7 +142,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters84e9b89449fe2573e51d08cc143e21116ed4608c6db56afffcb4ad85c8130709S3VersionKeyFA215BD6" + "Ref": "AssetParametersbe270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92824S3VersionKeyA28118BE" } ] } @@ -155,7 +155,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters84e9b89449fe2573e51d08cc143e21116ed4608c6db56afffcb4ad85c8130709S3VersionKeyFA215BD6" + "Ref": "AssetParametersbe270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92824S3VersionKeyA28118BE" } ] } @@ -197,7 +197,7 @@ "Properties": { "Content": { "S3Bucket": { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3Bucket59E5CFEF" }, "S3Key": { "Fn::Join": [ @@ -210,7 +210,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C" } ] } @@ -223,7 +223,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C" } ] } @@ -285,6 +285,9 @@ ] } ], + "SourceMarkers": [ + {} + ], "DestinationBucketName": { "Ref": "Destination920A3C57" }, @@ -378,10 +381,6 @@ "s3:List*", "s3:DeleteObject*", "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", "s3:Abort*" ], "Effect": "Allow", @@ -415,10 +414,6 @@ "s3:List*", "s3:DeleteObject*", "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", "s3:Abort*" ], "Effect": "Allow", @@ -452,10 +447,6 @@ "s3:List*", "s3:DeleteObject*", "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", "s3:Abort*" ], "Effect": "Allow", @@ -498,7 +489,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4cS3Bucket1BE31DB0" + "Ref": "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eS3BucketC3F9EAA2" }, "S3Key": { "Fn::Join": [ @@ -511,7 +502,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4cS3VersionKeyDC38E49C" + "Ref": "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eS3VersionKey030ACBFF" } ] } @@ -524,7 +515,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4cS3VersionKeyDC38E49C" + "Ref": "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eS3VersionKey030ACBFF" } ] } @@ -1073,7 +1064,7 @@ "Properties": { "Content": { "S3Bucket": { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3Bucket59E5CFEF" }, "S3Key": { "Fn::Join": [ @@ -1086,7 +1077,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C" } ] } @@ -1099,7 +1090,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C" } ] } @@ -1196,6 +1187,9 @@ ] } ], + "SourceMarkers": [ + {} + ], "DestinationBucketName": { "Ref": "Destination920A3C57" }, @@ -1540,10 +1534,6 @@ "s3:List*", "s3:DeleteObject*", "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", "s3:Abort*" ], "Effect": "Allow", @@ -1612,7 +1602,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4cS3Bucket1BE31DB0" + "Ref": "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eS3BucketC3F9EAA2" }, "S3Key": { "Fn::Join": [ @@ -1625,7 +1615,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4cS3VersionKeyDC38E49C" + "Ref": "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eS3VersionKey030ACBFF" } ] } @@ -1638,7 +1628,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4cS3VersionKeyDC38E49C" + "Ref": "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eS3VersionKey030ACBFF" } ] } @@ -1818,7 +1808,7 @@ "Properties": { "Content": { "S3Bucket": { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3Bucket59E5CFEF" }, "S3Key": { "Fn::Join": [ @@ -1831,7 +1821,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C" } ] } @@ -1844,7 +1834,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C" } ] } @@ -1906,6 +1896,9 @@ ] } ], + "SourceMarkers": [ + {} + ], "DestinationBucketName": { "Ref": "Destination281A09BDF" }, @@ -2008,7 +2001,7 @@ "Properties": { "Content": { "S3Bucket": { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3Bucket59E5CFEF" }, "S3Key": { "Fn::Join": [ @@ -2021,7 +2014,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C" } ] } @@ -2034,7 +2027,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C" } ] } @@ -2096,6 +2089,9 @@ ] } ], + "SourceMarkers": [ + {} + ], "DestinationBucketName": { "Ref": "Destination3E3DC043D" }, @@ -2119,7 +2115,7 @@ "Properties": { "Content": { "S3Bucket": { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3Bucket59E5CFEF" }, "S3Key": { "Fn::Join": [ @@ -2132,7 +2128,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C" } ] } @@ -2145,7 +2141,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C" } ] } @@ -2207,6 +2203,9 @@ ] } ], + "SourceMarkers": [ + {} + ], "DestinationBucketName": { "Ref": "Destination920A3C57" }, @@ -2221,7 +2220,7 @@ "Properties": { "Content": { "S3Bucket": { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3Bucket59E5CFEF" }, "S3Key": { "Fn::Join": [ @@ -2234,7 +2233,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C" } ] } @@ -2247,7 +2246,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + "Ref": "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C" } ] } @@ -2309,6 +2308,9 @@ ] } ], + "SourceMarkers": [ + {} + ], "DestinationBucketName": { "Ref": "Destination920A3C57" }, @@ -2323,41 +2325,41 @@ } }, "Parameters": { - "AssetParameters84e9b89449fe2573e51d08cc143e21116ed4608c6db56afffcb4ad85c8130709S3Bucket2C6C817C": { + "AssetParametersbe270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92824S3Bucket09A62232": { "Type": "String", - "Description": "S3 bucket for asset \"84e9b89449fe2573e51d08cc143e21116ed4608c6db56afffcb4ad85c8130709\"" + "Description": "S3 bucket for asset \"be270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92824\"" }, - "AssetParameters84e9b89449fe2573e51d08cc143e21116ed4608c6db56afffcb4ad85c8130709S3VersionKeyFA215BD6": { + "AssetParametersbe270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92824S3VersionKeyA28118BE": { "Type": "String", - "Description": "S3 key for asset version \"84e9b89449fe2573e51d08cc143e21116ed4608c6db56afffcb4ad85c8130709\"" + "Description": "S3 key for asset version \"be270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92824\"" }, - "AssetParameters84e9b89449fe2573e51d08cc143e21116ed4608c6db56afffcb4ad85c8130709ArtifactHash17D48178": { + "AssetParametersbe270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92824ArtifactHash76F8FCF2": { "Type": "String", - "Description": "Artifact hash for asset \"84e9b89449fe2573e51d08cc143e21116ed4608c6db56afffcb4ad85c8130709\"" + "Description": "Artifact hash for asset \"be270bbdebe0851c887569796e3997437cca54ce86893ed94788500448e92824\"" }, - "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7": { + "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3Bucket59E5CFEF": { "Type": "String", - "Description": "S3 bucket for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" + "Description": "S3 bucket for asset \"187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0\"" }, - "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F": { + "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0S3VersionKey7EE70F5C": { "Type": "String", - "Description": "S3 key for asset version \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" + "Description": "S3 key for asset version \"187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0\"" }, - "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68ArtifactHashD9A515C3": { + "AssetParameters187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0ArtifactHash8F73A2B0": { "Type": "String", - "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" + "Description": "Artifact hash for asset \"187e7a21dd5d55d36f1f45007ff6bbc5713cb0866ca86224c0f1f86b3d1e76a0\"" }, - "AssetParameters983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4cS3Bucket1BE31DB0": { + "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eS3BucketC3F9EAA2": { "Type": "String", - "Description": "S3 bucket for asset \"983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4c\"" + "Description": "S3 bucket for asset \"4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282e\"" }, - "AssetParameters983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4cS3VersionKeyDC38E49C": { + "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eS3VersionKey030ACBFF": { "Type": "String", - "Description": "S3 key for asset version \"983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4c\"" + "Description": "S3 key for asset version \"4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282e\"" }, - "AssetParameters983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4cArtifactHashBA6352EA": { + "AssetParameters4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282eArtifactHashE8052809": { "Type": "String", - "Description": "Artifact hash for asset \"983c442a2fe823a8b4ebb18d241a5150ae15103dacbf3f038c7c6343e565aa4c\"" + "Description": "Artifact hash for asset \"4e09e63403b235ffda9db09367996f2d4c9fe1f7aa19b402908d8221614a282e\"" }, "AssetParametersfc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222eS3Bucket9CD8B20A": { "Type": "String", From 7cf5c4965baf16e4aea7f75bc788b4b89207891a Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Wed, 2 Feb 2022 11:28:24 +0200 Subject: [PATCH 11/14] update snapshots --- ...eg.bucket-deployment-cloudfront.expected.json | 4 ++++ .../integ.bucket-deployment-data.expected.json | 4 ++++ .../test/integ.bucket-deployment.expected.json | 16 ++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json index 9684e61489f8e..0d9c0689fffc1 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json @@ -420,6 +420,10 @@ "s3:List*", "s3:DeleteObject*", "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", "s3:Abort*" ], "Effect": "Allow", diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.expected.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.expected.json index f85e5f0821aca..51b9a179e65c5 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.expected.json +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.expected.json @@ -369,6 +369,10 @@ "s3:List*", "s3:DeleteObject*", "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", "s3:Abort*" ], "Effect": "Allow", diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json index df07caa2ad1f7..43d16a936cb51 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json @@ -381,6 +381,10 @@ "s3:List*", "s3:DeleteObject*", "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", "s3:Abort*" ], "Effect": "Allow", @@ -414,6 +418,10 @@ "s3:List*", "s3:DeleteObject*", "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", "s3:Abort*" ], "Effect": "Allow", @@ -447,6 +455,10 @@ "s3:List*", "s3:DeleteObject*", "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", "s3:Abort*" ], "Effect": "Allow", @@ -1534,6 +1546,10 @@ "s3:List*", "s3:DeleteObject*", "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging", "s3:Abort*" ], "Effect": "Allow", From fd6f3a207ed3db0188388de3e5e63ace41185cc4 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Wed, 2 Feb 2022 14:14:25 +0200 Subject: [PATCH 12/14] Update packages/@aws-cdk/aws-s3-deployment/README.md Co-authored-by: Otavio Macedo --- packages/@aws-cdk/aws-s3-deployment/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-s3-deployment/README.md b/packages/@aws-cdk/aws-s3-deployment/README.md index eb0002593dbbe..2eada9dc86b1c 100644 --- a/packages/@aws-cdk/aws-s3-deployment/README.md +++ b/packages/@aws-cdk/aws-s3-deployment/README.md @@ -289,7 +289,7 @@ const appConfig = { }; new s3deploy.BucketDeployment(this, 'BucketDeployment', { - sources: [s3deploy.Source.jsonData('config.json', config)], + sources: [s3deploy.Source.jsonData('config.json', appConfig)], destinationBucket, }); ``` From 60a3aae1195d906c6ea2c21ded30d3a894ccf49e Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Wed, 2 Feb 2022 16:39:43 +0200 Subject: [PATCH 13/14] remove empty source markers --- .../lib/bucket-deployment.ts | 11 ++++------ .../test/bucket-deployment.test.ts | 21 +++++++++++++++++++ ...bucket-deployment-cloudfront.expected.json | 3 --- .../integ.bucket-deployment.expected.json | 18 ---------------- 4 files changed, 25 insertions(+), 28 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts index db2d883718cda..b08011866f7ea 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts +++ b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts @@ -323,12 +323,9 @@ export class BucketDeployment extends CoreConstruct { })); } - const markers: Record = {}; - for (const source of sources) { - for (const [k, v] of Object.entries(source.markers ?? {})) { - markers[k] = v; - } - } + // to avoid redundant stack updates, only include "SourceMarkers" if one of + // the sources actually has markers. + const hasMarkers = sources.some(source => source.markers); const crUniqueId = `CustomResource${this.renderUniqueId(props.memoryLimit, props.vpc)}`; const cr = new cdk.CustomResource(this, crUniqueId, { @@ -337,7 +334,7 @@ export class BucketDeployment extends CoreConstruct { properties: { SourceBucketNames: sources.map(source => source.bucket.bucketName), SourceObjectKeys: sources.map(source => source.zipObjectKey), - SourceMarkers: sources.map(source => source.markers ?? {}), + SourceMarkers: hasMarkers ? sources.map(source => source.markers ?? {}) : undefined, DestinationBucketName: props.destinationBucket.bucketName, DestinationBucketKeyPrefix: props.destinationKeyPrefix, RetainOnDelete: props.retainOnDelete, diff --git a/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts b/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts index 857cbbe369398..cff1a509c9b5b 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts +++ b/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts @@ -1062,6 +1062,26 @@ test('bucket has multiple deployments', () => { }); }); +test('"SourceMarkers" is not included if none of the sources have markers', () => { + const stack = new cdk.Stack(); + const bucket = new s3.Bucket(stack, 'Bucket'); + new s3deploy.BucketDeployment(stack, 'DeployWithVpc3', { + sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], + destinationBucket: bucket, + }); + + const map = Template.fromStack(stack).findResources('Custom::CDKBucketDeployment'); + expect(map).toBeDefined(); + const resource = map[Object.keys(map)[0]]; + expect(Object.keys(resource.Properties)).toStrictEqual([ + 'ServiceToken', + 'SourceBucketNames', + 'SourceObjectKeys', + 'DestinationBucketName', + 'Prune', + ]); +}); + test('Source.data() can be used to create a file with string contents', () => { const app = new cdk.App(); const stack = new cdk.Stack(app, 'Test'); @@ -1114,6 +1134,7 @@ test('Source.jsonData() can be used to create a file with a JSON object', () => }); }); + function readDataFile(casm: cxapi.CloudAssembly, assetId: string, filePath: string): string { const asset = casm.stacks[0].assets.find(a => a.id === assetId); if (!asset) { throw new Error('Asset not found'); } diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json index 0d9c0689fffc1..54fa70b01ed7f 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json @@ -318,9 +318,6 @@ ] } ], - "SourceMarkers": [ - {} - ], "DestinationBucketName": { "Ref": "Destination3E3DC043D" }, diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json index 43d16a936cb51..bf4c180c9b559 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json @@ -285,9 +285,6 @@ ] } ], - "SourceMarkers": [ - {} - ], "DestinationBucketName": { "Ref": "Destination920A3C57" }, @@ -1199,9 +1196,6 @@ ] } ], - "SourceMarkers": [ - {} - ], "DestinationBucketName": { "Ref": "Destination920A3C57" }, @@ -1912,9 +1906,6 @@ ] } ], - "SourceMarkers": [ - {} - ], "DestinationBucketName": { "Ref": "Destination281A09BDF" }, @@ -2105,9 +2096,6 @@ ] } ], - "SourceMarkers": [ - {} - ], "DestinationBucketName": { "Ref": "Destination3E3DC043D" }, @@ -2219,9 +2207,6 @@ ] } ], - "SourceMarkers": [ - {} - ], "DestinationBucketName": { "Ref": "Destination920A3C57" }, @@ -2324,9 +2309,6 @@ ] } ], - "SourceMarkers": [ - {} - ], "DestinationBucketName": { "Ref": "Destination920A3C57" }, From 5ea4a077373d63fcea0b30efd0150acecc0347ec Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Thu, 3 Feb 2022 10:41:15 +0200 Subject: [PATCH 14/14] update README --- packages/@aws-cdk/aws-s3-deployment/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/@aws-cdk/aws-s3-deployment/README.md b/packages/@aws-cdk/aws-s3-deployment/README.md index 2eada9dc86b1c..62595042f652a 100644 --- a/packages/@aws-cdk/aws-s3-deployment/README.md +++ b/packages/@aws-cdk/aws-s3-deployment/README.md @@ -280,6 +280,8 @@ references that will get resolved only during deployment. For example: ```ts +import * as sns from '@aws-cdk/aws-sns'; + declare const destinationBucket: s3.Bucket; declare const topic: sns.Topic;