-
Notifications
You must be signed in to change notification settings - Fork 4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: don't upload the same asset multiple times #1011
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
{ | ||
"Parameters": { | ||
"SampleAsset1S3Bucket469E18FF": { | ||
"Type": "String", | ||
"Description": "S3 bucket for asset \"aws-cdk-multi-assets/SampleAsset1\"" | ||
}, | ||
"SampleAsset1S3VersionKey63A628F0": { | ||
"Type": "String", | ||
"Description": "S3 key for asset version \"aws-cdk-multi-assets/SampleAsset1\"" | ||
}, | ||
"SampleAsset2S3BucketC94C651A": { | ||
"Type": "String", | ||
"Description": "S3 bucket for asset \"aws-cdk-multi-assets/SampleAsset2\"" | ||
}, | ||
"SampleAsset2S3VersionKey3A7E2CC4": { | ||
"Type": "String", | ||
"Description": "S3 key for asset version \"aws-cdk-multi-assets/SampleAsset2\"" | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
import cdk = require('@aws-cdk/cdk'); | ||
import path = require('path'); | ||
import assets = require('../lib'); | ||
|
||
class TestStack extends cdk.Stack { | ||
constructor(parent: cdk.App, name: string, props?: cdk.StackProps) { | ||
super(parent, name, props); | ||
|
||
// Check that the same asset added multiple times is | ||
// uploaded and copied. | ||
new assets.FileAsset(this, 'SampleAsset1', { | ||
path: path.join(__dirname, 'file-asset.txt') | ||
}); | ||
|
||
new assets.FileAsset(this, 'SampleAsset2', { | ||
path: path.join(__dirname, 'file-asset.txt') | ||
}); | ||
} | ||
} | ||
|
||
const app = new cdk.App(); | ||
new TestStack(app, 'aws-cdk-multi-assets'); | ||
app.run(); |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -21,6 +21,11 @@ export interface Uploaded { | |
} | ||
|
||
export class ToolkitInfo { | ||
/** | ||
* A cache of previous uploads done in this session | ||
*/ | ||
private readonly previousUploads: {[key: string]: Uploaded} = {}; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Rename |
||
|
||
constructor(private readonly props: { | ||
sdk: SDK, | ||
bucketName: string, | ||
|
@@ -60,17 +65,31 @@ export class ToolkitInfo { | |
return { filename, key, changed: false }; | ||
} | ||
|
||
debug(`${url}: uploading`); | ||
await s3.putObject({ | ||
Bucket: bucket, | ||
Key: key, | ||
Body: data, | ||
ContentType: props.contentType | ||
}).promise(); | ||
|
||
debug(`${url}: upload complete`); | ||
const uploaded = { filename, key, changed: true }; | ||
|
||
// Upload if it's new or server-side copy if it was already uploaded previously | ||
const previous = this.previousUploads[hash]; | ||
if (previous) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why not just reuse the same file that was already uploaded? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We're organizing the data in folders, by This is currently so that we can give permissions on the folder to constructs that require access to the asset, so they will be able to download all versions of it, so that even if we upload a new version and the Execution Role is updated to only allow downloading the latest version of the asset, old runs/definitions will not stop working. This is not required for Lambdas which have a point-in-time handover but it is required for CodeBuild script assets, for example, which download the file at an arbitrary later point in time. If we reused the same file between multiple assets, we'd either have to fall back to "only permissions to the latest version of the asset" or "permission to all assets in the bucket". |
||
debug(`${url}: copying`); | ||
await s3.copyObject({ | ||
Bucket: bucket, | ||
Key: key, | ||
CopySource: `${bucket}/${previous.key}` | ||
}).promise(); | ||
debug(`${url}: copy complete`); | ||
} else { | ||
debug(`${url}: uploading`); | ||
await s3.putObject({ | ||
Bucket: bucket, | ||
Key: key, | ||
Body: data, | ||
ContentType: props.contentType | ||
}).promise(); | ||
debug(`${url}: upload complete`); | ||
this.previousUploads[hash] = uploaded; | ||
} | ||
|
||
return { filename, key, changed: true }; | ||
return uploaded; | ||
} | ||
|
||
} | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Test for this?