From 2b64381b46986a6cbfb898ac5d1fe5cd161b8fc4 Mon Sep 17 00:00:00 2001 From: Chase Coalwell Date: Thu, 30 Jan 2020 15:30:35 -0800 Subject: [PATCH] feat: glacier checksum middleware (#789) --- clients/client-glacier/GlacierClient.ts | 9 + .../client-glacier/runtimeConfig.browser.ts | 4 +- clients/client-glacier/runtimeConfig.ts | 4 +- .../AddBodyChecksumGeneratorDependency.java | 97 +++++++++ .../aws/typescript/codegen/AwsDependency.java | 2 + .../AwsEndpointGeneratorIntegration.java | 2 +- ....codegen.integration.TypeScriptIntegration | 1 + packages/body-checksum-browser/.gitignore | 8 + packages/body-checksum-browser/.npmignore | 13 ++ packages/body-checksum-browser/CHANGELOG.md | 4 + packages/body-checksum-browser/LICENSE | 201 ++++++++++++++++++ packages/body-checksum-browser/README.md | 4 + packages/body-checksum-browser/jest.config.js | 5 + packages/body-checksum-browser/package.json | 29 +++ .../body-checksum-browser/src/index.spec.ts | 75 +++++++ packages/body-checksum-browser/src/index.ts | 40 ++++ packages/body-checksum-browser/tsconfig.json | 23 ++ .../body-checksum-browser/tsconfig.test.json | 11 + packages/body-checksum-node/.gitignore | 8 + packages/body-checksum-node/.npmignore | 13 ++ packages/body-checksum-node/CHANGELOG.md | 4 + packages/body-checksum-node/LICENSE | 201 ++++++++++++++++++ packages/body-checksum-node/README.md | 4 + packages/body-checksum-node/jest.config.js | 5 + packages/body-checksum-node/package.json | 29 +++ packages/body-checksum-node/src/index.spec.ts | 108 ++++++++++ packages/body-checksum-node/src/index.ts | 41 ++++ packages/body-checksum-node/tsconfig.json | 22 ++ .../body-checksum-node/tsconfig.test.json | 11 + packages/middleware-sdk-glacier/package.json | 10 +- .../src/account-id-default.ts | 3 +- .../src/add-checksum-headers.spec.ts | 99 +++++++++ .../src/add-checksum-headers.ts | 58 +++++ .../src/add-glacier-api-version.spec.ts | 15 +- .../src/configurations.ts | 27 ++- .../middleware-sdk-glacier/src/index.spec.ts | 5 + packages/middleware-sdk-glacier/src/index.ts | 1 + packages/middleware-sdk-glacier/tsconfig.json | 1 + 38 files changed, 1183 insertions(+), 14 deletions(-) create mode 100644 codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddBodyChecksumGeneratorDependency.java create mode 100644 packages/body-checksum-browser/.gitignore create mode 100644 packages/body-checksum-browser/.npmignore create mode 100644 packages/body-checksum-browser/CHANGELOG.md create mode 100644 packages/body-checksum-browser/LICENSE create mode 100644 packages/body-checksum-browser/README.md create mode 100644 packages/body-checksum-browser/jest.config.js create mode 100644 packages/body-checksum-browser/package.json create mode 100644 packages/body-checksum-browser/src/index.spec.ts create mode 100644 packages/body-checksum-browser/src/index.ts create mode 100644 packages/body-checksum-browser/tsconfig.json create mode 100644 packages/body-checksum-browser/tsconfig.test.json create mode 100644 packages/body-checksum-node/.gitignore create mode 100644 packages/body-checksum-node/.npmignore create mode 100644 packages/body-checksum-node/CHANGELOG.md create mode 100644 packages/body-checksum-node/LICENSE create mode 100644 packages/body-checksum-node/README.md create mode 100644 packages/body-checksum-node/jest.config.js create mode 100644 packages/body-checksum-node/package.json create mode 100644 packages/body-checksum-node/src/index.spec.ts create mode 100644 packages/body-checksum-node/src/index.ts create mode 100644 packages/body-checksum-node/tsconfig.json create mode 100644 packages/body-checksum-node/tsconfig.test.json create mode 100644 packages/middleware-sdk-glacier/src/add-checksum-headers.spec.ts create mode 100644 packages/middleware-sdk-glacier/src/add-checksum-headers.ts diff --git a/clients/client-glacier/GlacierClient.ts b/clients/client-glacier/GlacierClient.ts index 1db2416c9615..d31f1d5f3176 100644 --- a/clients/client-glacier/GlacierClient.ts +++ b/clients/client-glacier/GlacierClient.ts @@ -101,6 +101,7 @@ import { Encoder as __Encoder, HashConstructor as __HashConstructor, HttpHandlerOptions as __HttpHandlerOptions, + HttpRequest as __HttpRequest, MetadataBearer as __MetadataBearer, Provider as __Provider, StreamCollector as __StreamCollector, @@ -249,6 +250,14 @@ export interface ClientDefaults * Fetch related hostname, signing name or signing region with given region. */ regionInfoProvider?: RegionInfoProvider; + + /** + * Function that returns body checksums. + */ + bodyChecksumGenerator?: ( + request: __HttpRequest, + options: { sha256: __HashConstructor; utf8Decoder: __Decoder } + ) => Promise<[string, string]>; } export type GlacierClientConfig = Partial< diff --git a/clients/client-glacier/runtimeConfig.browser.ts b/clients/client-glacier/runtimeConfig.browser.ts index 41c40825b808..084e42253f53 100644 --- a/clients/client-glacier/runtimeConfig.browser.ts +++ b/clients/client-glacier/runtimeConfig.browser.ts @@ -1,3 +1,4 @@ +import { bodyChecksumGenerator } from "@aws-sdk/body-checksum-browser"; import { invalidFunction } from "@aws-sdk/invalid-dependency"; import { Sha256 } from "@aws-crypto/sha256-browser"; import { FetchHttpHandler } from "@aws-sdk/fetch-http-handler"; @@ -26,5 +27,6 @@ export const ClientDefaultValues: Required = { runtime: "browser", signingName: "glacier", credentialDefaultProvider: invalidFunction("Credential is missing") as any, - regionDefaultProvider: invalidFunction("Region is missing") as any + regionDefaultProvider: invalidFunction("Region is missing") as any, + bodyChecksumGenerator }; diff --git a/clients/client-glacier/runtimeConfig.ts b/clients/client-glacier/runtimeConfig.ts index d53ef49d09a5..62254998c202 100644 --- a/clients/client-glacier/runtimeConfig.ts +++ b/clients/client-glacier/runtimeConfig.ts @@ -1,3 +1,4 @@ +import { bodyChecksumGenerator } from "@aws-sdk/body-checksum-node"; import { defaultProvider as credentialDefaultProvider } from "@aws-sdk/credential-provider-node"; import { defaultProvider as regionDefaultProvider } from "@aws-sdk/region-provider"; import { Hash } from "@aws-sdk/hash-node"; @@ -27,5 +28,6 @@ export const ClientDefaultValues: Required = { runtime: "node", signingName: "glacier", credentialDefaultProvider, - regionDefaultProvider + regionDefaultProvider, + bodyChecksumGenerator }; diff --git a/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddBodyChecksumGeneratorDependency.java b/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddBodyChecksumGeneratorDependency.java new file mode 100644 index 000000000000..0cca18940a23 --- /dev/null +++ b/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddBodyChecksumGeneratorDependency.java @@ -0,0 +1,97 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.aws.typescript.codegen; + +import java.util.Set; +import java.util.logging.Logger; +import software.amazon.smithy.aws.traits.ServiceTrait; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.typescript.codegen.LanguageTarget; +import software.amazon.smithy.typescript.codegen.TypeScriptSettings; +import software.amazon.smithy.typescript.codegen.TypeScriptWriter; + +import software.amazon.smithy.typescript.codegen.integration.TypeScriptIntegration; +import software.amazon.smithy.utils.SetUtils; + +/** + * Adds blobReader dependency if needed. + */ +public class AddBodyChecksumGeneratorDependency implements TypeScriptIntegration { + private static final Set SERVICE_IDS = SetUtils.of("Glacier"); + + private static final Logger LOGGER = Logger.getLogger(AddBodyChecksumGeneratorDependency.class.getName()); + + @Override + public void addConfigInterfaceFields( + TypeScriptSettings settings, + Model model, + SymbolProvider symbolProvider, + TypeScriptWriter writer + ) { + if (!needsBodyChecksumGeneratorDep(settings.getService(model))) { + return; + } + writer.addImport("HttpRequest", "__HttpRequest", "@aws-sdk/types"); + writer.writeDocs("Function that returns body checksums."); + writer.write("bodyChecksumGenerator?: (request: __HttpRequest, options: { sha256: __HashConstructor; " + + "utf8Decoder: __Decoder }) => Promise<[string, string]>;\n"); +} + + @Override + public void addRuntimeConfigValues( + TypeScriptSettings settings, + Model model, + SymbolProvider symbolProvider, + TypeScriptWriter writer, + LanguageTarget target + ) { + if (!needsBodyChecksumGeneratorDep(settings.getService(model))) { + return; + } + + switch (target) { + case NODE: + writeNodeConfig(writer); + break; + case BROWSER: + writeBrowserConfig(writer); + break; + default: + LOGGER.info("Unknown JavaScript target: " + target); + } + } + + private void writeNodeConfig(TypeScriptWriter writer) { + writer.addDependency(AwsDependency.BODY_CHECKSUM_GENERATOR_NODE); + writer.addImport("bodyChecksumGenerator", "bodyChecksumGenerator", + AwsDependency.BODY_CHECKSUM_GENERATOR_NODE.packageName); + writer.write("bodyChecksumGenerator,"); + } + + private void writeBrowserConfig(TypeScriptWriter writer) { + writer.addDependency(AwsDependency.BODY_CHECKSUM_GENERATOR_BROWSER); + writer.addImport("bodyChecksumGenerator", "bodyChecksumGenerator", + AwsDependency.BODY_CHECKSUM_GENERATOR_BROWSER.packageName); + writer.write("bodyChecksumGenerator,"); + } + + private static boolean needsBodyChecksumGeneratorDep(ServiceShape service) { + String serviceId = service.getTrait(ServiceTrait.class).map(ServiceTrait::getSdkId).orElse(""); + return SERVICE_IDS.contains(serviceId); + } +} diff --git a/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AwsDependency.java b/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AwsDependency.java index 063723ba3b96..366d4f602786 100644 --- a/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AwsDependency.java +++ b/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AwsDependency.java @@ -50,6 +50,8 @@ public enum AwsDependency implements SymbolDependencyContainer { BODY_CHECKSUM(NORMAL_DEPENDENCY, "@aws-sdk/middleware-apply-body-checksum", "^1.0.0-alpha.1"), MIDDLEWARE_HOST_HEADER(NORMAL_DEPENDENCY, "@aws-sdk/middleware-host-header", "^1.0.0-alpha.1"), SQS_MIDDLEWARE(NORMAL_DEPENDENCY, "@aws-sdk/middleware-sdk-sqs", "^1.0.0-alpha.0"), + BODY_CHECKSUM_GENERATOR_BROWSER(NORMAL_DEPENDENCY, "@aws-sdk/body-checksum-browser", "^1.0.0-alpha.0"), + BODY_CHECKSUM_GENERATOR_NODE(NORMAL_DEPENDENCY, "@aws-sdk/body-checksum-node", "^1.0.0-alpha.0"), XML_BUILDER(NORMAL_DEPENDENCY, "@aws-sdk/xml-builder", "^1.0.0-alpha.1"), XML_PARSER(NORMAL_DEPENDENCY, "pixl-xml", "^1.0.13"), XML_PARSER_TYPES(DEV_DEPENDENCY, "@types/pixl-xml", "^1.0.1"), diff --git a/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AwsEndpointGeneratorIntegration.java b/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AwsEndpointGeneratorIntegration.java index a568b61a7584..0eb9ff39e22a 100644 --- a/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AwsEndpointGeneratorIntegration.java +++ b/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AwsEndpointGeneratorIntegration.java @@ -50,7 +50,7 @@ public void addConfigInterfaceFields( ) { writer.addImport("RegionInfoProvider", "RegionInfoProvider", TypeScriptDependency.AWS_SDK_TYPES.packageName); writer.writeDocs("Fetch related hostname, signing name or signing region with given region."); - writer.write("regionInfoProvider?: RegionInfoProvider;"); + writer.write("regionInfoProvider?: RegionInfoProvider;\n"); } @Override diff --git a/codegen/smithy-aws-typescript-codegen/src/main/resources/META-INF/services/software.amazon.smithy.typescript.codegen.integration.TypeScriptIntegration b/codegen/smithy-aws-typescript-codegen/src/main/resources/META-INF/services/software.amazon.smithy.typescript.codegen.integration.TypeScriptIntegration index 6df80fab9434..be07b3394c47 100644 --- a/codegen/smithy-aws-typescript-codegen/src/main/resources/META-INF/services/software.amazon.smithy.typescript.codegen.integration.TypeScriptIntegration +++ b/codegen/smithy-aws-typescript-codegen/src/main/resources/META-INF/services/software.amazon.smithy.typescript.codegen.integration.TypeScriptIntegration @@ -6,3 +6,4 @@ software.amazon.smithy.aws.typescript.codegen.AwsServiceIdIntegration software.amazon.smithy.aws.typescript.codegen.AwsPackageFixturesGeneratorIntegration software.amazon.smithy.aws.typescript.codegen.AddMd5HashDependency software.amazon.smithy.aws.typescript.codegen.AddStreamHasherDependency +software.amazon.smithy.aws.typescript.codegen.AddBodyChecksumGeneratorDependency diff --git a/packages/body-checksum-browser/.gitignore b/packages/body-checksum-browser/.gitignore new file mode 100644 index 000000000000..3d1714c9806e --- /dev/null +++ b/packages/body-checksum-browser/.gitignore @@ -0,0 +1,8 @@ +/node_modules/ +/build/ +/coverage/ +/docs/ +*.tsbuildinfo +*.tgz +*.log +package-lock.json diff --git a/packages/body-checksum-browser/.npmignore b/packages/body-checksum-browser/.npmignore new file mode 100644 index 000000000000..35b1cbfb769c --- /dev/null +++ b/packages/body-checksum-browser/.npmignore @@ -0,0 +1,13 @@ +/src/ +/coverage/ +/docs/ +tsconfig.test.json +*.tsbuildinfo + +*.spec.js +*.spec.d.ts +*.spec.js.map + +*.fixture.js +*.fixture.d.ts +*.fixture.js.map \ No newline at end of file diff --git a/packages/body-checksum-browser/CHANGELOG.md b/packages/body-checksum-browser/CHANGELOG.md new file mode 100644 index 000000000000..e9fb6ecf5930 --- /dev/null +++ b/packages/body-checksum-browser/CHANGELOG.md @@ -0,0 +1,4 @@ +# Change Log + +All notable changes to this project will be documented in this file. +See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. \ No newline at end of file diff --git a/packages/body-checksum-browser/LICENSE b/packages/body-checksum-browser/LICENSE new file mode 100644 index 000000000000..74d4e5c31f2e --- /dev/null +++ b/packages/body-checksum-browser/LICENSE @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/packages/body-checksum-browser/README.md b/packages/body-checksum-browser/README.md new file mode 100644 index 000000000000..30955d711a6c --- /dev/null +++ b/packages/body-checksum-browser/README.md @@ -0,0 +1,4 @@ +# @aws-sdk/body-checksum-browser + +[![NPM version](https://img.shields.io/npm/v/@aws-sdk/body-checksum-browser/preview.svg)](https://www.npmjs.com/package/@aws-sdk/body-checksum-browser) +[![NPM downloads](https://img.shields.io/npm/dm/@aws-sdk/body-checksum-browser.svg)](https://www.npmjs.com/package/@aws-sdk/body-checksum-browser) diff --git a/packages/body-checksum-browser/jest.config.js b/packages/body-checksum-browser/jest.config.js new file mode 100644 index 000000000000..498ea8304467 --- /dev/null +++ b/packages/body-checksum-browser/jest.config.js @@ -0,0 +1,5 @@ +const base = require("../../jest.config.base.js"); + +module.exports = { + ...base +}; diff --git a/packages/body-checksum-browser/package.json b/packages/body-checksum-browser/package.json new file mode 100644 index 000000000000..315c7ea80e44 --- /dev/null +++ b/packages/body-checksum-browser/package.json @@ -0,0 +1,29 @@ +{ + "name": "@aws-sdk/body-checksum-browser", + "version": "1.0.0-alpha.0", + "scripts": { + "prepublishOnly": "tsc", + "pretest": "tsc -p tsconfig.test.json", + "test": "jest" + }, + "main": "./build/index.js", + "types": "./build/index.d.ts", + "author": { + "name": "AWS SDK for JavaScript Team", + "url": "https://aws.amazon.com/javascript/" + }, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/chunked-blob-reader": "^1.0.0-alpha.2", + "@aws-sdk/protocol-http": "^1.0.0-alpha.6", + "@aws-sdk/sha256-tree-hash": "^1.0.0-alpha.1", + "@aws-sdk/types": "^1.0.0-alpha.4", + "@aws-sdk/util-hex-encoding": "^1.0.0-alpha.2", + "tslib": "^1.8.0" + }, + "devDependencies": { + "@types/jest": "^24.0.12", + "jest": "^24.7.1", + "typescript": "~3.4.0" + } +} diff --git a/packages/body-checksum-browser/src/index.spec.ts b/packages/body-checksum-browser/src/index.spec.ts new file mode 100644 index 000000000000..b0a4f490db7e --- /dev/null +++ b/packages/body-checksum-browser/src/index.spec.ts @@ -0,0 +1,75 @@ +import { bodyChecksumGenerator } from "."; +import { HttpRequest } from "@aws-sdk/protocol-http"; +import { fromUtf8 } from "@aws-sdk/util-utf8-browser"; +import { Sha256 } from "@aws-crypto/sha256-js"; +import { Readable } from "stream"; + +describe("bodyChecksumGenerator for browser", () => { + const sharedRequest = { + method: "POST", + protocol: "https:", + path: "/", + headers: {}, + hostname: "foo.us-east-1.amazonaws.com" + }; + const options = { + sha256: Sha256, + utf8Decoder: fromUtf8 + }; + + it("will calculate sha256 hashes when request body is a blob", async () => { + const data = new Uint8Array(5767168); + const blob = new Blob([data]); + + const request = new HttpRequest({ + ...sharedRequest, + body: blob + }); + + const [contentHash, treeHash] = await bodyChecksumGenerator( + request, + options + ); + + expect(contentHash).toBe( + "733cf513448ce6b20ad1bc5e50eb27c06aefae0c320713a5dd99f4e51bc1ca60" + ); + expect(treeHash).toBe( + "a3a82dbe3644dd6046be472f2e3ec1f8ef47f8f3adb86d0de4de7a254f255455" + ); + }); + + it("will calculate sha256 hashes when request body is a string", async () => { + const request = new HttpRequest({ + ...sharedRequest, + body: "bar" + }); + + const [contentHash, treeHash] = await bodyChecksumGenerator( + request, + options + ); + + expect(contentHash).toBe( + "fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9" + ); + expect(treeHash).toBe( + "fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9" + ); + }); + + it("will reject when request body is a non-blob stream", async () => { + const request = new HttpRequest({ + ...sharedRequest, + body: new Readable() + }); + + try { + await bodyChecksumGenerator(request, options); + } catch (e) { + expect(e).toEqual( + new Error("Unable to calculate checksums for non-blob streams.") + ); + } + }); +}); diff --git a/packages/body-checksum-browser/src/index.ts b/packages/body-checksum-browser/src/index.ts new file mode 100644 index 000000000000..38c7b59f8096 --- /dev/null +++ b/packages/body-checksum-browser/src/index.ts @@ -0,0 +1,40 @@ +import { TreeHash } from "@aws-sdk/sha256-tree-hash"; +import { Decoder, HttpRequest, HashConstructor } from "@aws-sdk/types"; +import { toHex } from "@aws-sdk/util-hex-encoding"; +import { blobReader } from "@aws-sdk/chunked-blob-reader"; + +const MiB = 1024 * 1024; + +export async function bodyChecksumGenerator( + request: HttpRequest, + options: { + sha256: HashConstructor; + utf8Decoder: Decoder; + } +): Promise<[string, string]> { + const contentHash = new options.sha256(); + const treeHash = new TreeHash(options.sha256, options.utf8Decoder); + const { body } = request; + if (typeof body === "string") { + contentHash.update(body); + treeHash.update(body); + } else { + if ( + Boolean(body) && + Object.prototype.toString.call(body) === "[object Blob]" + ) { + await blobReader( + body, + (chunk: any) => { + treeHash && treeHash.update(chunk); + contentHash && contentHash.update(chunk); + }, + MiB + ); + } else { + throw new Error("Unable to calculate checksums for non-blob streams."); + } + } + + return [toHex(await contentHash.digest()), toHex(await treeHash.digest())]; +} diff --git a/packages/body-checksum-browser/tsconfig.json b/packages/body-checksum-browser/tsconfig.json new file mode 100644 index 000000000000..d34b34707316 --- /dev/null +++ b/packages/body-checksum-browser/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "es5", + "module": "commonjs", + "declaration": true, + "strict": true, + "sourceMap": true, + "downlevelIteration": true, + "importHelpers": true, + "noEmitHelpers": true, + "lib": [ + "dom", + "es5", + "es2015.promise", + "es2015.collection", + "es2015.iterable", + "es2015.symbol.wellknown" + ], + "rootDir": "./src", + "outDir": "./build", + "incremental": true + } +} diff --git a/packages/body-checksum-browser/tsconfig.test.json b/packages/body-checksum-browser/tsconfig.test.json new file mode 100644 index 000000000000..17d0f1b7321f --- /dev/null +++ b/packages/body-checksum-browser/tsconfig.test.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "sourceMap": false, + "inlineSourceMap": true, + "inlineSources": true, + "rootDir": "./src", + "outDir": "./build", + "incremental": true + } +} diff --git a/packages/body-checksum-node/.gitignore b/packages/body-checksum-node/.gitignore new file mode 100644 index 000000000000..3d1714c9806e --- /dev/null +++ b/packages/body-checksum-node/.gitignore @@ -0,0 +1,8 @@ +/node_modules/ +/build/ +/coverage/ +/docs/ +*.tsbuildinfo +*.tgz +*.log +package-lock.json diff --git a/packages/body-checksum-node/.npmignore b/packages/body-checksum-node/.npmignore new file mode 100644 index 000000000000..35b1cbfb769c --- /dev/null +++ b/packages/body-checksum-node/.npmignore @@ -0,0 +1,13 @@ +/src/ +/coverage/ +/docs/ +tsconfig.test.json +*.tsbuildinfo + +*.spec.js +*.spec.d.ts +*.spec.js.map + +*.fixture.js +*.fixture.d.ts +*.fixture.js.map \ No newline at end of file diff --git a/packages/body-checksum-node/CHANGELOG.md b/packages/body-checksum-node/CHANGELOG.md new file mode 100644 index 000000000000..e9fb6ecf5930 --- /dev/null +++ b/packages/body-checksum-node/CHANGELOG.md @@ -0,0 +1,4 @@ +# Change Log + +All notable changes to this project will be documented in this file. +See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. \ No newline at end of file diff --git a/packages/body-checksum-node/LICENSE b/packages/body-checksum-node/LICENSE new file mode 100644 index 000000000000..74d4e5c31f2e --- /dev/null +++ b/packages/body-checksum-node/LICENSE @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/packages/body-checksum-node/README.md b/packages/body-checksum-node/README.md new file mode 100644 index 000000000000..21226366ebc9 --- /dev/null +++ b/packages/body-checksum-node/README.md @@ -0,0 +1,4 @@ +# @aws-sdk/body-checksum-node + +[![NPM version](https://img.shields.io/npm/v/@aws-sdk/body-checksum-node/preview.svg)](https://www.npmjs.com/package/@aws-sdk/body-checksum-node) +[![NPM downloads](https://img.shields.io/npm/dm/@aws-sdk/body-checksum-node.svg)](https://www.npmjs.com/package/@aws-sdk/body-checksum-node) diff --git a/packages/body-checksum-node/jest.config.js b/packages/body-checksum-node/jest.config.js new file mode 100644 index 000000000000..498ea8304467 --- /dev/null +++ b/packages/body-checksum-node/jest.config.js @@ -0,0 +1,5 @@ +const base = require("../../jest.config.base.js"); + +module.exports = { + ...base +}; diff --git a/packages/body-checksum-node/package.json b/packages/body-checksum-node/package.json new file mode 100644 index 000000000000..5b0b2ea0d621 --- /dev/null +++ b/packages/body-checksum-node/package.json @@ -0,0 +1,29 @@ +{ + "name": "@aws-sdk/body-checksum-node", + "version": "1.0.0-alpha.0", + "scripts": { + "prepublishOnly": "tsc", + "pretest": "tsc -p tsconfig.test.json", + "test": "jest" + }, + "main": "./build/index.js", + "types": "./build/index.d.ts", + "author": { + "name": "AWS SDK for JavaScript Team", + "url": "https://aws.amazon.com/javascript/" + }, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/is-array-buffer": "^1.0.0-alpha.2", + "@aws-sdk/protocol-http": "^1.0.0-alpha.6", + "@aws-sdk/sha256-tree-hash": "^1.0.0-alpha.1", + "@aws-sdk/types": "^1.0.0-alpha.4", + "@aws-sdk/util-hex-encoding": "^1.0.0-alpha.2", + "tslib": "^1.8.0" + }, + "devDependencies": { + "@types/jest": "^24.0.12", + "jest": "^24.7.1", + "typescript": "~3.4.0" + } +} diff --git a/packages/body-checksum-node/src/index.spec.ts b/packages/body-checksum-node/src/index.spec.ts new file mode 100644 index 000000000000..5c8b4612e1b0 --- /dev/null +++ b/packages/body-checksum-node/src/index.spec.ts @@ -0,0 +1,108 @@ +import { bodyChecksumGenerator } from "."; +import { HttpRequest } from "@aws-sdk/protocol-http"; +import { fromUtf8 } from "@aws-sdk/util-utf8-node"; +import { Sha256 } from "@aws-crypto/sha256-js"; +import { join } from "path"; +import { tmpdir } from "os"; +import { createReadStream, mkdtempSync, writeFileSync } from "fs"; +import { Readable } from "stream"; + +function createTemporaryFile(contents: string | Buffer): string { + const folder = mkdtempSync( + join(tmpdir(), "add-glacier-checksum-headers-node-") + ); + const fileLoc = join(folder, "test.txt"); + writeFileSync(fileLoc, contents); + + return fileLoc; +} + +describe("bodyChecksumGenerator for node", () => { + const sharedRequest = { + method: "POST", + protocol: "https:", + path: "/", + headers: {}, + hostname: "foo.us-east-1.amazonaws.com" + }; + const options = { + sha256: Sha256, + utf8Decoder: fromUtf8 + }; + + it("will calculate sha256 hashes when request body is Uint8Array", async () => { + const body = new Uint8Array(5767168); // 5.5 MiB + body.fill(0); + + const request = new HttpRequest({ + ...sharedRequest, + body + }); + + const [contentHash, treeHash] = await bodyChecksumGenerator( + request, + options + ); + + expect(contentHash).toBe( + "733cf513448ce6b20ad1bc5e50eb27c06aefae0c320713a5dd99f4e51bc1ca60" + ); + expect(treeHash).toBe( + "a3a82dbe3644dd6046be472f2e3ec1f8ef47f8f3adb86d0de4de7a254f255455" + ); + }); + + it("will calculate sha256 hashes when request body is a string", async () => { + const request = new HttpRequest({ + ...sharedRequest, + body: "bar" + }); + + const [contentHash, treeHash] = await bodyChecksumGenerator( + request, + options + ); + + expect(contentHash).toBe( + "fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9" + ); + expect(treeHash).toBe( + "fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9" + ); + }); + + it("will calculate sha256 hashes when request body is a file stream", async () => { + const temporaryFile = createTemporaryFile(Buffer.alloc(5767168, 0)); + const request = new HttpRequest({ + ...sharedRequest, + body: createReadStream(temporaryFile) + }); + + const [contentHash, treeHash] = await bodyChecksumGenerator( + request, + options + ); + + expect(contentHash).toBe( + "733cf513448ce6b20ad1bc5e50eb27c06aefae0c320713a5dd99f4e51bc1ca60" + ); + expect(treeHash).toBe( + "a3a82dbe3644dd6046be472f2e3ec1f8ef47f8f3adb86d0de4de7a254f255455" + ); + }); + + it("will reject when request body is a non-file stream", async () => { + const request = new HttpRequest({ + ...sharedRequest, + body: new Readable() + }); + + try { + await bodyChecksumGenerator(request, options); + } catch (e) { + expect(e).toEqual( + new Error("Unable to calculate checksums for non-file streams.") + ); + } + }); +}); diff --git a/packages/body-checksum-node/src/index.ts b/packages/body-checksum-node/src/index.ts new file mode 100644 index 000000000000..1a973d81366c --- /dev/null +++ b/packages/body-checksum-node/src/index.ts @@ -0,0 +1,41 @@ +import { createReadStream } from "fs"; +import { TreeHash } from "@aws-sdk/sha256-tree-hash"; +import { Decoder, HttpRequest, HashConstructor } from "@aws-sdk/types"; +import { isArrayBuffer } from "@aws-sdk/is-array-buffer"; +import { toHex } from "@aws-sdk/util-hex-encoding"; +import { streamReader } from "@aws-sdk/chunked-stream-reader-node"; + +export async function bodyChecksumGenerator( + request: HttpRequest, + options: { + sha256: HashConstructor; + utf8Decoder: Decoder; + } +): Promise<[string, string]> { + const contentHash = new options.sha256(); + const treeHash = new TreeHash(options.sha256, options.utf8Decoder); + const { body } = request; + if ( + typeof body === "string" || + ArrayBuffer.isView(body) || + isArrayBuffer(body) + ) { + contentHash && contentHash.update(body); + treeHash && treeHash.update(body); + } else { + if (typeof body.path !== "string") { + throw new Error("Unable to calculate checksums for non-file streams."); + } + const bodyTee = createReadStream(body.path, { + start: (body as any).start, + end: (body as any).end + }); + + await streamReader(bodyTee, (chunk: any) => { + contentHash && contentHash.update(chunk); + treeHash && treeHash.update(chunk); + }); + } + + return [toHex(await contentHash.digest()), toHex(await treeHash.digest())]; +} diff --git a/packages/body-checksum-node/tsconfig.json b/packages/body-checksum-node/tsconfig.json new file mode 100644 index 000000000000..38b94cda274e --- /dev/null +++ b/packages/body-checksum-node/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "target": "es5", + "module": "commonjs", + "declaration": true, + "strict": true, + "sourceMap": true, + "downlevelIteration": true, + "importHelpers": true, + "noEmitHelpers": true, + "lib": [ + "es5", + "es2015.promise", + "es2015.collection", + "es2015.iterable", + "es2015.symbol.wellknown" + ], + "rootDir": "./src", + "outDir": "./build", + "incremental": true + } +} diff --git a/packages/body-checksum-node/tsconfig.test.json b/packages/body-checksum-node/tsconfig.test.json new file mode 100644 index 000000000000..17d0f1b7321f --- /dev/null +++ b/packages/body-checksum-node/tsconfig.test.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "sourceMap": false, + "inlineSourceMap": true, + "inlineSources": true, + "rootDir": "./src", + "outDir": "./build", + "incremental": true + } +} diff --git a/packages/middleware-sdk-glacier/package.json b/packages/middleware-sdk-glacier/package.json index 901ccd2f6290..f6045d8582e4 100644 --- a/packages/middleware-sdk-glacier/package.json +++ b/packages/middleware-sdk-glacier/package.json @@ -13,14 +13,14 @@ "url": "https://aws.amazon.com/javascript/" }, "license": "Apache-2.0", - "devDependencies": { - "@types/jest": "^24.0.12", - "jest": "^24.7.1", - "typescript": "~3.4.0" - }, "dependencies": { "@aws-sdk/protocol-http": "^1.0.0-alpha.6", "@aws-sdk/types": "^1.0.0-alpha.4", "tslib": "^1.8.0" + }, + "devDependencies": { + "@types/jest": "^24.0.12", + "jest": "^24.7.1", + "typescript": "~3.4.0" } } diff --git a/packages/middleware-sdk-glacier/src/account-id-default.ts b/packages/middleware-sdk-glacier/src/account-id-default.ts index 8791771b08d9..76d2f321a9e5 100644 --- a/packages/middleware-sdk-glacier/src/account-id-default.ts +++ b/packages/middleware-sdk-glacier/src/account-id-default.ts @@ -4,8 +4,7 @@ import { InitializeHandlerOptions, InitializeHandlerOutput, InitializeMiddleware, - MetadataBearer, - Pluggable + MetadataBearer } from "@aws-sdk/types"; export function accountIdDefaultMiddleware(): InitializeMiddleware { diff --git a/packages/middleware-sdk-glacier/src/add-checksum-headers.spec.ts b/packages/middleware-sdk-glacier/src/add-checksum-headers.spec.ts new file mode 100644 index 000000000000..44d61b9444d9 --- /dev/null +++ b/packages/middleware-sdk-glacier/src/add-checksum-headers.spec.ts @@ -0,0 +1,99 @@ +import { addChecksumHeadersMiddleware } from "./add-checksum-headers"; +import { HttpRequest } from "@aws-sdk/protocol-http"; + +describe("addChecksumHeadersMiddleware", () => { + const bodyChecksumGenerator = jest + .fn() + .mockReturnValue([ + "733cf513448ce6b20ad1bc5e50eb27c06aefae0c320713a5dd99f4e51bc1ca60", + "733cf513448ce6b20ad1bc5e50eb27c06aefae0c320713a5dd99f4e51bc1ca60" + ]); + const sha256 = jest.fn(); + const utf8Decoder = jest.fn(); + const next = jest.fn(); + + const minimalRequest = { + method: "POST", + protocol: "https:", + path: "/", + headers: {}, + hostname: "foo.us-east-1.amazonaws.com" + }; + + const config = { + apiVersion: "1970-01-01", + bodyChecksumGenerator, + sha256, + utf8Decoder + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("will not set content-sha256 headers if request body is empty", async () => { + const handler = addChecksumHeadersMiddleware(config)(next, {} as any); + await handler({ + input: {}, + request: new HttpRequest({ + ...minimalRequest + }) + }); + + expect(next.mock.calls.length).toBe(1); + const { request } = next.mock.calls[0][0]; + expect(request.headers["x-amz-content-sha256"]).toBeUndefined(); + expect(request.headers["x-amz-sha-256-tree-hash"]).toBeUndefined(); + expect(bodyChecksumGenerator.mock.calls.length).toBe(0); + }); + + it("will not set sha256 tree header if header is already present", async () => { + const body = new Uint8Array(5767168); // 5.5 MiB + body.fill(0); + const handler = addChecksumHeadersMiddleware(config)(next, {} as any); + + await handler({ + input: {}, + request: new HttpRequest({ + ...minimalRequest, + headers: { + "x-amz-sha256-tree-hash": "foo" + }, + body: body + }) + }); + + expect(next.mock.calls.length).toBe(1); + const { request } = next.mock.calls[0][0]; + expect(request.headers["x-amz-content-sha256"]).toBe( + "733cf513448ce6b20ad1bc5e50eb27c06aefae0c320713a5dd99f4e51bc1ca60" + ); + expect(request.headers["x-amz-sha256-tree-hash"]).toBe("foo"); + expect(bodyChecksumGenerator.mock.calls.length).toBe(1); + }); + + it("will not set content sha256 header if header is already present", async () => { + const body = new Uint8Array(5767168); // 5.5 MiB + body.fill(0); + const handler = addChecksumHeadersMiddleware(config)(next, {} as any); + + await handler({ + input: {}, + request: new HttpRequest({ + ...minimalRequest, + headers: { + "x-amz-content-sha256": "foo" + }, + body: body + }) + }); + + expect(next.mock.calls.length).toBe(1); + const { request } = next.mock.calls[0][0]; + expect(request.headers["x-amz-sha256-tree-hash"]).toBe( + "733cf513448ce6b20ad1bc5e50eb27c06aefae0c320713a5dd99f4e51bc1ca60" + ); + expect(request.headers["x-amz-content-sha256"]).toBe("foo"); + expect(bodyChecksumGenerator.mock.calls.length).toBe(1); + }); +}); diff --git a/packages/middleware-sdk-glacier/src/add-checksum-headers.ts b/packages/middleware-sdk-glacier/src/add-checksum-headers.ts new file mode 100644 index 000000000000..f2007e8bdb01 --- /dev/null +++ b/packages/middleware-sdk-glacier/src/add-checksum-headers.ts @@ -0,0 +1,58 @@ +import { ResolvedGlacierMiddlewareConfig } from "./configurations"; +import { + BuildHandler, + BuildHandlerArguments, + BuildHandlerOptions, + BuildHandlerOutput, + BuildMiddleware, + MetadataBearer +} from "@aws-sdk/types"; +import { HttpRequest } from "@aws-sdk/protocol-http"; + +export function addChecksumHeadersMiddleware( + options: ResolvedGlacierMiddlewareConfig +): BuildMiddleware { + return ( + next: BuildHandler + ): BuildHandler => async ( + args: BuildHandlerArguments + ): Promise> => { + let request = args.request; + if (HttpRequest.isInstance(request)) { + let headers = request.headers; + const body = request.body; + if (body) { + const [contentHash, treeHash] = await options.bodyChecksumGenerator( + request, + options + ); + + for (const [headerName, hash] of >[ + ["x-amz-content-sha256", contentHash], + ["x-amz-sha256-tree-hash", treeHash] + ]) { + if (!(headerName in headers) && hash) { + headers = { + ...headers, + [headerName]: hash + }; + } + } + + // Update request headers with new set of headers. + request.headers = headers; + } + } + + return next({ + ...args, + request + }); + }; +} + +export const addChecksumHeadersMiddlewareOptions: BuildHandlerOptions = { + step: "build", + tags: ["SET_CHECKSUM_HEADERS"], + name: "addChecksumHeadersMiddleware" +}; diff --git a/packages/middleware-sdk-glacier/src/add-glacier-api-version.spec.ts b/packages/middleware-sdk-glacier/src/add-glacier-api-version.spec.ts index a06cb915baf3..ebcc26a82353 100644 --- a/packages/middleware-sdk-glacier/src/add-glacier-api-version.spec.ts +++ b/packages/middleware-sdk-glacier/src/add-glacier-api-version.spec.ts @@ -3,15 +3,24 @@ import { HttpRequest } from "@aws-sdk/protocol-http"; describe("addGlacierApiVersion", () => { const mockNextHandler = jest.fn(); + const unusedDep = jest.fn(); beforeEach(() => { jest.clearAllMocks(); }); + const config = { + apiVersion: "1970-01-01", + bodyChecksumGenerator: unusedDep, + sha256: unusedDep, + utf8Decoder: unusedDep + }; + it("sets the x-amz-glacier-version header", async () => { - const handler = addGlacierApiVersionMiddleware({ - apiVersion: "1970-01-01" - })(mockNextHandler, {} as any); + const handler = addGlacierApiVersionMiddleware(config)( + mockNextHandler, + {} as any + ); await handler({ input: {}, diff --git a/packages/middleware-sdk-glacier/src/configurations.ts b/packages/middleware-sdk-glacier/src/configurations.ts index ee74be92ec42..94ba1e804c5f 100644 --- a/packages/middleware-sdk-glacier/src/configurations.ts +++ b/packages/middleware-sdk-glacier/src/configurations.ts @@ -6,16 +6,37 @@ import { addGlacierApiVersionMiddleware, addGlacierApiVersionMiddlewareOptions } from "./add-glacier-api-version"; -import { Pluggable } from "@aws-sdk/types"; +import { + Decoder, + HashConstructor, + HttpRequest, + Pluggable +} from "@aws-sdk/types"; +import { + addChecksumHeadersMiddleware, + addChecksumHeadersMiddlewareOptions +} from "./add-checksum-headers"; export interface GlacierMiddlewareInputConfig {} interface PreviouslyResolved { apiVersion: string; + sha256: HashConstructor; + utf8Decoder: Decoder; + bodyChecksumGenerator: ( + request: HttpRequest, + Options: { sha256: HashConstructor; utf8Decoder: Decoder } + ) => Promise<[string, string]>; } export interface ResolvedGlacierMiddlewareConfig { apiVersion: string; + sha256: HashConstructor; + utf8Decoder: Decoder; + bodyChecksumGenerator: ( + request: HttpRequest, + Options: { sha256: HashConstructor; utf8Decoder: Decoder } + ) => Promise<[string, string]>; } export function resolveGlacierMiddlewareConfig( @@ -38,5 +59,9 @@ export const getGlacierPlugin = ( addGlacierApiVersionMiddleware(config), addGlacierApiVersionMiddlewareOptions ); + clientStack.add( + addChecksumHeadersMiddleware(config), + addChecksumHeadersMiddlewareOptions + ); } }); diff --git a/packages/middleware-sdk-glacier/src/index.spec.ts b/packages/middleware-sdk-glacier/src/index.spec.ts index 33fcc3383ecf..96f5600cc192 100644 --- a/packages/middleware-sdk-glacier/src/index.spec.ts +++ b/packages/middleware-sdk-glacier/src/index.spec.ts @@ -1,4 +1,5 @@ import { + addChecksumHeadersMiddleware, accountIdDefaultMiddleware, addGlacierApiVersionMiddleware, getGlacierPlugin, @@ -6,6 +7,10 @@ import { } from "./index"; describe("middleware-sdk-glacier package exports", () => { + it("addChecksumHeadersMiddleware", () => { + expect(typeof addChecksumHeadersMiddleware).toBe("function"); + }); + it("addGlacierApiVersionMiddleware", () => { expect(typeof addGlacierApiVersionMiddleware).toBe("function"); }); diff --git a/packages/middleware-sdk-glacier/src/index.ts b/packages/middleware-sdk-glacier/src/index.ts index 459e9cfa5cdc..4d3ca3e46da5 100644 --- a/packages/middleware-sdk-glacier/src/index.ts +++ b/packages/middleware-sdk-glacier/src/index.ts @@ -1,3 +1,4 @@ +export * from "./add-checksum-headers"; export * from "./add-glacier-api-version"; export * from "./account-id-default"; export * from "./configurations"; diff --git a/packages/middleware-sdk-glacier/tsconfig.json b/packages/middleware-sdk-glacier/tsconfig.json index 38b94cda274e..d34b34707316 100644 --- a/packages/middleware-sdk-glacier/tsconfig.json +++ b/packages/middleware-sdk-glacier/tsconfig.json @@ -9,6 +9,7 @@ "importHelpers": true, "noEmitHelpers": true, "lib": [ + "dom", "es5", "es2015.promise", "es2015.collection",