diff --git a/clients/client-s3/src/commands/GetObjectCommand.ts b/clients/client-s3/src/commands/GetObjectCommand.ts index b5e85a9f857b..1f0767dcfb76 100644 --- a/clients/client-s3/src/commands/GetObjectCommand.ts +++ b/clients/client-s3/src/commands/GetObjectCommand.ts @@ -1,5 +1,6 @@ // smithy-typescript generated code import { getFlexibleChecksumsPlugin } from "@aws-sdk/middleware-flexible-checksums"; +import { getS3ExpiresMiddlewarePlugin } from "@aws-sdk/middleware-sdk-s3"; import { getSsecPlugin } from "@aws-sdk/middleware-ssec"; import { getEndpointPlugin } from "@smithy/middleware-endpoint"; import { getSerdePlugin } from "@smithy/middleware-serde"; @@ -240,6 +241,7 @@ export interface GetObjectCommandOutput extends Omit, _ * // ContentRange: "STRING_VALUE", * // ContentType: "STRING_VALUE", * // Expires: new Date("TIMESTAMP"), + * // ExpiresString: "STRING_VALUE", * // WebsiteRedirectLocation: "STRING_VALUE", * // ServerSideEncryption: "AES256" || "aws:kms" || "aws:kms:dsse", * // Metadata: { // Metadata @@ -351,6 +353,7 @@ export class GetObjectCommand extends $Command getSerdePlugin(config, this.serialize, this.deserialize), getEndpointPlugin(config, Command.getEndpointParameterInstructions()), getSsecPlugin(config), + getS3ExpiresMiddlewarePlugin(config), getFlexibleChecksumsPlugin(config, { input: this.input, requestChecksumRequired: false, diff --git a/clients/client-s3/src/commands/HeadObjectCommand.ts b/clients/client-s3/src/commands/HeadObjectCommand.ts index 5be1de7b71c8..97b7eb482c08 100644 --- a/clients/client-s3/src/commands/HeadObjectCommand.ts +++ b/clients/client-s3/src/commands/HeadObjectCommand.ts @@ -1,4 +1,5 @@ // smithy-typescript generated code +import { getS3ExpiresMiddlewarePlugin } from "@aws-sdk/middleware-sdk-s3"; import { getSsecPlugin } from "@aws-sdk/middleware-ssec"; import { getEndpointPlugin } from "@smithy/middleware-endpoint"; import { getSerdePlugin } from "@smithy/middleware-serde"; @@ -215,6 +216,7 @@ export interface HeadObjectCommandOutput extends HeadObjectOutput, __MetadataBea * // ContentLanguage: "STRING_VALUE", * // ContentType: "STRING_VALUE", * // Expires: new Date("TIMESTAMP"), + * // ExpiresString: "STRING_VALUE", * // WebsiteRedirectLocation: "STRING_VALUE", * // ServerSideEncryption: "AES256" || "aws:kms" || "aws:kms:dsse", * // Metadata: { // Metadata @@ -289,6 +291,7 @@ export class HeadObjectCommand extends $Command getSerdePlugin(config, this.serialize, this.deserialize), getEndpointPlugin(config, Command.getEndpointParameterInstructions()), getSsecPlugin(config), + getS3ExpiresMiddlewarePlugin(config), ]; }) .s("AmazonS3", "HeadObject", {}) diff --git a/clients/client-s3/src/models/models_0.ts b/clients/client-s3/src/models/models_0.ts index 87b1afa64a05..91b0215e46b3 100644 --- a/clients/client-s3/src/models/models_0.ts +++ b/clients/client-s3/src/models/models_0.ts @@ -1,6 +1,5 @@ // smithy-typescript generated code import { ExceptionOptionType as __ExceptionOptionType, SENSITIVE_STRING } from "@smithy/smithy-client"; - import { StreamingBlobTypes } from "@smithy/types"; import { S3ServiceException as __BaseException } from "./S3ServiceException"; @@ -9037,10 +9036,18 @@ export interface GetObjectOutput { /** * @public - *

The date and time at which the object is no longer cacheable.

+ * @deprecated + * + * Deprecated in favor of ExpiresString. */ Expires?: Date; + /** + * @public + *

The date and time at which the object is no longer cacheable.

+ */ + ExpiresString?: string; + /** * @public *

If the bucket is configured as a website, redirects requests for this object to another @@ -10772,10 +10779,18 @@ export interface HeadObjectOutput { /** * @public - *

The date and time at which the object is no longer cacheable.

+ * @deprecated + * + * Deprecated in favor of ExpiresString. */ Expires?: Date; + /** + * @public + *

The date and time at which the object is no longer cacheable.

+ */ + ExpiresString?: string; + /** * @public *

If the bucket is configured as a website, redirects requests for this object to another diff --git a/clients/client-s3/src/models/models_1.ts b/clients/client-s3/src/models/models_1.ts index dc79c2b1ad48..4e7bd09a2305 100644 --- a/clients/client-s3/src/models/models_1.ts +++ b/clients/client-s3/src/models/models_1.ts @@ -1,6 +1,5 @@ // smithy-typescript generated code import { ExceptionOptionType as __ExceptionOptionType, SENSITIVE_STRING } from "@smithy/smithy-client"; - import { StreamingBlobTypes } from "@smithy/types"; import { @@ -28,7 +27,6 @@ import { StorageClass, Tag, } from "./models_0"; - import { S3ServiceException as __BaseException } from "./S3ServiceException"; /** diff --git a/clients/client-s3/src/protocols/Aws_restXml.ts b/clients/client-s3/src/protocols/Aws_restXml.ts index 86fbd3624689..2394812fcc44 100644 --- a/clients/client-s3/src/protocols/Aws_restXml.ts +++ b/clients/client-s3/src/protocols/Aws_restXml.ts @@ -4950,6 +4950,7 @@ export const de_GetObjectCommand = async ( [_CR]: [, output.headers[_cr]], [_CT]: [, output.headers[_ct]], [_E]: [() => void 0 !== output.headers[_e], () => __expectNonNull(__parseRfc7231DateTime(output.headers[_e]))], + [_ES]: [, output.headers[_ex]], [_WRL]: [, output.headers[_xawrl]], [_SSE]: [, output.headers[_xasse]], [_SSECA]: [, output.headers[_xasseca]], @@ -5440,6 +5441,7 @@ export const de_HeadObjectCommand = async ( [_CL]: [, output.headers[_cl]], [_CT]: [, output.headers[_ct]], [_E]: [() => void 0 !== output.headers[_e], () => __expectNonNull(__parseRfc7231DateTime(output.headers[_e]))], + [_ES]: [, output.headers[_ex]], [_WRL]: [, output.headers[_xawrl]], [_SSE]: [, output.headers[_xasse]], [_SSECA]: [, output.headers[_xasseca]], @@ -8359,7 +8361,7 @@ const se_LifecycleRule = (input: LifecycleRule, context: __SerdeContext): any => bn.c(se_LifecycleRuleFilter(input[_F], context).n(_F)); } if (input[_S] != null) { - bn.c(__XmlNode.of(_ES, input[_S]).n(_S)); + bn.c(__XmlNode.of(_ESx, input[_S]).n(_S)); } bn.l(input, "Transitions", "Transition", () => se_TransitionList(input[_Tr]!, context)); bn.l(input, "NoncurrentVersionTransitions", "NoncurrentVersionTransition", () => @@ -11772,8 +11774,9 @@ const _EODM = "ExpiredObjectDeleteMarker"; const _EOR = "ExistingObjectReplication"; const _EORS = "ExistingObjectReplicationStatus"; const _ERP = "EnableRequestProgress"; -const _ES = "ExpirationStatus"; +const _ES = "ExpiresString"; const _ESBO = "ExpectedSourceBucketOwner"; +const _ESx = "ExpirationStatus"; const _ET = "EncodingType"; const _ETa = "ETag"; const _ETn = "EncryptionType"; @@ -12116,6 +12119,7 @@ const _e = "expires"; const _en = "encryption"; const _et = "encoding-type"; const _eta = "etag"; +const _ex = "expiresstring"; const _fo = "fetch-owner"; const _i = "id"; const _im = "if-match"; diff --git a/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java b/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java index e069211d8b85..bb7df1bb9b3d 100644 --- a/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java +++ b/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java @@ -269,6 +269,11 @@ && containsInputMembers(m, o, BUCKET_ENDPOINT_INPUT_KEYS)) HAS_MIDDLEWARE) .servicePredicate((m, s) -> isS3(s)) .build(), + RuntimeClientPlugin.builder() + .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "S3ExpiresMiddleware", + HAS_MIDDLEWARE) + .operationPredicate((m, s, o) -> containsExpiresOutput(m, o)) + .build(), RuntimeClientPlugin.builder() .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "S3Express", HAS_MIDDLEWARE) @@ -288,6 +293,16 @@ private static boolean containsInputMembers( .isPresent(); } + private static boolean containsExpiresOutput( + Model model, + OperationShape operationShape + ) { + OperationIndex operationIndex = OperationIndex.of(model); + return operationIndex.getOutput(operationShape) + .filter(input -> input.getMemberNames().stream().anyMatch("Expires"::equals)) + .isPresent(); + } + private static boolean isS3(Shape serviceShape) { return serviceShape.getTrait(ServiceTrait.class).map(ServiceTrait::getSdkId).orElse("").equals("S3"); } diff --git a/packages/middleware-sdk-s3/src/index.ts b/packages/middleware-sdk-s3/src/index.ts index d7294c390d22..34f756c05b4a 100644 --- a/packages/middleware-sdk-s3/src/index.ts +++ b/packages/middleware-sdk-s3/src/index.ts @@ -1,6 +1,7 @@ export * from "./check-content-length-header"; export * from "./region-redirect-endpoint-middleware"; export * from "./region-redirect-middleware"; +export * from "./s3-expires-middleware"; export * from "./s3-express/index"; export * from "./s3Configuration"; export * from "./throw-200-exceptions"; diff --git a/packages/middleware-sdk-s3/src/s3-expires-middleware.e2e.spec.ts b/packages/middleware-sdk-s3/src/s3-expires-middleware.e2e.spec.ts new file mode 100644 index 000000000000..6b7ca0f78e8b --- /dev/null +++ b/packages/middleware-sdk-s3/src/s3-expires-middleware.e2e.spec.ts @@ -0,0 +1,126 @@ +import { S3 } from "@aws-sdk/client-s3"; +import { GetCallerIdentityCommandOutput, STS } from "@aws-sdk/client-sts"; + +jest.setTimeout(25000); + +describe("S3 Expires e2e test", () => { + const s3 = new S3({ + region: "us-west-2", + logger: { + trace() {}, + debug() {}, + info() {}, + warn: jest.fn(), + error() {}, + }, + }); + const stsClient = new STS({ region: "us-west-2" }); + + let callerID = null as unknown as GetCallerIdentityCommandOutput; + let Bucket: string; + + // random element limited to 2 letters to avoid concurrent IO, and + // to limit bucket count to 676 if there is failure to delete them. + const alphabet = "abcdefghijklmnopqrstuvwxyz"; + const randId = alphabet[(Math.random() * alphabet.length) | 0] + alphabet[(Math.random() * alphabet.length) | 0]; + + beforeAll(async () => { + callerID = await stsClient.getCallerIdentity({}); + Bucket = `${callerID.Account}-${randId}-s3-expires`; + await s3.createBucket({ + Bucket, + }); + }); + + afterAll(async () => { + await deleteBucket(s3, Bucket); + }); + + const staticDate = new Date(0); + const dateString = "Thu, 01 Jan 1970 00:00:00 GMT"; + + it("should parse Expires from response if it is valid date-time, and include ExpiresString", async () => { + await s3.putObject({ + Bucket, + Key: "good-expires", + Expires: staticDate, + Body: "good-expires", + }); + + const get = await s3.getObject({ + Bucket, + Key: "good-expires", + }); + await get.Body?.transformToByteArray(); // drain stream. + + expect(get.Expires?.getTime()).toEqual(staticDate.getTime()); + expect(get.ExpiresString).toEqual(dateString); + }); + + it("should fail with a non-blocking warning if Expires is not a valid date-time, and include the raw string in ExpiresString", async () => { + await s3.putObject({ + Bucket, + Key: "bad-expires", + Expires: new Date("invalid date"), + Body: "bad-expires", + }); + + const get = await s3.getObject({ + Bucket, + Key: "bad-expires", + }); + await get.Body?.transformToByteArray(); // drain stream. + + expect(get.Expires).toBeUndefined(); + expect(s3.config.logger.warn).toHaveBeenCalledWith( + `AWS SDK Warning for S3Client::GetObjectCommand response parsing (undefined, NaN undefined NaN NaN:NaN:NaN GMT): TypeError: Invalid RFC-7231 date-time value` + ); + expect(get.ExpiresString).toEqual("undefined, NaN undefined NaN NaN:NaN:NaN GMT"); + }); +}); + +async function deleteBucket(s3: S3, bucketName: string) { + const Bucket = bucketName; + + try { + await s3.headBucket({ + Bucket, + }); + } catch (e) { + return; + } + + const list = await s3 + .listObjects({ + Bucket, + }) + .catch((e) => { + if (!String(e).includes("NoSuchBucket")) { + throw e; + } + return { + Contents: [], + }; + }); + + const promises = [] as any[]; + for (const key of list.Contents ?? []) { + promises.push( + s3.deleteObject({ + Bucket, + Key: key.Key, + }) + ); + } + await Promise.all(promises); + + try { + return await s3.deleteBucket({ + Bucket, + }); + } catch (e) { + if (!String(e).includes("NoSuchBucket")) { + throw e; + } + } +} diff --git a/packages/middleware-sdk-s3/src/s3-expires-middleware.ts b/packages/middleware-sdk-s3/src/s3-expires-middleware.ts new file mode 100644 index 000000000000..17853a3350c0 --- /dev/null +++ b/packages/middleware-sdk-s3/src/s3-expires-middleware.ts @@ -0,0 +1,72 @@ +import { HttpResponse } from "@smithy/protocol-http"; +import { parseRfc7231DateTime } from "@smithy/smithy-client"; +import { + DeserializeHandler, + DeserializeHandlerArguments, + DeserializeHandlerOutput, + DeserializeMiddleware, + HandlerExecutionContext, + MetadataBearer, + Pluggable, + RelativeMiddlewareOptions, +} from "@smithy/types"; + +/** + * @internal + */ +interface PreviouslyResolved {} + +/** + * @internal + * + * From the S3 Expires compatibility spec. + * A model transform will ensure S3#Expires remains a timestamp shape, though + * it is deprecated. + * If a particular object has a non-date string set as the Expires value, + * the SDK will have the raw string as "ExpiresString" on the response. + * + */ +export const s3ExpiresMiddleware = (config: PreviouslyResolved): DeserializeMiddleware => { + return ( + next: DeserializeHandler, + context: HandlerExecutionContext + ): DeserializeHandler => + async (args: DeserializeHandlerArguments): Promise> => { + const result = await next(args); + const { response } = result; + if (HttpResponse.isInstance(response)) { + if (response.headers.expires) { + response.headers.expiresstring = response.headers.expires; + try { + parseRfc7231DateTime(response.headers.expires); + } catch (e) { + context.logger?.warn( + `AWS SDK Warning for ${context.clientName}::${context.commandName} response parsing (${response.headers.expires}): ${e}` + ); + delete response.headers.expires; + } + } + } + return result; + }; +}; + +/** + * @internal + */ +export const s3ExpiresMiddlewareOptions: RelativeMiddlewareOptions = { + tags: ["S3"], + name: "s3ExpiresMiddleware", + override: true, + relation: "after", + toMiddleware: "deserializerMiddleware", +}; + +/** + * @internal + */ +export const getS3ExpiresMiddlewarePlugin = (clientConfig: PreviouslyResolved): Pluggable => ({ + applyToStack: (clientStack) => { + clientStack.addRelativeTo(s3ExpiresMiddleware(clientConfig), s3ExpiresMiddlewareOptions); + }, +}); diff --git a/scripts/generate-clients/s3-hack.js b/scripts/generate-clients/s3-hack.js index 8050367d2543..4038ca3e4da0 100644 --- a/scripts/generate-clients/s3-hack.js +++ b/scripts/generate-clients/s3-hack.js @@ -10,6 +10,8 @@ const s3ModelObject = require(s3ModelLocation); /** * Activates a hack for S3-express Smithy suppression. + * And another one for S3 Expires. + * * @returns a function that undoes the hack. */ module.exports = function () { @@ -21,6 +23,53 @@ module.exports = function () { namespace: "com.amazonaws.s3", }); + const expiresShape = s3ModelObject.shapes["com.amazonaws.s3#Expires"]; + if (expiresShape) { + // enforce that Expires retains type timestamp. + expiresShape.type = "timestamp"; + + // add the ExpiresString string shape. + const newShapes = {}; + for (const [shapeId, shape] of Object.entries(s3ModelObject.shapes)) { + newShapes[shapeId] = shape; + if (shapeId === "com.amazonaws.s3#Expires") { + newShapes["com.amazonaws.s3#ExpiresString"] = { + type: "string", + }; + } + } + s3ModelObject.shapes = newShapes; + + // add ExpiresString alongside output shapes containing Expires. + for (const [shapeId, shape] of Object.entries(s3ModelObject.shapes)) { + if (shape?.traits?.["smithy.api#output"]) { + const newMembers = {}; + for (const [memberName, member] of Object.entries(shape.members)) { + newMembers[memberName] = member; + if (member.target === "com.amazonaws.s3#Expires") { + const existingDoc = member.traits["smithy.api#documentation"]; + if (!member.traits) { + member.traits = {}; + } + + newMembers.ExpiresString = { + target: "com.amazonaws.s3#ExpiresString", + traits: { + ...member.traits, + "smithy.api#httpHeader": "ExpiresString", + "smithy.api#documentation": existingDoc, + }, + }; + + member.traits["smithy.api#deprecated"] = {}; + member.traits["smithy.api#documentation"] = "Deprecated in favor of ExpiresString."; + } + } + shape.members = newMembers; + } + } + } + fs.writeFileSync(s3ModelLocation, JSON.stringify(s3ModelObject, null, 2)); return () => {