feat(client-firehose): Amazon Data Firehose / Features : Adds support…

… for a new DeliveryStreamType, DatabaseAsSource. DatabaseAsSource hoses allow customers to stream CDC events from their RDS and Amazon EC2 hosted databases, running MySQL and PostgreSQL database engines, to Iceberg Table destinations.
aws · Nov 8, 2024 · 9933561 · 9933561
1 parent 88e1c21
commit 9933561
Show file tree

Hide file tree

Showing 15 changed files with 2,002 additions and 582 deletions.
diff --git a/clients/client-firehose/src/commands/CreateDeliveryStreamCommand.ts b/clients/client-firehose/src/commands/CreateDeliveryStreamCommand.ts
@@ -32,30 +32,30 @@ export interface CreateDeliveryStreamCommandInput extends CreateDeliveryStreamIn
 export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamOutput, __MetadataBearer {}
 
 /**
- * <p>Creates a Firehose delivery stream.</p>
- *          <p>By default, you can create up to 50 delivery streams per Amazon Web Services
+ * <p>Creates a Firehose stream.</p>
+ *          <p>By default, you can create up to 50 Firehose streams per Amazon Web Services
  *          Region.</p>
  *          <p>This is an asynchronous operation that immediately returns. The initial status of the
- *          delivery stream is <code>CREATING</code>. After the delivery stream is created, its status
- *          is <code>ACTIVE</code> and it now accepts data. If the delivery stream creation fails, the
+ *          Firehose stream is <code>CREATING</code>. After the Firehose stream is created, its status
+ *          is <code>ACTIVE</code> and it now accepts data. If the Firehose stream creation fails, the
  *          status transitions to <code>CREATING_FAILED</code>. Attempts to send data to a delivery
  *          stream that is not in the <code>ACTIVE</code> state cause an exception. To check the state
- *          of a delivery stream, use <a>DescribeDeliveryStream</a>.</p>
- *          <p>If the status of a delivery stream is <code>CREATING_FAILED</code>, this status
+ *          of a Firehose stream, use <a>DescribeDeliveryStream</a>.</p>
+ *          <p>If the status of a Firehose stream is <code>CREATING_FAILED</code>, this status
  *          doesn't change, and you can't invoke <code>CreateDeliveryStream</code> again on it.
  *          However, you can invoke the <a>DeleteDeliveryStream</a> operation to delete
  *          it.</p>
- *          <p>A Firehose delivery stream can be configured to receive records directly
+ *          <p>A Firehose stream can be configured to receive records directly
  *          from providers using <a>PutRecord</a> or <a>PutRecordBatch</a>, or it
  *          can be configured to use an existing Kinesis stream as its source. To specify a Kinesis
  *          data stream as input, set the <code>DeliveryStreamType</code> parameter to
  *             <code>KinesisStreamAsSource</code>, and provide the Kinesis stream Amazon Resource Name
  *          (ARN) and role ARN in the <code>KinesisStreamSourceConfiguration</code>
  *          parameter.</p>
- *          <p>To create a delivery stream with server-side encryption (SSE) enabled, include <a>DeliveryStreamEncryptionConfigurationInput</a> in your request. This is
+ *          <p>To create a Firehose stream with server-side encryption (SSE) enabled, include <a>DeliveryStreamEncryptionConfigurationInput</a> in your request. This is
  *          optional. You can also invoke <a>StartDeliveryStreamEncryption</a> to turn on
- *          SSE for an existing delivery stream that doesn't have SSE enabled.</p>
- *          <p>A delivery stream is configured with a single destination, such as Amazon Simple
+ *          SSE for an existing Firehose stream that doesn't have SSE enabled.</p>
+ *          <p>A Firehose stream is configured with a single destination, such as Amazon Simple
  *          Storage Service (Amazon S3), Amazon Redshift, Amazon OpenSearch Service, Amazon OpenSearch
  *          Serverless, Splunk, and any custom HTTP endpoint or HTTP endpoints owned by or supported by
  *          third-party service providers, including Datadog, Dynatrace, LogicMonitor, MongoDB, New
@@ -107,7 +107,7 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
  * const client = new FirehoseClient(config);
  * const input = { // CreateDeliveryStreamInput
  *   DeliveryStreamName: "STRING_VALUE", // required
- *   DeliveryStreamType: "DirectPut" || "KinesisStreamAsSource" || "MSKAsSource",
+ *   DeliveryStreamType: "DirectPut" || "KinesisStreamAsSource" || "MSKAsSource" || "DatabaseAsSource",
  *   KinesisStreamSourceConfiguration: { // KinesisStreamSourceConfiguration
  *     KinesisStreamARN: "STRING_VALUE", // required
  *     RoleARN: "STRING_VALUE", // required
@@ -586,9 +586,22 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
  *         UniqueKeys: [
  *           "STRING_VALUE",
  *         ],
+ *         PartitionSpec: { // PartitionSpec
+ *           Identity: [ // PartitionFields
+ *             { // PartitionField
+ *               SourceName: "STRING_VALUE", // required
+ *             },
+ *           ],
+ *         },
  *         S3ErrorOutputPrefix: "STRING_VALUE",
  *       },
  *     ],
+ *     SchemaEvolutionConfiguration: { // SchemaEvolutionConfiguration
+ *       Enabled: true || false, // required
+ *     },
+ *     TableCreationConfiguration: { // TableCreationConfiguration
+ *       Enabled: true || false, // required
+ *     },
  *     BufferingHints: "<BufferingHints>",
  *     CloudWatchLoggingOptions: "<CloudWatchLoggingOptions>",
  *     ProcessingConfiguration: "<ProcessingConfiguration>",
@@ -599,9 +612,54 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
  *     RoleARN: "STRING_VALUE", // required
  *     CatalogConfiguration: { // CatalogConfiguration
  *       CatalogARN: "STRING_VALUE",
+ *       WarehouseLocation: "STRING_VALUE",
  *     },
  *     S3Configuration: "<S3DestinationConfiguration>", // required
  *   },
+ *   DatabaseSourceConfiguration: { // DatabaseSourceConfiguration
+ *     Type: "MySQL" || "PostgreSQL", // required
+ *     Endpoint: "STRING_VALUE", // required
+ *     Port: Number("int"), // required
+ *     SSLMode: "Disabled" || "Enabled",
+ *     Databases: { // DatabaseList
+ *       Include: [ // DatabaseIncludeOrExcludeList
+ *         "STRING_VALUE",
+ *       ],
+ *       Exclude: [
+ *         "STRING_VALUE",
+ *       ],
+ *     },
+ *     Tables: { // DatabaseTableList
+ *       Include: [ // DatabaseTableIncludeOrExcludeList
+ *         "STRING_VALUE",
+ *       ],
+ *       Exclude: [
+ *         "STRING_VALUE",
+ *       ],
+ *     },
+ *     Columns: { // DatabaseColumnList
+ *       Include: [ // DatabaseColumnIncludeOrExcludeList
+ *         "STRING_VALUE",
+ *       ],
+ *       Exclude: [
+ *         "STRING_VALUE",
+ *       ],
+ *     },
+ *     SurrogateKeys: [ // DatabaseSurrogateKeyList
+ *       "STRING_VALUE",
+ *     ],
+ *     SnapshotWatermarkTable: "STRING_VALUE", // required
+ *     DatabaseSourceAuthenticationConfiguration: { // DatabaseSourceAuthenticationConfiguration
+ *       SecretsManagerConfiguration: {
+ *         SecretARN: "STRING_VALUE",
+ *         RoleARN: "STRING_VALUE",
+ *         Enabled: true || false, // required
+ *       },
+ *     },
+ *     DatabaseSourceVPCConfiguration: { // DatabaseSourceVPCConfiguration
+ *       VpcEndpointServiceName: "STRING_VALUE", // required
+ *     },
+ *   },
  * };
  * const command = new CreateDeliveryStreamCommand(input);
  * const response = await client.send(command);
@@ -622,7 +680,7 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
  *
  * @throws {@link InvalidKMSResourceException} (client fault)
  *  <p>Firehose throws this exception when an attempt to put records or to start
- *          or stop delivery stream encryption fails. This happens when the KMS service throws one of
+ *          or stop Firehose stream encryption fails. This happens when the KMS service throws one of
  *          the following exception types: <code>AccessDeniedException</code>,
  *             <code>InvalidStateException</code>, <code>DisabledException</code>, or
  *             <code>NotFoundException</code>.</p>

diff --git a/clients/client-firehose/src/commands/DeleteDeliveryStreamCommand.ts b/clients/client-firehose/src/commands/DeleteDeliveryStreamCommand.ts
@@ -28,17 +28,17 @@ export interface DeleteDeliveryStreamCommandInput extends DeleteDeliveryStreamIn
 export interface DeleteDeliveryStreamCommandOutput extends DeleteDeliveryStreamOutput, __MetadataBearer {}
 
 /**
- * <p>Deletes a delivery stream and its data.</p>
- *          <p>You can delete a delivery stream only if it is in one of the following states:
+ * <p>Deletes a Firehose stream and its data.</p>
+ *          <p>You can delete a Firehose stream only if it is in one of the following states:
  *             <code>ACTIVE</code>, <code>DELETING</code>, <code>CREATING_FAILED</code>, or
- *             <code>DELETING_FAILED</code>. You can't delete a delivery stream that is in the
- *          <code>CREATING</code> state. To check the state of a delivery stream, use <a>DescribeDeliveryStream</a>. </p>
- *          <p>DeleteDeliveryStream is an asynchronous API. When an API request to DeleteDeliveryStream succeeds, the delivery stream is marked for deletion, and it goes into the
- *          <code>DELETING</code> state.While the delivery stream is in the <code>DELETING</code> state, the service might
+ *             <code>DELETING_FAILED</code>. You can't delete a Firehose stream that is in the
+ *          <code>CREATING</code> state. To check the state of a Firehose stream, use <a>DescribeDeliveryStream</a>. </p>
+ *          <p>DeleteDeliveryStream is an asynchronous API. When an API request to DeleteDeliveryStream succeeds, the Firehose stream is marked for deletion, and it goes into the
+ *          <code>DELETING</code> state.While the Firehose stream is in the <code>DELETING</code> state, the service might
  *          continue to accept records, but it doesn't make any guarantees with respect to delivering
  *          the data. Therefore, as a best practice, first stop any applications that are sending
- *          records before you delete a delivery stream.</p>
- *          <p>Removal of a delivery stream that is in the <code>DELETING</code> state is a low priority operation for the service. A stream may remain in the
+ *          records before you delete a Firehose stream.</p>
+ *          <p>Removal of a Firehose stream that is in the <code>DELETING</code> state is a low priority operation for the service. A stream may remain in the
  *          <code>DELETING</code> state for several minutes. Therefore, as a best practice, applications should not wait for streams in the <code>DELETING</code> state
  *          to be removed. </p>
  * @example

diff --git a/clients/client-firehose/src/commands/DescribeDeliveryStreamCommand.ts b/clients/client-firehose/src/commands/DescribeDeliveryStreamCommand.ts
@@ -32,10 +32,10 @@ export interface DescribeDeliveryStreamCommandInput extends DescribeDeliveryStre
 export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStreamOutput, __MetadataBearer {}
 
 /**
- * <p>Describes the specified delivery stream and its status. For example, after your
- *          delivery stream is created, call <code>DescribeDeliveryStream</code> to see whether the
- *          delivery stream is <code>ACTIVE</code> and therefore ready for data to be sent to it. </p>
- *          <p>If the status of a delivery stream is <code>CREATING_FAILED</code>, this status
+ * <p>Describes the specified Firehose stream and its status. For example, after your
+ *          Firehose stream is created, call <code>DescribeDeliveryStream</code> to see whether the
+ *          Firehose stream is <code>ACTIVE</code> and therefore ready for data to be sent to it. </p>
+ *          <p>If the status of a Firehose stream is <code>CREATING_FAILED</code>, this status
  *          doesn't change, and you can't invoke <a>CreateDeliveryStream</a> again on it.
  *          However, you can invoke the <a>DeleteDeliveryStream</a> operation to delete it.
  *          If the status is <code>DELETING_FAILED</code>, you can force deletion by invoking <a>DeleteDeliveryStream</a> again but with <a>DeleteDeliveryStreamInput$AllowForceDelete</a> set to true.</p>
@@ -58,19 +58,19 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
  * //     DeliveryStreamARN: "STRING_VALUE", // required
  * //     DeliveryStreamStatus: "CREATING" || "CREATING_FAILED" || "DELETING" || "DELETING_FAILED" || "ACTIVE", // required
  * //     FailureDescription: { // FailureDescription
- * //       Type: "RETIRE_KMS_GRANT_FAILED" || "CREATE_KMS_GRANT_FAILED" || "KMS_ACCESS_DENIED" || "DISABLED_KMS_KEY" || "INVALID_KMS_KEY" || "KMS_KEY_NOT_FOUND" || "KMS_OPT_IN_REQUIRED" || "CREATE_ENI_FAILED" || "DELETE_ENI_FAILED" || "SUBNET_NOT_FOUND" || "SECURITY_GROUP_NOT_FOUND" || "ENI_ACCESS_DENIED" || "SUBNET_ACCESS_DENIED" || "SECURITY_GROUP_ACCESS_DENIED" || "UNKNOWN_ERROR", // required
+ * //       Type: "VPC_ENDPOINT_SERVICE_NAME_NOT_FOUND" || "VPC_INTERFACE_ENDPOINT_SERVICE_ACCESS_DENIED" || "RETIRE_KMS_GRANT_FAILED" || "CREATE_KMS_GRANT_FAILED" || "KMS_ACCESS_DENIED" || "DISABLED_KMS_KEY" || "INVALID_KMS_KEY" || "KMS_KEY_NOT_FOUND" || "KMS_OPT_IN_REQUIRED" || "CREATE_ENI_FAILED" || "DELETE_ENI_FAILED" || "SUBNET_NOT_FOUND" || "SECURITY_GROUP_NOT_FOUND" || "ENI_ACCESS_DENIED" || "SUBNET_ACCESS_DENIED" || "SECURITY_GROUP_ACCESS_DENIED" || "UNKNOWN_ERROR", // required
  * //       Details: "STRING_VALUE", // required
  * //     },
  * //     DeliveryStreamEncryptionConfiguration: { // DeliveryStreamEncryptionConfiguration
  * //       KeyARN: "STRING_VALUE",
  * //       KeyType: "AWS_OWNED_CMK" || "CUSTOMER_MANAGED_CMK",
  * //       Status: "ENABLED" || "ENABLING" || "ENABLING_FAILED" || "DISABLED" || "DISABLING" || "DISABLING_FAILED",
  * //       FailureDescription: {
- * //         Type: "RETIRE_KMS_GRANT_FAILED" || "CREATE_KMS_GRANT_FAILED" || "KMS_ACCESS_DENIED" || "DISABLED_KMS_KEY" || "INVALID_KMS_KEY" || "KMS_KEY_NOT_FOUND" || "KMS_OPT_IN_REQUIRED" || "CREATE_ENI_FAILED" || "DELETE_ENI_FAILED" || "SUBNET_NOT_FOUND" || "SECURITY_GROUP_NOT_FOUND" || "ENI_ACCESS_DENIED" || "SUBNET_ACCESS_DENIED" || "SECURITY_GROUP_ACCESS_DENIED" || "UNKNOWN_ERROR", // required
+ * //         Type: "VPC_ENDPOINT_SERVICE_NAME_NOT_FOUND" || "VPC_INTERFACE_ENDPOINT_SERVICE_ACCESS_DENIED" || "RETIRE_KMS_GRANT_FAILED" || "CREATE_KMS_GRANT_FAILED" || "KMS_ACCESS_DENIED" || "DISABLED_KMS_KEY" || "INVALID_KMS_KEY" || "KMS_KEY_NOT_FOUND" || "KMS_OPT_IN_REQUIRED" || "CREATE_ENI_FAILED" || "DELETE_ENI_FAILED" || "SUBNET_NOT_FOUND" || "SECURITY_GROUP_NOT_FOUND" || "ENI_ACCESS_DENIED" || "SUBNET_ACCESS_DENIED" || "SECURITY_GROUP_ACCESS_DENIED" || "UNKNOWN_ERROR", // required
  * //         Details: "STRING_VALUE", // required
  * //       },
  * //     },
- * //     DeliveryStreamType: "DirectPut" || "KinesisStreamAsSource" || "MSKAsSource", // required
+ * //     DeliveryStreamType: "DirectPut" || "KinesisStreamAsSource" || "MSKAsSource" || "DatabaseAsSource", // required
  * //     VersionId: "STRING_VALUE", // required
  * //     CreateTimestamp: new Date("TIMESTAMP"),
  * //     LastUpdateTimestamp: new Date("TIMESTAMP"),
@@ -90,6 +90,60 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
  * //         DeliveryStartTimestamp: new Date("TIMESTAMP"),
  * //         ReadFromTimestamp: new Date("TIMESTAMP"),
  * //       },
+ * //       DatabaseSourceDescription: { // DatabaseSourceDescription
+ * //         Type: "MySQL" || "PostgreSQL",
+ * //         Endpoint: "STRING_VALUE",
+ * //         Port: Number("int"),
+ * //         SSLMode: "Disabled" || "Enabled",
+ * //         Databases: { // DatabaseList
+ * //           Include: [ // DatabaseIncludeOrExcludeList
+ * //             "STRING_VALUE",
+ * //           ],
+ * //           Exclude: [
+ * //             "STRING_VALUE",
+ * //           ],
+ * //         },
+ * //         Tables: { // DatabaseTableList
+ * //           Include: [ // DatabaseTableIncludeOrExcludeList
+ * //             "STRING_VALUE",
+ * //           ],
+ * //           Exclude: [
+ * //             "STRING_VALUE",
+ * //           ],
+ * //         },
+ * //         Columns: { // DatabaseColumnList
+ * //           Include: [ // DatabaseColumnIncludeOrExcludeList
+ * //             "STRING_VALUE",
+ * //           ],
+ * //           Exclude: [
+ * //             "STRING_VALUE",
+ * //           ],
+ * //         },
+ * //         SurrogateKeys: [
+ * //           "STRING_VALUE",
+ * //         ],
+ * //         SnapshotWatermarkTable: "STRING_VALUE",
+ * //         SnapshotInfo: [ // DatabaseSnapshotInfoList
+ * //           { // DatabaseSnapshotInfo
+ * //             Id: "STRING_VALUE", // required
+ * //             Table: "STRING_VALUE", // required
+ * //             RequestTimestamp: new Date("TIMESTAMP"), // required
+ * //             RequestedBy: "USER" || "FIREHOSE", // required
+ * //             Status: "IN_PROGRESS" || "COMPLETE" || "SUSPENDED", // required
+ * //             FailureDescription: "<FailureDescription>",
+ * //           },
+ * //         ],
+ * //         DatabaseSourceAuthenticationConfiguration: { // DatabaseSourceAuthenticationConfiguration
+ * //           SecretsManagerConfiguration: { // SecretsManagerConfiguration
+ * //             SecretARN: "STRING_VALUE",
+ * //             RoleARN: "STRING_VALUE",
+ * //             Enabled: true || false, // required
+ * //           },
+ * //         },
+ * //         DatabaseSourceVPCConfiguration: { // DatabaseSourceVPCConfiguration
+ * //           VpcEndpointServiceName: "STRING_VALUE", // required
+ * //         },
+ * //       },
  * //     },
  * //     Destinations: [ // DestinationDescriptionList // required
  * //       { // DestinationDescription
@@ -308,7 +362,7 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
  * //             },
  * //           },
  * //           CloudWatchLoggingOptions: "<CloudWatchLoggingOptions>",
- * //           SecretsManagerConfiguration: { // SecretsManagerConfiguration
+ * //           SecretsManagerConfiguration: {
  * //             SecretARN: "STRING_VALUE",
  * //             RoleARN: "STRING_VALUE",
  * //             Enabled: true || false, // required
@@ -548,9 +602,22 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
  * //               UniqueKeys: [
  * //                 "STRING_VALUE",
  * //               ],
+ * //               PartitionSpec: { // PartitionSpec
+ * //                 Identity: [ // PartitionFields
+ * //                   { // PartitionField
+ * //                     SourceName: "STRING_VALUE", // required
+ * //                   },
+ * //                 ],
+ * //               },
  * //               S3ErrorOutputPrefix: "STRING_VALUE",
  * //             },
  * //           ],
+ * //           SchemaEvolutionConfiguration: { // SchemaEvolutionConfiguration
+ * //             Enabled: true || false, // required
+ * //           },
+ * //           TableCreationConfiguration: { // TableCreationConfiguration
+ * //             Enabled: true || false, // required
+ * //           },
  * //           BufferingHints: "<BufferingHints>",
  * //           CloudWatchLoggingOptions: "<CloudWatchLoggingOptions>",
  * //           ProcessingConfiguration: "<ProcessingConfiguration>",
@@ -561,6 +628,7 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
  * //           RoleARN: "STRING_VALUE",
  * //           CatalogConfiguration: { // CatalogConfiguration
  * //             CatalogARN: "STRING_VALUE",
+ * //             WarehouseLocation: "STRING_VALUE",
  * //           },
  * //           S3DestinationDescription: "<S3DestinationDescription>",
  * //         },