diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/lib/index.ts b/packages/@aws-cdk/aws-stepfunctions-tasks/lib/index.ts index 9908c6bb4e422..2057acaf632e8 100644 --- a/packages/@aws-cdk/aws-stepfunctions-tasks/lib/index.ts +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/lib/index.ts @@ -20,3 +20,4 @@ export * from './emr-cancel-step'; export * from './emr-modify-instance-fleet-by-name'; export * from './emr-modify-instance-group-by-name'; export * from './run-glue-job-task'; +export * from './run-batch-job'; diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/lib/run-batch-job.ts b/packages/@aws-cdk/aws-stepfunctions-tasks/lib/run-batch-job.ts new file mode 100644 index 0000000000000..67868f402f44f --- /dev/null +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/lib/run-batch-job.ts @@ -0,0 +1,340 @@ +import * as batch from '@aws-cdk/aws-batch'; +import * as ec2 from '@aws-cdk/aws-ec2'; +import * as iam from '@aws-cdk/aws-iam'; +import * as sfn from '@aws-cdk/aws-stepfunctions'; +import { Duration, Stack } from '@aws-cdk/core'; +import { getResourceArn } from './resource-arn-suffix'; + +/** + * The overrides that should be sent to a container. + */ +export interface ContainerOverrides { + /** + * The command to send to the container that overrides + * the default command from the Docker image or the job definition. + * + * @default - No command overrides + */ + readonly command?: string[]; + + /** + * The environment variables to send to the container. + * You can add new environment variables, which are added to the container + * at launch, or you can override the existing environment variables from + * the Docker image or the job definition. + * + * @default - No environment overrides + */ + readonly environment?: { [key: string]: string }; + + /** + * The instance type to use for a multi-node parallel job. + * This parameter is not valid for single-node container jobs. + * + * @default - No instance type overrides + */ + readonly instanceType?: ec2.InstanceType; + + /** + * The number of MiB of memory reserved for the job. + * This value overrides the value set in the job definition. + * + * @default - No memory overrides + */ + readonly memory?: number; + + /** + * The number of physical GPUs to reserve for the container. + * The number of GPUs reserved for all containers in a job + * should not exceed the number of available GPUs on the compute + * resource that the job is launched on. + * + * @default - No GPU reservation + */ + readonly gpuCount?: number; + + /** + * The number of vCPUs to reserve for the container. + * This value overrides the value set in the job definition. + * + * @default - No vCPUs overrides + */ + readonly vcpus?: number; +} + +/** + * An object representing an AWS Batch job dependency. + */ +export interface JobDependency { + /** + * The job ID of the AWS Batch job associated with this dependency. + * + * @default - No jobId + */ + readonly jobId?: string; + + /** + * The type of the job dependency. + * + * @default - No type + */ + readonly type?: string; +} + +/** + * Properties for RunBatchJob + */ +export interface RunBatchJobProps { + /** + * The job definition used by this job. + */ + readonly jobDefinition: batch.IJobDefinition; + + /** + * The name of the job. + * The first character must be alphanumeric, and up to 128 letters (uppercase and lowercase), + * numbers, hyphens, and underscores are allowed. + */ + readonly jobName: string; + + /** + * The job queue into which the job is submitted. + */ + readonly jobQueue: batch.IJobQueue; + + /** + * The array size can be between 2 and 10,000. + * If you specify array properties for a job, it becomes an array job. + * For more information, see Array Jobs in the AWS Batch User Guide. + * + * @default - No array size + */ + readonly arraySize?: number; + + /** + * A list of container overrides in JSON format that specify the name of a container + * in the specified job definition and the overrides it should receive. + * + * @see https://docs.aws.amazon.com/batch/latest/APIReference/API_SubmitJob.html#Batch-SubmitJob-request-containerOverrides + * + * @default - No container overrides + */ + readonly containerOverrides?: ContainerOverrides; + + /** + * A list of dependencies for the job. + * A job can depend upon a maximum of 20 jobs. + * + * @see https://docs.aws.amazon.com/batch/latest/APIReference/API_SubmitJob.html#Batch-SubmitJob-request-dependsOn + * + * @default - No dependencies + */ + readonly dependsOn?: JobDependency[]; + + /** + * The payload to be passed as parametrs to the batch job + * + * @default - No parameters are passed + */ + readonly payload?: { [key: string]: any }; + + /** + * The number of times to move a job to the RUNNABLE status. + * You may specify between 1 and 10 attempts. + * If the value of attempts is greater than one, + * the job is retried on failure the same number of attempts as the value. + * + * @default - 1 + */ + readonly attempts?: number; + + /** + * The timeout configuration for this SubmitJob operation. + * The minimum value for the timeout is 60 seconds. + * + * @see https://docs.aws.amazon.com/batch/latest/APIReference/API_SubmitJob.html#Batch-SubmitJob-request-timeout + * + * @default - No timeout + */ + readonly timeout?: Duration; + + /** + * The service integration pattern indicates different ways to call TerminateCluster. + * + * The valid value is either FIRE_AND_FORGET or SYNC. + * + * @default SYNC + */ + readonly integrationPattern?: sfn.ServiceIntegrationPattern; +} + +/** + * A Step Functions Task to run AWS Batch + */ +export class RunBatchJob implements sfn.IStepFunctionsTask { + private readonly integrationPattern: sfn.ServiceIntegrationPattern; + + constructor(private readonly props: RunBatchJobProps) { + // validate integrationPattern + this.integrationPattern = + props.integrationPattern || sfn.ServiceIntegrationPattern.SYNC; + + const supportedPatterns = [ + sfn.ServiceIntegrationPattern.FIRE_AND_FORGET, + sfn.ServiceIntegrationPattern.SYNC + ]; + + if (!supportedPatterns.includes(this.integrationPattern)) { + throw new Error( + `Invalid Service Integration Pattern: ${this.integrationPattern} is not supported to call RunBatchJob.` + ); + } + + // validate arraySize limits + if ( + props.arraySize !== undefined && + (props.arraySize < 2 || props.arraySize > 10000) + ) { + throw new Error( + `Invalid value of arraySize. The array size can be between 2 and 10,000.` + ); + } + + // validate dependency size + if (props.dependsOn && props.dependsOn.length > 20) { + throw new Error( + `Invalid number of dependencies. A job can depend upon a maximum of 20 jobs.` + ); + } + + // validate attempts + if ( + props.attempts !== undefined && + (props.attempts < 1 || props.attempts > 10) + ) { + throw new Error( + `Invalid value of attempts. You may specify between 1 and 10 attempts.` + ); + } + + // validate timeout + if (props.timeout && props.timeout.toSeconds() < 60) { + throw new Error( + `Invalid value of timrout. The minimum value for the timeout is 60 seconds.` + ); + } + + // This is reuqired since environment variables must not start with AWS_BATCH; + // this naming convention is reserved for variables that are set by the AWS Batch service. + if (props.containerOverrides?.environment) { + Object.keys(props.containerOverrides.environment).forEach(key => { + if (key.match(/^AWS_BATCH/)) { + throw new Error( + `Invalid environment variable name: ${key}. Environment variable names starting with 'AWS_BATCH' are reserved.` + ); + } + }); + } + } + + public bind(_task: sfn.Task): sfn.StepFunctionsTaskConfig { + return { + resourceArn: getResourceArn( + 'batch', + 'submitJob', + this.integrationPattern + ), + policyStatements: this.configurePolicyStatements(_task), + parameters: { + JobDefinition: this.props.jobDefinition.jobDefinitionArn, + JobName: this.props.jobName, + JobQueue: this.props.jobQueue.jobQueueArn, + Parameters: this.props.payload, + + ArrayProperties: + this.props.arraySize !== undefined + ? { Size: this.props.arraySize } + : undefined, + + ContainerOverrides: this.props.containerOverrides + ? this.configureContainerOverrides(this.props.containerOverrides) + : undefined, + + DependsOn: this.props.dependsOn + ? this.props.dependsOn.map(jobDependency => ({ + JobId: jobDependency.jobId, + Type: jobDependency.type + })) + : undefined, + + RetryStrategy: + this.props.attempts !== undefined + ? { Attempts: this.props.attempts } + : undefined, + + Timeout: this.props.timeout + ? { AttemptDurationSeconds: this.props.timeout.toSeconds() } + : undefined + } + }; + } + + private configurePolicyStatements(task: sfn.Task): iam.PolicyStatement[] { + return [ + // Resource level access control for job-definition requires revision which batch does not support yet + // Using the alternative permissions as mentioned here: + // https://docs.aws.amazon.com/batch/latest/userguide/batch-supported-iam-actions-resources.html + new iam.PolicyStatement({ + resources: [ + Stack.of(task).formatArn({ + service: 'batch', + resource: 'job-definition', + resourceName: '*' + }), + this.props.jobQueue.jobQueueArn + ], + actions: ['batch:SubmitJob'] + }), + new iam.PolicyStatement({ + resources: [ + Stack.of(task).formatArn({ + service: 'events', + resource: 'rule/StepFunctionsGetEventsForBatchJobsRule' + }) + ], + actions: ['events:PutTargets', 'events:PutRule', 'events:DescribeRule'] + }) + ]; + } + + private configureContainerOverrides(containerOverrides: ContainerOverrides) { + let environment; + if (containerOverrides.environment) { + environment = Object.entries(containerOverrides.environment).map( + ([key, value]) => ({ + Name: key, + Value: value + }) + ); + } + + let resources; + if (containerOverrides.gpuCount) { + resources = [ + { + Type: 'GPU', + Value: `${containerOverrides.gpuCount}` + } + ]; + } + + return { + Command: containerOverrides.command, + Environment: environment, + InstanceType: containerOverrides.instanceType?.toString(), + Memory: containerOverrides.memory, + ResourceRequirements: resources, + Vcpus: containerOverrides.vcpus + }; + } +} diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/package.json b/packages/@aws-cdk/aws-stepfunctions-tasks/package.json index 7f29cd1fffd52..8ff438593eb38 100644 --- a/packages/@aws-cdk/aws-stepfunctions-tasks/package.json +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/package.json @@ -89,6 +89,7 @@ "dependencies": { "@aws-cdk/assets": "0.0.0", "@aws-cdk/aws-cloudwatch": "0.0.0", + "@aws-cdk/aws-batch": "0.0.0", "@aws-cdk/aws-ec2": "0.0.0", "@aws-cdk/aws-ecr": "0.0.0", "@aws-cdk/aws-ecr-assets": "0.0.0", @@ -106,6 +107,7 @@ "homepage": "https://github.com/aws/aws-cdk", "peerDependencies": { "@aws-cdk/assets": "0.0.0", + "@aws-cdk/aws-batch": "0.0.0", "@aws-cdk/aws-cloudwatch": "0.0.0", "@aws-cdk/aws-ec2": "0.0.0", "@aws-cdk/aws-ecr": "0.0.0", diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/test/batchjob-image/Dockerfile b/packages/@aws-cdk/aws-stepfunctions-tasks/test/batchjob-image/Dockerfile new file mode 100644 index 0000000000000..123b5670febc8 --- /dev/null +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/test/batchjob-image/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.6 +EXPOSE 8000 +WORKDIR /src +ADD . /src +CMD python3 index.py diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/test/batchjob-image/index.py b/packages/@aws-cdk/aws-stepfunctions-tasks/test/batchjob-image/index.py new file mode 100644 index 0000000000000..337ed86e5f2ec --- /dev/null +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/test/batchjob-image/index.py @@ -0,0 +1,6 @@ +#!/usr/bin/python +import os +import pprint + +print('Hello from Batch!') +pprint.pprint(dict(os.environ)) diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/test/integ.run-batch-job.expected.json b/packages/@aws-cdk/aws-stepfunctions-tasks/test/integ.run-batch-job.expected.json new file mode 100644 index 0000000000000..2959dd9d50551 --- /dev/null +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/test/integ.run-batch-job.expected.json @@ -0,0 +1,1036 @@ +{ + "Resources": { + "vpcA2121C38": { + "Type": "AWS::EC2::VPC", + "Properties": { + "CidrBlock": "10.0.0.0/16", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, + "InstanceTenancy": "default", + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc" + } + ] + } + }, + "vpcPublicSubnet1Subnet2E65531E": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.0.0/19", + "VpcId": { + "Ref": "vpcA2121C38" + }, + "AvailabilityZone": "test-region-1a", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc/PublicSubnet1" + }, + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + } + ] + } + }, + "vpcPublicSubnet1RouteTable48A2DF9B": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "vpcA2121C38" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc/PublicSubnet1" + } + ] + } + }, + "vpcPublicSubnet1RouteTableAssociation5D3F4579": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "vpcPublicSubnet1RouteTable48A2DF9B" + }, + "SubnetId": { + "Ref": "vpcPublicSubnet1Subnet2E65531E" + } + } + }, + "vpcPublicSubnet1DefaultRoute10708846": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "vpcPublicSubnet1RouteTable48A2DF9B" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "vpcIGWE57CBDCA" + } + }, + "DependsOn": [ + "vpcVPCGW7984C166" + ] + }, + "vpcPublicSubnet1EIPDA49DCBE": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc/PublicSubnet1" + } + ] + } + }, + "vpcPublicSubnet1NATGateway9C16659E": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "AllocationId": { + "Fn::GetAtt": [ + "vpcPublicSubnet1EIPDA49DCBE", + "AllocationId" + ] + }, + "SubnetId": { + "Ref": "vpcPublicSubnet1Subnet2E65531E" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc/PublicSubnet1" + } + ] + } + }, + "vpcPublicSubnet2Subnet009B674F": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.32.0/19", + "VpcId": { + "Ref": "vpcA2121C38" + }, + "AvailabilityZone": "test-region-1b", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc/PublicSubnet2" + }, + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + } + ] + } + }, + "vpcPublicSubnet2RouteTableEB40D4CB": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "vpcA2121C38" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc/PublicSubnet2" + } + ] + } + }, + "vpcPublicSubnet2RouteTableAssociation21F81B59": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "vpcPublicSubnet2RouteTableEB40D4CB" + }, + "SubnetId": { + "Ref": "vpcPublicSubnet2Subnet009B674F" + } + } + }, + "vpcPublicSubnet2DefaultRouteA1EC0F60": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "vpcPublicSubnet2RouteTableEB40D4CB" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "vpcIGWE57CBDCA" + } + }, + "DependsOn": [ + "vpcVPCGW7984C166" + ] + }, + "vpcPublicSubnet2EIP9B3743B1": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc/PublicSubnet2" + } + ] + } + }, + "vpcPublicSubnet2NATGateway9B8AE11A": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "AllocationId": { + "Fn::GetAtt": [ + "vpcPublicSubnet2EIP9B3743B1", + "AllocationId" + ] + }, + "SubnetId": { + "Ref": "vpcPublicSubnet2Subnet009B674F" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc/PublicSubnet2" + } + ] + } + }, + "vpcPublicSubnet3Subnet11B92D7C": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.64.0/19", + "VpcId": { + "Ref": "vpcA2121C38" + }, + "AvailabilityZone": "test-region-1c", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc/PublicSubnet3" + }, + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + } + ] + } + }, + "vpcPublicSubnet3RouteTableA3C00665": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "vpcA2121C38" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc/PublicSubnet3" + } + ] + } + }, + "vpcPublicSubnet3RouteTableAssociationD102D1C4": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "vpcPublicSubnet3RouteTableA3C00665" + }, + "SubnetId": { + "Ref": "vpcPublicSubnet3Subnet11B92D7C" + } + } + }, + "vpcPublicSubnet3DefaultRoute3F356A11": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "vpcPublicSubnet3RouteTableA3C00665" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "vpcIGWE57CBDCA" + } + }, + "DependsOn": [ + "vpcVPCGW7984C166" + ] + }, + "vpcPublicSubnet3EIP2C3B9D91": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc/PublicSubnet3" + } + ] + } + }, + "vpcPublicSubnet3NATGateway82F6CA9E": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "AllocationId": { + "Fn::GetAtt": [ + "vpcPublicSubnet3EIP2C3B9D91", + "AllocationId" + ] + }, + "SubnetId": { + "Ref": "vpcPublicSubnet3Subnet11B92D7C" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc/PublicSubnet3" + } + ] + } + }, + "vpcPrivateSubnet1Subnet934893E8": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.96.0/19", + "VpcId": { + "Ref": "vpcA2121C38" + }, + "AvailabilityZone": "test-region-1a", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc/PrivateSubnet1" + }, + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + } + ] + } + }, + "vpcPrivateSubnet1RouteTableB41A48CC": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "vpcA2121C38" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc/PrivateSubnet1" + } + ] + } + }, + "vpcPrivateSubnet1RouteTableAssociation67945127": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "vpcPrivateSubnet1RouteTableB41A48CC" + }, + "SubnetId": { + "Ref": "vpcPrivateSubnet1Subnet934893E8" + } + } + }, + "vpcPrivateSubnet1DefaultRoute1AA8E2E5": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "vpcPrivateSubnet1RouteTableB41A48CC" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "vpcPublicSubnet1NATGateway9C16659E" + } + } + }, + "vpcPrivateSubnet2Subnet7031C2BA": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.128.0/19", + "VpcId": { + "Ref": "vpcA2121C38" + }, + "AvailabilityZone": "test-region-1b", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc/PrivateSubnet2" + }, + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + } + ] + } + }, + "vpcPrivateSubnet2RouteTable7280F23E": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "vpcA2121C38" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc/PrivateSubnet2" + } + ] + } + }, + "vpcPrivateSubnet2RouteTableAssociation007E94D3": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "vpcPrivateSubnet2RouteTable7280F23E" + }, + "SubnetId": { + "Ref": "vpcPrivateSubnet2Subnet7031C2BA" + } + } + }, + "vpcPrivateSubnet2DefaultRouteB0E07F99": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "vpcPrivateSubnet2RouteTable7280F23E" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "vpcPublicSubnet2NATGateway9B8AE11A" + } + } + }, + "vpcPrivateSubnet3Subnet985AC459": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.160.0/19", + "VpcId": { + "Ref": "vpcA2121C38" + }, + "AvailabilityZone": "test-region-1c", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc/PrivateSubnet3" + }, + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + } + ] + } + }, + "vpcPrivateSubnet3RouteTable24DA79A0": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "vpcA2121C38" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc/PrivateSubnet3" + } + ] + } + }, + "vpcPrivateSubnet3RouteTableAssociationC58B3C2C": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "vpcPrivateSubnet3RouteTable24DA79A0" + }, + "SubnetId": { + "Ref": "vpcPrivateSubnet3Subnet985AC459" + } + } + }, + "vpcPrivateSubnet3DefaultRoute30C45F47": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "vpcPrivateSubnet3RouteTable24DA79A0" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "vpcPublicSubnet3NATGateway82F6CA9E" + } + } + }, + "vpcIGWE57CBDCA": { + "Type": "AWS::EC2::InternetGateway", + "Properties": { + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-integ/vpc" + } + ] + } + }, + "vpcVPCGW7984C166": { + "Type": "AWS::EC2::VPCGatewayAttachment", + "Properties": { + "VpcId": { + "Ref": "vpcA2121C38" + }, + "InternetGatewayId": { + "Ref": "vpcIGWE57CBDCA" + } + } + }, + "ComputeEnvEcsInstanceRoleCFB290F9": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": { + "Fn::Join": [ + "", + [ + "ec2.", + { + "Ref": "AWS::URLSuffix" + } + ] + ] + } + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role" + ] + ] + } + ] + }, + "DependsOn": [ + "vpcIGWE57CBDCA", + "vpcPrivateSubnet1DefaultRoute1AA8E2E5", + "vpcPrivateSubnet1RouteTableB41A48CC", + "vpcPrivateSubnet1RouteTableAssociation67945127", + "vpcPrivateSubnet1Subnet934893E8", + "vpcPrivateSubnet2DefaultRouteB0E07F99", + "vpcPrivateSubnet2RouteTable7280F23E", + "vpcPrivateSubnet2RouteTableAssociation007E94D3", + "vpcPrivateSubnet2Subnet7031C2BA", + "vpcPrivateSubnet3DefaultRoute30C45F47", + "vpcPrivateSubnet3RouteTable24DA79A0", + "vpcPrivateSubnet3RouteTableAssociationC58B3C2C", + "vpcPrivateSubnet3Subnet985AC459", + "vpcPublicSubnet1DefaultRoute10708846", + "vpcPublicSubnet1EIPDA49DCBE", + "vpcPublicSubnet1NATGateway9C16659E", + "vpcPublicSubnet1RouteTable48A2DF9B", + "vpcPublicSubnet1RouteTableAssociation5D3F4579", + "vpcPublicSubnet1Subnet2E65531E", + "vpcPublicSubnet2DefaultRouteA1EC0F60", + "vpcPublicSubnet2EIP9B3743B1", + "vpcPublicSubnet2NATGateway9B8AE11A", + "vpcPublicSubnet2RouteTableEB40D4CB", + "vpcPublicSubnet2RouteTableAssociation21F81B59", + "vpcPublicSubnet2Subnet009B674F", + "vpcPublicSubnet3DefaultRoute3F356A11", + "vpcPublicSubnet3EIP2C3B9D91", + "vpcPublicSubnet3NATGateway82F6CA9E", + "vpcPublicSubnet3RouteTableA3C00665", + "vpcPublicSubnet3RouteTableAssociationD102D1C4", + "vpcPublicSubnet3Subnet11B92D7C", + "vpcA2121C38", + "vpcVPCGW7984C166" + ] + }, + "ComputeEnvInstanceProfile81AFCCF2": { + "Type": "AWS::IAM::InstanceProfile", + "Properties": { + "Roles": [ + { + "Ref": "ComputeEnvEcsInstanceRoleCFB290F9" + } + ] + }, + "DependsOn": [ + "vpcIGWE57CBDCA", + "vpcPrivateSubnet1DefaultRoute1AA8E2E5", + "vpcPrivateSubnet1RouteTableB41A48CC", + "vpcPrivateSubnet1RouteTableAssociation67945127", + "vpcPrivateSubnet1Subnet934893E8", + "vpcPrivateSubnet2DefaultRouteB0E07F99", + "vpcPrivateSubnet2RouteTable7280F23E", + "vpcPrivateSubnet2RouteTableAssociation007E94D3", + "vpcPrivateSubnet2Subnet7031C2BA", + "vpcPrivateSubnet3DefaultRoute30C45F47", + "vpcPrivateSubnet3RouteTable24DA79A0", + "vpcPrivateSubnet3RouteTableAssociationC58B3C2C", + "vpcPrivateSubnet3Subnet985AC459", + "vpcPublicSubnet1DefaultRoute10708846", + "vpcPublicSubnet1EIPDA49DCBE", + "vpcPublicSubnet1NATGateway9C16659E", + "vpcPublicSubnet1RouteTable48A2DF9B", + "vpcPublicSubnet1RouteTableAssociation5D3F4579", + "vpcPublicSubnet1Subnet2E65531E", + "vpcPublicSubnet2DefaultRouteA1EC0F60", + "vpcPublicSubnet2EIP9B3743B1", + "vpcPublicSubnet2NATGateway9B8AE11A", + "vpcPublicSubnet2RouteTableEB40D4CB", + "vpcPublicSubnet2RouteTableAssociation21F81B59", + "vpcPublicSubnet2Subnet009B674F", + "vpcPublicSubnet3DefaultRoute3F356A11", + "vpcPublicSubnet3EIP2C3B9D91", + "vpcPublicSubnet3NATGateway82F6CA9E", + "vpcPublicSubnet3RouteTableA3C00665", + "vpcPublicSubnet3RouteTableAssociationD102D1C4", + "vpcPublicSubnet3Subnet11B92D7C", + "vpcA2121C38", + "vpcVPCGW7984C166" + ] + }, + "ComputeEnvResourceSecurityGroupB84CF86B": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "aws-stepfunctions-integ/ComputeEnv/Resource-Security-Group", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1" + } + ], + "VpcId": { + "Ref": "vpcA2121C38" + } + }, + "DependsOn": [ + "vpcIGWE57CBDCA", + "vpcPrivateSubnet1DefaultRoute1AA8E2E5", + "vpcPrivateSubnet1RouteTableB41A48CC", + "vpcPrivateSubnet1RouteTableAssociation67945127", + "vpcPrivateSubnet1Subnet934893E8", + "vpcPrivateSubnet2DefaultRouteB0E07F99", + "vpcPrivateSubnet2RouteTable7280F23E", + "vpcPrivateSubnet2RouteTableAssociation007E94D3", + "vpcPrivateSubnet2Subnet7031C2BA", + "vpcPrivateSubnet3DefaultRoute30C45F47", + "vpcPrivateSubnet3RouteTable24DA79A0", + "vpcPrivateSubnet3RouteTableAssociationC58B3C2C", + "vpcPrivateSubnet3Subnet985AC459", + "vpcPublicSubnet1DefaultRoute10708846", + "vpcPublicSubnet1EIPDA49DCBE", + "vpcPublicSubnet1NATGateway9C16659E", + "vpcPublicSubnet1RouteTable48A2DF9B", + "vpcPublicSubnet1RouteTableAssociation5D3F4579", + "vpcPublicSubnet1Subnet2E65531E", + "vpcPublicSubnet2DefaultRouteA1EC0F60", + "vpcPublicSubnet2EIP9B3743B1", + "vpcPublicSubnet2NATGateway9B8AE11A", + "vpcPublicSubnet2RouteTableEB40D4CB", + "vpcPublicSubnet2RouteTableAssociation21F81B59", + "vpcPublicSubnet2Subnet009B674F", + "vpcPublicSubnet3DefaultRoute3F356A11", + "vpcPublicSubnet3EIP2C3B9D91", + "vpcPublicSubnet3NATGateway82F6CA9E", + "vpcPublicSubnet3RouteTableA3C00665", + "vpcPublicSubnet3RouteTableAssociationD102D1C4", + "vpcPublicSubnet3Subnet11B92D7C", + "vpcA2121C38", + "vpcVPCGW7984C166" + ] + }, + "ComputeEnvResourceServiceInstanceRoleCF89E9E1": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "batch.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSBatchServiceRole" + ] + ] + } + ] + }, + "DependsOn": [ + "vpcIGWE57CBDCA", + "vpcPrivateSubnet1DefaultRoute1AA8E2E5", + "vpcPrivateSubnet1RouteTableB41A48CC", + "vpcPrivateSubnet1RouteTableAssociation67945127", + "vpcPrivateSubnet1Subnet934893E8", + "vpcPrivateSubnet2DefaultRouteB0E07F99", + "vpcPrivateSubnet2RouteTable7280F23E", + "vpcPrivateSubnet2RouteTableAssociation007E94D3", + "vpcPrivateSubnet2Subnet7031C2BA", + "vpcPrivateSubnet3DefaultRoute30C45F47", + "vpcPrivateSubnet3RouteTable24DA79A0", + "vpcPrivateSubnet3RouteTableAssociationC58B3C2C", + "vpcPrivateSubnet3Subnet985AC459", + "vpcPublicSubnet1DefaultRoute10708846", + "vpcPublicSubnet1EIPDA49DCBE", + "vpcPublicSubnet1NATGateway9C16659E", + "vpcPublicSubnet1RouteTable48A2DF9B", + "vpcPublicSubnet1RouteTableAssociation5D3F4579", + "vpcPublicSubnet1Subnet2E65531E", + "vpcPublicSubnet2DefaultRouteA1EC0F60", + "vpcPublicSubnet2EIP9B3743B1", + "vpcPublicSubnet2NATGateway9B8AE11A", + "vpcPublicSubnet2RouteTableEB40D4CB", + "vpcPublicSubnet2RouteTableAssociation21F81B59", + "vpcPublicSubnet2Subnet009B674F", + "vpcPublicSubnet3DefaultRoute3F356A11", + "vpcPublicSubnet3EIP2C3B9D91", + "vpcPublicSubnet3NATGateway82F6CA9E", + "vpcPublicSubnet3RouteTableA3C00665", + "vpcPublicSubnet3RouteTableAssociationD102D1C4", + "vpcPublicSubnet3Subnet11B92D7C", + "vpcA2121C38", + "vpcVPCGW7984C166" + ] + }, + "ComputeEnv2C40ACC2": { + "Type": "AWS::Batch::ComputeEnvironment", + "Properties": { + "ServiceRole": { + "Fn::GetAtt": [ + "ComputeEnvResourceServiceInstanceRoleCF89E9E1", + "Arn" + ] + }, + "Type": "MANAGED", + "ComputeResources": { + "AllocationStrategy": "BEST_FIT", + "InstanceRole": { + "Fn::GetAtt": [ + "ComputeEnvInstanceProfile81AFCCF2", + "Arn" + ] + }, + "InstanceTypes": [ + "optimal" + ], + "MaxvCpus": 256, + "MinvCpus": 0, + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "ComputeEnvResourceSecurityGroupB84CF86B", + "GroupId" + ] + } + ], + "Subnets": [ + { + "Ref": "vpcPrivateSubnet1Subnet934893E8" + }, + { + "Ref": "vpcPrivateSubnet2Subnet7031C2BA" + }, + { + "Ref": "vpcPrivateSubnet3Subnet985AC459" + } + ], + "Type": "EC2" + }, + "State": "ENABLED" + }, + "DependsOn": [ + "vpcIGWE57CBDCA", + "vpcPrivateSubnet1DefaultRoute1AA8E2E5", + "vpcPrivateSubnet1RouteTableB41A48CC", + "vpcPrivateSubnet1RouteTableAssociation67945127", + "vpcPrivateSubnet1Subnet934893E8", + "vpcPrivateSubnet2DefaultRouteB0E07F99", + "vpcPrivateSubnet2RouteTable7280F23E", + "vpcPrivateSubnet2RouteTableAssociation007E94D3", + "vpcPrivateSubnet2Subnet7031C2BA", + "vpcPrivateSubnet3DefaultRoute30C45F47", + "vpcPrivateSubnet3RouteTable24DA79A0", + "vpcPrivateSubnet3RouteTableAssociationC58B3C2C", + "vpcPrivateSubnet3Subnet985AC459", + "vpcPublicSubnet1DefaultRoute10708846", + "vpcPublicSubnet1EIPDA49DCBE", + "vpcPublicSubnet1NATGateway9C16659E", + "vpcPublicSubnet1RouteTable48A2DF9B", + "vpcPublicSubnet1RouteTableAssociation5D3F4579", + "vpcPublicSubnet1Subnet2E65531E", + "vpcPublicSubnet2DefaultRouteA1EC0F60", + "vpcPublicSubnet2EIP9B3743B1", + "vpcPublicSubnet2NATGateway9B8AE11A", + "vpcPublicSubnet2RouteTableEB40D4CB", + "vpcPublicSubnet2RouteTableAssociation21F81B59", + "vpcPublicSubnet2Subnet009B674F", + "vpcPublicSubnet3DefaultRoute3F356A11", + "vpcPublicSubnet3EIP2C3B9D91", + "vpcPublicSubnet3NATGateway82F6CA9E", + "vpcPublicSubnet3RouteTableA3C00665", + "vpcPublicSubnet3RouteTableAssociationD102D1C4", + "vpcPublicSubnet3Subnet11B92D7C", + "vpcA2121C38", + "vpcVPCGW7984C166" + ] + }, + "JobQueueEE3AD499": { + "Type": "AWS::Batch::JobQueue", + "Properties": { + "ComputeEnvironmentOrder": [ + { + "ComputeEnvironment": { + "Ref": "ComputeEnv2C40ACC2" + }, + "Order": 1 + } + ], + "Priority": 1, + "State": "ENABLED" + } + }, + "JobDefinition24FFE3ED": { + "Type": "AWS::Batch::JobDefinition", + "Properties": { + "Type": "container", + "ContainerProperties": { + "Image": { + "Fn::Join": [ + "", + [ + { + "Ref": "AWS::AccountId" + }, + ".dkr.ecr.", + { + "Ref": "AWS::Region" + }, + ".", + { + "Ref": "AWS::URLSuffix" + }, + "/aws-cdk/assets:4ba4a660dbcc1e71f0bf07105626a5bc65d95ae71724dc57bbb94c8e14202342" + ] + ] + }, + "Memory": 4, + "Privileged": false, + "ReadonlyRootFilesystem": false, + "Vcpus": 1 + }, + "RetryStrategy": { + "Attempts": 1 + }, + "Timeout": {} + } + }, + "StateMachineRoleB840431D": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": { + "Fn::Join": [ + "", + [ + "states.", + { + "Ref": "AWS::Region" + }, + ".amazonaws.com" + ] + ] + } + } + } + ], + "Version": "2012-10-17" + } + } + }, + "StateMachineRoleDefaultPolicyDF1E6607": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "batch:SubmitJob", + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":batch:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":job-definition/*" + ] + ] + }, + { + "Ref": "JobQueueEE3AD499" + } + ] + }, + { + "Action": [ + "events:PutTargets", + "events:PutRule", + "events:DescribeRule" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":events:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":rule/StepFunctionsGetEventsForBatchJobsRule" + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "StateMachineRoleDefaultPolicyDF1E6607", + "Roles": [ + { + "Ref": "StateMachineRoleB840431D" + } + ] + } + }, + "StateMachine2E01A3A5": { + "Type": "AWS::StepFunctions::StateMachine", + "Properties": { + "DefinitionString": { + "Fn::Join": [ + "", + [ + "{\"StartAt\":\"Start\",\"States\":{\"Start\":{\"Type\":\"Pass\",\"Result\":{\"bar\":\"SomeValue\"},\"Next\":\"Submit Job\"},\"Submit Job\":{\"End\":true,\"Parameters\":{\"JobDefinition\":\"", + { + "Ref": "JobDefinition24FFE3ED" + }, + "\",\"JobName\":\"MyJob\",\"JobQueue\":\"", + { + "Ref": "JobQueueEE3AD499" + }, + "\",\"Parameters\":{\"foo.$\":\"$.bar\"},\"ContainerOverrides\":{\"Environment\":[{\"Name\":\"key\",\"Value\":\"value\"}],\"Memory\":256,\"Vcpus\":1},\"RetryStrategy\":{\"Attempts\":3},\"Timeout\":{\"AttemptDurationSeconds\":60}},\"Type\":\"Task\",\"Resource\":\"arn:", + { + "Ref": "AWS::Partition" + }, + ":states:::batch:submitJob.sync\"}}}" + ] + ] + }, + "RoleArn": { + "Fn::GetAtt": [ + "StateMachineRoleB840431D", + "Arn" + ] + } + }, + "DependsOn": [ + "StateMachineRoleDefaultPolicyDF1E6607", + "StateMachineRoleB840431D" + ] + } + }, + "Outputs": { + "JobQueueArn": { + "Value": { + "Ref": "JobQueueEE3AD499" + } + }, + "StateMachineArn": { + "Value": { + "Ref": "StateMachine2E01A3A5" + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/test/integ.run-batch-job.ts b/packages/@aws-cdk/aws-stepfunctions-tasks/test/integ.run-batch-job.ts new file mode 100644 index 0000000000000..ecffd83190a13 --- /dev/null +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/test/integ.run-batch-job.ts @@ -0,0 +1,80 @@ +import * as batch from '@aws-cdk/aws-batch'; +import * as ec2 from '@aws-cdk/aws-ec2'; +import * as ecs from '@aws-cdk/aws-ecs'; +import * as sfn from '@aws-cdk/aws-stepfunctions'; +import * as cdk from '@aws-cdk/core'; +import * as path from 'path'; +import * as tasks from '../lib'; + +/* + * Stack verification steps: + * * aws stepfunctions start-execution --state-machine-arn : should return execution arn + * * aws batch list-jobs --job-queue --job-status RUNNABLE : should return jobs-list with size greater than 0 + * * + * * aws batch describe-jobs --jobs --query 'jobs[0].status': wait until the status is 'SUCCEEDED' + * * aws stepfunctions describe-execution --execution-arn --query 'status': should return status as SUCCEEDED + */ + +class RunBatchStack extends cdk.Stack { + constructor(scope: cdk.App, id: string, props: cdk.StackProps = {}) { + super(scope, id, props); + + const vpc = new ec2.Vpc(this, 'vpc'); + + const batchQueue = new batch.JobQueue(this, 'JobQueue', { + computeEnvironments: [ + { + order: 1, + computeEnvironment: new batch.ComputeEnvironment(this, 'ComputeEnv', { + computeResources: { vpc } + }) + } + ] + }); + + const batchJobDefinition = new batch.JobDefinition(this, 'JobDefinition', { + container: { + image: ecs.ContainerImage.fromAsset( + path.resolve(__dirname, 'batchjob-image') + ) + } + }); + + const submitJob = new sfn.Task(this, 'Submit Job', { + task: new tasks.RunBatchJob({ + jobDefinition: batchJobDefinition, + jobName: 'MyJob', + jobQueue: batchQueue, + containerOverrides: { + environment: { key: 'value' }, + memory: 256, + vcpus: 1 + }, + payload: { + foo: sfn.Data.stringAt('$.bar') + }, + attempts: 3, + timeout: cdk.Duration.seconds(60) + }) + }); + + const definition = new sfn.Pass(this, 'Start', { + result: sfn.Result.fromObject({ bar: 'SomeValue' }) + }).next(submitJob); + + const stateMachine = new sfn.StateMachine(this, 'StateMachine', { + definition + }); + + new cdk.CfnOutput(this, 'JobQueueArn', { + value: batchQueue.jobQueueArn + }); + new cdk.CfnOutput(this, 'StateMachineArn', { + value: stateMachine.stateMachineArn + }); + } +} + +const app = new cdk.App(); +new RunBatchStack(app, 'aws-stepfunctions-integ'); +app.synth(); diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/test/run-batch-job.test.ts b/packages/@aws-cdk/aws-stepfunctions-tasks/test/run-batch-job.test.ts new file mode 100644 index 0000000000000..1e6adba6038a7 --- /dev/null +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/test/run-batch-job.test.ts @@ -0,0 +1,253 @@ +import * as batch from '@aws-cdk/aws-batch'; +import * as ec2 from '@aws-cdk/aws-ec2'; +import * as ecs from '@aws-cdk/aws-ecs'; +import * as sfn from '@aws-cdk/aws-stepfunctions'; +import * as cdk from '@aws-cdk/core'; +import * as path from 'path'; +import * as tasks from '../lib'; + +let stack: cdk.Stack; +let batchJobDefinition: batch.IJobDefinition; +let batchJobQueue: batch.IJobQueue; + +beforeEach(() => { + // GIVEN + stack = new cdk.Stack(); + + batchJobDefinition = new batch.JobDefinition(stack, 'JobDefinition', { + container: { + image: ecs.ContainerImage.fromAsset( + path.join(__dirname, 'batchjob-image') + ) + } + }); + + batchJobQueue = new batch.JobQueue(stack, 'JobQueue', { + computeEnvironments: [ + { + order: 1, + computeEnvironment: new batch.ComputeEnvironment(stack, 'ComputeEnv', { + computeResources: { vpc: new ec2.Vpc(stack, 'vpc') } + }) + } + ] + }); +}); + +test('Task with only the required parameters', () => { + // WHEN + const task = new sfn.Task(stack, 'Task', { + task: new tasks.RunBatchJob({ + jobDefinition: batchJobDefinition, + jobName: 'JobName', + jobQueue: batchJobQueue + }) + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toEqual({ + Type: 'Task', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition' + }, + ':states:::batch:submitJob.sync' + ] + ] + }, + End: true, + Parameters: { + JobDefinition: { Ref: 'JobDefinition24FFE3ED' }, + JobName: 'JobName', + JobQueue: { Ref: 'JobQueueEE3AD499' } + } + }); +}); + +test('Task with all the parameters', () => { + // WHEN + const task = new sfn.Task(stack, 'Task', { + task: new tasks.RunBatchJob({ + jobDefinition: batchJobDefinition, + jobName: 'JobName', + jobQueue: batchJobQueue, + arraySize: 15, + containerOverrides: { + command: ['sudo', 'rm'], + environment: { key: 'value' }, + instanceType: new ec2.InstanceType('MULTI'), + memory: 1024, + gpuCount: 1, + vcpus: 10 + }, + dependsOn: [{ jobId: '1234', type: 'some_type' }], + payload: { + foo: sfn.Data.stringAt('$.bar') + }, + attempts: 3, + timeout: cdk.Duration.seconds(60), + integrationPattern: sfn.ServiceIntegrationPattern.FIRE_AND_FORGET + }) + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toEqual({ + Type: 'Task', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition' + }, + ':states:::batch:submitJob' + ] + ] + }, + End: true, + Parameters: { + JobDefinition: { Ref: 'JobDefinition24FFE3ED' }, + JobName: 'JobName', + JobQueue: { Ref: 'JobQueueEE3AD499' }, + ArrayProperties: { Size: 15 }, + ContainerOverrides: { + Command: ['sudo', 'rm'], + Environment: [{ Name: 'key', Value: 'value' }], + InstanceType: 'MULTI', + Memory: 1024, + ResourceRequirements: [{ Type: 'GPU', Value: '1' }], + Vcpus: 10 + }, + DependsOn: [{ JobId: '1234', Type: 'some_type' }], + Parameters: { 'foo.$': '$.bar' }, + RetryStrategy: { Attempts: 3 }, + Timeout: { AttemptDurationSeconds: 60 } + } + }); +}); + +test('Task throws if WAIT_FOR_TASK_TOKEN is supplied as service integration pattern', () => { + expect(() => { + new sfn.Task(stack, 'Task', { + task: new tasks.RunBatchJob({ + jobDefinition: batchJobDefinition, + jobName: 'JobName', + jobQueue: batchJobQueue, + integrationPattern: sfn.ServiceIntegrationPattern.WAIT_FOR_TASK_TOKEN + }) + }); + }).toThrow( + /Invalid Service Integration Pattern: WAIT_FOR_TASK_TOKEN is not supported to call RunBatchJob./i + ); +}); + +test('Task throws if environment in containerOverrides contain env with name starting with AWS_BATCH', () => { + expect(() => { + new sfn.Task(stack, 'Task', { + task: new tasks.RunBatchJob({ + jobDefinition: batchJobDefinition, + jobName: 'JobName', + jobQueue: batchJobQueue, + containerOverrides: { + environment: { AWS_BATCH_MY_NAME: 'MY_VALUE' } + } + }) + }); + }).toThrow( + /Invalid environment variable name: AWS_BATCH_MY_NAME. Environment variable names starting with 'AWS_BATCH' are reserved./i + ); +}); + +test('Task throws if arraySize is out of limits 2-10000', () => { + expect(() => { + new sfn.Task(stack, 'Task', { + task: new tasks.RunBatchJob({ + jobDefinition: batchJobDefinition, + jobName: 'JobName', + jobQueue: batchJobQueue, + arraySize: 1 + }) + }); + }).toThrow( + /Invalid value of arraySize. The array size can be between 2 and 10,000./i + ); + + expect(() => { + new sfn.Task(stack, 'Task', { + task: new tasks.RunBatchJob({ + jobDefinition: batchJobDefinition, + jobName: 'JobName', + jobQueue: batchJobQueue, + arraySize: 10001 + }) + }); + }).toThrow( + /Invalid value of arraySize. The array size can be between 2 and 10,000./i + ); +}); + +test('Task throws if dependencies exceeds 20', () => { + expect(() => { + new sfn.Task(stack, 'Task', { + task: new tasks.RunBatchJob({ + jobDefinition: batchJobDefinition, + jobName: 'JobName', + jobQueue: batchJobQueue, + dependsOn: [...Array(21).keys()].map(i => ({ + jobId: `${i}`, + type: `some_type-${i}` + })) + }) + }); + }).toThrow( + /Invalid number of dependencies. A job can depend upon a maximum of 20 jobs./i + ); +}); + +test('Task throws if attempts is out of limits 1-10', () => { + expect(() => { + new sfn.Task(stack, 'Task', { + task: new tasks.RunBatchJob({ + jobDefinition: batchJobDefinition, + jobName: 'JobName', + jobQueue: batchJobQueue, + attempts: 0 + }) + }); + }).toThrow( + /Invalid value of attempts. You may specify between 1 and 10 attempts./i + ); + + expect(() => { + new sfn.Task(stack, 'Task', { + task: new tasks.RunBatchJob({ + jobDefinition: batchJobDefinition, + jobName: 'JobName', + jobQueue: batchJobQueue, + attempts: 11 + }) + }); + }).toThrow( + /Invalid value of attempts. You may specify between 1 and 10 attempts./i + ); +}); + +test('Task throws if timeout is less than 60 sec', () => { + expect(() => { + new sfn.Task(stack, 'Task', { + task: new tasks.RunBatchJob({ + jobDefinition: batchJobDefinition, + jobName: 'JobName', + jobQueue: batchJobQueue, + timeout: cdk.Duration.seconds(59) + }) + }); + }).toThrow( + /Invalid value of timrout. The minimum value for the timeout is 60 seconds./i + ); +}); diff --git a/packages/@aws-cdk/aws-stepfunctions/README.md b/packages/@aws-cdk/aws-stepfunctions/README.md index 8768da56376bc..fd09f84de1270 100644 --- a/packages/@aws-cdk/aws-stepfunctions/README.md +++ b/packages/@aws-cdk/aws-stepfunctions/README.md @@ -127,6 +127,7 @@ couple of the tasks available are: * `tasks.InvokeActivity` -- start an Activity (Activities represent a work queue that you poll on a compute fleet you manage yourself) * `tasks.InvokeFunction` -- invoke a Lambda function with function ARN +* `tasks.RunBatchJob` -- run a Batch job * `tasks.RunLambdaTask` -- call Lambda as integrated service with magic ARN * `tasks.RunGlueJobTask` -- call Glue Job as integrated service * `tasks.PublishToTopic` -- publish a message to an SNS topic @@ -203,6 +204,39 @@ task.next(nextState); [Example CDK app](../aws-stepfunctions-tasks/test/integ.glue-task.ts) +#### Batch example + +```ts +import batch = require('@aws-cdk/aws-batch'); + +const batchQueue = new batch.JobQueue(this, 'JobQueue', { + computeEnvironments: [ + { + order: 1, + computeEnvironment: new batch.ComputeEnvironment(this, 'ComputeEnv', { + computeResources: { vpc } + }) + } + ] +}); + +const batchJobDefinition = new batch.JobDefinition(this, 'JobDefinition', { + container: { + image: ecs.ContainerImage.fromAsset( + path.resolve(__dirname, 'batchjob-image') + ) + } +}); + +const task = new sfn.Task(this, 'Submit Job', { + task: new tasks.RunBatchJob({ + jobDefinition: batchJobDefinition, + jobName: 'MyJob', + jobQueue: batchQueue + }) +}); +``` + #### SNS example ```ts