feat(stepfunctions-tasks): add step functions task to run glue job (#…

…6258)

* feat(stepfunctions-tasks): add step functions task to run glue job

* add new task RunGlueJobTask and associated unit tests
* since Job construct does not yet exist, uses job name as required parameter

closes #5266

* cleanup constructor properties, add integration test

* remove job run ID from props, update default prop descriptions

* add s3 assets package to module

* fix linting errors

* clean up documentation, add links to docs and glue task example

* add verification step to integration step, ensure job succeeds

* update expected integration test stack (asset names)

* add integ test verification comment about glue cold start

* cleaned up the note around cold start

* specify glue job ARN in state machine role permissions

* change state machine role permissions based on service integration pattern

Co-authored-by: Niranjan Jayakar <[email protected]>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

packages/@aws-cdk/aws-stepfunctions-tasks/lib/index.ts

@@ -19,3 +19,4 @@ export * from './emr-add-step';
 export * from './emr-cancel-step';
 export * from './emr-modify-instance-fleet-by-name';
 export * from './emr-modify-instance-group-by-name';
+export * from './run-glue-job-task';

packages/@aws-cdk/aws-stepfunctions-tasks/lib/run-glue-job-task.ts

@@ -0,0 +1,120 @@
+import * as iam from '@aws-cdk/aws-iam';
+import * as sfn from '@aws-cdk/aws-stepfunctions';
+import { Duration, Stack } from '@aws-cdk/core';
+import { getResourceArn } from './resource-arn-suffix';
+
+/**
+ * Properties for RunGlueJobTask
+ */
+export interface RunGlueJobTaskProps {
+
+  /**
+   * The service integration pattern indicates different ways to start the Glue job.
+   *
+   * The valid value for Glue is either FIRE_AND_FORGET or SYNC.
+   *
+   * @default FIRE_AND_FORGET
+   */
+  readonly integrationPattern?: sfn.ServiceIntegrationPattern;
+
+  /**
+   * The job arguments specifically for this run.
+   *
+   * For this job run, they replace the default arguments set in the job definition itself.
+   *
+   * @default - Default arguments set in the job definition
+   */
+  readonly arguments?: { [key: string]: string };
+
+  /**
+   * The job run timeout.
+   *
+   * This is the maximum time that a job run can consume resources before it is terminated and enters TIMEOUT status.
+   * Must be at least 1 minute.
+   *
+   * @default - Default timeout set in the job definition
+   */
+  readonly timeout?: Duration;
+
+  /**
+   * The name of the SecurityConfiguration structure to be used with this job run.
+   *
+   * This must match the Glue API
+   * [single-line string pattern](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-common.html#aws-glue-api-regex-oneLine).
+   *
+   * @default - Default configuration set in the job definition
+   */
+  readonly securityConfiguration?: string;
+
+  /**
+   * After a job run starts, the number of minutes to wait before sending a job run delay notification.
+   *
+   * Must be at least 1 minute.
+   *
+   * @default - Default delay set in the job definition
+   */
+  readonly notifyDelayAfter?: Duration;
+}
+
+/**
+ * Invoke a Glue job as a Task
+ *
+ * OUTPUT: the output of this task is a JobRun structure, for details consult
+ * https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-jobs-runs.html#aws-glue-api-jobs-runs-JobRun
+ *
+ * @see https://docs.aws.amazon.com/step-functions/latest/dg/connect-glue.html
+ */
+export class RunGlueJobTask implements sfn.IStepFunctionsTask {
+  private readonly integrationPattern: sfn.ServiceIntegrationPattern;
+
+  constructor(private readonly glueJobName: string, private readonly props: RunGlueJobTaskProps = {}) {
+    this.integrationPattern = props.integrationPattern || sfn.ServiceIntegrationPattern.FIRE_AND_FORGET;
+
+    const supportedPatterns = [
+      sfn.ServiceIntegrationPattern.FIRE_AND_FORGET,
+      sfn.ServiceIntegrationPattern.SYNC
+    ];
+
+    if (!supportedPatterns.includes(this.integrationPattern)) {
+      throw new Error(`Invalid Service Integration Pattern: ${this.integrationPattern} is not supported to call Glue.`);
+    }
+  }
+
+  public bind(task: sfn.Task): sfn.StepFunctionsTaskConfig {
+    const notificationProperty = this.props.notifyDelayAfter ? { NotifyDelayAfter: this.props.notifyDelayAfter.toMinutes() } : null;
+    let iamActions: string[] | undefined;
+    if (this.integrationPattern === sfn.ServiceIntegrationPattern.FIRE_AND_FORGET) {
+      iamActions = ["glue:StartJobRun"];
+    } else if (this.integrationPattern === sfn.ServiceIntegrationPattern.SYNC) {
+      iamActions = [
+        "glue:StartJobRun",
+        "glue:GetJobRun",
+        "glue:GetJobRuns",
+        "glue:BatchStopJobRun"
+      ];
+    }
+    return {
+      resourceArn: getResourceArn("glue", "startJobRun", this.integrationPattern),
+      policyStatements: [new iam.PolicyStatement({
+        resources: [
+          Stack.of(task).formatArn({
+            service: "glue",
+            resource: "job",
+            resourceName: this.glueJobName
+          })
+        ],
+        actions: iamActions
+      })],
+      metricPrefixSingular: 'GlueJob',
+      metricPrefixPlural: 'GlueJobs',
+      metricDimensions: { GlueJobName: this.glueJobName },
+      parameters: {
+        JobName: this.glueJobName,
+        Arguments: this.props.arguments,
+        Timeout: this.props.timeout?.toMinutes(),
+        SecurityConfiguration: this.props.securityConfiguration,
+        NotificationProperty: notificationProperty
+      }
+    };
+  }
+}

packages/@aws-cdk/aws-stepfunctions-tasks/package.json

@@ -80,6 +80,7 @@
   "license": "Apache-2.0",
   "devDependencies": {
     "@aws-cdk/assert": "0.0.0",
+    "@aws-cdk/aws-s3-assets": "0.0.0",
     "cdk-build-tools": "0.0.0",
     "cdk-integ-tools": "0.0.0",
     "jest": "^24.9.0",
@@ -92,6 +93,7 @@
     "@aws-cdk/aws-ecr": "0.0.0",
     "@aws-cdk/aws-ecr-assets": "0.0.0",
     "@aws-cdk/aws-ecs": "0.0.0",
+    "@aws-cdk/aws-glue": "0.0.0",
     "@aws-cdk/aws-iam": "0.0.0",
     "@aws-cdk/aws-kms": "0.0.0",
     "@aws-cdk/aws-lambda": "0.0.0",
@@ -109,6 +111,7 @@
     "@aws-cdk/aws-ecr": "0.0.0",
     "@aws-cdk/aws-ecr-assets": "0.0.0",
     "@aws-cdk/aws-ecs": "0.0.0",
+    "@aws-cdk/aws-glue": "0.0.0",
     "@aws-cdk/aws-iam": "0.0.0",
     "@aws-cdk/aws-kms": "0.0.0",
     "@aws-cdk/aws-lambda": "0.0.0",