diff --git a/packages/@aws-cdk/aws-glue-alpha/README.md b/packages/@aws-cdk/aws-glue-alpha/README.md index 331c0d8b0b431..8044d470c0b4d 100644 --- a/packages/@aws-cdk/aws-glue-alpha/README.md +++ b/packages/@aws-cdk/aws-glue-alpha/README.md @@ -493,3 +493,23 @@ new glue.Table(this, 'MyTable', { | array(itemType: Type) | Function | An array of some other type | | map(keyType: Type, valueType: Type) | Function | A map of some primitive key type to any value type | | struct(collumns: Column[]) | Function | Nested structure containing individually named and typed collumns | + +## Data Quality Ruleset + +A `DataQualityRuleset` specifies a data quality ruleset with DQDL rules applied to a specified AWS Glue table. For example, to create a data quality ruleset for a given table: + +```ts +new glue.DataQualityRuleset(this, 'MyDataQualityRuleset', { + clientToken: 'client_token', + description: 'description', + rulesetName: 'ruleset_name', + rulesetDqdl: 'ruleset_dqdl', + tags: { + key1: 'value1', + key2: 'value2', + }, + targetTable: new glue.DataQualityTargetTable('database_name', 'table_name'), +}); +``` + +For more information, see [AWS Glue Data Quality](https://docs.aws.amazon.com/glue/latest/dg/glue-data-quality.html). diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/data-quality-ruleset.ts b/packages/@aws-cdk/aws-glue-alpha/lib/data-quality-ruleset.ts new file mode 100644 index 0000000000000..5fdc194bc7415 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/lib/data-quality-ruleset.ts @@ -0,0 +1,139 @@ +import * as cdk from 'aws-cdk-lib'; +import * as constructs from 'constructs'; +import { IResource, Resource } from 'aws-cdk-lib/core'; +import { CfnDataQualityRuleset } from 'aws-cdk-lib/aws-glue'; + +/** + * Properties of a DataQualityTargetTable. + */ +export class DataQualityTargetTable { + /** + * The database name of the target table. + */ + readonly databaseName: string; + + /** + * The table name of the target table. + */ + readonly tableName: string; + + constructor(databaseName: string, tableName: string) { + this.databaseName = databaseName; + this.tableName = tableName; + } +} + +export interface IDataQualityRuleset extends IResource { + /** + * The ARN of the ruleset + * @attribute + */ + readonly rulesetArn: string; + + /** + * The name of the ruleset + * @attribute + */ + readonly rulesetName: string; +} + +/** + * Construction properties for `DataQualityRuleset` + */ +export interface DataQualityRulesetProps { + /** + * The name of the ruleset + * @default cloudformation generated name + */ + readonly rulesetName?: string; + + /** + * The client token of the ruleset + * @attribute + */ + readonly clientToken?: string; + + /** + * The description of the ruleset + * @attribute + */ + readonly description?: string; + + /** + * The dqdl of the ruleset + * @attribute + */ + readonly rulesetDqdl: string; + + /** + * Key-Value pairs that define tags for the ruleset. + * @default empty tags + */ + readonly tags?: { [key: string]: string }; + + /** + * The target table of the ruleset + * @attribute + */ + readonly targetTable: DataQualityTargetTable; +} + +/** + * A Glue Data Quality ruleset. + */ +export class DataQualityRuleset extends Resource implements IDataQualityRuleset { + public static fromRulesetArn(scope: constructs.Construct, id: string, rulesetArn: string): IDataQualityRuleset { + class Import extends Resource implements IDataQualityRuleset { + public rulesetArn = rulesetArn; + public rulesetName = cdk.Arn.extractResourceName(rulesetArn, 'dataqualityruleset'); + } + + return new Import(scope, id); + } + + public static fromRulesetName(scope: constructs.Construct, id: string, rulesetName: string): IDataQualityRuleset { + class Import extends Resource implements IDataQualityRuleset { + public rulesetArn = DataQualityRuleset.buildRulesetArn(scope, rulesetName); + public rulesetName = rulesetName; + } + + return new Import(scope, id); + } + + private static buildRulesetArn(scope: constructs.Construct, rulesetName: string) : string { + return cdk.Stack.of(scope).formatArn({ + service: 'glue', + resource: 'dataqualityruleset', + resourceName: rulesetName, + }); + } + + /** + * Name of this ruleset. + */ + public readonly rulesetName: string; + + /** + * ARN of this ruleset. + */ + public readonly rulesetArn: string; + + constructor(scope: constructs.Construct, id: string, props: DataQualityRulesetProps) { + super(scope, id, { + physicalName: props.rulesetName, + }); + + const rulesetResource = new CfnDataQualityRuleset(this, 'Resource', { + clientToken: props.clientToken, + description: props.description, + name: props.rulesetName, + ruleset: props.rulesetDqdl, + tags: props.tags, + targetTable: props.targetTable, + }); + + const resourceName = this.getResourceNameAttribute(rulesetResource.ref); + this.rulesetArn = DataQualityRuleset.buildRulesetArn(this, resourceName); + this.rulesetName = resourceName; + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/index.ts b/packages/@aws-cdk/aws-glue-alpha/lib/index.ts index f4895390b3081..8a416993bb5ba 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/index.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/index.ts @@ -2,6 +2,7 @@ export * from './connection'; export * from './data-format'; +export * from './data-quality-ruleset'; export * from './database'; export * from './job'; export * from './job-executable'; diff --git a/packages/@aws-cdk/aws-glue-alpha/package.json b/packages/@aws-cdk/aws-glue-alpha/package.json index a9932a32bf718..f18e376658f77 100644 --- a/packages/@aws-cdk/aws-glue-alpha/package.json +++ b/packages/@aws-cdk/aws-glue-alpha/package.json @@ -84,10 +84,11 @@ "@aws-cdk/cdk-build-tools": "0.0.0", "@aws-cdk/integ-runner": "0.0.0", "@aws-cdk/pkglint": "0.0.0", + "@aws-cdk/integ-tests-alpha": "0.0.0", "@types/jest": "^29.5.1", - "jest": "^29.5.0", "aws-cdk-lib": "0.0.0", - "constructs": "^10.0.0" + "constructs": "^10.0.0", + "jest": "^29.5.0" }, "dependencies": {}, "homepage": "https://github.com/aws/aws-cdk", diff --git a/packages/@aws-cdk/aws-glue-alpha/test/data-quality-ruleset.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/data-quality-ruleset.test.ts new file mode 100644 index 0000000000000..94db164230741 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/data-quality-ruleset.test.ts @@ -0,0 +1,75 @@ +import { Template } from 'aws-cdk-lib/assertions'; +import * as cdk from 'aws-cdk-lib'; +import * as glue from '../lib'; + +test('a data quality ruleset', () => { + const stack = new cdk.Stack(); + new glue.DataQualityRuleset(stack, 'DataQualityRuleset', { + description: 'description', + rulesetName: 'ruleset_name', + rulesetDqdl: 'ruleset_dqdl', + targetTable: new glue.DataQualityTargetTable('database_name', 'table_name'), + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::DataQualityRuleset', { + Description: 'description', + Name: 'ruleset_name', + Ruleset: 'ruleset_dqdl', + TargetTable: { + DatabaseName: 'database_name', + TableName: 'table_name', + }, + }); +}); + +test('a data quality ruleset with a client token', () => { + const stack = new cdk.Stack(); + new glue.DataQualityRuleset(stack, 'DataQualityRuleset', { + clientToken: 'client_token', + description: 'description', + rulesetName: 'ruleset_name', + rulesetDqdl: 'ruleset_dqdl', + targetTable: new glue.DataQualityTargetTable('database_name', 'table_name'), + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::DataQualityRuleset', { + ClientToken: 'client_token', + Description: 'description', + Name: 'ruleset_name', + Ruleset: 'ruleset_dqdl', + TargetTable: { + DatabaseName: 'database_name', + TableName: 'table_name', + }, + }); +}); + +test('a data quality ruleset with tags', () => { + const stack = new cdk.Stack(); + new glue.DataQualityRuleset(stack, 'DataQualityRuleset', { + clientToken: 'client_token', + description: 'description', + rulesetName: 'ruleset_name', + rulesetDqdl: 'ruleset_dqdl', + tags: { + key1: 'value1', + key2: 'value2', + }, + targetTable: new glue.DataQualityTargetTable('database_name', 'table_name'), + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::DataQualityRuleset', { + ClientToken: 'client_token', + Description: 'description', + Name: 'ruleset_name', + Ruleset: 'ruleset_dqdl', + Tags: { + key1: 'value1', + key2: 'value2', + }, + TargetTable: { + DatabaseName: 'database_name', + TableName: 'table_name', + }, + }); +}); diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/aws-glue-data-quality-ruleset.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/aws-glue-data-quality-ruleset.assets.json new file mode 100644 index 0000000000000..91e59d8665bf4 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/aws-glue-data-quality-ruleset.assets.json @@ -0,0 +1,19 @@ +{ + "version": "32.0.0", + "files": { + "b9515accbd6b765c36fbffa5adb190fc8f6d1f67573ab2655ede370368887799": { + "source": { + "path": "aws-glue-data-quality-ruleset.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "b9515accbd6b765c36fbffa5adb190fc8f6d1f67573ab2655ede370368887799.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": {} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/aws-glue-data-quality-ruleset.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/aws-glue-data-quality-ruleset.template.json new file mode 100644 index 0000000000000..bbe092536d594 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/aws-glue-data-quality-ruleset.template.json @@ -0,0 +1,140 @@ +{ + "Resources": { + "DataBucketE3889A50": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "MyDatabase1E2517DB": { + "Type": "AWS::Glue::Database", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseInput": { + "Name": "my_database" + } + } + }, + "CSVTableE499CABA": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "csv_table generated by CDK", + "Name": "csv_table", + "Parameters": { + "classification": "csv", + "has_encrypted_data": true + }, + "PartitionKeys": [ + { + "Name": "year", + "Type": "smallint" + }, + { + "Name": "month", + "Type": "bigint" + } + ], + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "string" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.apache.hadoop.hive.serde2.OpenCSVSerde" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "DataQualityRulesetBB177ADB": { + "Type": "AWS::Glue::DataQualityRuleset", + "Properties": { + "ClientToken": "client_token", + "Description": "my description", + "Name": "my_ruleset", + "Ruleset": "Rules = [RowCount > 10]", + "Tags": { + "key1": "value1", + "key2": "value2" + }, + "TargetTable": { + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableName": { + "Ref": "CSVTableE499CABA" + } + } + } + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/cdk.out new file mode 100644 index 0000000000000..f0b901e7c06e5 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/cdk.out @@ -0,0 +1 @@ +{"version":"32.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/gluedataqualityrulesetDefaultTestDeployAssertDE1B6FD7.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/gluedataqualityrulesetDefaultTestDeployAssertDE1B6FD7.assets.json new file mode 100644 index 0000000000000..e72c526027a5d --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/gluedataqualityrulesetDefaultTestDeployAssertDE1B6FD7.assets.json @@ -0,0 +1,19 @@ +{ + "version": "32.0.0", + "files": { + "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { + "source": { + "path": "gluedataqualityrulesetDefaultTestDeployAssertDE1B6FD7.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": {} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/gluedataqualityrulesetDefaultTestDeployAssertDE1B6FD7.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/gluedataqualityrulesetDefaultTestDeployAssertDE1B6FD7.template.json new file mode 100644 index 0000000000000..ad9d0fb73d1dd --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/gluedataqualityrulesetDefaultTestDeployAssertDE1B6FD7.template.json @@ -0,0 +1,36 @@ +{ + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/integ.json new file mode 100644 index 0000000000000..903d01ef677e0 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/integ.json @@ -0,0 +1,12 @@ +{ + "version": "32.0.0", + "testCases": { + "glue-data-quality-ruleset/DefaultTest": { + "stacks": [ + "aws-glue-data-quality-ruleset" + ], + "assertionStack": "glue-data-quality-ruleset/DefaultTest/DeployAssert", + "assertionStackName": "gluedataqualityrulesetDefaultTestDeployAssertDE1B6FD7" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/manifest.json new file mode 100644 index 0000000000000..200e21bae56bd --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/manifest.json @@ -0,0 +1,129 @@ +{ + "version": "32.0.0", + "artifacts": { + "aws-glue-data-quality-ruleset.assets": { + "type": "cdk:asset-manifest", + "properties": { + "file": "aws-glue-data-quality-ruleset.assets.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "aws-glue-data-quality-ruleset": { + "type": "aws:cloudformation:stack", + "environment": "aws://unknown-account/unknown-region", + "properties": { + "templateFile": "aws-glue-data-quality-ruleset.template.json", + "validateOnSynth": false, + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", + "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/b9515accbd6b765c36fbffa5adb190fc8f6d1f67573ab2655ede370368887799.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", + "additionalDependencies": [ + "aws-glue-data-quality-ruleset.assets" + ], + "lookupRole": { + "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}", + "requiresBootstrapStackVersion": 8, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "dependencies": [ + "aws-glue-data-quality-ruleset.assets" + ], + "metadata": { + "/aws-glue-data-quality-ruleset/DataBucket/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "DataBucketE3889A50" + } + ], + "/aws-glue-data-quality-ruleset/MyDatabase/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyDatabase1E2517DB" + } + ], + "/aws-glue-data-quality-ruleset/CSVTable/Table": [ + { + "type": "aws:cdk:logicalId", + "data": "CSVTableE499CABA" + } + ], + "/aws-glue-data-quality-ruleset/DataQualityRuleset/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "DataQualityRulesetBB177ADB" + } + ], + "/aws-glue-data-quality-ruleset/BootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "BootstrapVersion" + } + ], + "/aws-glue-data-quality-ruleset/CheckBootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "CheckBootstrapVersion" + } + ] + }, + "displayName": "aws-glue-data-quality-ruleset" + }, + "gluedataqualityrulesetDefaultTestDeployAssertDE1B6FD7.assets": { + "type": "cdk:asset-manifest", + "properties": { + "file": "gluedataqualityrulesetDefaultTestDeployAssertDE1B6FD7.assets.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "gluedataqualityrulesetDefaultTestDeployAssertDE1B6FD7": { + "type": "aws:cloudformation:stack", + "environment": "aws://unknown-account/unknown-region", + "properties": { + "templateFile": "gluedataqualityrulesetDefaultTestDeployAssertDE1B6FD7.template.json", + "validateOnSynth": false, + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", + "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", + "additionalDependencies": [ + "gluedataqualityrulesetDefaultTestDeployAssertDE1B6FD7.assets" + ], + "lookupRole": { + "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}", + "requiresBootstrapStackVersion": 8, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "dependencies": [ + "gluedataqualityrulesetDefaultTestDeployAssertDE1B6FD7.assets" + ], + "metadata": { + "/glue-data-quality-ruleset/DefaultTest/DeployAssert/BootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "BootstrapVersion" + } + ], + "/glue-data-quality-ruleset/DefaultTest/DeployAssert/CheckBootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "CheckBootstrapVersion" + } + ] + }, + "displayName": "glue-data-quality-ruleset/DefaultTest/DeployAssert" + }, + "Tree": { + "type": "cdk:tree", + "properties": { + "file": "tree.json" + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/tree.json new file mode 100644 index 0000000000000..79cf8fb9505da --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.js.snapshot/tree.json @@ -0,0 +1,274 @@ +{ + "version": "tree-0.1", + "tree": { + "id": "App", + "path": "", + "children": { + "aws-glue-data-quality-ruleset": { + "id": "aws-glue-data-quality-ruleset", + "path": "aws-glue-data-quality-ruleset", + "children": { + "DataBucket": { + "id": "DataBucket", + "path": "aws-glue-data-quality-ruleset/DataBucket", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-data-quality-ruleset/DataBucket/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::S3::Bucket", + "aws:cdk:cloudformation:props": {} + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" + } + }, + "MyDatabase": { + "id": "MyDatabase", + "path": "aws-glue-data-quality-ruleset/MyDatabase", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-data-quality-ruleset/MyDatabase/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Database", + "aws:cdk:cloudformation:props": { + "catalogId": { + "Ref": "AWS::AccountId" + }, + "databaseInput": { + "name": "my_database" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnDatabase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Database", + "version": "0.0.0" + } + }, + "CSVTable": { + "id": "CSVTable", + "path": "aws-glue-data-quality-ruleset/CSVTable", + "children": { + "Table": { + "id": "Table", + "path": "aws-glue-data-quality-ruleset/CSVTable/Table", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Table", + "aws:cdk:cloudformation:props": { + "catalogId": { + "Ref": "AWS::AccountId" + }, + "databaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "tableInput": { + "name": "csv_table", + "description": "csv_table generated by CDK", + "partitionKeys": [ + { + "name": "year", + "type": "smallint" + }, + { + "name": "month", + "type": "bigint" + } + ], + "parameters": { + "classification": "csv", + "has_encrypted_data": true + }, + "storageDescriptor": { + "location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "compressed": false, + "storedAsSubDirectories": false, + "columns": [ + { + "name": "col1", + "type": "string" + }, + { + "name": "col2", + "type": "string" + }, + { + "name": "col3", + "type": "string" + } + ], + "inputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "outputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "serdeInfo": { + "serializationLibrary": "org.apache.hadoop.hive.serde2.OpenCSVSerde" + } + }, + "tableType": "EXTERNAL_TABLE" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTable", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Table", + "version": "0.0.0" + } + }, + "DataQualityRuleset": { + "id": "DataQualityRuleset", + "path": "aws-glue-data-quality-ruleset/DataQualityRuleset", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-data-quality-ruleset/DataQualityRuleset/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::DataQualityRuleset", + "aws:cdk:cloudformation:props": { + "clientToken": "client_token", + "description": "my description", + "name": "my_ruleset", + "ruleset": "Rules = [RowCount > 10]", + "tags": { + "key1": "value1", + "key2": "value2" + }, + "targetTable": { + "databaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "tableName": { + "Ref": "CSVTableE499CABA" + } + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnDataQualityRuleset", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.DataQualityRuleset", + "version": "0.0.0" + } + }, + "BootstrapVersion": { + "id": "BootstrapVersion", + "path": "aws-glue-data-quality-ruleset/BootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnParameter", + "version": "0.0.0" + } + }, + "CheckBootstrapVersion": { + "id": "CheckBootstrapVersion", + "path": "aws-glue-data-quality-ruleset/CheckBootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnRule", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.Stack", + "version": "0.0.0" + } + }, + "glue-data-quality-ruleset": { + "id": "glue-data-quality-ruleset", + "path": "glue-data-quality-ruleset", + "children": { + "DefaultTest": { + "id": "DefaultTest", + "path": "glue-data-quality-ruleset/DefaultTest", + "children": { + "Default": { + "id": "Default", + "path": "glue-data-quality-ruleset/DefaultTest/Default", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.2.55" + } + }, + "DeployAssert": { + "id": "DeployAssert", + "path": "glue-data-quality-ruleset/DefaultTest/DeployAssert", + "children": { + "BootstrapVersion": { + "id": "BootstrapVersion", + "path": "glue-data-quality-ruleset/DefaultTest/DeployAssert/BootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnParameter", + "version": "0.0.0" + } + }, + "CheckBootstrapVersion": { + "id": "CheckBootstrapVersion", + "path": "glue-data-quality-ruleset/DefaultTest/DeployAssert/CheckBootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnRule", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.Stack", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/integ-tests-alpha.IntegTestCase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/integ-tests-alpha.IntegTest", + "version": "0.0.0" + } + }, + "Tree": { + "id": "Tree", + "path": "Tree", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.2.55" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.App", + "version": "0.0.0" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.ts new file mode 100644 index 0000000000000..0d4f1f2d9c2bc --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.data-quality-ruleset.ts @@ -0,0 +1,57 @@ +import * as s3 from 'aws-cdk-lib/aws-s3'; +import * as cdk from 'aws-cdk-lib'; +import { IntegTest } from '@aws-cdk/integ-tests-alpha'; +import * as glue from '../lib'; + +const app = new cdk.App(); + +const stack = new cdk.Stack(app, 'aws-glue-data-quality-ruleset'); + +const bucket = new s3.Bucket(stack, 'DataBucket'); +const database = new glue.Database(stack, 'MyDatabase', { + databaseName: 'my_database', +}); + +const columns = [{ + name: 'col1', + type: glue.Schema.STRING, +}, { + name: 'col2', + type: glue.Schema.STRING, +}, { + name: 'col3', + type: glue.Schema.STRING, +}]; + +const partitionKeys = [{ + name: 'year', + type: glue.Schema.SMALL_INT, +}, { + name: 'month', + type: glue.Schema.BIG_INT, +}]; + +const csvTable = new glue.Table(stack, 'CSVTable', { + database, + bucket, + tableName: 'csv_table', + columns, + partitionKeys, + dataFormat: glue.DataFormat.CSV, +}); + +new glue.DataQualityRuleset(stack, 'DataQualityRuleset', { + clientToken: 'client_token', + description: 'my description', + rulesetName: 'my_ruleset', + rulesetDqdl: 'Rules = [RowCount > 10]', + tags: { + key1: 'value1', + key2: 'value2', + }, + targetTable: new glue.DataQualityTargetTable(database.databaseName, csvTable.tableName), +}); + +new IntegTest(app, 'glue-data-quality-ruleset', { + testCases: [stack], +});