From 78ca63a9b1977af3b649038a40e84338e35e4e1a Mon Sep 17 00:00:00 2001 From: Carwyn Ellis <125277169+carwynmoj@users.noreply.github.com> Date: Tue, 30 May 2023 10:17:09 +0100 Subject: [PATCH 1/6] DPR-498 add contract registry name parameter to datahubjob --- terraform/environments/digital-prison-reporting/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/terraform/environments/digital-prison-reporting/main.tf b/terraform/environments/digital-prison-reporting/main.tf index 33f9dd6e435..f2ba08aabba 100644 --- a/terraform/environments/digital-prison-reporting/main.tf +++ b/terraform/environments/digital-prison-reporting/main.tf @@ -52,6 +52,7 @@ module "glue_reporting_hub_job" { "--enable-spark-ui" = false "--enable-job-insights" = true "--dpr.aws.kinesis.endpointUrl" = "https://kinesis.${local.account_region}.amazonaws.com" + "--dpr.contract.registryName" = "${module.glue_registry_avro.name}" } } @@ -684,4 +685,4 @@ module "s3_application_tf_state" { Resource_Type = "S3 Bucket" } ) -} \ No newline at end of file +} From 846954202aa3895a86155174e1b964777aa33b98 Mon Sep 17 00:00:00 2001 From: Carwyn Ellis <125277169+carwynmoj@users.noreply.github.com> Date: Tue, 30 May 2023 11:51:46 +0100 Subject: [PATCH 2/6] DPR-498 add specific output def for registry_name and use this in main.tf --- terraform/environments/digital-prison-reporting/main.tf | 2 +- .../digital-prison-reporting/modules/glue_registry/outputs.tf | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 terraform/environments/digital-prison-reporting/modules/glue_registry/outputs.tf diff --git a/terraform/environments/digital-prison-reporting/main.tf b/terraform/environments/digital-prison-reporting/main.tf index f2ba08aabba..19ea33cc84b 100644 --- a/terraform/environments/digital-prison-reporting/main.tf +++ b/terraform/environments/digital-prison-reporting/main.tf @@ -52,7 +52,7 @@ module "glue_reporting_hub_job" { "--enable-spark-ui" = false "--enable-job-insights" = true "--dpr.aws.kinesis.endpointUrl" = "https://kinesis.${local.account_region}.amazonaws.com" - "--dpr.contract.registryName" = "${module.glue_registry_avro.name}" + "--dpr.contract.registryName" = "${module.glue_registry_avro.registry_name}" } } diff --git a/terraform/environments/digital-prison-reporting/modules/glue_registry/outputs.tf b/terraform/environments/digital-prison-reporting/modules/glue_registry/outputs.tf new file mode 100644 index 00000000000..8b5a9fd3c6c --- /dev/null +++ b/terraform/environments/digital-prison-reporting/modules/glue_registry/outputs.tf @@ -0,0 +1,4 @@ +output "registry_name" { + description = "Registry Name" + value = aws_glue_registry.glue_registry[0].id +} From bf007ce88f1730b1195f21a362d8234903fb7c6c Mon Sep 17 00:00:00 2001 From: harichintala1 <109067588+harichintala1@users.noreply.github.com> Date: Wed, 31 May 2023 10:23:27 +0100 Subject: [PATCH 3/6] Split and Retrieve Name of the Resource from ID --- terraform/environments/digital-prison-reporting/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/digital-prison-reporting/main.tf b/terraform/environments/digital-prison-reporting/main.tf index 21c29b11500..43b36356883 100644 --- a/terraform/environments/digital-prison-reporting/main.tf +++ b/terraform/environments/digital-prison-reporting/main.tf @@ -52,7 +52,7 @@ module "glue_reporting_hub_job" { "--enable-spark-ui" = false "--enable-job-insights" = true "--dpr.aws.kinesis.endpointUrl" = "https://kinesis.${local.account_region}.amazonaws.com" - "--dpr.contract.registryName" = module.glue_registry_avro.registry_name + "--dpr.contract.registryName" = split("/", module.glue_registry_avro.registry_name) } } @@ -714,4 +714,4 @@ module "dynamo_tab_application_tf_state" { Resource_Type = "Dynamo Table" } ) -} \ No newline at end of file +} From faddebae37a33fcd95e5225e454ecec92ca06c40 Mon Sep 17 00:00:00 2001 From: harichintala1 <109067588+harichintala1@users.noreply.github.com> Date: Wed, 31 May 2023 10:57:49 +0100 Subject: [PATCH 4/6] Trim Prefix --- terraform/environments/digital-prison-reporting/main.tf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/terraform/environments/digital-prison-reporting/main.tf b/terraform/environments/digital-prison-reporting/main.tf index 43b36356883..6872ce57e90 100644 --- a/terraform/environments/digital-prison-reporting/main.tf +++ b/terraform/environments/digital-prison-reporting/main.tf @@ -3,6 +3,10 @@ ############################################### ## Glue Job, Reporting Hub ## Glue Cloud Platform Ingestion Job (Load, Reload, CDC) +locals { + glue_avro_registry = split("/", module.glue_registry_avro.registry_name) +} + module "glue_reporting_hub_job" { source = "./modules/glue_job" create_job = local.create_job @@ -52,7 +56,7 @@ module "glue_reporting_hub_job" { "--enable-spark-ui" = false "--enable-job-insights" = true "--dpr.aws.kinesis.endpointUrl" = "https://kinesis.${local.account_region}.amazonaws.com" - "--dpr.contract.registryName" = split("/", module.glue_registry_avro.registry_name) + "--dpr.contract.registryName" = trimprefix(local.glue_avro_registry, local.glue_avro_registry[0]) } } From ec4e5f458f3366e476671741bdbe1462d049c182 Mon Sep 17 00:00:00 2001 From: harichintala1 <109067588+harichintala1@users.noreply.github.com> Date: Wed, 31 May 2023 11:00:28 +0100 Subject: [PATCH 5/6] Fix the Registry Name --- terraform/environments/digital-prison-reporting/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/digital-prison-reporting/main.tf b/terraform/environments/digital-prison-reporting/main.tf index 6872ce57e90..f39553597d8 100644 --- a/terraform/environments/digital-prison-reporting/main.tf +++ b/terraform/environments/digital-prison-reporting/main.tf @@ -56,7 +56,7 @@ module "glue_reporting_hub_job" { "--enable-spark-ui" = false "--enable-job-insights" = true "--dpr.aws.kinesis.endpointUrl" = "https://kinesis.${local.account_region}.amazonaws.com" - "--dpr.contract.registryName" = trimprefix(local.glue_avro_registry, local.glue_avro_registry[0]) + "--dpr.contract.registryName" = trimprefix(module.glue_registry_avro.registry_name, local.glue_avro_registry[0]) } } From 2d4830f3e2de74e5bfcb5d3e8b5c56237dd0785f Mon Sep 17 00:00:00 2001 From: harichintala1 <109067588+harichintala1@users.noreply.github.com> Date: Wed, 31 May 2023 11:07:05 +0100 Subject: [PATCH 6/6] Add suffix / for trim --- terraform/environments/digital-prison-reporting/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/digital-prison-reporting/main.tf b/terraform/environments/digital-prison-reporting/main.tf index f39553597d8..a6c66ff5315 100644 --- a/terraform/environments/digital-prison-reporting/main.tf +++ b/terraform/environments/digital-prison-reporting/main.tf @@ -56,7 +56,7 @@ module "glue_reporting_hub_job" { "--enable-spark-ui" = false "--enable-job-insights" = true "--dpr.aws.kinesis.endpointUrl" = "https://kinesis.${local.account_region}.amazonaws.com" - "--dpr.contract.registryName" = trimprefix(module.glue_registry_avro.registry_name, local.glue_avro_registry[0]) + "--dpr.contract.registryName" = trimprefix(module.glue_registry_avro.registry_name, "${local.glue_avro_registry[0]}/") } }