From 2dc781509a86b9b5528dc3bd1d18a62d39f1ce46 Mon Sep 17 00:00:00 2001 From: matt-heery Date: Wed, 14 Aug 2024 12:57:12 +0100 Subject: [PATCH 1/7] initial implementation of a load zip file module --- .../lambdas_main.tf | 24 +++----------- .../unzipped_structure_extract/main.tf | 32 +++++++++++++++++++ .../unzipped_structure_extract/output.tf | 11 +++++++ .../unzipped_structure_extract/variables.tf | 29 +++++++++++++++++ 4 files changed, 76 insertions(+), 20 deletions(-) create mode 100644 terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/main.tf create mode 100644 terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/output.tf create mode 100644 terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/variables.tf diff --git a/terraform/environments/electronic-monitoring-data/lambdas_main.tf b/terraform/environments/electronic-monitoring-data/lambdas_main.tf index 90e59d7ad34..047775e6caa 100644 --- a/terraform/environments/electronic-monitoring-data/lambdas_main.tf +++ b/terraform/environments/electronic-monitoring-data/lambdas_main.tf @@ -287,27 +287,11 @@ module "load_json_table" { # Load json data from S3 to Athena #----------------------------------------------------------------------------------- -module "load_json_table_cadt" { - source = "./modules/lambdas" - function_name = "load_json_table_cadt" - is_image = true - role_name = aws_iam_role.load_json_table.name - role_arn = aws_iam_role.load_json_table.arn +module "load_g4s_atrium_unstructured" { + source = "./modules/unzipped_structure_extract" + iam_role = aws_iam_role.load_json_table memory_size = 2048 timeout = 900 - env_account_id = local.env_account_id - core_shared_services_id = local.environment_management.account_ids["core-shared-services-production"] - production_dev = local.is-production ? "prod" : "dev" - ecr_repo_name = "create-a-data-task" function_tag = "v0.0.0-884806f" - environment_variables = { - DLT_PROJECT_DIR : "/tmp" - DLT_DATA_DIR : "/tmp" - DLT_PIPELINE_DIR : "/tmp" - JSON_BUCKET_NAME = module.json-directory-structure-bucket.bucket.id - STANDARD_FILESYSTEM__QUERY_RESULT_BUCKET = "s3://${module.athena-s3-bucket.bucket.id}/output" - ATHENA_DUMP_BUCKET_NAME = module.metadata-s3-bucket.bucket.id - pipeline_name = "g4s_atrium_unstructured" - environment = local.is-production ? "prod" : "dev" - } + dataset_name = "g4s_atrium_unstructured" } diff --git a/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/main.tf b/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/main.tf new file mode 100644 index 00000000000..52b72a80264 --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/main.tf @@ -0,0 +1,32 @@ +locals { + dataset_name_parts = split("_", v) + SUPPLIER_NAME = local.dataset_name_parts[0] + SYSTEM_NAME = join("_", slice(local.dataset_name_parts, 1, length(local.dataset_name_parts))) +} + +module "this" { + source = "./modules/lambdas" + function_name = "load_${var.dataset_name}" + is_image = true + role_name = var.iam_role.name + role_arn = var.iam_role.arn + memory_size = var.memory_size + timeout = var.timeout + env_account_id = local.env_account_id + core_shared_services_id = local.environment_management.account_ids["core-shared-services-production"] + production_dev = local.is-production ? "prod" : "dev" + ecr_repo_name = "create-a-data-task" + function_tag = var.function_tag + environment_variables = { + DLT_PROJECT_DIR : "/tmp" + DLT_DATA_DIR : "/tmp" + DLT_PIPELINE_DIR : "/tmp" + JSON_BUCKET_NAME = module.json-directory-structure-bucket.bucket.id + STANDARD_FILESYSTEM__QUERY_RESULT_BUCKET = "s3://${module.athena-s3-bucket.bucket.id}/output" + ATHENA_DUMP_BUCKET_NAME = module.metadata-s3-bucket.bucket.id + pipeline_name = var.dataset_name + environment = local.is-production ? "prod" : "dev" + SUPPLIER_NAME = SUPPLIER_NAME + SYSTEM_NAME = SYSTEM_NAME + } +} diff --git a/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/output.tf b/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/output.tf new file mode 100644 index 00000000000..5e6d153d2ff --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/output.tf @@ -0,0 +1,11 @@ +output "lambda_function_name" { + value = module.this.function_name +} + +output "lambda_function_arn" { + value = module.this.arn +} + +output "lambda_function_invoke_arn" { + value = module.this.invoke_arn +} \ No newline at end of file diff --git a/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/variables.tf b/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/variables.tf new file mode 100644 index 00000000000..77fd18304e1 --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/variables.tf @@ -0,0 +1,29 @@ +variable "dataset_name" { + type = string + nullable = false +} + +variable "iam_role" { + type = object({ + name = string + arn = string + id = string + }) + nullable = false +} + +variable "memory_size" { + type = number + nullable = false + default = 240 +} +variable "timeout" { + type = number + nullable = false + default = 60 +} +variable "function_tag" { + type = string + nullable = false + default = "v0.0.0-884806f" +} From 3e6a85c121c9d9d2e7b4bd2f659515f85dd230ab Mon Sep 17 00:00:00 2001 From: matt-heery Date: Wed, 14 Aug 2024 12:59:25 +0100 Subject: [PATCH 2/7] correctly source --- .../modules/unzipped_structure_extract/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/main.tf b/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/main.tf index 52b72a80264..4449330d10b 100644 --- a/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/main.tf +++ b/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/main.tf @@ -5,7 +5,7 @@ locals { } module "this" { - source = "./modules/lambdas" + source = "../lambdas" function_name = "load_${var.dataset_name}" is_image = true role_name = var.iam_role.name From ca102f8b23cc83a42b225f0e86c82493de72c582 Mon Sep 17 00:00:00 2001 From: matt-heery Date: Wed, 14 Aug 2024 13:10:41 +0100 Subject: [PATCH 3/7] adding new variables --- .../lambdas_main.tf | 3 ++ .../unzipped_structure_extract/main.tf | 16 +++++----- .../unzipped_structure_extract/variables.tf | 31 +++++++++++++++++++ 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/terraform/environments/electronic-monitoring-data/lambdas_main.tf b/terraform/environments/electronic-monitoring-data/lambdas_main.tf index 047775e6caa..3037c665578 100644 --- a/terraform/environments/electronic-monitoring-data/lambdas_main.tf +++ b/terraform/environments/electronic-monitoring-data/lambdas_main.tf @@ -294,4 +294,7 @@ module "load_g4s_atrium_unstructured" { timeout = 900 function_tag = "v0.0.0-884806f" dataset_name = "g4s_atrium_unstructured" + production_dev = local.is-production ? "prod" : "dev" + json_bucket_name = module.json-directory-structure-bucket.bucket.id + athena_bucket_name = module.athena-s3-bucket.bucket.id } diff --git a/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/main.tf b/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/main.tf index 4449330d10b..c8fbc858106 100644 --- a/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/main.tf +++ b/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/main.tf @@ -1,5 +1,5 @@ locals { - dataset_name_parts = split("_", v) + dataset_name_parts = split("_", var.dataset_name) SUPPLIER_NAME = local.dataset_name_parts[0] SYSTEM_NAME = join("_", slice(local.dataset_name_parts, 1, length(local.dataset_name_parts))) } @@ -12,20 +12,20 @@ module "this" { role_arn = var.iam_role.arn memory_size = var.memory_size timeout = var.timeout - env_account_id = local.env_account_id - core_shared_services_id = local.environment_management.account_ids["core-shared-services-production"] - production_dev = local.is-production ? "prod" : "dev" + env_account_id = var.env_account_id + core_shared_services_id = var.core_shared_services_id + production_dev = var.production_dev ecr_repo_name = "create-a-data-task" function_tag = var.function_tag environment_variables = { DLT_PROJECT_DIR : "/tmp" DLT_DATA_DIR : "/tmp" DLT_PIPELINE_DIR : "/tmp" - JSON_BUCKET_NAME = module.json-directory-structure-bucket.bucket.id - STANDARD_FILESYSTEM__QUERY_RESULT_BUCKET = "s3://${module.athena-s3-bucket.bucket.id}/output" - ATHENA_DUMP_BUCKET_NAME = module.metadata-s3-bucket.bucket.id + JSON_BUCKET_NAME = var.json_bucket_name + STANDARD_FILESYSTEM__QUERY_RESULT_BUCKET = "s3://${var.athena_bucket_name}/output" + ATHENA_DUMP_BUCKET_NAME = "s3://${var.athena_bucket_name}/output" pipeline_name = var.dataset_name - environment = local.is-production ? "prod" : "dev" + environment = var.production_dev SUPPLIER_NAME = SUPPLIER_NAME SYSTEM_NAME = SYSTEM_NAME } diff --git a/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/variables.tf b/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/variables.tf index 77fd18304e1..a26d3627aed 100644 --- a/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/variables.tf +++ b/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/variables.tf @@ -27,3 +27,34 @@ variable "function_tag" { nullable = false default = "v0.0.0-884806f" } + +variable "env_account_id" { + description = "The account number of the aws account" + type = number +} + +variable "core_shared_services_id" { + description = "The account number of the core shared services account" + type = number + default = null + nullable = true +} + +variable "production_dev" { + description = "The environment the lambda is being deployed to" + type = string + nullable = true + default = null +} + +variable "json_bucket_name" { + description = "The bucket to pull json structure from" + type = string + nullable = false +} + +variable "athena_bucket_name" { + description = "Bucket to dump query output into" + type = string + nullable = false +} \ No newline at end of file From 7094e3b698508587a6dc87cb47c164c1f4242168 Mon Sep 17 00:00:00 2001 From: matt-heery Date: Wed, 14 Aug 2024 13:12:11 +0100 Subject: [PATCH 4/7] two missing args --- .../environments/electronic-monitoring-data/lambdas_main.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/terraform/environments/electronic-monitoring-data/lambdas_main.tf b/terraform/environments/electronic-monitoring-data/lambdas_main.tf index 3037c665578..7d0056b5d3c 100644 --- a/terraform/environments/electronic-monitoring-data/lambdas_main.tf +++ b/terraform/environments/electronic-monitoring-data/lambdas_main.tf @@ -294,6 +294,8 @@ module "load_g4s_atrium_unstructured" { timeout = 900 function_tag = "v0.0.0-884806f" dataset_name = "g4s_atrium_unstructured" + env_account_id = local.env_account_id + core_shared_services_id = local.environment_management.account_ids["core-shared-services-production"] production_dev = local.is-production ? "prod" : "dev" json_bucket_name = module.json-directory-structure-bucket.bucket.id athena_bucket_name = module.athena-s3-bucket.bucket.id From ed5ffec82f39dedd79ddfaf4f2ae524da467fa73 Mon Sep 17 00:00:00 2001 From: matt-heery Date: Wed, 14 Aug 2024 13:20:01 +0100 Subject: [PATCH 5/7] ENV VARS --- .../modules/unzipped_structure_extract/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/main.tf b/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/main.tf index c8fbc858106..d61d7f5ae73 100644 --- a/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/main.tf +++ b/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/main.tf @@ -26,7 +26,7 @@ module "this" { ATHENA_DUMP_BUCKET_NAME = "s3://${var.athena_bucket_name}/output" pipeline_name = var.dataset_name environment = var.production_dev - SUPPLIER_NAME = SUPPLIER_NAME - SYSTEM_NAME = SYSTEM_NAME + SUPPLIER_NAME = local.SUPPLIER_NAME + SYSTEM_NAME = local.SYSTEM_NAME } } From e6e693a71d70a59add168bb39de84e178a2128f9 Mon Sep 17 00:00:00 2001 From: matt-heery Date: Wed, 14 Aug 2024 13:27:27 +0100 Subject: [PATCH 6/7] lambda!!! --- .../modules/unzipped_structure_extract/output.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/output.tf b/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/output.tf index 5e6d153d2ff..62535b2acfc 100644 --- a/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/output.tf +++ b/terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/output.tf @@ -1,11 +1,11 @@ output "lambda_function_name" { - value = module.this.function_name + value = module.this.lambda_function_name } output "lambda_function_arn" { - value = module.this.arn + value = module.this.lambda_function_arn } output "lambda_function_invoke_arn" { - value = module.this.invoke_arn + value = module.this.lambda_function_invoke_arn } \ No newline at end of file From d6251e6e51709047b743bfbdd1e4010e74433def Mon Sep 17 00:00:00 2001 From: matt-heery Date: Wed, 14 Aug 2024 14:38:40 +0100 Subject: [PATCH 7/7] updating commit hash --- .../environments/electronic-monitoring-data/lambdas_main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/electronic-monitoring-data/lambdas_main.tf b/terraform/environments/electronic-monitoring-data/lambdas_main.tf index 7d0056b5d3c..e8ec4743750 100644 --- a/terraform/environments/electronic-monitoring-data/lambdas_main.tf +++ b/terraform/environments/electronic-monitoring-data/lambdas_main.tf @@ -292,7 +292,7 @@ module "load_g4s_atrium_unstructured" { iam_role = aws_iam_role.load_json_table memory_size = 2048 timeout = 900 - function_tag = "v0.0.0-884806f" + function_tag = "v0.0.0-c79d61f" dataset_name = "g4s_atrium_unstructured" env_account_id = local.env_account_id core_shared_services_id = local.environment_management.account_ids["core-shared-services-production"]