Merge pull request #7451 from ministryofjustice/dlt-generic-lambda

Load Zip File Module

terraform/environments/electronic-monitoring-data/lambdas_main.tf

@@ -287,27 +287,16 @@ module "load_json_table" {
 # Load json data from S3 to Athena
 #-----------------------------------------------------------------------------------
 
-module "load_json_table_cadt" {
-  source                  = "./modules/lambdas"
-  function_name           = "load_json_table_cadt"
-  is_image                = true
-  role_name               = aws_iam_role.load_json_table.name
-  role_arn                = aws_iam_role.load_json_table.arn
+module "load_g4s_atrium_unstructured" {
+  source                  = "./modules/unzipped_structure_extract"
+  iam_role                = aws_iam_role.load_json_table
   memory_size             = 2048
   timeout                 = 900
+  function_tag            = "v0.0.0-c79d61f"
+  dataset_name            = "g4s_atrium_unstructured"
   env_account_id          = local.env_account_id
   core_shared_services_id = local.environment_management.account_ids["core-shared-services-production"]
   production_dev          = local.is-production ? "prod" : "dev"
-  ecr_repo_name           = "create-a-data-task"
-  function_tag            = "v0.0.0-884806f"
-  environment_variables = {
-    DLT_PROJECT_DIR : "/tmp"
-    DLT_DATA_DIR : "/tmp"
-    DLT_PIPELINE_DIR : "/tmp"
-    JSON_BUCKET_NAME                         = module.json-directory-structure-bucket.bucket.id
-    STANDARD_FILESYSTEM__QUERY_RESULT_BUCKET = "s3://${module.athena-s3-bucket.bucket.id}/output"
-    ATHENA_DUMP_BUCKET_NAME                  = module.metadata-s3-bucket.bucket.id
-    pipeline_name                            = "g4s_atrium_unstructured"
-    environment                              = local.is-production ? "prod" : "dev"
-  }
+  json_bucket_name        = module.json-directory-structure-bucket.bucket.id
+  athena_bucket_name      = module.athena-s3-bucket.bucket.id
 }

terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/main.tf

@@ -0,0 +1,32 @@
+locals {
+  dataset_name_parts = split("_",  var.dataset_name)
+  SUPPLIER_NAME      = local.dataset_name_parts[0]
+  SYSTEM_NAME        = join("_", slice(local.dataset_name_parts, 1, length(local.dataset_name_parts)))
+}
+
+module "this" {
+  source                  = "../lambdas"
+  function_name           = "load_${var.dataset_name}"
+  is_image                = true
+  role_name               = var.iam_role.name
+  role_arn                = var.iam_role.arn
+  memory_size             = var.memory_size
+  timeout                 = var.timeout
+  env_account_id          = var.env_account_id
+  core_shared_services_id = var.core_shared_services_id
+  production_dev          = var.production_dev
+  ecr_repo_name           = "create-a-data-task"
+  function_tag            = var.function_tag
+  environment_variables = {
+    DLT_PROJECT_DIR : "/tmp"
+    DLT_DATA_DIR : "/tmp"
+    DLT_PIPELINE_DIR : "/tmp"
+    JSON_BUCKET_NAME                         = var.json_bucket_name
+    STANDARD_FILESYSTEM__QUERY_RESULT_BUCKET = "s3://${var.athena_bucket_name}/output"
+    ATHENA_DUMP_BUCKET_NAME                  = "s3://${var.athena_bucket_name}/output"
+    pipeline_name                            = var.dataset_name
+    environment                              = var.production_dev
+    SUPPLIER_NAME                            = local.SUPPLIER_NAME
+    SYSTEM_NAME                              = local.SYSTEM_NAME
+  }
+}

terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/output.tf

@@ -0,0 +1,11 @@
+output "lambda_function_name" {
+  value = module.this.lambda_function_name
+}
+
+output "lambda_function_arn" {
+  value = module.this.lambda_function_arn
+}
+
+output "lambda_function_invoke_arn" {
+  value = module.this.lambda_function_invoke_arn
+}

terraform/environments/electronic-monitoring-data/modules/unzipped_structure_extract/variables.tf

@@ -0,0 +1,60 @@
+variable "dataset_name" {
+  type    = string
+  nullable = false
+}
+
+variable "iam_role" {
+  type = object({
+    name = string
+    arn  = string
+    id   = string
+  })
+  nullable = false
+}
+
+variable "memory_size" {
+    type = number
+    nullable = false
+    default = 240
+}
+variable "timeout" {
+    type = number
+    nullable = false
+    default = 60
+}
+variable "function_tag" {
+    type = string
+    nullable = false
+    default = "v0.0.0-884806f"
+}
+
+variable "env_account_id" {
+  description = "The account number of the aws account"
+  type        = number
+}
+
+variable "core_shared_services_id" {
+  description = "The account number of the core shared services account"
+  type        = number
+  default     = null
+  nullable    = true
+}
+
+variable "production_dev" {
+  description = "The environment the lambda is being deployed to"
+  type        = string
+  nullable    = true
+  default     = null
+}
+
+variable "json_bucket_name" {
+  description = "The bucket to pull json structure from"
+  type = string
+  nullable = false
+}
+
+variable "athena_bucket_name" {
+  description = "Bucket to dump query output into"
+  type = string
+  nullable = false
+}