Merge pull request #3114 from ministryofjustice/glue-job-impvmnts

Add Spark Resource Parameters for Glue Job
ministryofjustice · Aug 17, 2023 · bb480bf · bb480bf
2 parents 4fe1f3e + 1d81226
commit bb480bf
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 0 deletions.
diff --git a/terraform/environments/digital-prison-reporting/application_variables.json b/terraform/environments/digital-prison-reporting/application_variables.json
@@ -6,6 +6,9 @@
       "db_description": "DPR Data Catalog",
       "create_job": true,
       "glue_job_name": "glue-job-ingester",
+      "reporting_hub_spark_driver_mem": "2g",
+      "reporting_hub_spark_executor_mem": "2g",
+      "reporting_hub_spark_log_level": "INFO",   
       "create_security_conf": true,
       "setup_buckets": true,
       "create_kinesis_streams": true,
@@ -36,6 +39,9 @@
       "db_description": "DPR Data Catalog",
       "create_job": true,
       "glue_job_name": "glue-job-ingester",
+      "reporting_hub_spark_driver_mem": "2g",
+      "reporting_hub_spark_executor_mem": "2g",
+      "reporting_hub_spark_log_level": "INFO",      
       "create_security_conf": true,
       "setup_buckets": true,
       "create_kinesis_streams": true,
@@ -66,6 +72,9 @@
       "db_description": "DPR Data Catalog",
       "create_job": true,
       "glue_job_name": "glue-job-ingester",
+      "reporting_hub_spark_driver_mem": "2g",
+      "reporting_hub_spark_executor_mem": "2g",
+      "reporting_hub_spark_log_level": "WARN",           
       "create_security_conf": true,
       "setup_buckets": true,
       "create_kinesis_streams": true,
@@ -96,6 +105,9 @@
       "db_description": "DPR Data Catalog",
       "create_job": true,
       "glue_job_name": "glue-job-ingester",
+      "reporting_hub_spark_driver_mem": "26g",
+      "reporting_hub_spark_executor_mem": "26g",
+      "reporting_hub_spark_log_level": "WARN",
       "create_security_conf": true,
       "setup_buckets": true,
       "create_kinesis_streams": true,

diff --git a/terraform/environments/digital-prison-reporting/locals.tf b/terraform/environments/digital-prison-reporting/locals.tf
@@ -39,6 +39,11 @@ locals {
   datamart_username        = jsondecode(data.aws_secretsmanager_secret_version.datamart.secret_string)["username"]
   datamart_password        = jsondecode(data.aws_secretsmanager_secret_version.datamart.secret_string)["password"]
 
+  # Glue Job parameters
+  reporting_hub_driver_mem    = local.application_data.accounts[local.environment].reporting_hub_spark_driver_mem
+  reporting_hub_executor_mem  = local.application_data.accounts[local.environment].reporting_hub_spark_executor_mem
+  reporting_hub_log_level     = local.application_data.accounts[local.environment].reporting_hub_spark_log_level
+
   # Common Policies
   kms_read_access_policy   = "${local.project}_kms_read_policy"
   s3_read_access_policy    = "${local.project}_s3_read_policy"

diff --git a/terraform/environments/digital-prison-reporting/main.tf b/terraform/environments/digital-prison-reporting/main.tf
@@ -64,6 +64,9 @@ module "glue_reporting_hub_job" {
     "--dpr.domain.catalog.db"                   = module.glue_data_domain_database.db_name
     "--dpr.redshift.secrets.name"               = "${local.project}-redshift-secret-${local.environment}"
     "--dpr.datamart.db.name"                    = "datamart"
+    "--spark.driver.memory"                     = local.reporting_hub_driver_mem
+    "--spark.executor.memory"                   = local.reporting_hub_executor_mem
+    "--dpr.log.level"                           = local.reporting_hub_log_level
   }
 }