Add Spark Log Level Support

ministryofjustice · Aug 17, 2023 · 1d81226 · 1d81226
1 parent c9bd167
commit 1d81226
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 4 deletions.
diff --git a/terraform/environments/digital-prison-reporting/application_variables.json b/terraform/environments/digital-prison-reporting/application_variables.json
@@ -7,7 +7,8 @@
       "create_job": true,
       "glue_job_name": "glue-job-ingester",
       "reporting_hub_spark_driver_mem": "2g",
-      "reporting_hub_spark_executor_mem": "2g",      
+      "reporting_hub_spark_executor_mem": "2g",
+      "reporting_hub_spark_log_level": "INFO",   
       "create_security_conf": true,
       "setup_buckets": true,
       "create_kinesis_streams": true,
@@ -40,6 +41,7 @@
       "glue_job_name": "glue-job-ingester",
       "reporting_hub_spark_driver_mem": "2g",
       "reporting_hub_spark_executor_mem": "2g",
+      "reporting_hub_spark_log_level": "INFO",      
       "create_security_conf": true,
       "setup_buckets": true,
       "create_kinesis_streams": true,
@@ -71,7 +73,8 @@
       "create_job": true,
       "glue_job_name": "glue-job-ingester",
       "reporting_hub_spark_driver_mem": "2g",
-      "reporting_hub_spark_executor_mem": "2g",      
+      "reporting_hub_spark_executor_mem": "2g",
+      "reporting_hub_spark_log_level": "WARN",           
       "create_security_conf": true,
       "setup_buckets": true,
       "create_kinesis_streams": true,
@@ -103,7 +106,8 @@
       "create_job": true,
       "glue_job_name": "glue-job-ingester",
       "reporting_hub_spark_driver_mem": "26g",
-      "reporting_hub_spark_executor_mem": "26g",      
+      "reporting_hub_spark_executor_mem": "26g",
+      "reporting_hub_spark_log_level": "WARN",
       "create_security_conf": true,
       "setup_buckets": true,
       "create_kinesis_streams": true,

diff --git a/terraform/environments/digital-prison-reporting/locals.tf b/terraform/environments/digital-prison-reporting/locals.tf
@@ -42,6 +42,7 @@ locals {
   # Glue Job parameters
   reporting_hub_driver_mem    = local.application_data.accounts[local.environment].reporting_hub_spark_driver_mem
   reporting_hub_executor_mem  = local.application_data.accounts[local.environment].reporting_hub_spark_executor_mem
+  reporting_hub_log_level     = local.application_data.accounts[local.environment].reporting_hub_spark_log_level
 
   # Common Policies
   kms_read_access_policy   = "${local.project}_kms_read_policy"

diff --git a/terraform/environments/digital-prison-reporting/main.tf b/terraform/environments/digital-prison-reporting/main.tf
@@ -65,7 +65,8 @@ module "glue_reporting_hub_job" {
     "--dpr.redshift.secrets.name"               = "${local.project}-redshift-secret-${local.environment}"
     "--dpr.datamart.db.name"                    = "datamart"
     "--spark.driver.memory"                     = local.reporting_hub_driver_mem
-    "--spark.executor.memory"                   = local.reporting_hub_executor_mem 
+    "--spark.executor.memory"                   = local.reporting_hub_executor_mem
+    "--dpr.log.level"                           = local.reporting_hub_log_level
   }
 }