Move read_jinja_template_at() to common cloudwatch module

cookbooks/aws-parallelcluster-environment/files/cloudwatch/cloudwatch_agent_common_utils.py

@@ -1,4 +1,6 @@
+import json
 import os
+import sys
 
 from jinja2 import FileSystemLoader
 from jinja2.sandbox import SandboxedEnvironment
@@ -15,3 +17,18 @@ def render_jinja_template(template_file_path):
     with open(template_file_path, "w", encoding="utf-8") as f:
         f.write(rendered_template)
     return template_file_path
+
+def read_jinja_template_at(path):
+    """Read the JSON file at path as a Jinja template."""
+    try:
+        with open(render_jinja_template(path), encoding="utf-8") as input_file:
+            return json.load(input_file)
+    except FileNotFoundError:
+        fail(f"No file exists at {path}")
+    except ValueError:
+        fail(f"File at {path} contains invalid JSON")
+    return None
+
+def fail(message):
+    """Exit nonzero with the given error message."""
+    sys.exit(message)

cookbooks/aws-parallelcluster-environment/files/cloudwatch/cloudwatch_agent_config_util.py

@@ -20,7 +20,7 @@
 import sys
 
 import jsonschema
-from cloudwatch_agent_common_utils import render_jinja_template
+from cloudwatch_agent_common_utils import fail, read_jinja_template_at
 
 DEFAULT_SCHEMA_PATH = os.path.realpath(os.path.join(os.path.curdir, "cloudwatch_agent_config_schema.json"))
 SCHEMA_PATH = os.environ.get("CW_LOGS_CONFIGS_SCHEMA_PATH", DEFAULT_SCHEMA_PATH)
@@ -29,11 +29,6 @@
 LOG_CONFIGS_BAK_PATH = f"{LOG_CONFIGS_PATH}.bak"
 
 
-def _fail(message):
-    """Exit nonzero with the given error message."""
-    sys.exit(message)
-
-
 def parse_args():
     """Parse command line args."""
     parser = argparse.ArgumentParser(
@@ -63,21 +58,9 @@ def _read_json_at(path):
         with open(path, encoding="utf-8") as input_file:
             return json.load(input_file)
     except FileNotFoundError:
-        _fail(f"No file exists at {path}")
-    except ValueError:
-        _fail(f"File at {path} contains invalid JSON")
-    return None
-
-
-def _read_jinja_template_at(path):
-    """Read the JSON file at path as a Jinja template."""
-    try:
-        with open(render_jinja_template(path), encoding="utf-8") as input_file:
-            return json.load(input_file)
-    except FileNotFoundError:
-        _fail(f"No file exists at {path}")
+        fail(f"No file exists at {path}")
     except ValueError:
-        _fail(f"File at {path} contains invalid JSON")
+        fail(f"File at {path} contains invalid JSON")
     return None
 
 
@@ -88,7 +71,7 @@ def _read_schema():
 
 def _read_log_configs():
     """Read the current version of the CloudWatch log configs file, cloudwatch_agent_config.json."""
-    return _read_jinja_template_at(LOG_CONFIGS_PATH)
+    return read_jinja_template_at(LOG_CONFIGS_PATH)
 
 
 def _validate_json_schema(input_json):
@@ -97,7 +80,7 @@ def _validate_json_schema(input_json):
     try:
         jsonschema.validate(input_json, schema)
     except jsonschema.exceptions.ValidationError as validation_err:
-        _fail(str(validation_err))
+        fail(str(validation_err))
 
 
 def _validate_timestamp_keys(input_json):
@@ -107,7 +90,7 @@ def _validate_timestamp_keys(input_json):
         valid_keys |= set(config.get("timestamp_formats").keys())
     for log_config in input_json.get("log_configs"):
         if log_config.get("timestamp_format_key") not in valid_keys:
-            _fail(
+            fail(
                 f"Log config with log_stream_name {log_config.get('log_stream_name')} and "
                 f"file_path {log_config.get('file_path'),} contains an invalid timestamp_format_key: "
                 f"{log_config.get('timestamp_format_key')}. Valid values are {', '.join(valid_keys),}"
@@ -126,7 +109,7 @@ def _validate_log_config_fields_uniqueness(input_json):
     for field in unique_fields:
         duplicates = _get_duplicate_values([config.get(field) for config in input_json.get("log_configs")])
         if duplicates:
-            _fail(f"The following {field} values are used multiple times: {', '.join(duplicates)}")
+            fail(f"The following {field} values are used multiple times: {', '.join(duplicates)}")
 
 
 def validate_json(input_json=None):

cookbooks/aws-parallelcluster-environment/files/cloudwatch/write_cloudwatch_agent_json.py

@@ -11,7 +11,7 @@
 import os
 import socket
 
-from cloudwatch_agent_common_utils import render_jinja_template
+from cloudwatch_agent_common_utils import render_jinja_template, read_jinja_template_at
 
 AWS_CLOUDWATCH_CFG_PATH = "/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json"
 DEFAULT_METRICS_COLLECTION_INTERVAL = 60
@@ -63,12 +63,6 @@ def add_instance_log_stream_prefixes(configs):
     return configs
 
 
-def read_data(config_path):
-    """Read in and render jinja template of the log configuration data from config_path."""
-    with open(render_jinja_template(config_path), encoding="utf-8") as infile:
-        return json.load(infile)
-
-
 def select_configs_for_scheduler(configs, scheduler):
     """Filter out from configs those entries whose 'schedulers' list does not contain scheduler."""
     return [config for config in configs if scheduler in config["schedulers"]]
@@ -223,7 +217,7 @@ def get_dict_value(value, attributes, default=None):
 def main():
     """Create cloudwatch agent config file."""
     args = parse_args()
-    config_data = read_data(args.config)
+    config_data = read_jinja_template_at(args.config)
     log_configs = select_logs(config_data["log_configs"], args)
     log_configs = add_timestamps(log_configs, config_data["timestamp_formats"])
     log_configs = add_log_group_name_params(args.log_group, log_configs)

cookbooks/aws-parallelcluster-environment/test/controls/cloudwatch_spec.rb

@@ -40,7 +40,7 @@
 
   describe file('/usr/local/bin/write_cloudwatch_agent_json.py') do
     it { should exist }
-    its('sha256sum') { should eq '54239b52d8cbae4b9da0efb3a73f06160e30b3580d9eba8e970fe2b81b973737' }
+    its('sha256sum') { should eq '2a7d52538d227300cf4cc787bcbb5190ee1b5981d804a0deceaf2684427d8e53' }
     its('owner') { should eq 'root' }
     its('group') { should eq 'root' }
     its('mode') { should cmp '0755' }
@@ -64,15 +64,15 @@
 
   describe file('/usr/local/bin/cloudwatch_agent_config_util.py') do
     it { should exist }
-    its('sha256sum') { should eq 'd50da5d7f6bdda90b41e18f713432c6eff8e03c6efe32f024b1c214cd9bb3eb8' }
+    its('sha256sum') { should eq '6e0332441117bc5a8ff5143c5067429864d9f4e653bcecdc7dea0ecd5f2fc26c' }
     its('owner') { should eq 'root' }
     its('group') { should eq 'root' }
     its('mode') { should cmp '0644' }
   end
 
   describe file('/usr/local/bin/cloudwatch_agent_common_utils.py') do
     it { should exist }
-    its('sha256sum') { should eq 'b65d53caf3d69f723324c4339f44cd8662a5c63ad8796118640738d7f6a63381' }
+    its('sha256sum') { should eq 'd032dc68645d506e3412acd54b04639868ca620d3f4a5a0a44b5d46be724dd07' }
     its('owner') { should eq 'root' }
     its('group') { should eq 'root' }
     its('mode') { should cmp '0755' }