From 36f6f9cceda32d7dc80bb68e76ac37cfe0a0ef63 Mon Sep 17 00:00:00 2001 From: Helena Greebe Date: Thu, 16 Nov 2023 08:52:33 -0500 Subject: [PATCH] Split the jinja rendering and json reading into two different functions --- .../cloudwatch_agent_common_utils.py | 19 ---------------- .../cloudwatch_agent_config_util.py | 22 +++++++++++++------ .../cloudwatch/write_cloudwatch_agent_json.py | 10 +++++++-- .../spec/unit/resources/cloudwatch_spec.rb | 11 ++++++++++ .../test/controls/cloudwatch_spec.rb | 6 ++--- 5 files changed, 37 insertions(+), 31 deletions(-) diff --git a/cookbooks/aws-parallelcluster-environment/files/cloudwatch/cloudwatch_agent_common_utils.py b/cookbooks/aws-parallelcluster-environment/files/cloudwatch/cloudwatch_agent_common_utils.py index 01fc7d01ef..be81ef142b 100644 --- a/cookbooks/aws-parallelcluster-environment/files/cloudwatch/cloudwatch_agent_common_utils.py +++ b/cookbooks/aws-parallelcluster-environment/files/cloudwatch/cloudwatch_agent_common_utils.py @@ -1,6 +1,4 @@ -import json import os -import sys from jinja2 import FileSystemLoader from jinja2.sandbox import SandboxedEnvironment @@ -17,20 +15,3 @@ def render_jinja_template(template_file_path): with open(template_file_path, "w", encoding="utf-8") as f: f.write(rendered_template) return template_file_path - - -def read_jinja_template_at(path): - """Read the JSON file at path as a Jinja template.""" - try: - with open(render_jinja_template(path), encoding="utf-8") as input_file: - return json.load(input_file) - except FileNotFoundError: - fail(f"No file exists at {path}") - except ValueError: - fail(f"File at {path} contains invalid JSON") - return None - - -def fail(message): - """Exit nonzero with the given error message.""" - sys.exit(message) diff --git a/cookbooks/aws-parallelcluster-environment/files/cloudwatch/cloudwatch_agent_config_util.py b/cookbooks/aws-parallelcluster-environment/files/cloudwatch/cloudwatch_agent_config_util.py index 61bbe0f23e..4598e64a0c 100644 --- a/cookbooks/aws-parallelcluster-environment/files/cloudwatch/cloudwatch_agent_config_util.py +++ b/cookbooks/aws-parallelcluster-environment/files/cloudwatch/cloudwatch_agent_config_util.py @@ -17,9 +17,10 @@ import json import os import shutil +import sys import jsonschema -from cloudwatch_agent_common_utils import fail, read_jinja_template_at +from cloudwatch_agent_common_utils import render_jinja_template DEFAULT_SCHEMA_PATH = os.path.realpath(os.path.join(os.path.curdir, "cloudwatch_agent_config_schema.json")) SCHEMA_PATH = os.environ.get("CW_LOGS_CONFIGS_SCHEMA_PATH", DEFAULT_SCHEMA_PATH) @@ -28,6 +29,11 @@ LOG_CONFIGS_BAK_PATH = f"{LOG_CONFIGS_PATH}.bak" +def _fail(message): + """Exit nonzero with the given error message.""" + sys.exit(message) + + def parse_args(): """Parse command line args.""" parser = argparse.ArgumentParser( @@ -57,9 +63,9 @@ def _read_json_at(path): with open(path, encoding="utf-8") as input_file: return json.load(input_file) except FileNotFoundError: - fail(f"No file exists at {path}") + _fail(f"No file exists at {path}") except ValueError: - fail(f"File at {path} contains invalid JSON") + _fail(f"File at {path} contains invalid JSON") return None @@ -70,7 +76,7 @@ def _read_schema(): def _read_log_configs(): """Read the current version of the CloudWatch log configs file, cloudwatch_agent_config.json.""" - return read_jinja_template_at(LOG_CONFIGS_PATH) + return _read_json_at(LOG_CONFIGS_PATH) def _validate_json_schema(input_json): @@ -79,7 +85,7 @@ def _validate_json_schema(input_json): try: jsonschema.validate(input_json, schema) except jsonschema.exceptions.ValidationError as validation_err: - fail(str(validation_err)) + _fail(str(validation_err)) def _validate_timestamp_keys(input_json): @@ -89,7 +95,7 @@ def _validate_timestamp_keys(input_json): valid_keys |= set(config.get("timestamp_formats").keys()) for log_config in input_json.get("log_configs"): if log_config.get("timestamp_format_key") not in valid_keys: - fail( + _fail( f"Log config with log_stream_name {log_config.get('log_stream_name')} and " f"file_path {log_config.get('file_path'),} contains an invalid timestamp_format_key: " f"{log_config.get('timestamp_format_key')}. Valid values are {', '.join(valid_keys),}" @@ -108,7 +114,7 @@ def _validate_log_config_fields_uniqueness(input_json): for field in unique_fields: duplicates = _get_duplicate_values([config.get(field) for config in input_json.get("log_configs")]) if duplicates: - fail(f"The following {field} values are used multiple times: {', '.join(duplicates)}") + _fail(f"The following {field} values are used multiple times: {', '.join(duplicates)}") def validate_json(input_json=None): @@ -166,6 +172,8 @@ def main(): input_json = get_input_json(args) validate_json(input_json) write_validated_json(input_json) + else: + render_jinja_template(LOG_CONFIGS_PATH) validate_json() except Exception: restore_backup() diff --git a/cookbooks/aws-parallelcluster-environment/files/cloudwatch/write_cloudwatch_agent_json.py b/cookbooks/aws-parallelcluster-environment/files/cloudwatch/write_cloudwatch_agent_json.py index 8f3a24fd34..caaaef089d 100644 --- a/cookbooks/aws-parallelcluster-environment/files/cloudwatch/write_cloudwatch_agent_json.py +++ b/cookbooks/aws-parallelcluster-environment/files/cloudwatch/write_cloudwatch_agent_json.py @@ -11,7 +11,7 @@ import os import socket -from cloudwatch_agent_common_utils import read_jinja_template_at +from cloudwatch_agent_common_utils import render_jinja_template AWS_CLOUDWATCH_CFG_PATH = "/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json" DEFAULT_METRICS_COLLECTION_INTERVAL = 60 @@ -63,6 +63,12 @@ def add_instance_log_stream_prefixes(configs): return configs +def read_data(config_path): + """Read in log configuration data from config_path.""" + with open(config_path, encoding="utf-8") as infile: + return json.load(infile) + + def select_configs_for_scheduler(configs, scheduler): """Filter out from configs those entries whose 'schedulers' list does not contain scheduler.""" return [config for config in configs if scheduler in config["schedulers"]] @@ -217,7 +223,7 @@ def get_dict_value(value, attributes, default=None): def main(): """Create cloudwatch agent config file.""" args = parse_args() - config_data = read_jinja_template_at(args.config) + config_data = read_data(render_jinja_template(args.config)) log_configs = select_logs(config_data["log_configs"], args) log_configs = add_timestamps(log_configs, config_data["timestamp_formats"]) log_configs = add_log_group_name_params(args.log_group, log_configs) diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/cloudwatch_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/cloudwatch_spec.rb index 737da9d14c..cf572eb671 100644 --- a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/cloudwatch_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/cloudwatch_spec.rb @@ -141,6 +141,7 @@ def self.configure(chef_run) cached(:config_schema_path) { '/usr/local/etc/cloudwatch_agent_config_schema.json' } cached(:config_data_path) { '/usr/local/etc/cloudwatch_agent_config.json' } cached(:validator_script_path) { '/usr/local/bin/cloudwatch_agent_config_util.py' } + cached(:common_module_path) { '/usr/local/bin/cloudwatch_agent_common_utils.py' } cached(:cookbook_venv_path) { 'cookbook/virtual/env/path' } cached(:log_group_name) { 'test_log_group_name' } cached(:scheduler) { 'test_scheduler' } @@ -176,6 +177,16 @@ def self.configure(chef_run) ) end + it 'creates cloudwatch_agent_common_utils.py' do + is_expected.to create_if_missing_cookbook_file('/usr/local/bin/cloudwatch_agent_common_utils.py').with( + source: 'cloudwatch/cloudwatch_agent_common_utils.py', + path: '/usr/local/bin/write_cloudwatch_agent_json.py', + user: 'root', + group: 'root', + mode: '0755' + ) + end + it 'creates cloudwatch_agent_config.json' do is_expected.to create_if_missing_cookbook_file('cloudwatch_agent_config.json').with( source: 'cloudwatch/cloudwatch_agent_config.json', diff --git a/cookbooks/aws-parallelcluster-environment/test/controls/cloudwatch_spec.rb b/cookbooks/aws-parallelcluster-environment/test/controls/cloudwatch_spec.rb index 6d56a24461..5afbf1bfc0 100644 --- a/cookbooks/aws-parallelcluster-environment/test/controls/cloudwatch_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/test/controls/cloudwatch_spec.rb @@ -40,7 +40,7 @@ describe file('/usr/local/bin/write_cloudwatch_agent_json.py') do it { should exist } - its('sha256sum') { should eq '5a2ec92e78c49b2064fea88564b0ffe8ccfda92798de1d8bf8a94158e8c1f366' } + its('sha256sum') { should eq 'f24283f824072456394ab01cd014d37a977fc63fe951c9dd4a7df982bf6e5a82' } its('owner') { should eq 'root' } its('group') { should eq 'root' } its('mode') { should cmp '0755' } @@ -64,7 +64,7 @@ describe file('/usr/local/bin/cloudwatch_agent_config_util.py') do it { should exist } - its('sha256sum') { should eq '68b2a5e96a87eeb4da90cbbb1a4d3796f410b2c2e2c69f084e6515149b6bd29b' } + its('sha256sum') { should eq '57a69fe5ceebeb98c067ab5975bf566e610659a4751810da18f441f583791c07' } its('owner') { should eq 'root' } its('group') { should eq 'root' } its('mode') { should cmp '0644' } @@ -72,7 +72,7 @@ describe file('/usr/local/bin/cloudwatch_agent_common_utils.py') do it { should exist } - its('sha256sum') { should eq '2cedb9bf8d430717afe638d989b3f1b97632dfeb89a730c812c4382bc94920ee' } + its('sha256sum') { should eq 'b65d53caf3d69f723324c4339f44cd8662a5c63ad8796118640738d7f6a63381' } its('owner') { should eq 'root' } its('group') { should eq 'root' } its('mode') { should cmp '0755' }