From 83cfea933d3c395f628ebca333ff454999bef098 Mon Sep 17 00:00:00 2001 From: Andrei Date: Fri, 25 Oct 2024 09:26:49 +0200 Subject: [PATCH 1/2] Use coreyaml loader --- README.md | 5 ++++ make_argocd_fly/resource.py | 29 +++++++++++-------- make_argocd_fly/utils.py | 4 +-- requirements.txt | 1 + tests/manual/config.yml | 5 ++++ .../app_5/configmap_prometheus-config.yml | 5 ++++ .../app_5/configmap_prometheus-config.yml | 5 ++++ tests/manual/source/app_5/configmap.yml.j2 | 5 ++++ 8 files changed, 45 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index f2e4daa..2f38fe8 100644 --- a/README.md +++ b/README.md @@ -303,3 +303,8 @@ python -m twine upload dist/* python -m cProfile -o profile.pstats main.py --root-dir gprof2dot log.pstats [-z ] | dot -Tsvg -o profile.svg ``` + +### Code coverage HTML report +``` +pytest --cov --cov-report=html:coverage_re tests/* +``` diff --git a/make_argocd_fly/resource.py b/make_argocd_fly/resource.py index 8856adb..f5d4595 100644 --- a/make_argocd_fly/resource.py +++ b/make_argocd_fly/resource.py @@ -5,29 +5,34 @@ import yaml import yaml.composer try: - from yaml import CSafeLoader as SafeLoader + from yamlcore import CCoreLoader as CoreLoader except ImportError: - from yaml import SafeLoader + from yamlcore import CoreLoader +from yamlcore import CoreDumper from make_argocd_fly.exceptions import MissingSourceResourcesError log = logging.getLogger(__name__) -class SafeDumper(yaml.SafeDumper): +class YamlDumper(CoreDumper): def increase_indent(self, flow=False, *args, **kwargs): return super().increase_indent(flow=flow, indentless=False) -def str_presenter(dumper, data): - """configures yaml for dumping multiline strings - Ref: https://stackoverflow.com/questions/8640959/how-can-i-control-what-scalar-form-pyyaml-uses-for-my-data""" - if data.count('\n') > 0: - return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') - return dumper.represent_scalar('tag:yaml.org,2002:str', data) +def represent_str(dumper, data): + """configures yaml for dumping multiline strings + Ref: https://stackoverflow.com/questions/8640959/how-can-i-control-what-scalar-form-pyyaml-uses-for-my-data""" + if data.count('\n') > 0: + return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') + return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='') -yaml.add_representer(str, str_presenter, Dumper=SafeDumper) +yaml.add_representer(str, represent_str, Dumper=YamlDumper) + + +class YamlLoader(CoreLoader): + pass class ResourceViewer: @@ -138,13 +143,13 @@ async def _write_resource(self, file_path: str, resource_yml: str) -> None: os.makedirs(path, exist_ok=True) try: - yaml_obj = yaml.load(resource_yml, Loader=SafeLoader) + yaml_obj = yaml.load(resource_yml, Loader=YamlLoader) except yaml.composer.ComposerError: log.error('Error parsing yaml to write as file {}. Yaml:\n{}'.format(file_path, resource_yml)) raise with open(os.path.join(self.output_dir_abs_path, file_path), 'w') as f: - yaml.dump(yaml_obj, f, Dumper=SafeDumper, + yaml.dump(yaml_obj, f, Dumper=YamlDumper, default_flow_style=False, sort_keys=False, allow_unicode=True, diff --git a/make_argocd_fly/utils.py b/make_argocd_fly/utils.py index 0025563..b1bcf59 100644 --- a/make_argocd_fly/utils.py +++ b/make_argocd_fly/utils.py @@ -187,8 +187,8 @@ def latest_version_check(): return if current_version < latest_version: - log.warning('You are running {} ({}) but there is a newer version of the package available ({})'.format(package_name, current_version, - latest_version)) + log.warning('You are running {} ({}) but there is a newer version of the package available ({}).'.format(package_name, current_version, + latest_version)) confirm_dialog() else: log.info('You are using the latest version of {} ({})'.format(package_name, current_version)) diff --git a/requirements.txt b/requirements.txt index 15e4f12..fe4af74 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ PyYAML==6.0.1 +yamlcore==0.0.4 Jinja2==3.1.3 jinja2-ansible-filters==1.3.2 yamllint==1.35.1 diff --git a/tests/manual/config.yml b/tests/manual/config.yml index 2bdc3dc..cdb3f24 100644 --- a/tests/manual/config.yml +++ b/tests/manual/config.yml @@ -101,6 +101,11 @@ vars: double_reference_version: ${reference_version} reference_version: ${app[version]} resource: Deployment_thanos.yml + number: 01239 + number_oct: 0123 + number_string: "01239" + number_string_2: "123" + number_oct_string: "0123" app: resource: ${resource} version: 0.1.0 diff --git a/tests/manual/output/external_1/app_5/configmap_prometheus-config.yml b/tests/manual/output/external_1/app_5/configmap_prometheus-config.yml index ee2f699..b787d8e 100644 --- a/tests/manual/output/external_1/app_5/configmap_prometheus-config.yml +++ b/tests/manual/output/external_1/app_5/configmap_prometheus-config.yml @@ -9,3 +9,8 @@ data: "test": "json" } ip: 127.0.0.1 + number: 1239 + number_oct: 83 + number_string: '01239' + number_string_2: '123' + number_oct_string: '0123' diff --git a/tests/manual/output/management/app_5/configmap_prometheus-config.yml b/tests/manual/output/management/app_5/configmap_prometheus-config.yml index f51f6f3..dd600b1 100644 --- a/tests/manual/output/management/app_5/configmap_prometheus-config.yml +++ b/tests/manual/output/management/app_5/configmap_prometheus-config.yml @@ -9,3 +9,8 @@ data: "test": "json_management" } ip: 127.0.0.1 + number: 1239 + number_oct: 83 + number_string: '01239' + number_string_2: '123' + number_oct_string: '0123' diff --git a/tests/manual/source/app_5/configmap.yml.j2 b/tests/manual/source/app_5/configmap.yml.j2 index 57ff3de..d9bc4be 100644 --- a/tests/manual/source/app_5/configmap.yml.j2 +++ b/tests/manual/source/app_5/configmap.yml.j2 @@ -10,3 +10,8 @@ data: {% endfilter %} ip: {{ 'localhost' | dig }} + number: {{ number }} + number_oct: {{ number_oct }} + number_string: "{{ number_string }}" + number_string_2: "{{ number_string_2 }}" + number_oct_string: "{{ number_oct_string }}" From 38d1024b39e1596384e20f39d216cc6f786b1a43 Mon Sep 17 00:00:00 2001 From: Andrei Date: Fri, 25 Oct 2024 14:31:49 +0200 Subject: [PATCH 2/2] misc: detect and quote numbers starting with zepo --- make_argocd_fly/resource.py | 17 ++++++++++++----- requirements.txt | 1 - tests/manual/config.yml | 2 +- .../app_5/configmap_prometheus-config.yml | 3 ++- .../app_5/configmap_prometheus-config.yml | 3 ++- tests/manual/source/app_5/configmap.yml.j2 | 1 + 6 files changed, 18 insertions(+), 9 deletions(-) diff --git a/make_argocd_fly/resource.py b/make_argocd_fly/resource.py index f5d4595..eda720e 100644 --- a/make_argocd_fly/resource.py +++ b/make_argocd_fly/resource.py @@ -5,17 +5,17 @@ import yaml import yaml.composer try: - from yamlcore import CCoreLoader as CoreLoader + from yaml import CSafeLoader as SafeLoader except ImportError: - from yamlcore import CoreLoader -from yamlcore import CoreDumper + from yaml import SafeLoader +from yaml import SafeDumper from make_argocd_fly.exceptions import MissingSourceResourcesError log = logging.getLogger(__name__) -class YamlDumper(CoreDumper): +class YamlDumper(SafeDumper): def increase_indent(self, flow=False, *args, **kwargs): return super().increase_indent(flow=flow, indentless=False) @@ -25,13 +25,20 @@ def represent_str(dumper, data): Ref: https://stackoverflow.com/questions/8640959/how-can-i-control-what-scalar-form-pyyaml-uses-for-my-data""" if data.count('\n') > 0: return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') + if data.startswith('0'): + try: + int(data[1:]) + return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='\'') + except (SyntaxError, ValueError): + pass + return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='') yaml.add_representer(str, represent_str, Dumper=YamlDumper) -class YamlLoader(CoreLoader): +class YamlLoader(SafeLoader): pass diff --git a/requirements.txt b/requirements.txt index fe4af74..15e4f12 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ PyYAML==6.0.1 -yamlcore==0.0.4 Jinja2==3.1.3 jinja2-ansible-filters==1.3.2 yamllint==1.35.1 diff --git a/tests/manual/config.yml b/tests/manual/config.yml index cdb3f24..d31e171 100644 --- a/tests/manual/config.yml +++ b/tests/manual/config.yml @@ -101,7 +101,7 @@ vars: double_reference_version: ${reference_version} reference_version: ${app[version]} resource: Deployment_thanos.yml - number: 01239 + number: 01239 # numbers cannot start with 0, it will be quoted number_oct: 0123 number_string: "01239" number_string_2: "123" diff --git a/tests/manual/output/external_1/app_5/configmap_prometheus-config.yml b/tests/manual/output/external_1/app_5/configmap_prometheus-config.yml index b787d8e..d33082a 100644 --- a/tests/manual/output/external_1/app_5/configmap_prometheus-config.yml +++ b/tests/manual/output/external_1/app_5/configmap_prometheus-config.yml @@ -9,8 +9,9 @@ data: "test": "json" } ip: 127.0.0.1 - number: 1239 + number: '01239' number_oct: 83 number_string: '01239' number_string_2: '123' number_oct_string: '0123' + boolean_string: 'yes' diff --git a/tests/manual/output/management/app_5/configmap_prometheus-config.yml b/tests/manual/output/management/app_5/configmap_prometheus-config.yml index dd600b1..1f14c7a 100644 --- a/tests/manual/output/management/app_5/configmap_prometheus-config.yml +++ b/tests/manual/output/management/app_5/configmap_prometheus-config.yml @@ -9,8 +9,9 @@ data: "test": "json_management" } ip: 127.0.0.1 - number: 1239 + number: '01239' number_oct: 83 number_string: '01239' number_string_2: '123' number_oct_string: '0123' + boolean_string: 'yes' diff --git a/tests/manual/source/app_5/configmap.yml.j2 b/tests/manual/source/app_5/configmap.yml.j2 index d9bc4be..2d89433 100644 --- a/tests/manual/source/app_5/configmap.yml.j2 +++ b/tests/manual/source/app_5/configmap.yml.j2 @@ -15,3 +15,4 @@ data: number_string: "{{ number_string }}" number_string_2: "{{ number_string_2 }}" number_oct_string: "{{ number_oct_string }}" + boolean_string: "yes"