Support for original output coloring (#3220)

* Add click package

* Support for original output coloring

* Add click package to CI pipeline

* Fix CI task

* Use human friendly color codes

* Fix naming style

* Do not detect log level for colored loggers

* Apply --no-color option for epicli output formatter

* Highlight info on Ansible commands

* Fix UncolorJsonFormatter

* Update changelog

* Update pylint_score_cli_threshold

* Better formatting

* Add support for NO_COLOR env var

* Use 'python3 -m pip' instead of pip

* Ensure click

* Restore higher threshold

* Use python3 -m pylint

* Fix pylint_score_cli_threshold

.devcontainer/pyproject.toml

@@ -14,6 +14,7 @@ python-json-logger = "*"
 "ruamel.yaml" = "*"
 ansible = "5.2.0"
 azure-cli = "2.32.0"
+click = "*"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

.devcontainer/requirements.txt

@@ -104,6 +104,7 @@ certifi==2022.5.18.1; python_full_version >= "3.6.0" and python_version < "4" an
 cffi==1.15.0; python_version >= "3.8" and python_full_version >= "3.6.0" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6")
 chardet==3.0.4; python_full_version >= "3.6.0"
 charset-normalizer==2.0.12; python_version >= "3.6" and python_full_version >= "3.6.0"
+click==8.1.3; python_version >= "3.7"
 colorama==0.4.4; python_full_version >= "3.6.0"
 cryptography==37.0.2
 deprecated==1.2.13; python_version >= "3.6" and python_full_version >= "3.6.0"

ci/pipelines/build.yaml

@@ -31,7 +31,8 @@ jobs:
         inputs:
           targetType: inline
           script: |
-            pip install boto3 jinja2 jsonschema pytest pytest_mock python-json-logger pyyaml ruamel.yaml setuptools twine wheel
+            python3 -m pip install boto3 click jinja2 jsonschema pytest pytest_mock python-json-logger pyyaml \
+                                   ruamel.yaml setuptools twine wheel
 
       - task: Bash@3
         displayName: Run unit tests

ci/pipelines/linters.yaml

@@ -14,7 +14,7 @@ pool:
 
 variables:
   ansible_lint_error_threshold: 338
-  pylint_score_cli_threshold: 9.44
+  pylint_score_cli_threshold: 9.50
   pylint_score_tests_threshold: 9.78
   rubocop_linter_threshold: 183
 
@@ -34,7 +34,7 @@ jobs:
         inputs:
           targetType: inline
           script: |
-            pip install --upgrade ansible==5.2.0 ansible-lint ansible-lint-junit==0.16 lxml pip setuptools
+            python3 -m pip install --upgrade ansible==5.2.0 ansible-lint ansible-lint-junit==0.16 lxml pip setuptools
 
       - task: Bash@3
         displayName: Run Ansible Lint
@@ -67,14 +67,16 @@ jobs:
         inputs:
           targetType: inline
           script: |
-            pip install --upgrade pylint pylint-fail-under pylint-junit
+            # epicli deps: click
+            python3 -m pip install --upgrade pylint pylint-fail-under pylint-junit \
+                                             click
 
       - task: Bash@3
         displayName: Run Pylint on CLI code
         inputs:
           targetType: inline
           script: |
-            pylint ./cli \
+            python3 -m pylint ./cli \
             --rcfile .pylintrc \
             --fail-under=$(pylint_score_cli_threshold) \
             --output cli_code_results.xml
@@ -91,7 +93,7 @@ jobs:
         inputs:
           targetType: inline
           script: |
-            pylint ./tests \
+            python3 -m pylint ./tests \
             --rcfile .pylintrc \
             --fail-under=$(pylint_score_tests_threshold) \
             --output test_code_results.xml \

cli/epicli.py

@@ -39,30 +39,32 @@ def main():
         formatter_class=argparse.RawTextHelpFormatter)
 
     # setup some root arguments
-    parser.add_argument('--version', action='version', help='Shows the CLI version', version=VERSION)
-    parser.add_argument('--licenses', action='version',
-                        help='Shows the third party packages and their licenses the CLI is using.',
-                        version=json.dumps(LICENSES, indent=4))
+    parser.add_argument('--auto-approve', dest='auto_approve', action="store_true",
+                        help='Auto approve any user input queries asked by epicli.')
+    parser.add_argument('--licenses', action='version', version=json.dumps(LICENSES, indent=4),
+                        help='Shows the third party packages and their licenses the CLI is using.')
+    parser.add_argument('--log-count', dest='log_count', type=str,
+                        help='Roleover count where each CLI run will generate a new log.')
+    parser.add_argument('--log-date-format', dest='log_date_format', type=str,
+                        help='''Format for the logging date/time. Uses the default Python strftime formatting,
+more information here: https://docs.python.org/3.7/library/time.html#time.strftime''')
     parser.add_argument('-l', '--log-file', dest='log_name', type=str,
-                        help='The name of the log file written to the output directory')
+                        help='The name of the log file written to the output directory.')
     parser.add_argument('--log-format', dest='log_format', type=str,
                         help='''Format for the logging string. Uses the default Python log formatting,
 more information here: https://docs.python.org/3.7/library/logging.html''')
-    parser.add_argument('--log-date-format', dest='log_date_format', type=str,
-                        help='''Format for the logging date/time. Uses the default Python strftime formatting,
-more information here: https://docs.python.org/3.7/library/time.html#time.strftime''')
-    parser.add_argument('--log-count', dest='log_count', type=str,
-                        help='Roleover count where each CLI run will generate a new log.')
-    parser.add_argument('--log-type', choices=['plain', 'json'], default='plain',
-                        dest='log_type', action='store', help='''Type of logs that will be written to the output file.
+    parser.add_argument('--log-type', choices=['plain', 'json'], default='plain', dest='log_type', action='store',
+                        help='''Type of logs that will be written to the output file.
 Currently supported formats are plain text or JSON''')
+    parser.add_argument('--no-color', dest='no_color', action="store_true",
+                        help='Disables output coloring.')
     parser.add_argument('--validate-certs', choices=['true', 'false'], default='true', action='store',
                         dest='validate_certs',
                         help='''[Experimental]: Disables certificate checks for certain Ansible operations
 which might have issues behind proxies (https://github.com/ansible/ansible/issues/32750).
 Should NOT be used in production for security reasons.''')
-    parser.add_argument('--auto-approve', dest='auto_approve', action="store_true",
-                        help='Auto approve any user input queries asked by Epicli')
+    parser.add_argument('--version', action='version', version=VERSION,
+                        help='Shows the CLI version.')
 
     # set debug verbosity level.
     def debug_level(x):
@@ -122,6 +124,7 @@ def debug_level(x):
         config.upgrade_components = args.upgrade_components
     config.debug = args.debug
     config.auto_approve = args.auto_approve
+    config.no_color = args.no_color or os.getenv('NO_COLOR', '') != ''
 
     try:
         return args.func(args)

cli/licenses.py

@@ -1101,7 +1101,7 @@
         "Author": "Kenneth Reitz",
         "License": "Other",
         "License URL": "https://api.github.com/repos/certifi/python-certifi/license",
-        "License repo": "This package contains a modified version of ca-bundle.crt:\n\nca-bundle.crt -- Bundle of CA Root Certificates\n\nCertificate data from Mozilla as of: Thu Nov  3 19:04:19 2011#\nThis is a bundle of X.509 certificates of public Certificate Authorities\n(CA). These were automatically extracted from Mozilla's root certificates\nfile (certdata.txt).  This file can be found in the mozilla source tree:\nhttp://mxr.mozilla.org/mozilla/source/security/nss/lib/ckfw/builtins/certdata.txt?raw=1#\nIt contains the certificates in PEM format and therefore\ncan be directly used with curl / libcurl / php_curl, or with\nan Apache+mod_ssl webserver for SSL client authentication.\nJust configure this file as the SSLCACertificateFile.#\n\n***** BEGIN LICENSE BLOCK *****\nThis Source Code Form is subject to the terms of the Mozilla Public License,\nv. 2.0. If a copy of the MPL was not distributed with this file, You can obtain\none at http://mozilla.org/MPL/2.0/.\n\n***** END LICENSE BLOCK *****\n@(#) $RCSfile: certdata.txt,v $ $Revision: 1.80 $ $Date: 2011/11/03 15:11:58 $\n"
+        "License repo": "This package contains a modified version of ca-bundle.crt:\n\nca-bundle.crt -- Bundle of CA Root Certificates\n\nCertificate data from Mozilla as of: Thu Nov  3 19:04:19 2011#\nThis is a bundle of X.509 certificates of public Certificate Authorities\n(CA). These were automatically extracted from Mozilla's root certificates\nfile (certdata.txt).  This file can be found in the mozilla source tree:\nhttps://hg.mozilla.org/mozilla-central/file/tip/security/nss/lib/ckfw/builtins/certdata.txt\nIt contains the certificates in PEM format and therefore\ncan be directly used with curl / libcurl / php_curl, or with\nan Apache+mod_ssl webserver for SSL client authentication.\nJust configure this file as the SSLCACertificateFile.#\n\n***** BEGIN LICENSE BLOCK *****\nThis Source Code Form is subject to the terms of the Mozilla Public License,\nv. 2.0. If a copy of the MPL was not distributed with this file, You can obtain\none at http://mozilla.org/MPL/2.0/.\n\n***** END LICENSE BLOCK *****\n@(#) $RCSfile: certdata.txt,v $ $Revision: 1.80 $ $Date: 2011/11/03 15:11:58 $\n"
     },
     {
         "Name": "cffi",
@@ -1133,6 +1133,14 @@
         "License repo": "MIT License\n\nCopyright (c) 2019 TAHRI Ahmed R.\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.",
         "License text": "MIT License\n\nCopyright (c) [year] [fullname]\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"
     },
+    {
+        "Name": "click",
+        "Version": "8.1.3",
+        "Summary": "Composable command line interface toolkit",
+        "Home-page": "https://palletsprojects.com/p/click/",
+        "Author": "Armin Ronacher",
+        "License": "BSD-3-Clause"
+    },
     {
         "Name": "colorama",
         "Version": "0.4.4",

cli/src/Config.py

@@ -42,6 +42,7 @@ def __init__(self):
             self._wait_for_pods = False
             self._upgrade_components = []
             self._vault_password_location = os.path.join(expanduser("~"), '.epicli/vault.cfg')
+            self._no_color: bool = False
 
         @property
         def full_download(self) -> bool:
@@ -188,6 +189,14 @@ def upgrade_components(self):
         def upgrade_components(self, upgrade_components):
             self._upgrade_components = upgrade_components
 
+        @property
+        def no_color(self) -> bool:
+            return self._no_color
+
+        @no_color.setter
+        def no_color(self, no_color: bool):
+            self._no_color = no_color
+
     instance = None
 
     def __new__(cls):

cli/src/Log.py

@@ -3,25 +3,20 @@
 import os
 import threading
 
+import click
 from pythonjsonlogger import jsonlogger
 
 from cli.src.Config import Config
 from cli.src.helpers.build_io import get_output_path
 
 
 class ColorFormatter(logging.Formatter):
-    grey = '\x1b[38;21m'
-    yellow = '\x1b[33;21m'
-    red = '\x1b[31;21m'
-    bold_red = '\x1b[31;1m'
-    reset = '\x1b[0m'
-
     FORMATS = {
-        logging.DEBUG: grey + 'format' + reset,
-        logging.INFO: grey + 'format' + reset,
-        logging.WARNING: yellow + 'format' + reset,
-        logging.ERROR: red + 'format' + reset,
-        logging.CRITICAL: bold_red + 'format' + reset
+        logging.DEBUG:    click.style('format', fg='bright_black'), # grey
+        logging.INFO:     click.style('format'),
+        logging.WARNING:  click.style('format', fg='yellow'),
+        logging.ERROR:    click.style('format', fg='red'),
+        logging.CRITICAL: click.style('format', fg='red', bold=True)
     }
 
     def format(self, record):
@@ -32,6 +27,24 @@ def format(self, record):
         return formatter.format(record)
 
 
+class UncolorFormatter(logging.Formatter):
+    """
+    Formatter that removes ANSI styling information (escape sequences).
+    """
+    def format(self, record: logging.LogRecord) -> str:
+        return click.unstyle(super().format(record))
+
+
+class UncolorJsonFormatter(jsonlogger.JsonFormatter):
+    """
+    JSON formatter that removes ANSI styling information (escape sequences).
+    """
+    def format(self, record: logging.LogRecord) -> str:
+        if isinstance(record.msg, str):
+            record.msg = click.unstyle(record.msg)
+        return super().format(record)
+
+
 class Log:
     class __LogBase:
         stream_handler = None
@@ -42,8 +55,9 @@ def __init__(self):
 
             # create stream handler with color formatter
             self.stream_handler = logging.StreamHandler()
-            color_formatter = ColorFormatter()
-            self.stream_handler.setFormatter(color_formatter)
+            formatter = logging.Formatter(config.log_format,
+                                          datefmt=config.log_date_format) if config.no_color else ColorFormatter()
+            self.stream_handler.setFormatter(formatter)
 
             # create file handler
             log_path = os.path.join(get_output_path(), config.log_file)
@@ -55,10 +69,10 @@ def __init__(self):
 
             # attach propper formatter to file_handler (plain|json)
             if config.log_type == 'plain':
-                file_formatter = logging.Formatter(config.log_format, datefmt=config.log_date_format)
+                file_formatter = UncolorFormatter(config.log_format, datefmt=config.log_date_format)
                 self.file_handler.setFormatter(file_formatter)
             elif config.log_type == 'json':
-                json_formatter = jsonlogger.JsonFormatter(config.log_format, datefmt=config.log_date_format)
+                json_formatter = UncolorJsonFormatter(config.log_format, datefmt=config.log_date_format)
                 self.file_handler.setFormatter(json_formatter)
 
 
@@ -80,27 +94,38 @@ def __init__(self, logger_name):
         threading.Thread.__init__(self)
         self.logger = Log(logger_name)
         self.daemon = False
-        self.fdRead, self.fdWrite = os.pipe()
-        self.pipeReader = os.fdopen(self.fdRead)
+        self.fd_read, self.fd_write = os.pipe()
+        self.pipe_reader = os.fdopen(self.fd_read)
         self.start()
-        self.errorStrings = ['error', 'Error', 'ERROR', 'fatal', 'FAILED']
-        self.warningStrings = ['warning', 'warning', 'WARNING']
-        self.stderrstrings = []
+        self.error_strings = ['error', 'Error', 'ERROR', 'fatal', 'FAILED']
+        self.warning_strings = ['warning', 'warning', 'WARNING']
+        self.output_error_lines = []
 
     def fileno(self):
-        return self.fdWrite
+        return self.fd_write
 
     def run(self):
-        for line in iter(self.pipeReader.readline, ''):
+        """Run thread logging everything."""
+        colored_loggers = ['AnsibleCommand', 'SpecCommand', 'TerraformCommand']
+        logger_short_name = self.logger.name.split('.')[-1]
+        with_error_detection = logger_short_name in ['TerraformCommand']
+        with_level_detection = logger_short_name not in colored_loggers
+
+        for line in iter(self.pipe_reader.readline, ''):
             line = line.strip('\n')
-            if any([substring in line for substring in self.errorStrings]):
-                self.stderrstrings.append(line)
-                self.logger.error(line)
-            elif any([substring in line for substring in self.warningStrings]):
+            if with_error_detection and any(string in line for string in self.error_strings):
+                self.output_error_lines.append(line)
+            if with_level_detection:
+                if any(string in line for string in self.error_strings):
+                    self.logger.error(line)
+                elif any(string in line for string in self.warning_strings):
                     self.logger.warning(line)
+                else:
+                    self.logger.info(line)
             else:
                 self.logger.info(line)
-        self.pipeReader.close()
+
+        self.pipe_reader.close()
 
     def close(self):
-        os.close(self.fdWrite)
+        os.close(self.fd_write)