Merge branch 'develop' into feature/change-vault-docs

CHANGELOG-0.7.md

@@ -1,16 +1,23 @@
 # Changelog 0.7
 
-## [0.7.1] 2020-07-XX
+## [0.7.1] 2020-07-xx
+
+### Added
+
+- Minor logging improvements added while fixing issue [#1424](https://github.com/epiphany-platform/epiphany/issues/1424).
+- [#1438](https://github.com/epiphany-platform/epiphany/pull/1438) - Rename Terraform plugin vendor in VSCode recommendations
 
 ### Fixed
 
+- [#1424](https://github.com/epiphany-platform/epiphany/issues/1424) - Terraform returning an error during deployments on Azure ("A retryable error occurred.")
 - [#1399](https://github.com/epiphany-platform/epiphany/issues/1399) - Epicli upgrade: Kubernetes upgrade may hang
 - [#1398](https://github.com/epiphany-platform/epiphany/issues/1398) - Vault installation fails when using canal/calico network plugin
 - [#1412](https://github.com/epiphany-platform/epiphany/issues/1412) - Certificate in Vault is also generated or copied even if flag in configuration tls_disable is set to true
 
 ### Added
 
 - [#1413](https://github.com/epiphany-platform/epiphany/issues/1413) - Set protocol for Vault only in one place in configuration
+- [#1423](https://github.com/epiphany-platform/epiphany/issues/1423) - Error reading generated service principal
 
 ## [0.7.0] 2020-06-30
 

core/src/epicli/.devcontainer/Pipfile

@@ -11,7 +11,7 @@ jsonschema = "*"
 python-json-logger = "*"
 ansible = "==2.8.8"
 terraform-bin = "*"
-azure-cli = "==2.6.0"
+azure-cli = "==2.8.0"
 skopeo-bin = "*"
 "ruamel.yaml" = "*"
 

core/src/epicli/.devcontainer/requirements.txt

@@ -1,18 +1,18 @@
-adal==1.2.3
+adal==1.2.4
 ansible==2.8.8
 antlr4-python3-runtime==4.7.2
 applicationinsights==0.11.9
 argcomplete==1.11.1
 attrs==19.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 azure-batch==9.0.0
 azure-cli-command-modules-nspkg==2.0.3
-azure-cli-core==2.6.0
+azure-cli-core==2.8.0
 azure-cli-nspkg==3.0.4
 azure-cli-telemetry==1.0.4
-azure-cli==2.6.0
+azure-cli==2.8.0
 azure-common==1.1.25
-azure-core==1.5.0
-azure-cosmos==3.1.2
+azure-core==1.7.0
+azure-cosmos==3.2.0
 azure-datalake-store==0.0.48
 azure-functions-devops-build==0.0.22
 azure-graphrbac==0.60.0
@@ -23,13 +23,13 @@ azure-mgmt-apimanagement==0.1.0
 azure-mgmt-appconfiguration==0.4.0
 azure-mgmt-applicationinsights==0.1.1
 azure-mgmt-authorization==0.52.0
-azure-mgmt-batch==7.0.0
+azure-mgmt-batch==9.0.0
 azure-mgmt-batchai==2.0.0
 azure-mgmt-billing==0.2.0
 azure-mgmt-botservice==0.2.0
 azure-mgmt-cdn==4.1.0rc1
-azure-mgmt-cognitiveservices==5.0.0
-azure-mgmt-compute==12.0.0
+azure-mgmt-cognitiveservices==6.2.0
+azure-mgmt-compute==12.1.0
 azure-mgmt-consumption==2.0.0
 azure-mgmt-containerinstance==1.5.0
 azure-mgmt-containerregistry==3.0.0rc13
@@ -44,24 +44,24 @@ azure-mgmt-deploymentmanager==0.2.0
 azure-mgmt-devtestlabs==4.0.0
 azure-mgmt-dns==2.1.0
 azure-mgmt-eventgrid==2.2.0
-azure-mgmt-eventhub==3.0.0
+azure-mgmt-eventhub==4.0.0
 azure-mgmt-hdinsight==1.4.0
-azure-mgmt-imagebuilder==0.2.1
+azure-mgmt-imagebuilder==0.4.0
 azure-mgmt-iotcentral==3.0.0
 azure-mgmt-iothub==0.12.0
 azure-mgmt-iothubprovisioningservices==0.2.0
 azure-mgmt-keyvault==2.2.0
 azure-mgmt-kusto==0.3.0
-azure-mgmt-loganalytics==0.5.0
+azure-mgmt-loganalytics==0.6.0
 azure-mgmt-managedservices==1.0.0
 azure-mgmt-managementgroups==0.2.0
 azure-mgmt-maps==0.1.0
 azure-mgmt-marketplaceordering==0.2.1
-azure-mgmt-media==2.1.0
+azure-mgmt-media==2.2.0
 azure-mgmt-monitor==0.9.0
 azure-mgmt-msi==0.2.0
 azure-mgmt-netapp==0.8.0
-azure-mgmt-network==10.1.0
+azure-mgmt-network==10.2.0
 azure-mgmt-nspkg==3.0.2
 azure-mgmt-policyinsights==0.4.0
 azure-mgmt-privatedns==0.1.0
@@ -72,39 +72,39 @@ azure-mgmt-redhatopenshift==0.1.0
 azure-mgmt-redis==7.0.0rc1
 azure-mgmt-relay==0.1.0
 azure-mgmt-reservations==0.6.0
-azure-mgmt-resource==9.0.0
+azure-mgmt-resource==10.0.0
 azure-mgmt-search==2.1.0
-azure-mgmt-security==0.1.0
+azure-mgmt-security==0.4.1
 azure-mgmt-servicebus==0.6.0
 azure-mgmt-servicefabric==0.4.0
 azure-mgmt-signalr==0.3.0
 azure-mgmt-sql==0.18.0
 azure-mgmt-sqlvirtualmachine==0.5.0
-azure-mgmt-storage==9.0.0
+azure-mgmt-storage==11.0.0
 azure-mgmt-trafficmanager==0.51.0
 azure-mgmt-web==0.46.0
-azure-multiapi-storage==0.3.2
+azure-multiapi-storage==0.3.5
 azure-nspkg==3.0.2
 azure-storage-blob==1.5.0
 azure-storage-common==1.4.2
 bcrypt==3.1.7; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-boto3==1.13.19
-botocore==1.16.19
-certifi==2020.4.5.1
+boto3==1.14.19
+botocore==1.17.19
+certifi==2020.6.20
 cffi==1.14.0
 chardet==3.0.4
 colorama==0.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 cryptography==2.9.2
-docutils==0.15.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
+docutils==0.15.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
 fabric==2.5.0
 humanfriendly==8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-idna==2.9
-importlib-metadata==1.6.0; python_version == '3.7' and python_version < '3.8'
+idna==2.10
+importlib-metadata==1.7.0; python_version == '3.7' and python_version < '3.8'
 invoke==1.4.1
 isodate==0.6.0
 javaproperties==0.5.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'
 jinja2==2.11.2
-jmespath==0.10.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
+jmespath==0.10.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
 jsmin==2.2.2
 jsondiff==1.2.0
 jsonschema==3.2.0
@@ -113,8 +113,8 @@ markupsafe==1.1.1; python_version >= '2.7' and python_version not in '3.0, 3.1,
 mock==4.0.2; python_version >= '3.6'
 msal-extensions==0.1.3
 msal==1.0.0
-msrest==0.6.14
-msrestazure==0.6.3
+msrest==0.6.17
+msrestazure==0.6.4
 oauthlib==3.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 paramiko==2.7.1
 pkginfo==1.5.0.1
@@ -125,17 +125,17 @@ pyjwt[crypto]==1.7.1
 pynacl==1.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 pyopenssl==19.1.0
 pyrsistent==0.16.0
-python-dateutil==2.8.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
+python-dateutil==2.8.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 python-json-logger==0.1.11
 pytz==2019.1
 pyyaml==5.3.1
 requests-oauthlib==1.3.0
-requests==2.23.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-ruamel.yaml.clib==0.2.0; platform_python_implementation == 'CPython' and python_version < '3.9'
+requests==2.24.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+ruamel.yaml.clib==0.2.0; python_version < '3.9' and platform_python_implementation == 'CPython'
 ruamel.yaml==0.16.10
 s3transfer==0.3.3
 scp==0.13.2
-six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
+six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 skopeo-bin==1.0.3
 sshtunnel==0.1.5
 tabulate==0.8.7

core/src/epicli/cli/engine/ansible/AnsibleCommand.py

@@ -50,7 +50,7 @@ def run_task_with_retries(self, inventory, module, hosts, retries, timeout=10, a
                 break
             except Exception as e:
                 self.logger.error(e)
-                self.logger.info('Retry running task: ' + str(i + 1) + '/' + str(retries))
+                self.logger.warning('Retry running task: ' + str(i + 1) + '/' + str(retries))
                 time.sleep(timeout)
         else:
             raise Exception(f'Failed running task after {str(retries)} retries')
@@ -88,7 +88,7 @@ def run_playbook_with_retries(self, inventory, playbook_path, retries, timeout=1
                 break
             except Exception as e:
                 self.logger.error(e)
-                self.logger.info('Retry running playbook: ' + str(i + 1) + '/' + str(retries))
+                self.logger.warning('Retry running playbook: ' + str(i + 1) + '/' + str(retries))
                 time.sleep(timeout)
         else:
             raise Exception(f'Failed running playbook after {str(retries)} retries')

core/src/epicli/cli/engine/providers/azure/APIProxy.py

@@ -34,7 +34,7 @@ def login_sp(self, sp_data):
         name = sp_data['name']
         password = sp_data['password']
         tenant = sp_data['tenant']
-        return self.run(self, f'az login --service-principal -u \'{name}\' -p "{password}" --tenant \'{tenant}\'', False)      
+        return self.run(self, f'az login --service-principal -u \'{name}\' -p \'{password}\' --tenant \'{tenant}\'', False)      
 
     def set_active_subscribtion(self, subscription_id):
         self.run(self, f'az account set --subscription {subscription_id}')  

core/src/epicli/cli/engine/terraform/TerraformCommand.py

@@ -16,7 +16,7 @@ def __init__(self, working_directory=os.path.dirname(__file__)):
         self.working_directory = working_directory
 
     def apply(self, auto_approve=False, env=os.environ.copy()):
-        self.run(self, self.APPLY_COMMAND, auto_approve=auto_approve, env=env)
+        self.run(self, self.APPLY_COMMAND, auto_approve=auto_approve, env=env, auto_retries=3)
 
     def destroy(self, auto_approve=False, env=os.environ.copy()):
         self.run(self, self.DESTROY_COMMAND, auto_approve=auto_approve, env=env)
@@ -28,7 +28,7 @@ def init(self, env=os.environ.copy()):
         self.run(self, self.INIT_COMMAND, env=env)
 
     @staticmethod
-    def run(self, command, env, auto_approve=False):
+    def run(self, command, env, auto_approve=False, auto_retries=1):
         cmd = ['terraform', command]
 
         if auto_approve:
@@ -37,6 +37,8 @@ def run(self, command, env, auto_approve=False):
         if command == self.APPLY_COMMAND or command == self.DESTROY_COMMAND:
             cmd.append(f'-state={self.working_directory}/terraform.tfstate')
 
+        cmd.append('-no-color')
+
         cmd.append(self.working_directory)
 
         cmd = ' '.join(cmd)
@@ -45,9 +47,16 @@ def run(self, command, env, auto_approve=False):
         if Config().debug > 0:
             env['TF_LOG'] = terraform_verbosity[Config().debug]
 
-        logpipe = LogPipe(__name__)
-        with subprocess.Popen(cmd, stdout=logpipe, stderr=logpipe, env=env,  shell=True) as sp:
-            logpipe.close()
+        retries = 1
+        do_retry = True
+        while ((retries <= auto_retries) and do_retry):
+            logpipe = LogPipe(__name__)
+            with subprocess.Popen(cmd, stdout=logpipe, stderr=logpipe, env=env,  shell=True) as sp:
+                logpipe.close()
+            retries = retries + 1
+            do_retry = next((True for s in logpipe.stderrstrings if 'RetryableError' in s), False)
+            if do_retry and retries <= auto_retries:
+                self.logger.warning(f'Terraform failed with "RetryableError" error. Retry: ' + str(retries) + '/' + str(auto_retries))
 
         if sp.returncode != 0:
             raise Exception(f'Error running: "{cmd}"')

core/src/epicli/cli/helpers/Log.py

@@ -2,39 +2,66 @@
 import logging.handlers
 import threading
 import os
+import enum
 from pythonjsonlogger import jsonlogger
 from cli.helpers.build_saver import get_output_path
 from cli.helpers.Config import Config
 
+class ColorFormatter(logging.Formatter):
+    grey = '\x1b[38;21m'
+    yellow = '\x1b[33;21m'
+    red = '\x1b[31;21m'
+    bold_red = '\x1b[31;1m'
+    reset = '\x1b[0m'
+    lineformat =  Config().log_format
+
+    FORMATS = {
+        logging.DEBUG: grey + lineformat + reset,
+        logging.INFO: grey + lineformat + reset,
+        logging.WARNING: yellow + lineformat + reset,
+        logging.ERROR: red + lineformat + reset,
+        logging.CRITICAL: bold_red + lineformat + reset
+    }
+
+    def format(self, record):
+        log_fmt = self.FORMATS.get(record.levelno)
+        formatter = logging.Formatter(log_fmt, datefmt=Config().log_date_format)
+        return formatter.format(record)
+
 
 class Log:
     class __LogBase:
+        stream_handler = None
         json_file_handler = None
-        json_stream_handler = None
 
         def __init__(self):
             config = Config()
+
+            json_formatter = jsonlogger.JsonFormatter(config.log_format, datefmt=config.log_date_format)
+            color_formater = ColorFormatter()
+
             log_path = os.path.join(get_output_path(), config.log_file)
-            logging.basicConfig(level=logging.INFO, format=config.log_format, datefmt=config.log_date_format)
-            formatter = jsonlogger.JsonFormatter(config.log_format, datefmt=config.log_date_format)
             should_roll_over = os.path.isfile(log_path)
-            handler = logging.handlers.RotatingFileHandler(log_path, backupCount=config.log_count)
+            self.json_file_handler = logging.handlers.RotatingFileHandler(log_path, backupCount=config.log_count)
+            self.json_file_handler.setLevel(level=logging.INFO)
             if should_roll_over:
-                handler.doRollover()
-            self.json_file_handler = handler
-            self.json_file_handler.setFormatter(formatter)
-            self.json_stream_handler = logging.StreamHandler()
-            self.json_stream_handler.setFormatter(formatter)
+                self.json_file_handler.doRollover()
+            self.json_file_handler.setFormatter(json_formatter)
+
+            self.stream_handler = logging.StreamHandler()
+            if config.log_type == 'json':
+                self.stream_handler.setFormatter(json_formatter)
+            else:
+                self.stream_handler.setFormatter(color_formater)
 
     instance = None
 
     def __new__(cls, logger_name):
         if Log.instance is None:
             Log.instance = Log.__LogBase()
-        config = Config()
         logger = logging.getLogger(logger_name)
-        if config.log_type == 'json':
-            logger.addHandler(Log.instance.json_stream_handler)
+        logger.setLevel(level=logging.INFO)
+        logger.addHandler(Log.instance.stream_handler)
         logger.addHandler(Log.instance.json_file_handler)
         return logger
 
@@ -48,13 +75,23 @@ def __init__(self, logger_name):
         self.fdRead, self.fdWrite = os.pipe()
         self.pipeReader = os.fdopen(self.fdRead)
         self.start()
+        self.errorStrings = ['error', 'Error', 'ERROR', 'fatal', 'FAILED']
+        self.warningStrings = ['warning', 'warning', 'WARNING']
+        self.stderrstrings = []
 
     def fileno(self):
         return self.fdWrite
 
     def run(self):
         for line in iter(self.pipeReader.readline, ''):
-            self.logger.info(line.strip('\n'))
+            line = line.strip('\n')
+            if any([substring in line for substring in self.errorStrings]):
+                self.stderrstrings.append(line)
+                self.logger.error(line)
+            elif any([substring in line for substring in self.warningStrings]):
+                    self.logger.warning(line)
+            else:
+                self.logger.info(line)
         self.pipeReader.close()
 
     def close(self):