AzureCLI integration (hitachienergy#427)

* - Added login to Azure-CLI and selection of subscription.
- Added creation of service principle.
- Added creation of the resource group.

* - Added posibility to run without service principal

* - Changed text outputs

* - Fixed typo.

core/src/epicli/Pipfile

@@ -17,6 +17,7 @@ jsonschema = "*"
 python-json-logger = "*"
 ansible = "*"
 terraform-bin = "*"
+azure-cli = "==2.0.67"
 
 [requires]
 python_version = "3.7"

core/src/epicli/cli/engine/EpiphanyEngine.py

@@ -103,7 +103,7 @@ def apply(self):
                     template_generator.run()
 
                 # Run Terraform to create infrastructure
-                with TerraformRunner(self.cluster_model.specification.name) as tf_runner:
+                with TerraformRunner(self.cluster_model) as tf_runner:
                     tf_runner.run()
 
             self.process_configuration_docs()

core/src/epicli/cli/engine/TerraformCommand.py

@@ -13,37 +13,39 @@ def __init__(self, working_directory=os.path.dirname(__file__)):
         self.INIT_COMMAND = "init"
         self.working_directory = working_directory
 
-    def apply(self, auto_approve=False):
-        self.run(self, self.APPLY_COMMAND, auto_approve=auto_approve)
+    def apply(self, auto_approve=False, env=os.environ.copy()):
+        self.run(self, self.APPLY_COMMAND, auto_approve=auto_approve, env=env)
 
-    def destroy(self, auto_approve=False):
-        self.run(self, self.DESTROY_COMMAND, auto_approve=auto_approve)
+    def destroy(self, auto_approve=False, env=os.environ.copy()):
+        self.run(self, self.DESTROY_COMMAND, auto_approve=auto_approve, env=env)
 
-    def plan(self):
-        self.run(self, self.PLAN_COMMAND)
+    def plan(self, env=os.environ.copy()):
+        self.run(self, self.PLAN_COMMAND, env=env)
 
-    def init(self):
-        self.run(self, self.INIT_COMMAND)
+    def init(self, env=os.environ.copy()):
+        self.run(self, self.INIT_COMMAND, env=env)
 
     @staticmethod
-    def run(self, command, auto_approve=False):
+    def run(self, command, env, auto_approve=False):
         cmd = ['terraform', command]
 
         if auto_approve:
             cmd.append('--auto-approve')
 
         if command == self.APPLY_COMMAND:
-            cmd.append('-state=' + self.working_directory + '/terraform.tfstate')
+            cmd.append(f'-state={self.working_directory}/terraform.tfstate')
 
         cmd.append(self.working_directory)
 
         self.logger.info('Running: "' + ' '.join(cmd) + '"')
 
+        cmd = ' '.join(cmd)
+
         logpipe = LogPipe(__name__)
-        with subprocess.Popen(cmd, stdout=logpipe, stderr=logpipe) as sp:
+        with subprocess.Popen(cmd, stdout=logpipe, stderr=logpipe, env=env,  shell=True) as sp:
             logpipe.close()
 
         if sp.returncode != 0:
-            raise Exception('Error running: "' + ' '.join(cmd) + '"')
+            raise Exception(f'Error running: "{cmd}"')
         else:
-            self.logger.info('Done running "' + ' '.join(cmd) + '"')
+            self.logger.info(f'Done running "{cmd}"')

core/src/epicli/cli/engine/TerraformRunner.py

@@ -1,19 +1,45 @@
+import os
 from cli.engine.TerraformCommand import TerraformCommand
+from cli.engine.azure.AzureCommand import AzureCommand
 from cli.helpers.Step import Step
-from cli.helpers.build_saver import get_terraform_path
+from cli.helpers.build_saver import get_terraform_path, save_sp, SP_FILE_NAME
+from cli.helpers.data_loader import load_yaml_file
 
 
 class TerraformRunner(Step):
 
-    def __init__(self, cluster_name):
+    def __init__(self, cluster_model):
         super().__init__(__name__)
-        self.terraform = TerraformCommand(get_terraform_path(cluster_name))
+        self.cluster_model = cluster_model
+        self.terraform = TerraformCommand(get_terraform_path(self.cluster_model.specification.name))
+        self.azure_cli = AzureCommand()
 
     def __enter__(self):
         super().__enter__()
         return self
 
     def run(self):
-        self.terraform.init()
-        self.terraform.apply(auto_approve=True)
+        new_env = os.environ.copy()
+        self.terraform.init(env=new_env)
 
+        #if the provider is Azure we need to login and setup service principle.
+        if self.cluster_model.provider == 'azure':
+            subscription = self.azure_cli.login(self.cluster_model.specification.cloud.subscription_name)
+
+            if self.cluster_model.specification.cloud.use_service_principal:
+                sp_file = os.path.join(get_terraform_path(self.cluster_model.specification.name), SP_FILE_NAME)
+                if not os.path.exists(sp_file):
+                    self.logger.info('Creating service principle')
+                    sp = self.azure_cli.create_sp(self.cluster_model.specification.cloud.resource_group_name, subscription['id'])
+                    save_sp(sp, self.cluster_model.specification.name)
+                else:
+                    self.logger.info('Using service principle from file')
+                    sp = load_yaml_file(sp_file)
+
+                #Setup environment variables for Terraform when working with Azure.
+                new_env['ARM_SUBSCRIPTION_ID'] = subscription['id']
+                new_env['ARM_TENANT_ID'] = sp['tenant']
+                new_env['ARM_CLIENT_ID'] = sp['appId']
+                new_env['ARM_CLIENT_SECRET'] = sp['password']
+
+        self.terraform.apply(auto_approve=True, env=new_env)

core/src/epicli/cli/engine/azure/APIProxy.py

@@ -7,5 +7,4 @@ def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
-        pass
-
+        pass

core/src/epicli/cli/engine/azure/AzureCommand.py

@@ -0,0 +1,49 @@
+import json
+import re
+import time
+from subprocess import Popen, PIPE
+from cli.helpers.Log import LogPipe, Log
+from cli.helpers.doc_list_helpers import select_first
+
+
+class AzureCommand:
+    def __init__(self):
+        self.logger = Log(__name__)
+
+    def login(self, subscription_name):
+        all_subscription = self.run(self, 'az login')
+        subscription = select_first(all_subscription, lambda x: x['name'] == subscription_name)
+        if subscription is None:
+            raise Exception(f'User does not have access to subscription: "{subscription_name}"')
+        self.run(self, f'az account set --subscription {subscription["id"]}')
+        return subscription
+
+    def create_sp(self, app_name, subscription_id):
+        #TODO: make role configurable?
+        sp = self.run(self, f'az ad sp create-for-rbac -n "{app_name}" --role="Contributor" --scopes="/subscriptions/{subscription_id}"')
+        # Sleep for a while. Sometimes the call returns before the rights of the SP are finished creating.
+        for x in range(0, 20):
+            self.logger.info(f'Waiting 20 seconds...{x}')
+            time.sleep(1)
+        return sp
+
+    @staticmethod
+    def run(self, cmd):
+        self.logger.info('Running: "' + cmd + '"')
+
+        logpipe = LogPipe(__name__)
+        with Popen(cmd, stdout=PIPE, stderr=logpipe, shell=True) as sp:
+            logpipe.close()
+            try:
+                data = sp.stdout.read().decode('utf-8')
+                data = re.sub(r'\s+', '', data)
+                data = re.sub(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]', '', data)
+                output = json.loads(data)
+            except:
+                output = {}
+
+        if sp.returncode != 0:
+            raise Exception(f'Error running: "{cmd}"')
+        else:
+            self.logger.info(f'Done running "{cmd}"')
+            return output

core/src/epicli/cli/engine/azure/InfrastructureBuilder.py

@@ -1,10 +1,13 @@
 from cli.helpers.Step import Step
 
 
-class ConfigBuilder(Step):
-    def __init__(self):
+class InfrastructureBuilder(Step):
+    def __init__(self, docs):
         super().__init__(__name__)
+        self.docs = docs
 
     def run(self):
-        raise NotImplementedError()
+        infrastructure = []
+
+        return infrastructure
 

core/src/epicli/cli/helpers/build_saver.py

@@ -3,11 +3,12 @@
 import os
 from distutils import dir_util
 from cli.helpers.data_loader import load_template_file, types
-from cli.helpers.yaml_helpers import dump_all
+from cli.helpers.yaml_helpers import dump_all, dump
 from cli.helpers.Config import Config
 
 TERRAFORM_OUTPUT_DIR = 'terraform/'
 MANIFEST_FILE_NAME = 'manifest.yml'
+SP_FILE_NAME = 'sp.yml'
 INVENTORY_FILE_NAME = 'inventory'
 ANSIBLE_OUTPUT_DIR = 'ansible/'
 
@@ -20,6 +21,14 @@ def save_manifest(docs, cluster_name, manifest_name=MANIFEST_FILE_NAME):
     return path
 
 
+def save_sp(service_principle, cluster_name):
+    terraform_dir = get_terraform_path(cluster_name)
+    path = os.path.join(terraform_dir, SP_FILE_NAME)
+    with open(path, 'w') as stream:
+        dump(service_principle, stream)
+    return path
+
+
 def save_inventory(inventory, cluster_model):
     cluster_name = cluster_model.specification.name
     build_dir = get_build_path(cluster_name)

core/src/epicli/cli/helpers/data_loader.py

@@ -29,12 +29,17 @@ def load_yaml_obj(file_type, provider, kind):
     script_dir = os.path.dirname(__file__)
     path_to_file = os.path.join(script_dir, DATA_FOLDER_PATH, provider, file_type, kind+'.yml')
     if os.path.isfile(path_to_file):
+        return load_yaml_file(path_to_file)
         with open(path_to_file, 'r') as stream:
             return safe_load(stream)
     else:
         path_to_file = os.path.join(script_dir, DATA_FOLDER_PATH, 'common', file_type, kind + '.yml')
-        with open(path_to_file, 'r') as stream:
-            return safe_load(stream)
+        return load_yaml_file(path_to_file)
+
+
+def load_yaml_file(path_to_file):
+    with open(path_to_file, 'r') as stream:
+        return safe_load(stream)
 
 
 def load_all_yaml_objs(file_type, provider, kind):

core/src/epicli/data/azure/terraform/epiphany-cluster.j2

@@ -1 +1,19 @@
-# TODO: Fill template
+#####################################################
+# DO NOT Modify by hand - Manage by Automation
+#####################################################
+#####################################################
+# This file can be used as a base template to build other Terraform files. It attempts to use as much
+# Terraform interprolation as possible by creating Terraform variables instead of changing inline
+# this approach provides an easier way to do creative looping, fetch IDs of created resources etc.
+#####################################################
+#####################################################
+# {{ specification.name }}
+#####################################################
+
+provider "azurerm" {
+}
+
+resource "azurerm_resource_group" "rg" {
+    name = "{{ specification.cloud.resource_group_name }}"
+    location = "{{ specification.cloud.region }}"
+}

core/src/epicli/data/common/defaults/epiphany-cluster.yml

@@ -11,8 +11,10 @@ specification:
     key_path: /root/.ssh/epiphany-operations/id_rsa # YOUR-SSH-KEY-PATH
   cloud:
     subscription_name: YOUR-SUB-NAME
+    resource_group_name: YOUR-RESOURCE-GROUP-NAME
     vnet_address_pool: 10.1.0.0/20
     use_public_ips: False # When not using public IPs you have to provide connectivity via private IPs (VPN)
+    use_service_principal: False
     region: eu-west-2
     credentials: # todo change it to get credentials from vault
       key: 3124-4124-4124