Feature/az resource group (#437)

- Moved resource group creation to separate model
- Added creation of VNET
- Fixed env variables Azure with SP login
- Fixed some typos

core/src/epicli/cli/engine/providers/azure/InfrastructureBuilder.py

@@ -1,13 +1,48 @@
 from cli.helpers.Step import Step
-
+from cli.helpers.naming_helpers import resource_name
+from cli.helpers.doc_list_helpers import select_single, select_all
+from cli.helpers.doc_list_helpers import select_first
+from cli.helpers.data_loader import load_yaml_obj, types
 
 class InfrastructureBuilder(Step):
     def __init__(self, docs):
         super().__init__(__name__)
+        self.cluster_model = select_single(docs, lambda x: x.kind == 'epiphany-cluster')
+        self.cluster_name = self.cluster_model.specification.name.lower()
+        self.cluster_prefix = self.cluster_model.specification.prefix.lower()
+        self.resource_group_name = resource_name(self.cluster_prefix, self.cluster_name, 'rg')
+        self.region = self.cluster_model.specification.cloud.region
         self.docs = docs
 
     def run(self):
         infrastructure = []
 
+        resource_group = self.get_resource_group()
+        infrastructure.append(resource_group)        
+
+        vnet = self.get_virtual_network()
+        infrastructure.append(vnet)
+
         return infrastructure
 
+    def get_resource_group(self):
+        resource_group = self.get_config_or_default(self.docs, 'infrastructure/resource-group')
+        resource_group.specification.name = self.resource_group_name
+        resource_group.specification.region = self.cluster_model.specification.cloud.region
+        return resource_group    
+
+    def get_virtual_network(self):
+        vnet = self.get_config_or_default(self.docs, 'infrastructure/vnet')
+        vnet.specification.name = resource_name(self.cluster_prefix, self.cluster_name, 'vnet')
+        vnet.specification.address_space = self.cluster_model.specification.cloud.vnet_address_pool
+        vnet.specification.resource_group_name = self.resource_group_name
+        vnet.specification.location = self.cluster_model.specification.cloud.region
+        return vnet            
+
+    @staticmethod
+    def get_config_or_default(docs, kind):
+        config = select_first(docs, lambda x: x.kind == kind)
+        if config is None:
+            return load_yaml_obj(types.DEFAULT, 'azure', kind)
+        return config        
+

core/src/epicli/cli/engine/terraform/TerraformRunner.py

@@ -4,6 +4,7 @@
 from cli.helpers.Step import Step
 from cli.helpers.build_saver import get_terraform_path, save_sp, SP_FILE_NAME
 from cli.helpers.data_loader import load_yaml_file
+from cli.helpers.naming_helpers import resource_name
 
 
 class TerraformRunner(Step):
@@ -13,6 +14,7 @@ def __init__(self, cluster_model, config_docs):
         self.cluster_model = cluster_model
         self.config_docs = config_docs
         self.terraform = TerraformCommand(get_terraform_path(self.cluster_model.specification.name))
+        self.new_env = os.environ.copy()
 
     def __enter__(self):
         super().__enter__()
@@ -22,17 +24,15 @@ def run(self):
         pass
 
     def build(self):
-        new_env = os.environ.copy()
-        self.terraform.init(env=new_env)
+        self.terraform.init(env=self.new_env)
         if self.cluster_model.provider == 'azure':
             self.azure_login()
-        self.terraform.apply(auto_approve=True, env=new_env)
+        self.terraform.apply(auto_approve=True, env=self.new_env)
 
     def delete(self):
-        new_env = os.environ.copy()
         if self.cluster_model.provider == 'azure':
             self.azure_login()
-        self.terraform.destroy(auto_approve=True, env=new_env)
+        self.terraform.destroy(auto_approve=True, env=self.new_env)
 
     def azure_login(self):
         # From the 4 methods terraform provides to login to 
@@ -47,14 +47,17 @@ def azure_login(self):
             sp_file = os.path.join(get_terraform_path(self.cluster_model.specification.name), SP_FILE_NAME)
             if not os.path.exists(sp_file):
                 self.logger.info('Creating service principal')
-                sp = apiproxy.create_sp(self.cluster_model.specification.cloud.resource_group_name, subscription['id'])
+                cluster_name = self.cluster_model.specification.name.lower()
+                cluster_prefix = self.cluster_model.specification.prefix.lower()               
+                resource_group_name = resource_name(cluster_prefix, cluster_name, 'rg')
+                sp = apiproxy.create_sp(resource_group_name, subscription['id'])
                 save_sp(sp, self.cluster_model.specification.name)
             else:
                 self.logger.info('Using service principal from file')
                 sp = load_yaml_file(sp_file)
 
             #Setup environment variables for Terraform when working with Azure and service principal.
-            new_env['ARM_SUBSCRIPTION_ID'] = subscription['id']
-            new_env['ARM_TENANT_ID'] = sp['tenant']
-            new_env['ARM_CLIENT_ID'] = sp['appId']
-            new_env['ARM_CLIENT_SECRET'] = sp['password']
+            self.new_env['ARM_SUBSCRIPTION_ID'] = subscription['id']
+            self.new_env['ARM_TENANT_ID'] = sp['tenant']
+            self.new_env['ARM_CLIENT_ID'] = sp['appId']
+            self.new_env['ARM_CLIENT_SECRET'] = sp['password']

core/src/epicli/data/aws/terraform/epiphany-cluster.j2

@@ -1,5 +1,5 @@
 #####################################################
-# DO NOT Modify by hand - Manage by Automation
+# DO NOT Modify by hand - Managed by Automation
 #####################################################
 #####################################################
 # This file can be used as a base template to build other Terraform files. It attempts to use as much

core/src/epicli/data/aws/terraform/infrastructure/default-security-group.j2

@@ -1,5 +1,5 @@
 #####################################################
-# DO NOT Modify by hand - Manage by Automation
+# DO NOT Modify by hand - Managed by Automation
 #####################################################
 #####################################################
 # This file can be used as a base template to build other Terraform files. It attempts to use as much

core/src/epicli/data/aws/terraform/infrastructure/efs-storage.j2

@@ -1,5 +1,5 @@
 #####################################################
-# DO NOT Modify by hand - Manage by Automation
+# DO NOT Modify by hand - Managed by Automation
 #####################################################
 #####################################################
 # This file can be used as a base template to build other Terraform files. It attempts to use as much

core/src/epicli/data/aws/terraform/infrastructure/internet-gateway.j2

@@ -1,5 +1,5 @@
 #####################################################
-# DO NOT Modify by hand - Manage by Automation
+# DO NOT Modify by hand - Managed by Automation
 #####################################################
 #####################################################
 # This file can be used as a base template to build other Terraform files. It attempts to use as much

core/src/epicli/data/aws/terraform/infrastructure/launch-configuration.j2

@@ -1,5 +1,5 @@
 #####################################################
-# DO NOT Modify by hand - Manage by Automation
+# DO NOT Modify by hand - Managed by Automation
 #####################################################
 #####################################################
 # This file can be used as a base template to build other Terraform files. It attempts to use as much

core/src/epicli/data/aws/terraform/infrastructure/public-key.j2

@@ -1,5 +1,5 @@
 #####################################################
-# DO NOT Modify by hand - Manage by Automation
+# DO NOT Modify by hand - Managed by Automation
 #####################################################
 #####################################################
 # This file can be used as a base template to build other Terraform files. It attempts to use as much

core/src/epicli/data/aws/terraform/infrastructure/resource-group.j2

@@ -1,5 +1,5 @@
 #####################################################
-# DO NOT Modify by hand - Manage by Automation
+# DO NOT Modify by hand - Managed by Automation
 #####################################################
 #####################################################
 # This file can be used as a base template to build other Terraform files. It attempts to use as much

core/src/epicli/data/aws/terraform/infrastructure/route-table-association.j2

@@ -1,5 +1,5 @@
 #####################################################
-# DO NOT Modify by hand - Manage by Automation
+# DO NOT Modify by hand - Managed by Automation
 #####################################################
 #####################################################
 # This file can be used as a base template to build other Terraform files. It attempts to use as much

core/src/epicli/data/aws/terraform/infrastructure/route-table.j2

@@ -1,5 +1,5 @@
 #####################################################
-# DO NOT Modify by hand - Manage by Automation
+# DO NOT Modify by hand - Managed by Automation
 #####################################################
 #####################################################
 # This file can be used as a base template to build other Terraform files. It attempts to use as much

core/src/epicli/data/aws/terraform/infrastructure/security-group-rule.j2

@@ -1,5 +1,5 @@
 #####################################################
-# DO NOT Modify by hand - Manage by Automation
+# DO NOT Modify by hand - Managed by Automation
 #####################################################
 #####################################################
 # This file can be used as a base template to build other Terraform files. It attempts to use as much

core/src/epicli/data/aws/terraform/infrastructure/security-group.j2

@@ -1,5 +1,5 @@
 #####################################################
-# DO NOT Modify by hand - Manage by Automation
+# DO NOT Modify by hand - Managed by Automation
 #####################################################
 #####################################################
 # This file can be used as a base template to build other Terraform files. It attempts to use as much

core/src/epicli/data/aws/terraform/infrastructure/subnet.j2

@@ -1,5 +1,5 @@
 #####################################################
-# DO NOT Modify by hand - Manage by Automation
+# DO NOT Modify by hand - Managed by Automation
 #####################################################
 #####################################################
 # This file can be used as a base template to build other Terraform files. It attempts to use as much

core/src/epicli/data/aws/terraform/infrastructure/virtual-machine.j2

@@ -1,5 +1,5 @@
 #####################################################
-# DO NOT Modify by hand - Manage by Automation
+# DO NOT Modify by hand - Managed by Automation
 #####################################################
 #####################################################
 # This file can be used as a base template to build other Terraform files. It attempts to use as much

core/src/epicli/data/aws/terraform/infrastructure/vpc.j2

@@ -1,5 +1,5 @@
 #####################################################
-# DO NOT Modify by hand - Manage by Automation
+# DO NOT Modify by hand - Managed by Automation
 #####################################################
 #####################################################
 # This file can be used as a base template to build other Terraform files. It attempts to use as much

core/src/epicli/data/azure/defaults/infrastructure/resource-group.yml

@@ -0,0 +1,8 @@
+kind: infrastructure/resource-group
+version: 0.3.0
+title: "Resource Group"
+provider: azure
+name: default
+specification:
+  name: SET_BY_AUTOMATION
+  region: SET_BY_AUTOMATION

core/src/epicli/data/azure/defaults/infrastructure/vnet.yml

@@ -0,0 +1,10 @@
+kind: infrastructure/vnet
+version: 0.3.0
+title: "VNET Config"
+provider: azure
+name: default
+specification:
+  name: SET_BY_AUTOMATION
+  address_space: SET_BY_AUTOMATION
+  location: SET_BY_AUTOMATION
+  resource_group_name: SET_BY_AUTOMATION

core/src/epicli/data/azure/terraform/epiphany-cluster.j2

@@ -1,5 +1,5 @@
 #####################################################
-# DO NOT Modify by hand - Manage by Automation
+# DO NOT Modify by hand - Managed by Automation
 #####################################################
 #####################################################
 # This file can be used as a base template to build other Terraform files. It attempts to use as much
@@ -12,8 +12,3 @@
 
 provider "azurerm" {
 }
-
-resource "azurerm_resource_group" "rg" {
-    name = "{{ specification.cloud.resource_group_name }}"
-    location = "{{ specification.cloud.region }}"
-}

core/src/epicli/data/azure/terraform/infrastructure/resource-group.j2

@@ -0,0 +1,17 @@
+#####################################################
+# DO NOT Modify by hand - Managed by Automation
+#####################################################
+#####################################################
+# This file can be used as a base template to build other Terraform files. It attempts to use as much
+# Terraform interprolation as possible by creating Terraform variables instead of changing inline
+# this approach provides an easier way to do creative looping, fetch IDs of created resources etc.
+#####################################################
+#####################################################
+# {{ specification.name }}
+#####################################################
+
+
+resource "azurerm_resource_group" "rg" {
+    name = "{{ specification.name }}"
+    location = "{{ specification.region }}"
+}

core/src/epicli/data/azure/terraform/infrastructure/vnet.j2

@@ -0,0 +1,18 @@
+#####################################################
+# DO NOT Modify by hand - Managed by Automation
+#####################################################
+#####################################################
+# This file can be used as a base template to build other Terraform files. It attempts to use as much
+# Terraform interprolation as possible by creating Terraform variables instead of changing inline
+# this approach provides an easier way to do creative looping, fetch IDs of created resources etc.
+#####################################################
+#####################################################
+# {{ specification.name }}
+#####################################################
+
+resource "azurerm_virtual_network" "vnet" {
+  name                = "{{ specification.name }}"
+  address_space       = ["{{ specification.address_space }}"]
+  location            = "{{ specification.location }}"
+  resource_group_name = "{{ specification.resource_group_name }}"
+}

core/src/epicli/data/azure/validation/infrastructure/vnet.yml

@@ -0,0 +1 @@
+$ref: '#/definitions/unvalidated_specification'

core/src/epicli/data/common/defaults/epiphany-cluster.yml

@@ -11,7 +11,6 @@ specification:
     key_path: /root/.ssh/epiphany-operations/id_rsa # YOUR-SSH-KEY-PATH
   cloud:
     subscription_name: YOUR-SUB-NAME
-    resource_group_name: YOUR-RESOURCE-GROUP-NAME
     vnet_address_pool: 10.1.0.0/20
     use_public_ips: False # When not using public IPs you have to provide connectivity via private IPs (VPN)
     use_service_principal: False