diff --git a/README.md b/README.md index 125729e2..01142b84 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ describing the target cloud infrastructure. | GCloud | Compute Engine - VM |:white_check_mark:| | GCloud | CloudSQL |:white_check_mark:| | GCloud | AlloyDB |:white_check_mark:| -| Azure | VM | :x: | +| Azure | VM |:white_check_mark:| | Azure | Database - Flexible | :x: | | Azure | CosmosDB | :x: | @@ -332,11 +332,12 @@ $ export GOOGLE_PROJECT= ``` #### Terraform +Install manually or [use terraform for latest instructions](https://developer.hashicorp.com/terraform/downloads) ```console $ sudo apt install unzip -y -$ wget https://releases.hashicorp.com/terraform/1.2.6/terraform_1.2.6_linux_amd64.zip -$ unzip terraform_1.2.6_linux_amd64.zip +$ wget https://releases.hashicorp.com/terraform/1.3.6/terraform_1.3.6_linux_amd64.zip +$ unzip terraform_1.3.6_linux_amd64.zip $ sudo install terraform /usr/bin ``` diff --git a/edbterraform/data/templates/azure/agreements.tf.j2 b/edbterraform/data/templates/azure/agreements.tf.j2 new file mode 100644 index 00000000..4b00c415 --- /dev/null +++ b/edbterraform/data/templates/azure/agreements.tf.j2 @@ -0,0 +1,7 @@ +module "agreements" { + source = "./modules/agreement" + + publisher = var.operating_system.publisher + offer = var.operating_system.offer + plan = var.operating_system.sku +} diff --git a/edbterraform/data/templates/azure/key_pair.tf.j2 b/edbterraform/data/templates/azure/key_pair.tf.j2 new file mode 100644 index 00000000..8657c029 --- /dev/null +++ b/edbterraform/data/templates/azure/key_pair.tf.j2 @@ -0,0 +1,14 @@ +module "key_pair_{{ region_ }}" { + source = "./modules/key_pair" + + name = "{{ region }}-${random_id.apply.hex}" + resource_name = module.vpc_{{ region_ }}.resource_name + region = module.vpc_{{ region_ }}.region + public_key = var.ssh_pub_key + + depends_on = [module.vpc_{{ region_ }}] + + providers = { + azurerm = azurerm.{{ region_ }} + } +} diff --git a/edbterraform/data/templates/azure/machine.tf.j2 b/edbterraform/data/templates/azure/machine.tf.j2 new file mode 100644 index 00000000..72700f1d --- /dev/null +++ b/edbterraform/data/templates/azure/machine.tf.j2 @@ -0,0 +1,38 @@ +module "machine_{{ region_ }}" { + source = "./modules/machine" + + for_each = { + for rm in lookup(local.region_machines, "{{ region }}", []) : + rm.name => rm + } + + resource_name = module.vpc_{{ region_ }}.resource_name + subnet_id = module.network_{{ region_ }}[each.value.spec.zone].subnet_id + operating_system = var.operating_system + cluster_name = var.cluster_name + machine = ( + merge( + each.value.spec, + {name = each.value.name}, + {zone = tostring(each.value.spec.zone) == "0" ? null : each.value.spec.zone}, + ) + ) + additional_volumes = try(each.value.spec.additional_volumes, null) + ssh_user = var.ssh_user + private_key = var.ssh_priv_key + public_key_name = module.key_pair_{{ region_ }}.name + name_id = random_id.apply.hex + + depends_on = [ + module.vpc_{{ region_ }}, + module.key_pair_{{ region_ }}, + module.network_{{ region_ }}, + module.security_{{ region_ }}, + module.agreements, + ] + + providers = { + azurerm = azurerm.{{ region_ }} + } + +} diff --git a/edbterraform/data/templates/azure/main.tf.j2 b/edbterraform/data/templates/azure/main.tf.j2 new file mode 100644 index 00000000..08f3f0bc --- /dev/null +++ b/edbterraform/data/templates/azure/main.tf.j2 @@ -0,0 +1,80 @@ +locals { + region_az_networks = { + for region, region_spec in var.regions: region => { + for zone, network in try(region_spec.zones, {}): zone => network + } + } + region_machines = { + for name, machine_spec in var.machines: machine_spec.region => { + name = name + spec = machine_spec + }... + } +} + +resource "random_id" "apply" { + byte_length = 4 +} + +{% if has_machines %} +{% include "agreements.tf.j2" %} +{% endif %} + +{% for region in regions.keys() %} +{% set region_ = region | replace('-', '_') %} + +{% if has_regions %} +{% include "network.tf.j2" %} +{% endif %} + +{% if has_machines %} +{% include "key_pair.tf.j2" %} + +{% include "machine.tf.j2" %} +{% endif %} + +{% endfor %} + +{% if has_region_peering %} +{% include "region_peering.tf.j2" %} +{% endif %} + +resource "local_file" "servers_yml" { + filename = "${abspath(path.root)}/servers.yml" + file_permission = "0600" + content = <<-EOT +--- +servers: +{% set boxes = { + 'machines': { + 'active': has_machines, + 'regions': machine_regions, + 'module_base': 'module.machine_', + }, +} %} +{% for type, attributes in boxes.items() if attributes["active"] %} + {{type}}: +{% for region in attributes["regions"] -%} +{% set module = attributes["module_base"] ~ region | replace('-', '_') %} +%{ for key, value in {{ module }} ~} + ${key}: +%{ for name, item in value ~} + ${name}: ${try(jsonencode(item), "Error, unsupported type",)} +%{ endfor ~} +%{ endfor ~} +{% endfor %} +{% endfor %} + EOT +} + +{% for type, attributes in boxes.items() if attributes["active"] %} +output "{{type}}" { + value = [ +{% for region in attributes["regions"] -%} +{% set module = attributes["module_base"] ~ region | replace('-', '_') %} + {{ module }}[*], +{% endfor %} + ] + sensitive = true +} +{% endfor %} diff --git a/edbterraform/data/templates/azure/network.tf.j2 b/edbterraform/data/templates/azure/network.tf.j2 new file mode 100644 index 00000000..2bcd7bba --- /dev/null +++ b/edbterraform/data/templates/azure/network.tf.j2 @@ -0,0 +1,56 @@ +module "vpc_{{ region_ }}"{ + source = "./modules/vpc" + + name = "${var.vpc_tag}-{{ region }}-${random_id.apply.hex}" + cidr_blocks = [ lookup(lookup(var.regions, "{{ region }}"), "cidr_block") ] + region = "{{ region }}" + + providers = { + azurerm = azurerm.{{ region_ }} + } +} + +module "network_{{ region_ }}" { + source = "./modules/network" + + for_each = lookup(local.region_az_networks, "{{ region }}", null) + + resource_name = module.vpc_{{ region_ }}.resource_name + network_name = module.vpc_{{ region_ }}.network_name + region = module.vpc_{{ region_ }}.region + zone = tostring(each.key) == "0" ? null : each.key + ip_cidr_range = [ each.value ] + name = "{{region}}-${each.key}-${random_id.apply.hex}" + + depends_on = [module.vpc_{{ region_ }}] + + providers = { + azurerm = azurerm.{{ region_ }} + } +} + +module "security_{{ region_ }}" { + source = "./modules/security" + + for_each = lookup(local.region_az_networks, "{{ region }}", null) + + subnet_id = module.network_{{ region_ }}[each.key].subnet_id + region = module.vpc_{{ region_ }}.region + resource_name = module.vpc_{{ region_ }}.resource_name + service_name = "service-{{ region }}-${each.key}-${random_id.apply.hex}" + service_ports = lookup(lookup(var.regions, "{{ region }}", null), "service_ports", []) + public_cidrblock = var.public_cidrblock + region_name = "region-{{ region }}-${each.key}-${random_id.apply.hex}" + region_ports = lookup(lookup(var.regions, "{{ region }}", null), "region_ports", []) + region_cidrblocks = flatten([ + for region in try(var.regions, []) : [ + for ip_cidr in try(region.zones, []) : ip_cidr + ] + ]) + + depends_on = [module.vpc_{{ region_ }}, module.network_{{ region_ }}] + + providers = { + azurerm = azurerm.{{ region_ }} + } +} diff --git a/edbterraform/data/templates/azure/providers.tf.j2 b/edbterraform/data/templates/azure/providers.tf.j2 new file mode 100644 index 00000000..88a4df08 --- /dev/null +++ b/edbterraform/data/templates/azure/providers.tf.j2 @@ -0,0 +1,16 @@ +provider "azurerm" { + features {} +} + +{% for region in regions.keys() %} +{% set region_ = region | replace('-', '_') %} +provider "azurerm" { + features { + resource_group { + prevent_deletion_if_contains_resources = false + } + } + alias = "{{ region_ }}" +} + +{% endfor %} diff --git a/edbterraform/data/templates/azure/region_peering.tf.j2 b/edbterraform/data/templates/azure/region_peering.tf.j2 new file mode 100644 index 00000000..6a1b52e3 --- /dev/null +++ b/edbterraform/data/templates/azure/region_peering.tf.j2 @@ -0,0 +1,41 @@ +{% set previous_created = [] %} +{% for (requester, accepter) in peers %} +{% set requester_ = requester|replace('-', '_') %} +{% set accepter_ = accepter|replace('-', '_') %} +module "vpc_peering_{{ requester_ }}_{{ accepter_ }}" { + source = "./modules/vpc_peering" + + peering_name = "peer-{{ requester }}-{{ accepter }}-${random_id.apply.hex}" + resource_name = module.vpc_{{ requester_ }}.resource_name + network_name = module.vpc_{{ requester_ }}.network_name + peer_network_id = module.vpc_{{ accepter_ }}.network_id + + depends_on = [ + module.network_{{ requester_ }}, + module.network_{{ accepter_ }}, + {% if previous_created %}{{ previous_created[-1] }},{% endif %} + ] + + providers = { + azurerm = azurerm.{{ requester_ }} + } + +} + +module "vpc_peering_{{ accepter_ }}_{{ requester_ }}" { + source = "./modules/vpc_peering" + + peering_name = "peer-{{ accepter }}-{{ requester }}-${random_id.apply.hex}" + resource_name = module.vpc_{{ accepter_ }}.resource_name + network_name = module.vpc_{{ accepter_ }}.network_name + peer_network_id = module.vpc_{{ requester_ }}.network_id + + depends_on = [module.vpc_peering_{{ requester_ }}_{{ accepter_ }}] + + providers = { + azurerm = azurerm.{{ accepter_ }} + } + +} +{% set dummy = previous_created.append("module.vpc_peering_" + accepter_ + "_" + requester_) %} +{% endfor %} diff --git a/edbterraform/data/terraform/azure/modules/agreement/main.tf b/edbterraform/data/terraform/azure/modules/agreement/main.tf new file mode 100644 index 00000000..c6585a32 --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/agreement/main.tf @@ -0,0 +1,5 @@ +resource "azurerm_marketplace_agreement" "image" { + publisher = var.publisher + offer = var.offer + plan = var.plan +} diff --git a/edbterraform/data/terraform/azure/modules/agreement/providers.tf b/edbterraform/data/terraform/azure/modules/agreement/providers.tf new file mode 100644 index 00000000..e8df5670 --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/agreement/providers.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 3.37.0" + } + } + required_version = ">= 1.3.6" +} diff --git a/edbterraform/data/terraform/azure/modules/agreement/variables.tf b/edbterraform/data/terraform/azure/modules/agreement/variables.tf new file mode 100644 index 00000000..f1ec46b4 --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/agreement/variables.tf @@ -0,0 +1,3 @@ +variable "publisher" {} +variable "offer" {} +variable "plan" {} diff --git a/edbterraform/data/terraform/azure/modules/key_pair/main.tf b/edbterraform/data/terraform/azure/modules/key_pair/main.tf new file mode 100644 index 00000000..1511ec58 --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/key_pair/main.tf @@ -0,0 +1,6 @@ +resource "azurerm_ssh_public_key" "main" { + name = var.name + resource_group_name = var.resource_name + location = var.region + public_key = file(var.public_key) +} diff --git a/edbterraform/data/terraform/azure/modules/key_pair/outputs.tf b/edbterraform/data/terraform/azure/modules/key_pair/outputs.tf new file mode 100644 index 00000000..0196fc9e --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/key_pair/outputs.tf @@ -0,0 +1,11 @@ +output "name" { + value = azurerm_ssh_public_key.main.name +} + +output "public_key" { + value = azurerm_ssh_public_key.main.public_key +} + +output "id" { + value = azurerm_ssh_public_key.main.id +} diff --git a/edbterraform/data/terraform/azure/modules/key_pair/providers.tf b/edbterraform/data/terraform/azure/modules/key_pair/providers.tf new file mode 100644 index 00000000..e8df5670 --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/key_pair/providers.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 3.37.0" + } + } + required_version = ">= 1.3.6" +} diff --git a/edbterraform/data/terraform/azure/modules/key_pair/variables.tf b/edbterraform/data/terraform/azure/modules/key_pair/variables.tf new file mode 100644 index 00000000..662f4a4f --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/key_pair/variables.tf @@ -0,0 +1,4 @@ +variable "name" {} +variable "resource_name" {} +variable "public_key" {} +variable "region" {} diff --git a/edbterraform/data/terraform/azure/modules/machine/main.tf b/edbterraform/data/terraform/azure/modules/machine/main.tf new file mode 100644 index 00000000..c50cd0ce --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/machine/main.tf @@ -0,0 +1,160 @@ +data "azurerm_ssh_public_key" "main" { + name = var.public_key_name + resource_group_name = var.resource_name +} + +resource "azurerm_public_ip" "main" { + name = "ip-${var.machine.name}-${var.name_id}" + resource_group_name = var.resource_name + location = var.machine.region + allocation_method = "Static" + zones = local.zones + sku = local.public_ip_sku +} + +resource "azurerm_network_interface" "internal" { + name = "internal-${var.machine.name}-${var.name_id}" + resource_group_name = var.resource_name + location = var.machine.region + + ip_configuration { + name = "internal" + private_ip_address_version = "IPv4" + private_ip_address_allocation = "Dynamic" + subnet_id = var.subnet_id + public_ip_address_id = azurerm_public_ip.main.id + } + + depends_on = [azurerm_public_ip.main, ] +} + +resource "azurerm_linux_virtual_machine" "main" { + name = format("%s-%s-%s", var.cluster_name, var.machine.name, var.name_id) + resource_group_name = var.resource_name + location = var.machine.region + zone = var.machine.zone + size = var.machine.instance_type + + admin_username = var.ssh_user + admin_ssh_key { + username = var.ssh_user + public_key = data.azurerm_ssh_public_key.main.public_key + } + disable_password_authentication = true + network_interface_ids = [azurerm_network_interface.internal.id] + + os_disk { + caching = var.machine.volume.caching + storage_account_type = var.machine.volume.type + disk_size_gb = var.machine.volume.size_gb + } + + additional_capabilities { + ultra_ssd_enabled = local.ultra_ssd_enabled + } + + plan { + name = var.operating_system.sku + product = var.operating_system.offer + publisher = var.operating_system.publisher + } + + source_image_reference { + publisher = var.operating_system.publisher + offer = var.operating_system.offer + sku = var.operating_system.sku + version = var.operating_system.version + } + + depends_on = [azurerm_network_interface.internal, ] +} + +resource "azurerm_managed_disk" "volume" { + for_each = local.additional_volumes + + name = format("%s-%s-%s-%s", var.machine.name, var.cluster_name, var.name_id, each.key) + resource_group_name = var.resource_name + location = var.machine.region + zone = var.machine.zone + storage_account_type = each.value.type + create_option = "Empty" + disk_size_gb = each.value.size_gb + disk_iops_read_write = each.value.iops + + lifecycle { + precondition { + condition = ( + each.value.type != local.premium_ssd.value || + contains(local.premium_ssd.regions, var.machine.region) + ) + error_message = <<-EOT + ${var.machine.name} not a valid configuration. + Premium SSD v2 only availiable in: ${jsonencode(local.premium_ssd.regions)} + For more information visit: + https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#regional-availability + EOT + } + } + + depends_on = [ + azurerm_linux_virtual_machine.main, + ] +} + +resource "azurerm_virtual_machine_data_disk_attachment" "attached_volumes" { + for_each = local.additional_volumes + + virtual_machine_id = azurerm_linux_virtual_machine.main.id + managed_disk_id = azurerm_managed_disk.volume[each.key].id + lun = 10 + tonumber(each.key) + caching = each.value.caching + + depends_on = [ + azurerm_managed_disk.volume, + ] +} + +resource "null_resource" "copy_setup_volume_script" { + count = local.volume_script_count + + provisioner "file" { + content = file("${abspath(path.module)}/setup_volume.sh") + destination = "/tmp/setup_volume.sh" + + # Requires firewall access to ssh port + connection { + type = "ssh" + user = var.ssh_user + host = azurerm_linux_virtual_machine.main.public_ip_address + private_key = file(var.private_key) + } + } + + depends_on = [ + azurerm_virtual_machine_data_disk_attachment.attached_volumes, + ] + +} + +resource "null_resource" "setup_volume" { + for_each = local.additional_volumes + + provisioner "remote-exec" { + inline = [ + "chmod a+x /tmp/setup_volume.sh", + "/tmp/setup_volume.sh ${element(local.linux_device_names, tonumber(each.key))} ${each.value.mount_point} ${length(lookup(var.machine, "additional_volumes", [])) + 1} >> /tmp/mount.log 2>&1" + ] + + # Requires firewall access to ssh port + connection { + type = "ssh" + user = var.ssh_user + host = azurerm_linux_virtual_machine.main.public_ip_address + private_key = file(var.private_key) + } + } + + depends_on = [ + null_resource.copy_setup_volume_script + ] +} diff --git a/edbterraform/data/terraform/azure/modules/machine/outputs.tf b/edbterraform/data/terraform/azure/modules/machine/outputs.tf new file mode 100644 index 00000000..035f72cb --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/machine/outputs.tf @@ -0,0 +1,18 @@ +output "type" { + value = var.machine.type +} +output "instance_type" { + value = azurerm_linux_virtual_machine.main.size +} +output "zone" { + value = azurerm_linux_virtual_machine.main.zone +} +output "region" { + value = azurerm_linux_virtual_machine.main.location +} +output "public_ip" { + value = azurerm_linux_virtual_machine.main.public_ip_address +} +output "private_ip" { + value = azurerm_linux_virtual_machine.main.private_ip_address +} diff --git a/edbterraform/data/terraform/azure/modules/machine/providers.tf b/edbterraform/data/terraform/azure/modules/machine/providers.tf new file mode 100644 index 00000000..e8df5670 --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/machine/providers.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 3.37.0" + } + } + required_version = ">= 1.3.6" +} diff --git a/edbterraform/data/terraform/azure/modules/machine/setup_volume.sh b/edbterraform/data/terraform/azure/modules/machine/setup_volume.sh new file mode 100644 index 00000000..7514fb97 --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/machine/setup_volume.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +COUNTER=0 +DEVICE=$1 +MOUNTPOINT=$2 + +while [ ! -b ${DEVICE} ]; do + sleep 2 + COUNTER=$((COUNTER + 1)) + if [ $COUNTER -ge 10 ]; then + exit 2 + fi +done + + +sudo mkfs.xfs "${DEVICE}" +sudo mkdir -p "${MOUNTPOINT}" +DEVICE_UUID="$(sudo /sbin/wipefs -i -O UUID ${DEVICE})" +sudo mount -t xfs ${DEVICE} ${MOUNTPOINT} +echo "UUID=${DEVICE_UUID} ${MOUNTPOINT} xfs noatime 0 0" | sudo tee -a /etc/fstab diff --git a/edbterraform/data/terraform/azure/modules/machine/variables.tf b/edbterraform/data/terraform/azure/modules/machine/variables.tf new file mode 100644 index 00000000..9defb260 --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/machine/variables.tf @@ -0,0 +1,105 @@ +variable "operating_system" { + type = object({ + sku = string + offer = string + publisher = string + version = string + }) +} +variable "machine" { + type = object({ + name = string + type = string + region = string + zone = optional(string) + instance_type = string + volume = object({ + caching = optional(string, "None") + size_gb = number + type = string + }) + }) +} +variable "cluster_name" {} +locals { + zones = var.machine.zone == null ? null : [var.machine.zone] + public_ip_sku = var.machine.zone == null ? "Basic" : "Standard" +} +variable "resource_name" { + type = string +} +variable "name_id" { + type = string +} +variable "subnet_id" {} +variable "ssh_user" {} +variable "public_key_name" {} +variable "private_key" {} +variable "additional_volumes" { + type = list(object({ + mount_point = string + size_gb = number + type = string + caching = optional(string, "None") + iops = optional(number) + })) + + validation { + condition = alltrue([ + for volume in var.additional_volumes : + volume.iops == null || + contains(["UltraSSD_LRS", "PremiumV2_LRS", ], volume.type) + ]) + error_message = <<-EOT + IOPs is only configurable with the following disk types: + UltraSSD PremiumV2 + EOT + } + + validation { + condition = alltrue([ + for volume in var.additional_volumes : + volume.caching == "None" || + volume.type != "UltraSSD_LRS" + ]) + error_message = <<-EOT + Caching must be set to "None" when using UltraSSD_LRS + EOT + } + + default = [] + nullable = false +} + +locals { + additional_volumes = { + for key, value in var.additional_volumes : + key => value + } + + volume_script_count = length(var.additional_volumes) > 0 ? 1 : 0 + + premium_ssd = { + regions = ["eastus", "westeurope", ] + value = "PremiumV2_LRS" + } + + # Must be enabled in virtual machine resource when in use + ultra_ssd_enabled = anytrue([ + for volume in var.additional_volumes : + volume.type == "UltraSSD_LRS" + ]) + + # /dev/sda /dev/sdb are mounted by default + # azure automatically sets additional mount points starting with /dev/sdc + linux_device_names = [ + "/dev/sdc", + "/dev/sdd", + "/dev/sde", + "/dev/sdf", + "/dev/sdg", + "/dev/sdh", + "/dev/sdi", + "/dev/sdj", + ] +} diff --git a/edbterraform/data/terraform/azure/modules/network/main.tf b/edbterraform/data/terraform/azure/modules/network/main.tf new file mode 100644 index 00000000..93a5ee5e --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/network/main.tf @@ -0,0 +1,21 @@ +resource "azurerm_subnet" "internal" { + name = var.name + resource_group_name = var.resource_name + virtual_network_name = var.network_name + address_prefixes = var.ip_cidr_range + + lifecycle { + precondition { + condition = ( + var.zone == null || + contains(local.regions_with_zones, var.region) + ) + error_message = <<-EOT + Regions with zones are restricted due to limited availability. + Change zone to 0 or select a region with zones: ${jsonencode(local.regions_with_zones)} + For up-to-date regions with available zone support, please visit: + https://learn.microsoft.com/en-us/azure/reliability/availability-zones-service-support#azure-regions-with-availability-zone-support + EOT + } + } +} diff --git a/edbterraform/data/terraform/azure/modules/network/outputs.tf b/edbterraform/data/terraform/azure/modules/network/outputs.tf new file mode 100644 index 00000000..d13ef70a --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/network/outputs.tf @@ -0,0 +1,3 @@ +output "subnet_id" { + value = azurerm_subnet.internal.id +} diff --git a/edbterraform/data/terraform/azure/modules/network/providers.tf b/edbterraform/data/terraform/azure/modules/network/providers.tf new file mode 100644 index 00000000..e8df5670 --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/network/providers.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 3.37.0" + } + } + required_version = ">= 1.3.6" +} diff --git a/edbterraform/data/terraform/azure/modules/network/variables.tf b/edbterraform/data/terraform/azure/modules/network/variables.tf new file mode 100644 index 00000000..4b5b910a --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/network/variables.tf @@ -0,0 +1,83 @@ +variable "name" { + type = string + default = "name" + nullable = false +} +variable "resource_name" { + type = string + default = "resource" + nullable = false +} +variable "network_name" { + type = string + default = "network" + nullable = false +} +variable "ip_cidr_range" { + type = list(string) + default = [] + description = <<-EOT + From Terraform + NOTE: Currently only a single address prefix can be set + as the Multiple Subnet Address Prefixes Feature + is not yet in public preview or general availability. + EOT + nullable = false +} +variable "region" { + type = string +} +variable "zone" { + type = string + default = null + nullable = true + + validation { + condition = ( + # try(..., false) must be used since var.location.zone might be null, + # null is not allowed in contains() function and fails during terraform plan + var.zone == null || + try(contains(["1", "2", "3", ], var.zone), false) + ) + error_message = <<-EOT + Zone must be: 1 - 3 + EOT + } +} + +locals { + regions_with_zones = [ + # Americas + "brazilsouth", + "canadacentral", + "centralus", + "eastus", + "eastus2", + "southcentralus", + # US Gov Virginia not available + "westus2", + "westus3", + # Europe + "francecentral", + "germanywestcentral", + "northeurope", + "norwayeast", + "uksouth", + "westeurope", + "swedencentral", + "switzerlandnorth", + # Middle East + "qatarcentral", + "uaenorth", + # Africa + "southafricanorth", + # Asia Pacific + "australiaeast", + "centralindia", + "japaneast", + "koreacentral", + "southeastasia", + # China North 3 not available + "eastasia", + ] +} diff --git a/edbterraform/data/terraform/azure/modules/security/main.tf b/edbterraform/data/terraform/azure/modules/security/main.tf new file mode 100644 index 00000000..0126fce9 --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/security/main.tf @@ -0,0 +1,52 @@ +resource "azurerm_network_security_group" "firewall" { + name = var.service_name + resource_group_name = var.resource_name + location = var.region +} + +resource "azurerm_network_security_rule" "services" { + count = length(var.service_ports) + + resource_group_name = var.resource_name + network_security_group_name = azurerm_network_security_group.firewall.name + + name = try(var.service_ports[count.index].name, "none") + description = var.service_ports[count.index].description + protocol = var.service_ports[count.index].protocol + destination_port_range = length(tostring(try(var.service_ports[count.index].port, ""))) > 0 ? var.service_ports[count.index].port : "*" + destination_address_prefix = "*" + source_port_range = "*" + source_address_prefix = "*" + access = "Allow" + direction = "Inbound" + priority = 100 + count.index +} + +resource "azurerm_network_security_rule" "regions" { + count = length(var.region_ports) + + resource_group_name = var.resource_name + network_security_group_name = azurerm_network_security_group.firewall.name + + name = try(var.region_ports[count.index].name, "none") + description = var.region_ports[count.index].description + protocol = var.region_ports[count.index].protocol + destination_port_range = length(tostring(try(var.region_ports[count.index].port, ""))) > 0 ? var.region_ports[count.index].port : "*" + destination_address_prefixes = var.region_cidrblocks + source_port_range = "*" + source_address_prefixes = var.region_cidrblocks + access = "Allow" + direction = "Inbound" + priority = 200 + count.index +} + +resource "azurerm_subnet_network_security_group_association" "firewall" { + count = length(try(var.service_ports, var.region_ports, [])) > 0 ? 1 : 0 + + subnet_id = var.subnet_id + network_security_group_id = azurerm_network_security_group.firewall.id + + depends_on = [ + azurerm_network_security_group.firewall + ] +} diff --git a/edbterraform/data/terraform/azure/modules/security/providers.tf b/edbterraform/data/terraform/azure/modules/security/providers.tf new file mode 100644 index 00000000..11b9fef1 --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/security/providers.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 3.37.0" + } + } + required_version = ">= 0.13" +} \ No newline at end of file diff --git a/edbterraform/data/terraform/azure/modules/security/variables.tf b/edbterraform/data/terraform/azure/modules/security/variables.tf new file mode 100644 index 00000000..09b21f9f --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/security/variables.tf @@ -0,0 +1,9 @@ +variable "subnet_id" {} +variable "resource_name" {} +variable "region" {} +variable "service_name" {} +variable "service_ports" {} +variable "region_name" {} +variable "public_cidrblock" {} +variable "region_ports" {} +variable "region_cidrblocks" {} diff --git a/edbterraform/data/terraform/azure/modules/vpc/main.tf b/edbterraform/data/terraform/azure/modules/vpc/main.tf new file mode 100644 index 00000000..08fa0bb2 --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/vpc/main.tf @@ -0,0 +1,13 @@ +resource "azurerm_resource_group" "main" { + name = var.name + location = var.region +} + +resource "azurerm_virtual_network" "main" { + name = "vpc-${var.name}" + location = azurerm_resource_group.main.location + resource_group_name = azurerm_resource_group.main.name + address_space = var.cidr_blocks + + depends_on = [azurerm_resource_group.main] +} diff --git a/edbterraform/data/terraform/azure/modules/vpc/outputs.tf b/edbterraform/data/terraform/azure/modules/vpc/outputs.tf new file mode 100644 index 00000000..8bded5c0 --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/vpc/outputs.tf @@ -0,0 +1,15 @@ +output "resource_name" { + value = azurerm_resource_group.main.name +} + +output "network_name" { + value = azurerm_virtual_network.main.name +} + +output "network_id" { + value = azurerm_virtual_network.main.id +} + +output "region" { + value = azurerm_resource_group.main.location +} diff --git a/edbterraform/data/terraform/azure/modules/vpc/providers.tf b/edbterraform/data/terraform/azure/modules/vpc/providers.tf new file mode 100644 index 00000000..e6896d02 --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/vpc/providers.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 3.37.0" + } + } + required_version = ">= 0.13" +} diff --git a/edbterraform/data/terraform/azure/modules/vpc/variables.tf b/edbterraform/data/terraform/azure/modules/vpc/variables.tf new file mode 100644 index 00000000..202e235c --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/vpc/variables.tf @@ -0,0 +1,7 @@ +variable "region" {} +variable "cidr_blocks" { + type = list(string) + default = [] + nullable = true +} +variable "name" {} diff --git a/edbterraform/data/terraform/azure/modules/vpc_peering/main.tf b/edbterraform/data/terraform/azure/modules/vpc_peering/main.tf new file mode 100644 index 00000000..f622bf2d --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/vpc_peering/main.tf @@ -0,0 +1,6 @@ +resource "azurerm_virtual_network_peering" "peering" { + name = var.peering_name + resource_group_name = var.resource_name + virtual_network_name = var.network_name + remote_virtual_network_id = var.peer_network_id +} diff --git a/edbterraform/data/terraform/azure/modules/vpc_peering/providers.tf b/edbterraform/data/terraform/azure/modules/vpc_peering/providers.tf new file mode 100644 index 00000000..e8df5670 --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/vpc_peering/providers.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 3.37.0" + } + } + required_version = ">= 1.3.6" +} diff --git a/edbterraform/data/terraform/azure/modules/vpc_peering/variables.tf b/edbterraform/data/terraform/azure/modules/vpc_peering/variables.tf new file mode 100644 index 00000000..c0453177 --- /dev/null +++ b/edbterraform/data/terraform/azure/modules/vpc_peering/variables.tf @@ -0,0 +1,4 @@ +variable "peering_name" {} +variable "resource_name" {} +variable "network_name" {} +variable "peer_network_id" {} diff --git a/edbterraform/data/terraform/azure/variables.tf b/edbterraform/data/terraform/azure/variables.tf new file mode 100644 index 00000000..c48879ee --- /dev/null +++ b/edbterraform/data/terraform/azure/variables.tf @@ -0,0 +1,61 @@ +variable "regions" {} +variable "machines" {} +variable "cluster_name" {} +variable "ssh_pub_key" {} +variable "ssh_priv_key" {} +variable "ssh_user" {} +variable "operating_system" {} + +variable "source_ranges" { + default = "0.0.0.0/0" +} + +variable "public_cidrblock" { + default = "0.0.0.0/0" +} + +# Tags +variable "project_name" { + default = "edb" +} + +variable "environment" { + description = "Environment name" + type = string + default = "azure-dev" +} + +variable "resource_tags" { + description = "A map of tags to add to all resources." + type = map(string) + default = {} +} + +variable "subnetwork_name" { + # Must have network_name tag as a prefix + default = "edb-network-subnetwork" +} + +variable "project_tag" { + type = string + default = "edb-gcloud-deployment" +} + +# Subnets +variable "subnet_name" { + default = "edb-public-subnet" +} + +variable "subnet_tag" { + default = "edb-public-subnet" +} + +variable "vpc_tag" { + default = "edb-vpc" +} + +variable "created_by" { + type = string + description = "EDB terraform Azure" + default = "EDB terraform Azure" +} diff --git a/edbterraform/lib.py b/edbterraform/lib.py index 6b8c19ae..0dc0bf21 100644 --- a/edbterraform/lib.py +++ b/edbterraform/lib.py @@ -85,8 +85,9 @@ def generate_ssh_key_pair(dir): def create_project_dir(dir, csp): - # Creates a new terraform project (directory) and copy terraform modules - # into this directory. + # Creates a new terraform project directory and + # copies terraform modules with variable.tf from + # cloud service provider directory into this project directory. if os.path.exists(dir): sys.exit("ERROR: directory %s already exists" % dir) @@ -265,7 +266,7 @@ def new_project_main(): '--cloud-service-provider', '-c', metavar='CLOUD_SERVICE_PROVIDER', dest='csp', - choices=['aws', 'gcloud'], + choices=['aws', 'gcloud', 'azure'], default='aws', help="Cloud Service Provider. Default: %(default)s" ) diff --git a/infrastructure-examples/azure-vms.yml b/infrastructure-examples/azure-vms.yml new file mode 100644 index 00000000..539fa829 --- /dev/null +++ b/infrastructure-examples/azure-vms.yml @@ -0,0 +1,60 @@ +--- +cluster_name: azure-infra +azure: + ssh_user: rocky + operating_system: + publisher: "erockyenterprisesoftwarefoundationinc1653071250513" + offer: "rockylinux" + sku: "free" + version: "8.6.0" + regions: + westus: + cidr_block: 10.1.0.0/16 + zones: + 0: 10.1.20.0/24 + service_ports: + - name: ssh_access + port: 22 + protocol: Tcp + description: "SSH" + westus3: + cidr_block: 10.3.0.0/16 + zones: + 2: 10.3.20.0/24 + 3: 10.3.30.0/24 + service_ports: + - name: ssh_access + port: 22 + protocol: Tcp + description: "SSH" + region_ports: + - name: ping_access + protocol: Icmp + description: "ping" + machines: + dbt2-driver: + type: dbt2-driver + region: westus + zone: 0 + instance_type: Standard_D2as_v4 + volume: + type: StandardSSD_LRS + size_gb: 50 + pg1: + type: postgres + region: westus3 + zone: 2 + instance_type: Standard_D2as_v4 + volume: + type: StandardSSD_LRS + size_gb: 50 + additional_volumes: + - mount_point: /opt/pg_data + type: UltraSSD_LRS + size_gb: 50 + iops: 1000 + - mount_point: /opt/pg_wal + type: UltraSSD_LRS + size_gb: 50 + iops: 1000 +