From 3f2b1b1073eef4a5531772342946e3e670630f8f Mon Sep 17 00:00:00 2001 From: MiguelNdeCarvalho Date: Thu, 12 May 2022 15:43:05 +0100 Subject: [PATCH 01/16] terrabutler: settings.py: Add name to default environment --- terrabutler/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/terrabutler/settings.py b/terrabutler/settings.py index c1cc609..498197d 100644 --- a/terrabutler/settings.py +++ b/terrabutler/settings.py @@ -22,6 +22,7 @@ "environments": { "default": { "domain": str, + "name": str, "profile_name": str, "region": str }, From b0b157d70d310c7ba7092438f067c4207ba70a3a Mon Sep 17 00:00:00 2001 From: MiguelNdeCarvalho Date: Thu, 12 May 2022 15:45:03 +0100 Subject: [PATCH 02/16] terrabutler: env.py: Use default env name and org variable from settings --- terrabutler/env.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/terrabutler/env.py b/terrabutler/env.py index c78ed10..a6ccdb5 100644 --- a/terrabutler/env.py +++ b/terrabutler/env.py @@ -6,7 +6,9 @@ import os import subprocess -# Values from Config +# Values from Settings +org = get_settings()["general"]["organization"] +default_env_name = get_settings()["environments"]["default"]["name"] backend_dir = os.path.realpath(get_settings()["locations"]["backend_dir"]) environment_file = os.path.realpath(get_settings() ["locations"]["environment_file"]) @@ -55,7 +57,6 @@ def delete_env(env, confirmation, destroy, s3): from terrabutler.tf import terraform_destroy_all_sites available_envs = get_available_envs(s3) current_env = get_current_env() - org = get_settings()["general"]["organization"] permanent_environments = get_settings()["environments"]["permanent"] if env not in available_envs: @@ -146,7 +147,8 @@ def get_available_envs(s3): directory = inception_dir subprocess.run(args=["terraform", "init", "-reconfigure", "-backend-config", - f"{environment_file}/pl-dev-inception.tfvars"], + f"{backend_dir}/{org}-{default_env_name}-" + "inception.tfvars"], cwd=directory, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) From 8a27a9c729cb19dfaaaafd41f655eabc15b4c168 Mon Sep 17 00:00:00 2001 From: MiguelNdeCarvalho Date: Thu, 12 May 2022 15:53:42 +0100 Subject: [PATCH 03/16] terrabutler: env.py: Use variables on get_available_envs() --- terrabutler/env.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/terrabutler/env.py b/terrabutler/env.py index a6ccdb5..2a7245f 100644 --- a/terrabutler/env.py +++ b/terrabutler/env.py @@ -131,9 +131,10 @@ def get_available_envs(s3): # Get Environments by accessing S3 if s3: - dev_env = boto3.session.Session(profile_name="pl-dev") + dev_env = boto3.session.Session(profile_name=f"{org}" + f"-{default_env_name}") s3 = dev_env.resource("s3") - bucket = s3.Bucket("pl-dev-site-inception-tfstate") + bucket = s3.Bucket(f"{org}-{default_env_name}-site-inception-tfstate") envs = [] From 1677e2f95a4f43bf96bfd3f7881477965b3da3c9 Mon Sep 17 00:00:00 2001 From: MiguelNdeCarvalho Date: Thu, 12 May 2022 16:11:45 +0100 Subject: [PATCH 04/16] terrabutler: inception.py: Fix hardcodes --- terrabutler/inception.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/terrabutler/inception.py b/terrabutler/inception.py index 8394842..8673cb6 100644 --- a/terrabutler/inception.py +++ b/terrabutler/inception.py @@ -23,21 +23,24 @@ def inception_init_needed(): def inception_init(): from terrabutler.env import reload_direnv from terrabutler.settings import get_settings - site_dir = path.realpath(get_settings()["locations"]["inception_dir"]) + org = get_settings()["general"]["organization"] + default_env_name = get_settings()["environments"]["default"]["name"] + inception_dir = path.realpath(get_settings()["locations"]["inception_dir"]) backend_dir = path.realpath(get_settings()["locations"]["backend_dir"]) if not inception_init_check(): try: subprocess.run(args=["terraform", "init", "-backend-config", - f"{backend_dir}/pl-dev-inception.tfvars"], - cwd=site_dir, stdout=subprocess.DEVNULL, + f"{backend_dir}/{org}-{default_env_name}" + "-inception.tfvars"], + cwd=inception_dir, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) except subprocess.CalledProcessError: print(Fore.RED + "There was an error while doing the initializing") exit(1) try: - with open(f"{site_dir}/.terraform/environment", "w") as f: + with open(f"{inception_dir}/.terraform/environment", "w") as f: f.write("dev") except FileNotFoundError: print(Fore.RED + "The file that manages the environments could not" From 78ec2dbaae1d402e0a3da76d55d16bce8a726165 Mon Sep 17 00:00:00 2001 From: MiguelNdeCarvalho Date: Thu, 12 May 2022 16:56:40 +0100 Subject: [PATCH 05/16] terrabutler: inception.py: Use variables --- terrabutler/inception.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/terrabutler/inception.py b/terrabutler/inception.py index 8673cb6..81b4c1b 100644 --- a/terrabutler/inception.py +++ b/terrabutler/inception.py @@ -1,13 +1,14 @@ +from terrabutler.settings import get_settings from colorama import Fore from os import path import subprocess def inception_init_check(): - site_dir = path.realpath("site_inception") + dir = path.realpath(get_settings()["locations"]["inception_dir"]) - if (path.exists(f"{site_dir}/.terraform") and - path.exists(f"{site_dir}/.terraform/environment")): + if (path.exists(f"{dir}/.terraform") and + path.exists(f"{dir}/.terraform/environment")): return True return False From c952a83b60035c3f2ab2ec236f2c435d58a748d0 Mon Sep 17 00:00:00 2001 From: MiguelNdeCarvalho Date: Thu, 12 May 2022 16:57:15 +0100 Subject: [PATCH 06/16] terrabutler: More hardcode fixes --- terrabutler/tf.py | 9 +++++---- terrabutler/variables.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/terrabutler/tf.py b/terrabutler/tf.py index d2c1f37..85fef5f 100644 --- a/terrabutler/tf.py +++ b/terrabutler/tf.py @@ -5,6 +5,7 @@ from terrabutler.settings import get_settings # Values from Config +org = get_settings()["general"]["organization"] backend_dir = os.path.realpath(get_settings()["locations"]["backend_dir"]) environment_file = os.path.realpath(get_settings() ["locations"]["environment_file"]) @@ -51,15 +52,15 @@ def terraform_args_builder(needed_args, site, backend_dir, var_dir): if needed_args == "backend": if site == "inception": return ["-backend-config", - f"{backend_dir}/pl-dev-inception.tfvars"] + f"{backend_dir}/{org}-dev-inception.tfvars"] else: return ["-backend-config", - f"{backend_dir}/pl-{env}-{site}.tfvars"] + f"{backend_dir}/{org}-{env}-{site}.tfvars"] elif needed_args == "var": return ["-var-file", f"{variables_dir}/global.tfvars", - "-var-file", f"{variables_dir}/pl-{env}.tfvars", - "-var-file", f"{variables_dir}/pl-{env}-{site}.tfvars" + "-var-file", f"{variables_dir}/{org}-{env}.tfvars", + "-var-file", f"{variables_dir}/{org}-{env}-{site}.tfvars" ] return [] diff --git a/terrabutler/variables.py b/terrabutler/variables.py index 745f7bf..d72c56d 100644 --- a/terrabutler/variables.py +++ b/terrabutler/variables.py @@ -68,7 +68,7 @@ def encrypt_password(password): """ Encrypt password with AWS KMS """ - environment = boto3.session.Session(profile_name="pl-dev", + environment = boto3.session.Session(profile_name=f"{ORG}-dev", region_name=REGION) kms = environment.client("kms") encrypted = kms.encrypt(KeyId=KEY_ID, Plaintext=password) From 4ffbc8cc71095f6106e512011de86702d59cd3b9 Mon Sep 17 00:00:00 2001 From: MiguelNdeCarvalho Date: Thu, 12 May 2022 17:35:59 +0100 Subject: [PATCH 07/16] terrabutler: Add utils.py --- terrabutler/utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 terrabutler/utils.py diff --git a/terrabutler/utils.py b/terrabutler/utils.py new file mode 100644 index 0000000..a44b8bd --- /dev/null +++ b/terrabutler/utils.py @@ -0,0 +1,12 @@ +from os import getenv + +ROOT_PATH = getenv("TERRABUTLER_ROOT") + +paths = { + "root": ROOT_PATH, + "backends": ROOT_PATH + "/configs/backends", + "templates": ROOT_PATH + "/configs/templates", + "variables": ROOT_PATH + "/configs/variables", + "environment": ROOT_PATH + "/site_inception/.terraform/environment", + "inception": ROOT_PATH + "/inception", +} From 23610a5632591c4992cce81c22ab69666c954527 Mon Sep 17 00:00:00 2001 From: MiguelNdeCarvalho Date: Thu, 12 May 2022 17:37:02 +0100 Subject: [PATCH 08/16] terrabutler: utils.py: Fix inception path --- terrabutler/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terrabutler/utils.py b/terrabutler/utils.py index a44b8bd..b0edaec 100644 --- a/terrabutler/utils.py +++ b/terrabutler/utils.py @@ -8,5 +8,5 @@ "templates": ROOT_PATH + "/configs/templates", "variables": ROOT_PATH + "/configs/variables", "environment": ROOT_PATH + "/site_inception/.terraform/environment", - "inception": ROOT_PATH + "/inception", + "inception": ROOT_PATH + "/site_inception", } From 376abc9a45b34715bc72a9136755acce4a9399eb Mon Sep 17 00:00:00 2001 From: MiguelNdeCarvalho Date: Thu, 12 May 2022 17:40:03 +0100 Subject: [PATCH 09/16] terrabutler: env.py: Use paths for locations --- terrabutler/env.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/terrabutler/env.py b/terrabutler/env.py index 2a7245f..c26b0d6 100644 --- a/terrabutler/env.py +++ b/terrabutler/env.py @@ -1,5 +1,6 @@ from terrabutler.settings import get_settings from terrabutler.tf import terraform_init_all_sites +from terrabutler.utils import paths from click import confirm from colorama import Fore import boto3 @@ -9,12 +10,6 @@ # Values from Settings org = get_settings()["general"]["organization"] default_env_name = get_settings()["environments"]["default"]["name"] -backend_dir = os.path.realpath(get_settings()["locations"]["backend_dir"]) -environment_file = os.path.realpath(get_settings() - ["locations"]["environment_file"]) -inception_dir = os.path.realpath(get_settings()["locations"]["inception_dir"]) -templates_dir = os.path.realpath(get_settings()["locations"]["templates_dir"]) -variables_dir = os.path.realpath(get_settings()["locations"]["variables_dir"]) def create_env(env, confirmation, temporary, apply, s3): @@ -31,7 +26,7 @@ def create_env(env, confirmation, temporary, apply, s3): " environment?", default=False): try: subprocess.run(args=['terraform', 'workspace', 'new', env], - cwd=inception_dir, stdout=subprocess.DEVNULL, + cwd=paths["inception"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True) except subprocess.CalledProcessError: print(Fore.RED + "There was an error while creating the new" @@ -77,12 +72,12 @@ def delete_env(env, confirmation, destroy, s3): " environment?", default=False): if destroy and not is_protected_env(env): terraform_destroy_all_sites() # Destroy all sites - for file in os.listdir(variables_dir): + for file in os.listdir(paths["variables"]): if file.startswith(f"{org}-{env}"): - os.remove(os.path.join(variables_dir, file)) + os.remove(os.path.join(paths["variables"], file)) try: subprocess.run(args=['terraform', 'workspace', 'delete', env], - cwd=inception_dir, stdout=subprocess.DEVNULL, + cwd=paths["inception"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True) except subprocess.CalledProcessError: print(Fore.RED + f"There was an error while deleting the '{env}'" @@ -95,7 +90,7 @@ def delete_env(env, confirmation, destroy, s3): def get_current_env(): - with open(environment_file, 'r') as f: + with open(paths["environment"], 'r') as f: return f.read() @@ -113,7 +108,7 @@ def set_current_env(env, s3): exit(1) else: try: - with open(environment_file, "w") as f: + with open(paths["environment"], "w") as f: f.write(env) except FileNotFoundError: print(Fore.RED + "The file that manages the environments could not" @@ -145,10 +140,10 @@ def get_available_envs(s3): return envs # Get Environments by accessing the .terraform/environment file - directory = inception_dir + directory = paths["inception"] subprocess.run(args=["terraform", "init", "-reconfigure", "-backend-config", - f"{backend_dir}/{org}-{default_env_name}-" + f"{paths['backends']}/{org}-{default_env_name}-" "inception.tfvars"], cwd=directory, stdout=subprocess.DEVNULL, From 6f7915d2f364af3a36311edba44c007ed2a41019 Mon Sep 17 00:00:00 2001 From: MiguelNdeCarvalho Date: Thu, 12 May 2022 17:42:04 +0100 Subject: [PATCH 10/16] terrabutler: inception.py: Use paths --- terrabutler/inception.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/terrabutler/inception.py b/terrabutler/inception.py index 81b4c1b..59de398 100644 --- a/terrabutler/inception.py +++ b/terrabutler/inception.py @@ -1,11 +1,11 @@ -from terrabutler.settings import get_settings +from terrabutler.utils import paths from colorama import Fore from os import path import subprocess def inception_init_check(): - dir = path.realpath(get_settings()["locations"]["inception_dir"]) + dir = paths["inception"] if (path.exists(f"{dir}/.terraform") and path.exists(f"{dir}/.terraform/environment")): @@ -26,8 +26,8 @@ def inception_init(): from terrabutler.settings import get_settings org = get_settings()["general"]["organization"] default_env_name = get_settings()["environments"]["default"]["name"] - inception_dir = path.realpath(get_settings()["locations"]["inception_dir"]) - backend_dir = path.realpath(get_settings()["locations"]["backend_dir"]) + inception_dir = paths["inception"] + backend_dir = paths["backends"] if not inception_init_check(): try: From d4c057322cfb6096ea5b627669c2abfd4de940d6 Mon Sep 17 00:00:00 2001 From: MiguelNdeCarvalho Date: Thu, 12 May 2022 17:43:37 +0100 Subject: [PATCH 11/16] terrabutler: utils.py: Add settings --- terrabutler/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/terrabutler/utils.py b/terrabutler/utils.py index b0edaec..0901eb9 100644 --- a/terrabutler/utils.py +++ b/terrabutler/utils.py @@ -3,10 +3,11 @@ ROOT_PATH = getenv("TERRABUTLER_ROOT") paths = { - "root": ROOT_PATH, "backends": ROOT_PATH + "/configs/backends", - "templates": ROOT_PATH + "/configs/templates", - "variables": ROOT_PATH + "/configs/variables", "environment": ROOT_PATH + "/site_inception/.terraform/environment", "inception": ROOT_PATH + "/site_inception", + "root": ROOT_PATH, + "settings": ROOT_PATH + "configs/settings.yml", + "templates": ROOT_PATH + "/configs/templates", + "variables": ROOT_PATH + "/configs/variables" } From ad99a03500f8cd7b35692067d936aa74cdbb8385 Mon Sep 17 00:00:00 2001 From: MiguelNdeCarvalho Date: Thu, 12 May 2022 17:44:51 +0100 Subject: [PATCH 12/16] terrabutler: settings.py: Use paths --- terrabutler/settings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/terrabutler/settings.py b/terrabutler/settings.py index 498197d..c932fe3 100644 --- a/terrabutler/settings.py +++ b/terrabutler/settings.py @@ -1,9 +1,10 @@ +from terrabutler import paths from colorama import Fore -from os import getenv, path +from os import path from schema import Schema, SchemaError import yaml -PATH = path.realpath(getenv("TERRABUTLER_ROOT") + "configs/settings.yml") +PATH = paths["settings"] SCHEMA = Schema({ "general": { "organization": str, From af041592273c253db5b3e7551700b84d54eed47d Mon Sep 17 00:00:00 2001 From: MiguelNdeCarvalho Date: Thu, 12 May 2022 17:48:47 +0100 Subject: [PATCH 13/16] terrabutler: tf.py: Use paths function --- terrabutler/tf.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/terrabutler/tf.py b/terrabutler/tf.py index 85fef5f..8b86562 100644 --- a/terrabutler/tf.py +++ b/terrabutler/tf.py @@ -3,15 +3,10 @@ import subprocess from colorama import Fore from terrabutler.settings import get_settings +from terrabutler.utils import paths # Values from Config org = get_settings()["general"]["organization"] -backend_dir = os.path.realpath(get_settings()["locations"]["backend_dir"]) -environment_file = os.path.realpath(get_settings() - ["locations"]["environment_file"]) -inception_dir = os.path.realpath(get_settings()["locations"]["inception_dir"]) -templates_dir = os.path.realpath(get_settings()["locations"]["templates_dir"]) -variables_dir = os.path.realpath(get_settings()["locations"]["variables_dir"]) def setup_tfenv(site): @@ -37,8 +32,8 @@ def terraform_args_print(command, site): elif command == "plan" or command == "apply": needed_args = "var" - args = terraform_args_builder(needed_args, site, backend_dir, - variables_dir) + args = terraform_args_builder(needed_args, site, paths["backends"], + paths["variables"]) return " ".join(args) @@ -58,9 +53,9 @@ def terraform_args_builder(needed_args, site, backend_dir, var_dir): f"{backend_dir}/{org}-{env}-{site}.tfvars"] elif needed_args == "var": - return ["-var-file", f"{variables_dir}/global.tfvars", - "-var-file", f"{variables_dir}/{org}-{env}.tfvars", - "-var-file", f"{variables_dir}/{org}-{env}-{site}.tfvars" + return ["-var-file", f"{paths['variables']}/global.tfvars", + "-var-file", f"{paths['variables']}/{org}-{env}.tfvars", + "-var-file", f"{paths['variables']}/{org}-{env}-{site}.tfvars" ] return [] @@ -91,7 +86,7 @@ def terraform_command_runner(command, args, needed_args, site): setup_tfenv(site_dir) command = terraform_command_builder(command, args, needed_args, site, - backend_dir, variables_dir) + paths["backends"], paths["variables"]) try: p = subprocess.Popen(args=command, cwd=site_dir) p.wait() From 702f56185755fe13b8a194c9a48d55a785b9cd54 Mon Sep 17 00:00:00 2001 From: MiguelNdeCarvalho Date: Thu, 12 May 2022 17:50:38 +0100 Subject: [PATCH 14/16] terrabutler: variables.py: Use paths --- terrabutler/variables.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/terrabutler/variables.py b/terrabutler/variables.py index d72c56d..e514e4f 100644 --- a/terrabutler/variables.py +++ b/terrabutler/variables.py @@ -14,11 +14,10 @@ digits ) from terrabutler.settings import get_settings +from terrabutler.utils import paths REGION = get_settings()["environments"]["default"]["region"] -TEMPLATES_DIR = path.realpath(get_settings()["locations"]["templates_dir"]) -VARIABLES_DIR = path.realpath(get_settings()["locations"]["variables_dir"]) ORG = get_settings()["general"]["organization"] KEY_ID = get_settings()["general"]["secrets_key_id"] @@ -27,8 +26,8 @@ def generate_var_files(env): """ Create a variables files for a given environment """ - templates = listdir(TEMPLATES_DIR) - file_loader = FileSystemLoader(TEMPLATES_DIR) + templates = listdir(paths["templates"]) + file_loader = FileSystemLoader(paths["templates"]) environment = Environment(loader=file_loader) sites = list(get_settings()["sites"]["ordered"]) firebase_credentials = (get_settings()["environments"]["temporary"] @@ -48,10 +47,10 @@ def generate_var_files(env): firebase_credentials=firebase_credentials) name = template.replace(".j2", "") if name == 'env': - with open(f"{VARIABLES_DIR}/{ORG}-{env}.tfvars", "w") as fh: + with open(f"{paths['variables']}/{ORG}-{env}.tfvars", "w") as fh: fh.write(output) else: - with open(f"{VARIABLES_DIR}/{ORG}-{env}-{name}.tfvars", "w")as fh: + with open(f"{paths['variables']}/{ORG}-{env}-{name}.tfvars", "w")as fh: fh.write(output) From 0b8d00967cb96daaf3c4ac83dad0d8cce4bdf2f8 Mon Sep 17 00:00:00 2001 From: MiguelNdeCarvalho Date: Thu, 12 May 2022 17:51:14 +0100 Subject: [PATCH 15/16] terrabutler: settings.py: Remove locations from YAML --- terrabutler/settings.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/terrabutler/settings.py b/terrabutler/settings.py index c932fe3..30c406e 100644 --- a/terrabutler/settings.py +++ b/terrabutler/settings.py @@ -10,13 +10,6 @@ "organization": str, "secrets_key_id": str }, - "locations": { - "backend_dir": str, - "environment_file": str, - "inception_dir": str, - "templates_dir": str, - "variables_dir": str - }, "sites": { "ordered": list }, From 66d5e8d6c5e7f862997cac39ef6aec9e92c4cd3e Mon Sep 17 00:00:00 2001 From: MiguelNdeCarvalho Date: Thu, 12 May 2022 17:53:12 +0100 Subject: [PATCH 16/16] terrabutler: variables.py: Fix unused import and line too long --- terrabutler/variables.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/terrabutler/variables.py b/terrabutler/variables.py index e514e4f..bb8e70d 100644 --- a/terrabutler/variables.py +++ b/terrabutler/variables.py @@ -1,7 +1,4 @@ -from os import ( - listdir, - path -) +from os import listdir import boto3 from base64 import b64encode from jinja2 import ( @@ -50,7 +47,8 @@ def generate_var_files(env): with open(f"{paths['variables']}/{ORG}-{env}.tfvars", "w") as fh: fh.write(output) else: - with open(f"{paths['variables']}/{ORG}-{env}-{name}.tfvars", "w")as fh: + with open(f"{paths['variables']}/{ORG}-{env}-{name}" + ".tfvars", "w") as fh: fh.write(output)