Merge pull request #44 from nyarly/init-access-the-data

Init access the data

documentation/terraform-add-new-project.md

@@ -0,0 +1,22 @@
+# Adding a new Project to Terraform
+
+* Fork Incubator (if you haven't)
+* Pull main branch
+* Create feature branch
+
+```shell
+> mkdir -p terraform-incubator/{projectname}/project terraform-incubator/{projectname}/dev
+```
+
+...
+
+* Commit
+* Push
+* Create PR to Incubator
+
+```
+aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 035866691871.dkr.ecr.us-west-2.amazonaws.com
+```
+
+
+## ACM?

flake.nix

@@ -17,6 +17,7 @@
         terraform
         terragrunt
         tfautomv
+        ssm-session-manager-plugin
       ];
       GIT_TEMPLATE_DIR="";
     };

terraform-incubator/access-the-data/main.tf

@@ -0,0 +1,148 @@
+locals {
+  // we use tf to create the zone, but other projects might
+  // have an existing zone and get it with a data block
+  zone_id = module.zone.zone_id
+
+  envs = {
+    dev = {
+      environment = "dev"
+      host_names  = ["dev"]
+      container_env = {
+        CKAN_SITE_URL = "https://dev.accessthedata.org"
+      }
+    }
+  }
+}
+
+module "zone" {
+  source = "../../terraform-modules/project-zone"
+
+  zone_name            = "accessthedata.org"
+  github_at_apex       = true
+  shared_configuration = local.shared_configuration
+}
+
+module "database" {
+  for_each = local.envs
+
+  source = "../../terraform-modules/database"
+
+  shared_configuration = local.shared_configuration
+  environment          = each.value.environment
+  db_name              = "accessthedata"
+  owner_name           = "ckan"
+}
+
+module "datastore_database" {
+  for_each = local.envs
+
+  source = "../../terraform-modules/database"
+
+  shared_configuration = local.shared_configuration
+  environment          = each.value.environment
+  db_name              = "accessthedata_datastore"
+  owner_name           = "ckands"
+  viewer_name          = "ckands_ro"
+}
+
+module "secrets" {
+  for_each     = local.envs
+  source       = "../../terraform-modules/cheap-secrets"
+  scope-name   = "ckan-${each.key}"
+  secret-names = ["csrf", "admin-password"]
+}
+
+module "access-the-data" {
+  for_each = local.envs
+
+  source = "../../terraform-modules/multi-container-service"
+
+  shared_configuration = local.shared_configuration
+
+  region           = "us-west-2"
+  project_name     = "access-the-data"
+  application_type = "fullstack"
+  environment      = each.value.environment
+  zone_id          = local.zone_id
+
+  vpc_cidr = "10.10.0.0/16"
+
+  containers = {
+    ckan = {
+      tag    = "latest"
+      cpu    = 256
+      memory = 512
+      port   = 80
+
+      subdomains    = each.value.host_names
+      path_patterns = ["/*"]
+      env_vars = merge({
+        DATABASE      = "postgres"
+        POSTGRES_HOST = module.database[each.key].host
+        POSTGRES_PORT = module.database[each.key].port
+
+        // SQLALCHEMY has been set up in the container =
+        // we don't know the PG password, so we can't build the URLs
+
+        # Taken verbatim from .env
+        CKAN_DB                  = module.database[each.key].database
+        CKAN_DB_USER             = module.database[each.key].owner
+        CKAN_DATASTORE_DB        = module.datastore_database[each.key].database
+        CKAN_DATASTORE_DB_RWUSER = module.datastore_database[each.key].owner
+        CKAN_DATASTORE_DB_ROUSER = module.datastore_database[each.key].viewer
+        CKAN_VERSION             = "2.10.0"
+        CKAN_SITE_ID             = "default"
+
+        CKAN_PORT      = "5000"
+        CKAN_PORT_HOST = "5000"
+
+        CKAN_SYSADMIN_NAME  = "ckan_admin"
+        CKAN_SYSADMIN_EMAIL = "[email protected]"
+        CKAN_STORAGE_PATH   = "/var/lib/ckan"
+
+        CKAN_SMTP_SERVER    = "smtp.hackforla.org:25"
+        CKAN_SMTP_STARTTLS  = "True"
+        CKAN_SMTP_USER      = "user"
+        CKAN_SMTP_PASSWORD  = "pass"
+        CKAN_SMTP_MAIL_FROM = "ckan@localhost"
+
+        CKAN_SOLR_URL                       = "http://solr:8983/solr/ckan"
+        CKAN_REDIS_URL                      = "redis://redis:6379/1"
+        CKAN_DATAPUSHER_URL                 = "http://datapusher:8800"
+        CKAN__DATAPUSHER__CALLBACK_URL_BASE = "http://ckan:5000"
+        CKAN__HARVEST__MQ__HOSTNAME         = "redis"
+
+        CKAN__PLUGINS               = "envvars image_view text_view recline_view datastore datapusher ckanext_hack4laatd"
+        CKAN__HARVEST__MQ__TYPE     = "redis"
+        CKAN__HARVEST__MQ__PORT     = "6379"
+        CKAN__HARVEST__MQ__REDIS_DB = "1"
+        CKAN__FAVICON               = "favicon.png"
+      }, lookup(each.value.container_env, "ckan", {}))
+      secrets = {
+        CKAN_DB_PASSWORD               = module.database[each.key].owner_password_arn
+        CKAN_DATASTORE_DB_RWPASSWORD   = module.datastore_database[each.key].owner_password_arn
+        CKAN_DATASTORE_DB_ROPASSWORD   = module.datastore_database[each.key].viewer_password_arn
+        CKAN___BEAKER__SESSION__SECRET = module.secrets[each.key].arn["csrf"]
+        CKAN_SYSADMIN_PASSWORD         = module.secrets[each.key].arn["admin-password"]
+      }
+    }
+
+    datapusher = {
+      tag    = "latest"
+      cpu    = 256
+      memory = 512
+    }
+
+    solr = {
+      tag    = "latest"
+      cpu    = 512
+      memory = 4096
+    }
+
+    redis = {
+      tag    = "latest"
+      cpu    = 256
+      memory = 512
+    }
+  }
+}

terraform-incubator/access-the-data/moves.tf

@@ -0,0 +1,8 @@
+moved {
+  from = aws_route53_record.apex
+  to   = module.zone.aws_route53_record.apex
+}
+moved {
+  from = aws_route53_zone.this
+  to   = module.zone.aws_route53_zone.this
+}

terraform-incubator/access-the-data/versions.tf

@@ -0,0 +1,42 @@
+// Get configuration from the shared infrastructure
+data "terraform_remote_state" "shared" {
+  backend = "s3"
+
+  config = {
+    bucket         = "hlfa-incubator-terragrunt"
+    dynamodb_table = "terraform-locks"
+    encrypt        = true
+    key            = "terragrunt-states/incubator/./terraform.tfstate"
+    region         = "us-west-2"
+  }
+}
+
+locals {
+  shared_configuration = data.terraform_remote_state.shared.outputs.configuration
+}
+
+provider "aws" {
+  region = "us-west-2"
+}
+
+// Set up Postgres provider to create the database
+terraform {
+  required_providers {
+    postgresql = {
+      source  = "cyrilgdn/postgresql"
+      version = "~> 1.21.0"
+    }
+  }
+}
+data "aws_ssm_parameter" "rds_credentials" {
+  name = "rds_credentials"
+}
+data "aws_db_instance" "shared" {
+  db_instance_identifier = local.shared_configuration.db_identifier
+}
+provider "postgresql" {
+  host      = data.aws_db_instance.shared.address
+  password  = data.aws_ssm_parameter.rds_credentials.value
+  username  = "postgres"
+  superuser = false
+}

terraform-incubator/people-depot/project/main.tf

@@ -16,7 +16,7 @@ module "people_depot" {
   container_cpu   = 256
   aws_managed_dns = false
   container_env_vars = {
-    SQL_HOST          = "incubator-prod-database.cewewwrvdqjn.us-west-2.rds.amazonaws.com"
+    SQL_HOST          = data.terraform_remote_state.shared.outputs.db_instance_endpoint
     COGNITO_USER_POOL = "us-west-2_Fn4rkZpuB"
 
     COGNITO_AWS_REGION   = "us-west-2"

terraform-incubator/shared_resources/acm/main.tf

@@ -0,0 +1,25 @@
+terraform {
+  backend "s3" {
+    bucket         = "hlfa-incubator-terragrunt"
+    dynamodb_table = "terraform-locks"
+    encrypt        = true
+    key            = "terragrunt-states/incubator/acm/terraform.tfstate"
+    region         = "us-west-2"
+  }
+}
+
+provider "aws" {
+  region = "us-west-2"
+}
+
+module "acm" {
+  source = "../../../terraform-modules/acm"
+
+  #domain_names = ["ballotnav.org", "civictechindex.org", "vrms.io", "homeunite.us"]
+  domain_names = ["ballotnav.org", "civictechindex.org", "vrms.io"]
+  tags         = { terraform_managed = "true", last_changed = formatdate("EEE YYYY-MMM-DD hh:mm:ss", timestamp()) }
+}
+
+output "acm_certificate_arns" {
+  value = module.acm.acm_certificate_arns
+}

terraform-incubator/shared_resources/alb/main.tf

@@ -0,0 +1,74 @@
+terraform {
+  backend "s3" {
+    bucket         = "hlfa-incubator-terragrunt"
+    dynamodb_table = "terraform-locks"
+    encrypt        = true
+    key            = "terragrunt-states/incubator/alb/terraform.tfstate"
+    region         = "us-west-2"
+  }
+}
+
+provider "aws" {
+  region = "us-west-2"
+}
+
+data "terraform_remote_state" "shared" {
+  for_each = toset(["network", "acm"])
+  backend  = "s3"
+
+  config = {
+    bucket         = "hlfa-incubator-terragrunt"
+    dynamodb_table = "terraform-locks"
+    encrypt        = true
+    key            = "terragrunt-states/incubator/${each.key}/terraform.tfstate"
+    region         = "us-west-2"
+  }
+}
+
+module "alb" {
+  source = "../../../terraform-modules/applicationlb"
+
+  vpc_id               = data.terraform_remote_state.shared["network"].outputs.vpc_id
+  public_subnet_ids    = data.terraform_remote_state.shared["network"].outputs.public_subnet_ids
+  acm_certificate_arns = data.terraform_remote_state.shared["acm"].outputs.acm_certificate_arns
+
+  // Input from Variables
+  environment     = "prod"
+  region          = "us-west-2"
+  resource_name   = "incubator"
+  default_alb_url = "www.hackforla.org"
+
+  tags = { terraform_managed = "true", last_changed = formatdate("EEE YYYY-MMM-DD hh:mm:ss", timestamp()) }
+}
+
+output "alb_id" {
+  value = module.alb.alb_id
+}
+
+output "security_group_id" {
+  value = module.alb.security_group_id
+}
+
+output "lb_dns_name" {
+  value = module.alb.lb_dns_name
+}
+
+output "lb_zone_id" {
+  value = module.alb.lb_zone_id
+}
+
+output "lb_arn" {
+  value = module.alb.lb_arn
+}
+
+output "alb_target_group_arn" {
+  value = module.alb.alb_target_group_arn
+}
+
+output "alb_target_group_id" {
+  value = module.alb.alb_target_group_arn
+}
+
+output "alb_https_listener_arn" {
+  value = module.alb.alb_https_listener_arn
+}

terraform-incubator/shared_resources/alb/moves.tf

@@ -0,0 +1,20 @@
+moved {
+  from = aws_lb.alb
+  to   = module.alb.aws_lb.alb
+}
+moved {
+  from = aws_lb_listener.http_redirect
+  to   = module.alb.aws_lb_listener.http_redirect
+}
+moved {
+  from = aws_lb_listener.ssl
+  to   = module.alb.aws_lb_listener.ssl
+}
+moved {
+  from = aws_lb_listener_certificate.example["arn:aws:acm:us-west-2:035866691871:certificate/4db5d979-9797-4689-a9e9-58b7ac55c79d"]
+  to   = module.alb.aws_lb_listener_certificate.example["arn:aws:acm:us-west-2:035866691871:certificate/4db5d979-9797-4689-a9e9-58b7ac55c79d"]
+}
+moved {
+  from = aws_security_group.alb
+  to   = module.alb.aws_security_group.alb
+}