Create template files for the setting up a new project on Incubator

[robinglov]

Overview

Create template files for the setting up a new project on Incubator.

Action Items

• Write template versions.tf and main.tf file templates
  • Copy text to a comment and ask Judson or other senior engineer to edit
  • edit draft comments
  • remove label: role: Dev Ops Engineer
  • add label role: product
  • unassign yourself
  • add the issue to the questions/review column of project board
• Set up a folder called examples/sample-project as a child folder of https://github.com/hackforla/incubator/tree/main/terraform-incubator
• Add tf templates to the sample-project folder

Resources/Instructions

Refer to these files:
https://github.com/hackforla/incubator/blob/main/terraform-incubator/access-the-data/main.tf
https://github.com/hackforla/incubator/blob/main/terraform-incubator/access-the-data/versions.tf

[ExperimentsInHonesty]

Examples from access-the-data

main.tf

locals {
// we use tf to create the zone, but other projects might
// have an existing zone and get it with a data block
zone_id = module.zone.zone_id

envs = {
dev = {
environment = "dev"
host_names = ["dev"]
container_env = {
CKAN_SITE_URL = "https://dev.accessthedata.org"
}
}
}
}

module "zone" {
source = "../../terraform-modules/project-zone"

zone_name = "accessthedata.org"
github_at_apex = true
shared_configuration = local.shared_configuration
}

module "database" {
for_each = local.envs

source = "../../terraform-modules/database"

shared_configuration = local.shared_configuration
environment = each.value.environment
db_name = "accessthedata"
owner_name = "ckan"
}

module "datastore_database" {
for_each = local.envs

source = "../../terraform-modules/database"

shared_configuration = local.shared_configuration
environment = each.value.environment
db_name = "accessthedata_datastore"
owner_name = "ckands"
viewer_name = "ckands_ro"
}

module "secrets" {
for_each = local.envs
source = "../../terraform-modules/cheap-secrets"
scope-name = "ckan-${each.key}"
secret-names = ["csrf", "admin-password"]
}

module "access-the-data" {
for_each = local.envs

source = "../../terraform-modules/multi-container-service"

shared_configuration = local.shared_configuration

region = "us-west-2"
project_name = "access-the-data"
application_type = "fullstack"
environment = each.value.environment
zone_id = local.zone_id

vpc_cidr = "10.10.0.0/16"

containers = {
ckan = {
tag = "latest"
cpu = 256
memory = 512
port = 80

  subdomains    = each.value.host_names
  path_patterns = ["/*"]
  env_vars = merge({
    DATABASE      = "postgres"
    POSTGRES_HOST = module.database[each.key].host
    POSTGRES_PORT = module.database[each.key].port

    // SQLALCHEMY has been set up in the container =
    // we don't know the PG password, so we can't build the URLs

    # Taken verbatim from .env
    CKAN_DB                  = module.database[each.key].database
    CKAN_DB_USER             = module.database[each.key].owner
    CKAN_DATASTORE_DB        = module.datastore_database[each.key].database
    CKAN_DATASTORE_DB_RWUSER = module.datastore_database[each.key].owner
    CKAN_DATASTORE_DB_ROUSER = module.datastore_database[each.key].viewer
    CKAN_VERSION             = "2.10.0"
    CKAN_SITE_ID             = "default"

    CKAN_PORT      = "5000"
    CKAN_PORT_HOST = "5000"

    CKAN_SYSADMIN_NAME  = "ckan_admin"
    CKAN_SYSADMIN_EMAIL = "[email protected]"
    CKAN_STORAGE_PATH   = "/var/lib/ckan"

    CKAN_SMTP_SERVER    = "smtp.hackforla.org:25"
    CKAN_SMTP_STARTTLS  = "True"
    CKAN_SMTP_USER      = "user"
    CKAN_SMTP_PASSWORD  = "pass"
    CKAN_SMTP_MAIL_FROM = "ckan@localhost"

    CKAN_SOLR_URL                       = "http://solr:8983/solr/ckan"
    CKAN_REDIS_URL                      = "redis://redis:6379/1"
    CKAN_DATAPUSHER_URL                 = "http://datapusher:8800"
    CKAN__DATAPUSHER__CALLBACK_URL_BASE = "http://ckan:5000"
    CKAN__HARVEST__MQ__HOSTNAME         = "redis"

    CKAN__PLUGINS               = "envvars image_view text_view recline_view datastore datapusher ckanext_hack4laatd"
    CKAN__HARVEST__MQ__TYPE     = "redis"
    CKAN__HARVEST__MQ__PORT     = "6379"
    CKAN__HARVEST__MQ__REDIS_DB = "1"
    CKAN__FAVICON               = "favicon.png"
  }, lookup(each.value.container_env, "ckan", {}))
  secrets = {
    CKAN_DB_PASSWORD               = module.database[each.key].owner_password_arn
    CKAN_DATASTORE_DB_RWPASSWORD   = module.datastore_database[each.key].owner_password_arn
    CKAN_DATASTORE_DB_ROPASSWORD   = module.datastore_database[each.key].viewer_password_arn
    CKAN___BEAKER__SESSION__SECRET = module.secrets[each.key].arn["csrf"]
    CKAN_SYSADMIN_PASSWORD         = module.secrets[each.key].arn["admin-password"]
  }
}

datapusher = {
  tag    = "latest"
  cpu    = 256
  memory = 512
}

solr = {
  tag    = "latest"
  cpu    = 512
  memory = 4096
}

redis = {
  tag    = "latest"
  cpu    = 256
  memory = 512
}

}
}

versions.tf

// Get configuration from the shared infrastructure
data "terraform_remote_state" "shared" {
backend = "s3"

config = {
bucket = "hlfa-incubator-terragrunt"
dynamodb_table = "terraform-locks"
encrypt = true
key = "terragrunt-states/incubator/./terraform.tfstate"
region = "us-west-2"
}
}

locals {
shared_configuration = data.terraform_remote_state.shared.outputs.configuration
}

provider "aws" {
region = "us-west-2"
}

// Set up Postgres provider to create the database
terraform {
required_providers {
postgresql = {
source = "cyrilgdn/postgresql"
version = "~> 1.21.0"
}
}
}
data "aws_ssm_parameter" "rds_credentials" {
name = "rds_credentials"
}
data "aws_db_instance" "shared" {
db_instance_identifier = local.shared_configuration.db_identifier
}
provider "postgresql" {
host = data.aws_db_instance.shared.address
password = data.aws_ssm_parameter.rds_credentials.value
username = "postgres"
superuser = false
}

[ExperimentsInHonesty]

main.tf draft

locals {
// we use tf to create the zone, but other projects might
// have an existing zone and get it with a data block
zone_id = module.zone.zone_id

envs = {
dev = {
environment = "dev"
host_names = ["dev"]
container_env = {
CKAN_SITE_URL = "https://dev.accessthedata.org"
}
}
}
}

module "zone" {
source = "../../terraform-modules/project-zone"

zone_name = "accessthedata.org"
github_at_apex = true
shared_configuration = local.shared_configuration
}

module "database" {
for_each = local.envs

source = "../../terraform-modules/database"

shared_configuration = local.shared_configuration
environment = each.value.environment
db_name = "accessthedata"
owner_name = "ckan"
}

module "datastore_database" {
for_each = local.envs

source = "../../terraform-modules/database"

shared_configuration = local.shared_configuration
environment = each.value.environment
db_name = "accessthedata_datastore"
owner_name = "ckands"
viewer_name = "ckands_ro"
}

module "secrets" {
for_each = local.envs
source = "../../terraform-modules/cheap-secrets"
scope-name = "ckan-${each.key}"
secret-names = ["csrf", "admin-password"]
}

module "access-the-data" {
for_each = local.envs

source = "../../terraform-modules/multi-container-service"

shared_configuration = local.shared_configuration

region = "us-west-2"
project_name = "access-the-data"
application_type = "fullstack"
environment = each.value.environment
zone_id = local.zone_id

vpc_cidr = "10.10.0.0/16"

containers = {
ckan = {
tag = "latest"
cpu = 256
memory = 512
port = 80

  subdomains    = each.value.host_names
  path_patterns = ["/*"]
  env_vars = merge({
    DATABASE      = "postgres"
    POSTGRES_HOST = module.database[each.key].host
    POSTGRES_PORT = module.database[each.key].port

    // SQLALCHEMY has been set up in the container =
    // we don't know the PG password, so we can't build the URLs

    # Taken verbatim from .env
    CKAN_DB                  = module.database[each.key].database
    CKAN_DB_USER             = module.database[each.key].owner
    CKAN_DATASTORE_DB        = module.datastore_database[each.key].database
    CKAN_DATASTORE_DB_RWUSER = module.datastore_database[each.key].owner
    CKAN_DATASTORE_DB_ROUSER = module.datastore_database[each.key].viewer
    CKAN_VERSION             = "2.10.0"
    CKAN_SITE_ID             = "default"

    CKAN_PORT      = "5000"
    CKAN_PORT_HOST = "5000"

    CKAN_SYSADMIN_NAME  = "ckan_admin"
    CKAN_SYSADMIN_EMAIL = "[email protected]"
    CKAN_STORAGE_PATH   = "/var/lib/ckan"

    CKAN_SMTP_SERVER    = "smtp.hackforla.org:25"
    CKAN_SMTP_STARTTLS  = "True"
    CKAN_SMTP_USER      = "user"
    CKAN_SMTP_PASSWORD  = "pass"
    CKAN_SMTP_MAIL_FROM = "ckan@localhost"

    CKAN_SOLR_URL                       = "http://solr:8983/solr/ckan"
    CKAN_REDIS_URL                      = "redis://redis:6379/1"
    CKAN_DATAPUSHER_URL                 = "http://datapusher:8800"
    CKAN__DATAPUSHER__CALLBACK_URL_BASE = "http://ckan:5000"
    CKAN__HARVEST__MQ__HOSTNAME         = "redis"

    CKAN__PLUGINS               = "envvars image_view text_view recline_view datastore datapusher ckanext_hack4laatd"
    CKAN__HARVEST__MQ__TYPE     = "redis"
    CKAN__HARVEST__MQ__PORT     = "6379"
    CKAN__HARVEST__MQ__REDIS_DB = "1"
    CKAN__FAVICON               = "favicon.png"
  }, lookup(each.value.container_env, "ckan", {}))
  secrets = {
    CKAN_DB_PASSWORD               = module.database[each.key].owner_password_arn
    CKAN_DATASTORE_DB_RWPASSWORD   = module.datastore_database[each.key].owner_password_arn
    CKAN_DATASTORE_DB_ROPASSWORD   = module.datastore_database[each.key].viewer_password_arn
    CKAN___BEAKER__SESSION__SECRET = module.secrets[each.key].arn["csrf"]
    CKAN_SYSADMIN_PASSWORD         = module.secrets[each.key].arn["admin-password"]
  }
}

datapusher = {
  tag    = "latest"
  cpu    = 256
  memory = 512
}

solr = {
  tag    = "latest"
  cpu    = 512
  memory = 4096
}

redis = {
  tag    = "latest"
  cpu    = 256
  memory = 512
}

}
}

[ExperimentsInHonesty]

version.tf draft

// Get configuration from the shared infrastructure
data "terraform_remote_state" "shared" {
backend = "s3"

config = {
bucket = "hlfa-incubator-terragrunt"
dynamodb_table = "terraform-locks"
encrypt = true
key = "terragrunt-states/incubator/./terraform.tfstate"
region = "us-west-2"
}
}

locals {
shared_configuration = data.terraform_remote_state.shared.outputs.configuration
}

provider "aws" {
region = "us-west-2"
}

// Set up Postgres provider to create the database
terraform {
required_providers {
postgresql = {
source = "cyrilgdn/postgresql"
version = "~> 1.21.0"
}
}
}
data "aws_ssm_parameter" "rds_credentials" {
name = "rds_credentials"
}
data "aws_db_instance" "shared" {
db_instance_identifier = local.shared_configuration.db_identifier
}
provider "postgresql" {
host = data.aws_db_instance.shared.address
password = data.aws_ssm_parameter.rds_credentials.value
username = "postgres"
superuser = false
}

[ExperimentsInHonesty]

@nyarly if you have time to edit some draft main.tf and versions.tf files we have set them up for you in this issue. The checkboxes that apply to you in this issue are

• edit draft comments
• remove label: role: Dev Ops Engineer
• add label role: product
• unassign yourself
• add the issue to the questions/review column of project board

If you don't have time, let us know and we will ask Tyson.

[robinglov]

@Tyson-miller would you like to take this on?