From d341ef72aa6b74e54c11aeb455516b2a69a8a664 Mon Sep 17 00:00:00 2001 From: dan sweeting Date: Tue, 23 Feb 2021 17:10:33 +0000 Subject: [PATCH] feat: add terraform infra as code for AWS fargate (#1987) --- .gitignore | 2 ++ terraform/README.md | 32 +++++++++++++++++++ terraform/dashboard.json.template | 40 ++++++++++++++++++++++++ terraform/provider.tf | 14 +++++++++ terraform/resource-ecs.tf | 42 +++++++++++++++++++++++++ terraform/resource-logging.tf | 51 +++++++++++++++++++++++++++++++ terraform/resource-vpc.tf | 40 ++++++++++++++++++++++++ terraform/taskdef.json.template | 18 +++++++++++ terraform/terraform.tfvars | 8 +++++ terraform/variables.tf | 40 ++++++++++++++++++++++++ 10 files changed, 287 insertions(+) create mode 100644 terraform/README.md create mode 100644 terraform/dashboard.json.template create mode 100644 terraform/provider.tf create mode 100644 terraform/resource-ecs.tf create mode 100644 terraform/resource-logging.tf create mode 100644 terraform/resource-vpc.tf create mode 100644 terraform/taskdef.json.template create mode 100644 terraform/terraform.tfvars create mode 100644 terraform/variables.tf diff --git a/.gitignore b/.gitignore index 973352e6c2..3829afeae4 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ success-*.png desktop.ini twitch.json +terraform/terraform.tfstate +terraform/terraform.tfstate.backup diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 0000000000..4c78c978f9 --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,32 @@ +# Terraform for AWS Fargate + +Here's some configurable terraform to get you up and running with the streetmerchant docker image in AWS ECS Fargate. + +Running on cloud infrastructure, your mileage may vary, and you'll need to integrate with one of the chat notifications rather than having your local browser navigate to a url for you. + +The author's findings were that it worked ok; running the container from within EU-West-2 region was suficient to get a timely alert for PS5 stock on amazon, and follow the link to a successful basket add. + +Dependencies: +- Terraform 14 + +##Getting started + +There's an example tfvars file to start you off; rename this with your own preferences. Anything you can set in the `dotenv` file you'll need to set in terraform.tfvars to get the env vars into your fargate container. + +Authenticate yourself with your own AWS account as with any aws commandline tool. + +If you wish, add a specific section to your aws credentials file and set that profile name in `terraform.tfvars`. + +Then you can: +```shell +cd ./terraform +terraform init + +terraform plan +terraform apply +``` + +## What's included + +- container, running streetmerchant, with your chosen config +- cloud metrics and a dashboard tracking 'out of stock' and 'error' responses from your configured stores diff --git a/terraform/dashboard.json.template b/terraform/dashboard.json.template new file mode 100644 index 0000000000..57a1287001 --- /dev/null +++ b/terraform/dashboard.json.template @@ -0,0 +1,40 @@ +{ + "widgets": [ + { + "type": "metric", + "x": 0, + "y": 0, + "width": 18, + "height": 12, + "properties": { + "metrics": ${out_of_stock}, + "view": "timeSeries", + "stacked": false, + "region": "${region}", + "start": "-PT1H", + "end": "P0D", + "stat": "Sum", + "period": 300, + "title": "out of stock" + } + }, + { + "type": "metric", + "x": 0, + "y": 0, + "width": 18, + "height": 12, + "properties": { + "metrics": ${error}, + "view": "timeSeries", + "stacked": false, + "region": "${region}", + "start": "-PT1H", + "end": "P0D", + "stat": "Sum", + "period": 300, + "title": "error" + } + } + ] +} diff --git a/terraform/provider.tf b/terraform/provider.tf new file mode 100644 index 0000000000..12d405eb48 --- /dev/null +++ b/terraform/provider.tf @@ -0,0 +1,14 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 3.0" + } + } +} + +provider "aws" { + region = var.region + shared_credentials_file = var.credential_file + profile = var.credential_profile +} diff --git a/terraform/resource-ecs.tf b/terraform/resource-ecs.tf new file mode 100644 index 0000000000..e484cfa9ba --- /dev/null +++ b/terraform/resource-ecs.tf @@ -0,0 +1,42 @@ +resource "aws_ecs_cluster" "main" { + name = "${var.app_name}-ecs-cluster" +} + +resource "aws_ecs_service" "main" { + name = "${var.app_name}-ecs-service" + cluster = aws_ecs_cluster.main.id + task_definition = aws_ecs_task_definition.main.id + desired_count = 1 + network_configuration { + subnets = [ + aws_subnet.aws-subnet.id + ] + assign_public_ip = true + } + launch_type = "FARGATE" +} + +data "aws_iam_role" "ecs_task_execution_role" { + name = "ecsTaskExecutionRole" +} + +locals { + container_env = [for k, v in var.streetmerchant_env : { name: k, value: v}] +} + +resource "aws_ecs_task_definition" "main" { + container_definitions = templatefile("taskdef.json.template", { + "name": var.app_name + "awslogs-group": aws_cloudwatch_log_group.main.name + "region": var.region + "cpu": var.cpu + "memory": parseint(var.memory,10) + "environment": jsonencode(local.container_env) + }) + family = var.app_name + requires_compatibilities = ["FARGATE"] + network_mode = "awsvpc" + cpu = var.cpu + memory = var.memory + execution_role_arn = data.aws_iam_role.ecs_task_execution_role.arn +} diff --git a/terraform/resource-logging.tf b/terraform/resource-logging.tf new file mode 100644 index 0000000000..c3d456b207 --- /dev/null +++ b/terraform/resource-logging.tf @@ -0,0 +1,51 @@ +resource "aws_cloudwatch_log_group" "main" { + name = var.app_name + retention_in_days = 3 +} + +locals { + stores = split(",",var.streetmerchant_env["STORES"]) + metrics = { + out_of_stock = [for store in local.stores : ["${var.app_name}-out-of-stock", store]] + error = [for store in local.stores : ["${var.app_name}-error", store]] + } +} + +resource "aws_cloudwatch_log_metric_filter" "out_of_stock" { + for_each = toset(local.stores) + + log_group_name = aws_cloudwatch_log_group.main.name + name = "${each.key}-out-of-stock" + + pattern = "${each.key} \"OUT OF STOCK\"" + metric_transformation { + name = each.key + namespace = "${var.app_name}-out-of-stock" + value = 1 + default_value = 0 + } +} + +resource "aws_cloudwatch_log_metric_filter" "error" { + for_each = toset(local.stores) + + log_group_name = aws_cloudwatch_log_group.main.name + name = "${each.key}-error" + + pattern = "${each.key} \"ERROR\"" + metric_transformation { + name = each.key + namespace = "${var.app_name}-error" + value = 1 + default_value = 0 + } +} + +resource "aws_cloudwatch_dashboard" "main" { + dashboard_name = "${var.app_name}-dashboard" + dashboard_body = templatefile("dashboard.json.template", { + out_of_stock = jsonencode(local.metrics.out_of_stock) + error = jsonencode(local.metrics.error) + region = var.region + }) +} diff --git a/terraform/resource-vpc.tf b/terraform/resource-vpc.tf new file mode 100644 index 0000000000..4bf3ba0d89 --- /dev/null +++ b/terraform/resource-vpc.tf @@ -0,0 +1,40 @@ +resource "aws_vpc" "main" { + enable_dns_support = true + cidr_block = "10.0.0.0/16" + tags = { + app = "ps5" + } +} + +resource "aws_internet_gateway" "main" { + vpc_id = aws_vpc.main.id + tags = { + app = "ps5" + } +} + +resource "aws_subnet" "aws-subnet" { + vpc_id = aws_vpc.main.id + cidr_block = aws_vpc.main.cidr_block + map_public_ip_on_launch = true + tags = { + app = "ps5" + } +} + +resource "aws_route_table" "main" { + vpc_id = aws_vpc.main.id + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.main.id + } + tags = { + app = "ps5" + } +} + +resource "aws_main_route_table_association" "main" { + route_table_id = aws_route_table.main.id + vpc_id = aws_vpc.main.id +} + diff --git a/terraform/taskdef.json.template b/terraform/taskdef.json.template new file mode 100644 index 0000000000..e95b2fd796 --- /dev/null +++ b/terraform/taskdef.json.template @@ -0,0 +1,18 @@ +[ + { + "name": "${name}-task", + "image": "ghcr.io/jef/streetmerchant:latest", + "cpu": ${cpu}, + "memory": ${memory}, + "essential": true, + "environment": ${environment}, + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "${awslogs-group}", + "awslogs-region": "${region}", + "awslogs-stream-prefix": "ecs" + } + } + } +] diff --git a/terraform/terraform.tfvars b/terraform/terraform.tfvars new file mode 100644 index 0000000000..7c4840dfd3 --- /dev/null +++ b/terraform/terraform.tfvars @@ -0,0 +1,8 @@ +credential_profile = "ps5" + +streetmerchant_env = { + "STORES" = "amazon-uk,game,argos,box,currys,johnlewis,shopto,smythstoys,very,amazon-it,amazon-nl" + "SHOW_ONLY_SERIES" = "sonyps5c,sonyps5de" + "SLACK_TOKEN" = "your slack api token" + "SLACK_CHANNEL" = "your slack channel name" +} diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000000..0da31aa452 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,40 @@ +variable "credential_file" { + type = string + description = "your aws credentials file" + default = "~/.aws/credentials" +} + +variable "credential_profile" { + type = string + description = "the section in ~/.aws/credentials with your desired aws_access_key_id and aws_secret_access_key values" + default = "default" +} + +variable "region" { + type = string + description = "aws region" + default = "eu-west-2" +} + +variable "app_name" { + type = string + default = "streetmerchant" +} + +variable "memory" { + type = string + default = "2048" + description = "ecs task memory" +} + +variable "cpu" { + type = number + default = 1024 + description = "ecs task cpu" +} + +variable "streetmerchant_env" { + type = map + description = "name/value pairs for .env values" + default = {} +}