elk upgrade

elk-service/data.tf

@@ -0,0 +1,16 @@
+data "aws_iam_policy_document" "es_cloudwatch_policy" {
+  statement {
+    actions = [
+      "logs:CreateLogStream",
+      "logs:PutLogEvents",
+      "logs:PutLogEventsBatch",
+    ]
+
+    resources = ["arn:aws:logs:*"]
+
+    principals {
+      identifiers = ["es.amazonaws.com"]
+      type        = "Service"
+    }
+  }
+}

elk-service/domain.tf

@@ -0,0 +1,103 @@
+resource "aws_cloudwatch_log_group" "es" {
+  name              = local.common_name
+  retention_in_days = var.cloudwatch_log_retention
+  kms_key_id        = local.logs_kms_arn
+  tags = merge(
+    local.tags,
+    {
+      "Name" = local.common_name
+    },
+  )
+}
+
+resource "aws_iam_service_linked_role" "es" {
+  aws_service_name = "es.amazonaws.com"
+}
+
+resource "aws_cloudwatch_log_resource_policy" "example" {
+  policy_name     = "${local.common_name}-logs-pol"
+  policy_document = data.aws_iam_policy_document.es_cloudwatch_policy.json
+}
+
+resource "aws_elasticsearch_domain" "es" {
+  domain_name           = local.common_name
+  elasticsearch_version = lookup(var.alf_elk_service_props, "elasticsearch_version", "6.8")
+
+  cluster_config {
+    instance_type            = lookup(var.alf_elk_service_props, "instance_type", "t2.medium.elasticsearch")
+    dedicated_master_enabled = lookup(var.alf_elk_service_props, "dedicated_master_enabled", true)
+    dedicated_master_count   = lookup(var.alf_elk_service_props, "dedicated_master_count", 3)
+    dedicated_master_type    = lookup(var.alf_elk_service_props, "dedicated_master_type", "t2.medium.elasticsearch")
+    zone_awareness_enabled   = lookup(var.alf_elk_service_props, "zone_awareness_enabled", true)
+    instance_count           = lookup(var.alf_elk_service_props, "instance_count", 3)
+    zone_awareness_config {
+      availability_zone_count = lookup(var.alf_elk_service_props, "availability_zone_count", 3)
+    }
+  }
+
+  ebs_options {
+    ebs_enabled = lookup(var.alf_elk_service_props, "es_ebs_enabled", true)
+    volume_type = lookup(var.alf_elk_service_props, "es_ebs_type", "gp2")
+    volume_size = lookup(var.alf_elk_service_props, "es_ebs_size", 10)
+    iops        = lookup(var.alf_elk_service_props, "iops", 0)
+  }
+
+  vpc_options {
+    subnet_ids         = flatten(local.private_subnet_ids)
+    security_group_ids = [aws_security_group.es.id]
+  }
+
+  access_policies = templatefile(
+    "${path.module}/templates/iam/es_access_policy.conf",
+    {
+      domain_name = local.common_name
+      region      = var.region
+      account_id  = local.account_id
+    }
+  )
+
+  snapshot_options {
+    automated_snapshot_start_hour = lookup(var.alf_elk_service_props, "automated_snapshot_start_hour", 23)
+  }
+
+  encrypt_at_rest {
+    enabled = lookup(var.alf_elk_service_props, "encrypt_at_rest", true)
+  }
+
+  node_to_node_encryption {
+    enabled = lookup(var.alf_elk_service_props, "node_to_node_encryption", true)
+  }
+
+  domain_endpoint_options {
+    enforce_https       = true
+    tls_security_policy = lookup(var.alf_elk_service_props, "tls_security_policy", "Policy-Min-TLS-1-2-2019-07")
+  }
+
+  log_publishing_options {
+    enabled                  = true
+    cloudwatch_log_group_arn = aws_cloudwatch_log_group.es.arn
+    log_type                 = "INDEX_SLOW_LOGS"
+  }
+
+  log_publishing_options {
+    enabled                  = true
+    cloudwatch_log_group_arn = aws_cloudwatch_log_group.es.arn
+    log_type                 = "SEARCH_SLOW_LOGS"
+  }
+
+  log_publishing_options {
+    enabled                  = true
+    cloudwatch_log_group_arn = aws_cloudwatch_log_group.es.arn
+    log_type                 = "ES_APPLICATION_LOGS"
+  }
+
+  tags = merge(
+    local.tags,
+    {
+      "Name" = "${local.common_name}-sg"
+    },
+  )
+
+  depends_on = [aws_iam_service_linked_role.es]
+
+}

elk-service/main.tf

@@ -0,0 +1,139 @@
+terraform {
+  # The configuration for this backend will be filled in by Terragrunt
+  # The configuration for this backend will be filled in by Terragrunt
+  backend "s3" {
+  }
+}
+
+####################################################
+# DATA SOURCE MODULES FROM OTHER TERRAFORM BACKENDS
+####################################################
+#-------------------------------------------------------------
+### Getting the common details
+#-------------------------------------------------------------
+data "terraform_remote_state" "common" {
+  backend = "s3"
+
+  config = {
+    bucket = var.remote_state_bucket_name
+    key    = "alfresco/common/terraform.tfstate"
+    region = var.region
+  }
+}
+
+#-------------------------------------------------------------
+### Getting the s3 details
+#-------------------------------------------------------------
+data "terraform_remote_state" "s3bucket" {
+  backend = "s3"
+
+  config = {
+    bucket = var.remote_state_bucket_name
+    key    = "alfresco/s3buckets/terraform.tfstate"
+    region = var.region
+  }
+}
+
+#-------------------------------------------------------------
+### Getting the IAM details
+#-------------------------------------------------------------
+data "terraform_remote_state" "iam" {
+  backend = "s3"
+
+  config = {
+    bucket = var.remote_state_bucket_name
+    key    = "alfresco/iam/terraform.tfstate"
+    region = var.region
+  }
+}
+
+#-------------------------------------------------------------
+### Getting the shared monitoring details
+#-------------------------------------------------------------
+data "terraform_remote_state" "monitoring" {
+  backend = "s3"
+
+  config = {
+    bucket = var.remote_state_bucket_name
+    key    = "shared-monitoring/terraform.tfstate"
+    region = var.region
+  }
+}
+
+#-------------------------------------------------------------
+### Getting the security groups details
+#-------------------------------------------------------------
+data "terraform_remote_state" "security-groups" {
+  backend = "s3"
+
+  config = {
+    bucket = var.remote_state_bucket_name
+    key    = "alfresco/security-groups/terraform.tfstate"
+    region = var.region
+  }
+}
+
+#-------------------------------------------------------------
+### Getting the network security groups details
+#-------------------------------------------------------------
+data "terraform_remote_state" "network-security-groups" {
+  backend = "s3"
+
+  config = {
+    bucket = var.remote_state_bucket_name
+    key    = "security-groups/terraform.tfstate"
+    region = var.region
+  }
+}
+
+#-------------------------------------------------------------
+### Getting ACM Cert
+#-------------------------------------------------------------
+data "aws_acm_certificate" "cert" {
+  domain      = "*.${data.terraform_remote_state.common.outputs.external_domain}"
+  types       = ["AMAZON_ISSUED"]
+  most_recent = true
+}
+
+####################################################
+# Locals
+####################################################
+
+locals {
+  account_id                   = data.terraform_remote_state.common.outputs.common_account_id
+  vpc_id                       = data.terraform_remote_state.common.outputs.vpc_id
+  vpc_cidr_block               = data.terraform_remote_state.common.outputs.vpc_cidr_block
+  internal_domain              = data.terraform_remote_state.common.outputs.internal_domain
+  private_zone_id              = data.terraform_remote_state.common.outputs.private_zone_id
+  public_zone_id               = data.terraform_remote_state.common.outputs.public_zone_id
+  external_domain              = data.terraform_remote_state.common.outputs.external_domain
+  environment_identifier       = data.terraform_remote_state.common.outputs.environment_identifier
+  application                  = "alf-elk-svc"
+  common_name                  = "${data.terraform_remote_state.common.outputs.short_environment_identifier}-${local.application}"
+  short_environment_identifier = data.terraform_remote_state.common.outputs.short_environment_identifier
+  region                       = var.region
+  environment                  = data.terraform_remote_state.common.outputs.environment
+  tags                         = data.terraform_remote_state.common.outputs.common_tags
+  instance_profile             = data.terraform_remote_state.iam.outputs.iam_instance_es_admin_profile_name
+  ssh_deployer_key             = data.terraform_remote_state.common.outputs.common_ssh_deployer_key
+  s3bucket                     = data.terraform_remote_state.s3bucket.outputs.s3bucket
+  bastion_inventory            = var.bastion_inventory
+  logs_kms_arn                 = data.terraform_remote_state.common.outputs.kms_arn
+  config-bucket                = data.terraform_remote_state.common.outputs.common_s3-config-bucket
+  certificate_arn              = data.aws_acm_certificate.cert.arn
+  public_subnet_ids            = data.terraform_remote_state.common.outputs.public_subnet_ids
+  private_subnet_ids           = data.terraform_remote_state.common.outputs.private_subnet_ids
+  elk_bucket_name              = data.terraform_remote_state.s3bucket.outputs.elk_backups_bucket_name
+  storage_s3bucket             = data.terraform_remote_state.s3bucket.outputs.s3bucket
+  backups_bucket               = data.terraform_remote_state.s3bucket.outputs.alf_backups_bucket_name
+  storage_kms_arn              = data.terraform_remote_state.s3bucket.outputs.s3bucket_kms_arn
+  mon_jenkins_sg               = data.terraform_remote_state.security-groups.outputs.security_groups_map["mon_jenkins"]
+
+  monitoring_groups = [
+    data.terraform_remote_state.network-security-groups.outputs.sg_ssh_bastion_in_id,
+    data.terraform_remote_state.network-security-groups.outputs.sg_mon_efs,
+    data.terraform_remote_state.network-security-groups.outputs.sg_monitoring,
+    data.terraform_remote_state.network-security-groups.outputs.sg_elasticsearch,
+  ]
+}
+

elk-service/output.tf

@@ -0,0 +1,5 @@
+output "elk_service" {
+  value = {
+    es_sg_id = aws_security_group.es.id
+  }
+}

elk-service/security-groups.tf

@@ -0,0 +1,39 @@
+resource "aws_security_group" "es" {
+  name        = "${local.common_name}-sg"
+  description = "${local.common_name}-sg"
+  vpc_id      = local.vpc_id
+  tags = merge(
+    local.tags,
+    {
+      "Name" = "${local.common_name}-sg"
+    },
+  )
+}
+
+resource "aws_security_group_rule" "ingress_self" {
+  security_group_id = aws_security_group.es.id
+  type              = "ingress"
+  from_port         = 0
+  to_port           = 0
+  protocol          = -1
+  self              = true
+}
+
+resource "aws_security_group_rule" "egress_self" {
+  security_group_id = aws_security_group.es.id
+  type              = "egress"
+  from_port         = 0
+  to_port           = 0
+  protocol          = -1
+  self              = true
+}
+
+resource "aws_security_group_rule" "ingress_https" {
+  security_group_id = aws_security_group.es.id
+  type              = "ingress"
+  from_port         = 443
+  to_port           = 443
+  protocol          = "tcp"
+  cidr_blocks       = [local.vpc_cidr_block]
+  description       = "${local.common_name}-https"
+}

elk-service/templates/iam/es_access_policy.conf

@@ -0,0 +1,11 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Action": "es:*",
+            "Principal": "*",
+            "Effect": "Allow",
+            "Resource": "arn:aws:es:${region}:${account_id}:domain/${domain_name}/*"
+        }
+    ]
+}

elk-service/terragrunt.hcl

@@ -0,0 +1,14 @@
+include {
+  path = "${find_in_parent_folders()}"
+}
+
+dependencies {
+  paths = [
+    "../common",
+    "../s3buckets",
+    "../iam",
+    "../security-groups",
+    "../rds",
+    "../efs"
+  ]
+}

elk-service/variables.tf

@@ -0,0 +1,57 @@
+variable "region" {
+}
+
+variable "role_arn" {
+}
+
+variable "remote_state_bucket_name" {
+  description = "Terraform remote state bucket name"
+}
+
+variable "cloudwatch_log_retention" {
+}
+
+variable "bastion_inventory" {
+  default = "dev"
+}
+
+variable "environment_name" {
+}
+
+# Elasticsearch
+
+variable "es_admin_instance_type" {
+  default = "t2.large"
+}
+
+# ELasticsearch snapshot name
+variable "es_snapshot_name" {
+  default = "snapshot_1"
+}
+
+variable "es_s3_repo_name" {
+  default = "alfresco_s3_repo"
+}
+
+# Restore mode
+variable "alf_restore_status" {
+  default = "no-restore"
+}
+
+variable "availability_zone" {
+  description = "List of the three AZs we want to use"
+  type        = map(string)
+}
+
+variable "alf_elk_service_props" {
+  type = map(string)
+  default = {
+    elasticsearch_version         = "6.8"
+    instance_type                 = "t2.medium.elasticsearch"
+    automated_snapshot_start_hour = 23
+    encrypt_at_rest               = false
+  }
+}
+
+variable "alf_cloudwatch_log_retention" {
+}

elk-service/versions.tf

@@ -0,0 +1,4 @@
+
+terraform {
+  required_version = ">= 0.12"
+}