Merge pull request #259 from ministryofjustice/fix/adding-glue-crawle…

…r-to-module

initial code for glue fix

.github/workflows/documentation.yml

@@ -4,6 +4,8 @@ on:
   pull_request:
     branches:
       - main
+
+permissions: read-all
 
 jobs:
   docs:

README.md

@@ -171,11 +171,18 @@ If you're looking to raise an issue with this module, please create a new issue
 | [aws_athena_database.lb-access-logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/athena_database) | resource |
 | [aws_athena_named_query.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/athena_named_query) | resource |
 | [aws_athena_workgroup.lb-access-logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/athena_workgroup) | resource |
+| [aws_glue_crawler.ssm_resource_sync](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/glue_crawler) | resource |
+| [aws_iam_policy.lb_glue_crawler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_role.lb_glue_crawler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role_policy_attachment.lb_glue_crawler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_role_policy_attachment.lb_glue_servicec](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_lb.loadbalancer](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb) | resource |
 | [aws_lb_target_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb_target_group) | resource |
 | [aws_security_group.lb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource |
 | [aws_elb_service_account.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/elb_service_account) | data source |
 | [aws_iam_policy_document.bucket_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.lb_glue_crawler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.lb_glue_crawler_assume](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_vpc.shared](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source |
 
 ## Inputs
@@ -194,6 +201,7 @@ If you're looking to raise an issue with this module, please create a new issue
 | <a name="input_load_balancer_type"></a> [load\_balancer\_type](#input\_load\_balancer\_type) | application or network | `string` | `"application"` | no |
 | <a name="input_loadbalancer_egress_rules"></a> [loadbalancer\_egress\_rules](#input\_loadbalancer\_egress\_rules) | Create new security group with these egress rules for the loadbalancer.  Or use the security\_groups var to attach existing group(s) | <pre>map(object({<br>    description     = string<br>    from_port       = number<br>    to_port         = number<br>    protocol        = string<br>    security_groups = list(string)<br>    cidr_blocks     = list(string)<br>  }))</pre> | `{}` | no |
 | <a name="input_loadbalancer_ingress_rules"></a> [loadbalancer\_ingress\_rules](#input\_loadbalancer\_ingress\_rules) | Create new security group with these ingress rules for the loadbalancer.  Or use the security\_groups var to attach existing group(s) | <pre>map(object({<br>    description     = string<br>    from_port       = number<br>    to_port         = number<br>    protocol        = string<br>    security_groups = list(string)<br>    cidr_blocks     = list(string)<br>  }))</pre> | `{}` | no |
+| <a name="input_log_schedule"></a> [log\_schedule](#input\_log\_schedule) | n/a | `string` | `"cron(15 1 ? * MON *)"` | no |
 | <a name="input_public_subnets"></a> [public\_subnets](#input\_public\_subnets) | Public subnets | `list(string)` | n/a | yes |
 | <a name="input_region"></a> [region](#input\_region) | AWS Region where resources are to be created | `string` | n/a | yes |
 | <a name="input_s3_versioning"></a> [s3\_versioning](#input\_s3\_versioning) | A boolean that determines whether s3 will have versioning | `bool` | `true` | no |

main.tf

@@ -279,3 +279,63 @@ resource "aws_lb_target_group" "this" {
     },
   )
 }
+
+
+# Glue crawler to update Athena Table
+# Role for crawler
+resource "aws_iam_role" "lb_glue_crawler" {
+  name               = "ssm-glue-crawler"
+  assume_role_policy = data.aws_iam_policy_document.lb_glue_crawler_assume.json
+}
+
+data "aws_iam_policy_document" "lb_glue_crawler_assume" {
+  statement {
+    effect  = "Allow"
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["glue.amazonaws.com"]
+    }
+  }
+}
+
+resource "aws_iam_policy" "lb_glue_crawler" {
+  name   = "LbGlueCrawler"
+  policy = data.aws_iam_policy_document.lb_glue_crawler.json
+}
+
+data "aws_iam_policy_document" "lb_glue_crawler" {
+  statement {
+    effect = "Allow"
+    actions = [
+      "s3:GetObject",
+      "s3:PutObject"
+    ]
+    resources = [var.existing_bucket_name != "" ? "arn:aws:s3:::${var.existing_bucket_name}/${var.application_name}/AWSLogs/${var.account_number}/*" : "${module.s3-bucket[0].bucket.arn}/${var.application_name}/AWSLogs/${var.account_number}/*"]
+  }
+}
+
+# Glue Crawler Policy
+resource "aws_iam_role_policy_attachment" "lb_glue_crawler" {
+  role       = aws_iam_role.lb_glue_crawler.name
+  policy_arn = aws_iam_policy.lb_glue_crawler.arn
+}
+
+resource "aws_iam_role_policy_attachment" "lb_glue_service" {
+  role       = aws_iam_role.lb_glue_crawler.id
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
+}
+
+# Glue Crawler
+resource "aws_glue_crawler" "ssm_resource_sync" {
+  #checkov:skip=CKV_AWS_195
+  database_name = aws_athena_database.lb-access-logs[0].name
+  name          = "lb_resource_sync"
+  role          = aws_iam_role.lb_glue_crawler.arn
+  schedule      = var.log_schedule
+
+  s3_target {
+    path = "s3://${var.existing_bucket_name}"
+  }
+}

test/unit-test/main.tf

@@ -76,4 +76,5 @@ module "lb_access_logs_enabled" {
   idle_timeout               = 60
   force_destroy_bucket       = true
   lb_target_groups           = local.lb_target_groups
+  log_schedule               = "cron(15 1 ? * MON *)"
 }

variables.tf

@@ -118,4 +118,9 @@ variable "lb_target_groups" {
     })), [])
   }))
   default = {}
+}
+variable "log_schedule" {
+  type    = string
+  default = "cron(15 1 ? * MON *)"
+
 }