From d70f571a407518d83d7c8ba89c4075a6f78e2abd Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Thu, 25 Jan 2024 17:54:30 +0100 Subject: [PATCH] [ci] Add linter and format terraform files (#991) * [ci] Add linter for terraform files * [ci] Format terraform files * [ci] Update README * Fix hygiene ci step --- .github/workflows/CI.md | 2 ++ .github/workflows/ci.yml | 4 ++- Makefile | 8 +++-- deploy/README.md | 6 ++++ .../terraform-aws-kubernetes/cluster.tf | 4 +-- .../terraform-aws-kubernetes/iam.tf | 30 +++++++++---------- .../terraform-aws-kubernetes/main.tf | 2 +- .../terraform-aws-kubernetes/network_lb.tf | 2 +- .../terraform-commons-dss/main.tf | 24 +++++++-------- .../terraform-commons-dss/variables.tf | 2 +- .../terraform.dev.example.tfvars | 2 +- .../modules/terraform-aws-dss/variables.tf | 2 +- .../terraform.dev.example.tfvars | 2 +- .../modules/terraform-google-dss/variables.tf | 2 +- .../utils/definitions/image_pull_secret.tf | 2 +- .../ci/aws-1/kubernetes_admin_access.tf | 10 +++---- deploy/operations/ci/aws-1/terraform.tfvars | 6 ++-- deploy/operations/ci/aws-1/variables.tf | 2 +- 18 files changed, 63 insertions(+), 49 deletions(-) diff --git a/.github/workflows/CI.md b/.github/workflows/CI.md index bab9857b9..a4747876b 100644 --- a/.github/workflows/CI.md +++ b/.github/workflows/CI.md @@ -12,6 +12,8 @@ Before a pull request can be merged into the master branch, it must pass all aut ### Go lint (`make go-lint`) +### Terraform lint (`make terraform-lint`) + ## DSS tests (`make check-dss`) ### Deployment infrastructure tests (`make evaluate-tanka`) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e8f98bd8..9f88ba019 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,11 +24,13 @@ jobs: - name: Python lint run: make python-lint - name: Automated hygiene verification - run: make check-hygiene + run: make hygiene - name: Shell lint run: make shell-lint - name: Go lint run: make go-lint + - name: Terraform lint + run: make terraform-lint dss-tests: name: DSS tests diff --git a/Makefile b/Makefile index 6f1db5817..3dc9a16e1 100644 --- a/Makefile +++ b/Makefile @@ -37,10 +37,10 @@ format: gofmt -s -w . .PHONY: lint -lint: python-lint shell-lint go-lint +lint: python-lint shell-lint go-lint terraform-lint .PHONY: check-hygiene -check-hygiene: python-lint hygiene shell-lint go-lint +check-hygiene: python-lint hygiene shell-lint go-lint terraform-lint .PHONY: python-lint python-lint: @@ -58,6 +58,10 @@ shell-lint: go-lint: echo "===== Checking Go lint (except for *.gen.go files) =====" && docker run --rm -v $(CURDIR):/dss -w /dss golangci/golangci-lint:v1.50.1 golangci-lint run --timeout 5m --skip-dirs /dss/build/workspace --skip-files '.*\.gen\.go' -v -E gofmt,bodyclose,rowserrcheck,misspell,golint,staticcheck,vet +.PHONY: terraform-lint +terraform-lint: + docker run --rm -w /opt/dss -v ./deploy:/opt/dss/deploy -e TF_LOG=TRACE hashicorp/terraform fmt -recursive -check + # This mirrors the hygiene-tests continuous integration workflow job (.github/workflows/ci.yml) .PHONY: hygiene-tests hygiene-tests: check-hygiene diff --git a/deploy/README.md b/deploy/README.md index edf4f4d08..552765e6c 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -38,3 +38,9 @@ If you wish to deploy a DSS from scratch, "Getting Started" instructions can be - [Google (GKE)](infrastructure/modules/terraform-google-dss/README.md#Getting-started) For a real use case, you can look into the configurations of the [CI job](../.github/workflows/dss-deploy.yml) in operations: [ci](operations/ci) + +## Development + +### Formatting + +Terraform files must be formatted using `terraform fmt -recursive` command to pass the CI linter check. diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/cluster.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/cluster.tf index b4e408cc7..6a8d52077 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/cluster.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/cluster.tf @@ -5,7 +5,7 @@ resource "aws_eks_cluster" "kubernetes_cluster" { vpc_config { subnet_ids = aws_subnet.dss[*].id endpoint_public_access = true - public_access_cidrs = [ + public_access_cidrs = [ "0.0.0.0/0" ] } @@ -32,7 +32,7 @@ resource "aws_eks_node_group" "eks_node_group" { node_role_arn = aws_iam_role.dss-cluster-node-group.arn disk_size = 100 node_group_name_prefix = aws_eks_cluster.kubernetes_cluster.name - instance_types = [ + instance_types = [ var.aws_instance_type ] diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf index 00131e28c..0f85bbd58 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf @@ -11,17 +11,17 @@ resource "aws_iam_role" "dss-cluster" { name = "${var.cluster_name}-dss-cluster" assume_role_policy = jsonencode( - { - "Version" : "2012-10-17", - "Statement" : [ - { - "Effect" : "Allow", - "Principal" : { - "Service" : "eks.amazonaws.com" - }, - "Action" : "sts:AssumeRole" - } - ] + { + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Principal" : { + "Service" : "eks.amazonaws.com" + }, + "Action" : "sts:AssumeRole" + } + ] }) permissions_boundary = var.aws_iam_permissions_boundary @@ -41,14 +41,14 @@ resource "aws_iam_role" "dss-cluster-node-group" { assume_role_policy = jsonencode({ Statement = [ { - Action = "sts:AssumeRole" - Effect = "Allow" + Action = "sts:AssumeRole" + Effect = "Allow" Principal = { Service = "ec2.amazonaws.com" } } ] - Version = "2012-10-17" + Version = "2012-10-17" }) permissions_boundary = var.aws_iam_permissions_boundary @@ -84,7 +84,7 @@ resource "aws_iam_role" "AmazonEKS_EBS_CSI_DriverRole" { // Policies resource "aws_iam_policy" "AWSLoadBalancerControllerPolicy" { - name = "${var.cluster_name}-AWSLoadBalancerControllerPolicy" + name = "${var.cluster_name}-AWSLoadBalancerControllerPolicy" # Source: https://docs.aws.amazon.com/eks/latest/userguide/aws-load-balancer-controller.html # Template: https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.4.4/docs/install/iam_policy.json diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/main.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/main.tf index fc14c133f..4255b4e69 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/main.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/main.tf @@ -31,6 +31,6 @@ provider "helm" { kubernetes { host = aws_eks_cluster.kubernetes_cluster.endpoint cluster_ca_certificate = base64decode(aws_eks_cluster.kubernetes_cluster.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.kubernetes_cluster.token + token = data.aws_eks_cluster_auth.kubernetes_cluster.token } } diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf index a78aaf124..5e13c698d 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf @@ -30,7 +30,7 @@ resource "aws_acm_certificate" "app_hostname" { resource "aws_acm_certificate_validation" "app_hostname_cert" { count = var.aws_route53_zone_id == "" ? 0 : 1 certificate_arn = aws_acm_certificate.app_hostname.arn - validation_record_fqdns = [for name in aws_acm_certificate.app_hostname.domain_validation_options.*.resource_record_name: trimsuffix(name, ".")] + validation_record_fqdns = [for name in aws_acm_certificate.app_hostname.domain_validation_options.*.resource_record_name : trimsuffix(name, ".")] } output "app_hostname_cert_arn" { diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/main.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/main.tf index 680f57fe5..09f3cc2d6 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/main.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/main.tf @@ -1,11 +1,11 @@ locals { - workspace_folder = replace(replace(var.kubernetes_context_name, "/", "_"), ":", "_") + workspace_folder = replace(replace(var.kubernetes_context_name, "/", "_"), ":", "_") # Replace ':' and '/' characters from folder name by underscores. Those characters are used by AWS for contexts. workspace_location = abspath("${path.module}/../../../../build/workspace/${local.workspace_folder}") } resource "local_file" "tanka_config_main" { - content = templatefile("${path.module}/templates/main.jsonnet.tmp", { + content = templatefile("${path.module}/templates/main.jsonnet.tmp", { root_path = path.module VAR_NAMESPACE = var.kubernetes_namespace VAR_CLUSTER_CONTEXT = var.kubernetes_context_name @@ -33,7 +33,7 @@ resource "local_file" "tanka_config_main" { } resource "local_file" "tanka_config_spec" { - content = templatefile("${path.module}/templates/spec.json.tmp", { + content = templatefile("${path.module}/templates/spec.json.tmp", { root_path = path.module namespace = var.kubernetes_namespace cluster_context = var.kubernetes_context_name @@ -43,7 +43,7 @@ resource "local_file" "tanka_config_spec" { } resource "local_file" "make_certs" { - content = templatefile("${path.module}/templates/make-certs.sh.tmp", { + content = templatefile("${path.module}/templates/make-certs.sh.tmp", { cluster_context = var.kubernetes_context_name namespace = var.kubernetes_namespace node_address = join(" ", var.crdb_internal_nodes[*].dns) @@ -53,7 +53,7 @@ resource "local_file" "make_certs" { } resource "local_file" "apply_certs" { - content = templatefile("${path.module}/templates/apply-certs.sh.tmp", { + content = templatefile("${path.module}/templates/apply-certs.sh.tmp", { cluster_context = var.kubernetes_context_name namespace = var.kubernetes_namespace }) @@ -61,7 +61,7 @@ resource "local_file" "apply_certs" { } resource "local_file" "get_credentials" { - content = templatefile("${path.module}/templates/get-credentials.sh.tmp", { + content = templatefile("${path.module}/templates/get-credentials.sh.tmp", { get_credentials_cmd = var.kubernetes_get_credentials_cmd }) filename = "${local.workspace_location}/get-credentials.sh" @@ -96,11 +96,11 @@ resource "local_file" "helm_chart_values" { loadBalancers = { cockroachdbNodes = [ - for ip in var.crdb_internal_nodes[*].ip : - { - ip = ip - subnet = var.workload_subnet - } + for ip in var.crdb_internal_nodes[*].ip : + { + ip = ip + subnet = var.workload_subnet + } ] dssGateway = { @@ -114,7 +114,7 @@ resource "local_file" "helm_chart_values" { image = local.image conf = { - pubKeys = [ + pubKeys = [ "/test-certs/auth2.pem" ] jwksEndpoint = var.authorization.jwks != null ? var.authorization.jwks.endpoint : "" diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.tf index e91131f4c..360d2dc41 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.tf @@ -56,7 +56,7 @@ variable "image_pull_secret" { Example: docker-registry EOT - default = "" + default = "" } variable "authorization" { diff --git a/deploy/infrastructure/modules/terraform-aws-dss/terraform.dev.example.tfvars b/deploy/infrastructure/modules/terraform-aws-dss/terraform.dev.example.tfvars index 745bf84c6..f00be4f19 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/terraform.dev.example.tfvars +++ b/deploy/infrastructure/modules/terraform-aws-dss/terraform.dev.example.tfvars @@ -5,7 +5,7 @@ aws_region = "eu-west-1" # DNS Management -aws_route53_zone_id = "Z01551234567890123456" +aws_route53_zone_id = "Z01551234567890123456" # Hostnames app_hostname = "dss.interuss.example.com" diff --git a/deploy/infrastructure/modules/terraform-aws-dss/variables.tf b/deploy/infrastructure/modules/terraform-aws-dss/variables.tf index 3d276e351..a4a71745d 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/variables.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/variables.tf @@ -132,7 +132,7 @@ variable "image_pull_secret" { Example: docker-registry EOT - default = "" + default = "" } variable "authorization" { diff --git a/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars b/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars index 0721910b9..12f9edb4f 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars +++ b/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars @@ -18,7 +18,7 @@ google_machine_type = "e2-medium" google_kubernetes_storage_class = "standard" # DSS configuration -image = "latest" +image = "latest" image_pull_secret = "" authorization = { public_key_pem_path = "/test-certs/auth2.pem" diff --git a/deploy/infrastructure/modules/terraform-google-dss/variables.tf b/deploy/infrastructure/modules/terraform-google-dss/variables.tf index 0eeaab1d9..a131be3c0 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/variables.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/variables.tf @@ -123,7 +123,7 @@ variable "image_pull_secret" { Example: docker-registry EOT - default = "" + default = "" } variable "authorization" { diff --git a/deploy/infrastructure/utils/definitions/image_pull_secret.tf b/deploy/infrastructure/utils/definitions/image_pull_secret.tf index 7bdbff4f0..de99f4dc1 100644 --- a/deploy/infrastructure/utils/definitions/image_pull_secret.tf +++ b/deploy/infrastructure/utils/definitions/image_pull_secret.tf @@ -17,5 +17,5 @@ variable "image_pull_secret" { Example: docker-registry EOT - default = "" + default = "" } \ No newline at end of file diff --git a/deploy/operations/ci/aws-1/kubernetes_admin_access.tf b/deploy/operations/ci/aws-1/kubernetes_admin_access.tf index 8a6c10899..2f5ff7ef9 100644 --- a/deploy/operations/ci/aws-1/kubernetes_admin_access.tf +++ b/deploy/operations/ci/aws-1/kubernetes_admin_access.tf @@ -4,7 +4,7 @@ resource "local_file" "aws-auth-config-map" { content = yamlencode({ apiVersion = "v1" - kind = "ConfigMap" + kind = "ConfigMap" metadata = { name = "aws-auth" namespace = "kube-system" @@ -12,7 +12,7 @@ resource "local_file" "aws-auth-config-map" { data = { mapRoles = yamlencode([ { - groups = [ + groups = [ "system:bootstrappers", "system:nodes" ] @@ -20,14 +20,14 @@ resource "local_file" "aws-auth-config-map" { username = "system:node:{{EC2PrivateDNSName}}" }, { - groups = [ + groups = [ "system:masters" ] rolearn = var.aws_iam_administrator_role username = "interuss-aws-administrator" }, { - groups = [ + groups = [ "system:masters" ] rolearn = var.aws_iam_ci_role @@ -37,5 +37,5 @@ resource "local_file" "aws-auth-config-map" { } }) - filename = "${module.terraform-aws-dss.workspace_location}/aws_auth_config_map.yml" + filename = "${module.terraform-aws-dss.workspace_location}/aws_auth_config_map.yml" } diff --git a/deploy/operations/ci/aws-1/terraform.tfvars b/deploy/operations/ci/aws-1/terraform.tfvars index 90a5e8458..cbccb61d3 100644 --- a/deploy/operations/ci/aws-1/terraform.tfvars +++ b/deploy/operations/ci/aws-1/terraform.tfvars @@ -7,7 +7,7 @@ aws_region = "us-east-1" aws_route53_zone_id = "Z03377073HUSGB4L9FKEK" # Hostnames -app_hostname = "dss.ci.aws-interuss.uspace.dev" +app_hostname = "dss.ci.aws-interuss.uspace.dev" crdb_hostname_suffix = "db.ci.aws-interuss.uspace.dev" # Kubernetes configuration @@ -26,5 +26,5 @@ crdb_locality = "interuss_dss-ci-aws-ue1" crdb_external_nodes = [] aws_iam_permissions_boundary = "arn:aws:iam::301042233698:policy/GithubCIPermissionBoundaries20231130225039606500000001" -aws_iam_administrator_role = "arn:aws:iam::301042233698:role/AWSReservedSSO_AdministratorAccess_9b637c80b830ea2c" -aws_iam_ci_role = "arn:aws:iam::301042233698:role/InterUSSGithubCI" +aws_iam_administrator_role = "arn:aws:iam::301042233698:role/AWSReservedSSO_AdministratorAccess_9b637c80b830ea2c" +aws_iam_ci_role = "arn:aws:iam::301042233698:role/InterUSSGithubCI" diff --git a/deploy/operations/ci/aws-1/variables.tf b/deploy/operations/ci/aws-1/variables.tf index 3d276e351..a4a71745d 100644 --- a/deploy/operations/ci/aws-1/variables.tf +++ b/deploy/operations/ci/aws-1/variables.tf @@ -132,7 +132,7 @@ variable "image_pull_secret" { Example: docker-registry EOT - default = "" + default = "" } variable "authorization" {