[ci] Test deployment to AWS (#974)

* 874: Automate creation of the infrastructure for the first ci environment * Fix login with session token * Add aws_iam_path variable to scope resources created * Add support for permissions boundary * Add AWS_SESSION_TOKEN to docker container * Only plan * Remove path from iam resources * Expose node group arn as output * Grant administrator access to kubernetes * Add README with debugging instructions * Trigger on PR * Update workflow * Update workflow * Update workflow * Update workflow * Update workflow * Update test script * exit code from container * Add kubernetes provider configuration * Grant explicitely GithubCI role to K8s * Keep only generated files output * Add title * Remove unnecessary version * Clean up * Manage operations/ci/aws-1/variables.tf with utility * new line end of file * Update documentation * Improve README
interuss · Dec 11, 2023 · 456d0cc · 456d0cc
1 parent d0a6b32
commit 456d0cc
Show file tree

Hide file tree

Showing 27 changed files with 717 additions and 71 deletions.
diff --git a/.github/workflows/dss-deploy.yml b/.github/workflows/dss-deploy.yml
@@ -0,0 +1,49 @@
+name: Deploy DSS
+on:
+  workflow_dispatch: {}
+  pull_request: {}
+jobs:
+  deploy:
+    name: Deploy DSS to AWS
+    runs-on: ubuntu-latest
+    if: github.repository == 'interuss/dss' || github.repository == 'Orbitalize/dss'
+    concurrency:
+      group: dss-deploy-aws
+      cancel-in-progress: false
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Job information
+        run: |
+          echo "Job information"
+          echo "Trigger: ${{ github.event_name }}"
+          echo "Host: ${{ runner.os }}"
+          echo "Repository: ${{ github.repository }}"
+          echo "Branch: ${{ github.ref }}"
+          docker images
+
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::301042233698:role/InterUSSGithubCI
+          aws-region: us-east-1
+          mask-aws-account-id: true
+          role-duration-seconds: 1800
+
+      - name: Caller Id
+        run: |
+          aws sts get-caller-identity
+
+      - name: Test Deployment Scenario AWS-1
+        shell: bash
+        working-directory: ./deploy/operations/
+        env:
+          COMPOSE_PROFILES: aws-1
+        run: |
+          docker compose up --exit-code-from ci-aws-1
diff --git a/.gitignore b/.gitignore
@@ -128,4 +128,7 @@ test/e2e_test_result
 go
 
 # vscode files
-.vscode
+.vscode
+
+# terraform
+.terraform*
diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/cluster.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/cluster.tf
@@ -1,11 +1,11 @@
 resource "aws_eks_cluster" "kubernetes_cluster" {
   name     = var.cluster_name
   role_arn = aws_iam_role.dss-cluster.arn
-
+  
   vpc_config {
     subnet_ids             = aws_subnet.dss[*].id
     endpoint_public_access = true
-    public_access_cidrs = [
+    public_access_cidrs    = [
       "0.0.0.0/0"
     ]
   }
@@ -26,7 +26,7 @@ resource "aws_eks_node_group" "eks_node_group" {
   node_role_arn          = aws_iam_role.dss-cluster-node-group.arn
   disk_size              = 100
   node_group_name_prefix = aws_eks_cluster.kubernetes_cluster.name
-  instance_types = [
+  instance_types         = [
     var.aws_instance_type
   ]
 

diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/ebs.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/ebs.tf
@@ -1,12 +1,3 @@
-data "tls_certificate" "cluster_oidc_provider" {
-  url = aws_eks_cluster.kubernetes_cluster.identity[0].oidc[0].issuer
-}
-
-resource "aws_iam_openid_connect_provider" "cluster_provider" {
-  client_id_list  = ["sts.amazonaws.com"]
-  thumbprint_list = data.tls_certificate.cluster_oidc_provider.certificates[*].sha1_fingerprint
-  url             = data.tls_certificate.cluster_oidc_provider.url
-}
 
 resource "aws_eks_addon" "aws-ebs-csi-driver" {
   addon_name               = "aws-ebs-csi-driver"

diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf
@@ -7,22 +7,24 @@ locals {
 }
 
 resource "aws_iam_role" "dss-cluster" {
+  // EKS does not support a path in the role arn
   name = "${var.cluster_name}-dss-cluster"
 
-  assume_role_policy = <<POLICY
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Principal": {
-        "Service": "eks.amazonaws.com"
-      },
-      "Action": "sts:AssumeRole"
-    }
-  ]
-}
-POLICY
+  assume_role_policy = jsonencode(
+  {
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        "Effect" : "Allow",
+        "Principal" : {
+          "Service" : "eks.amazonaws.com"
+        },
+        "Action" : "sts:AssumeRole"
+      }
+    ]
+  })
+
+  permissions_boundary = var.aws_iam_permissions_boundary
 }
 
 # Policy used by internal kubernetes services to access AWS resources.
@@ -31,30 +33,66 @@ resource "aws_iam_role_policy_attachment" "dss-cluster-service" {
   role       = aws_iam_role.dss-cluster.name
 }
 
+# Roles
+
 resource "aws_iam_role" "dss-cluster-node-group" {
   name = "${var.cluster_name}-cluster-node-group"
 
   assume_role_policy = jsonencode({
     Statement = [
       {
-        Action = "sts:AssumeRole"
-        Effect = "Allow"
+        Action    = "sts:AssumeRole"
+        Effect    = "Allow"
         Principal = {
           Service = "ec2.amazonaws.com"
         }
       }
     ]
-    Version = "2012-10-17"
+    Version   = "2012-10-17"
   })
+
+  permissions_boundary = var.aws_iam_permissions_boundary
 }
 
+// EBS
+
+resource "aws_iam_role" "AmazonEKS_EBS_CSI_DriverRole" {
+  name = "${var.cluster_name}-AmazonEKS_EBS_CSI_DriverRole"
+
+  assume_role_policy = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        "Effect" : "Allow",
+        "Principal" : {
+          "Federated" : format("arn:aws:iam::${local.aws_account_id}:%s", replace(local.aws_cluster_oidc_issuer, "https://", "oidc-provider/")),
+        },
+        "Action" : "sts:AssumeRoleWithWebIdentity",
+        "Condition" : {
+          "StringEquals" : {
+            format("%s:aud", replace(local.aws_cluster_oidc_issuer, "https://", "")) : "sts.amazonaws.com",
+            format("%s:sub", replace(local.aws_cluster_oidc_issuer, "https://", "")) : "system:serviceaccount:kube-system:ebs-csi-controller-sa"
+          }
+        }
+      }
+    ]
+  })
+
+  permissions_boundary = var.aws_iam_permissions_boundary
+}
+
+// Policies
+
 resource "aws_iam_policy" "AWSLoadBalancerControllerPolicy" {
-  name = "${var.cluster_name}-AWSLoadBalancerControllerPolicy"
+  name   = "${var.cluster_name}-AWSLoadBalancerControllerPolicy"
+
   # Source: https://docs.aws.amazon.com/eks/latest/userguide/aws-load-balancer-controller.html
   # Template: https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.4.4/docs/install/iam_policy.json
   policy = file("${path.module}/AWSLoadBalancerControllerPolicy.json")
 }
 
+// Attachments
+
 resource "aws_iam_role_policy_attachment" "AWSLoadBalancerControllerPolicy" {
   policy_arn = aws_iam_policy.AWSLoadBalancerControllerPolicy.arn
   role       = aws_iam_role.dss-cluster-node-group.name
@@ -70,35 +108,11 @@ resource "aws_iam_role_policy_attachment" "AmazonEKS_CNI_Policy" {
   role       = aws_iam_role.dss-cluster-node-group.name
 }
 
-## Docker registry
 resource "aws_iam_role_policy_attachment" "AmazonEC2ContainerRegistryReadOnly" {
   policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
   role       = aws_iam_role.dss-cluster-node-group.name
 }
 
-## EBS
-resource "aws_iam_role" "AmazonEKS_EBS_CSI_DriverRole" {
-  name = "${var.cluster_name}-AmazonEKS_EBS_CSI_DriverRole"
-
-  assume_role_policy = jsonencode({
-    "Version" : "2012-10-17",
-    "Statement" : [
-      {
-        "Effect" : "Allow",
-        "Principal" : {
-          "Federated" : format("arn:aws:iam::${local.aws_account_id}:%s", replace(local.aws_cluster_oidc_issuer, "https://", "oidc-provider/")),
-        },
-        "Action" : "sts:AssumeRoleWithWebIdentity",
-        "Condition" : {
-          "StringEquals" : {
-            format("%s:aud", replace(local.aws_cluster_oidc_issuer, "https://", "")) : "sts.amazonaws.com",
-            format("%s:sub", replace(local.aws_cluster_oidc_issuer, "https://", "")) : "system:serviceaccount:kube-system:ebs-csi-controller-sa"
-          }
-        }
-      }
-    ]
-  })
-}
 
 resource "aws_iam_role_policy_attachment" "AmazonEKS_EBS_CSI_DriverRole" {
   policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy"

diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/main.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/main.tf
@@ -7,6 +7,9 @@ terraform {
     tls = {
       source = "hashicorp/tls"
     }
+    helm = {
+      source = "hashicorp/helm"
+    }
   }
 }
 
@@ -20,14 +23,14 @@ provider "aws" {
   }
 }
 
+data "aws_eks_cluster_auth" "kubernetes_cluster" {
+  name = aws_eks_cluster.kubernetes_cluster.name
+}
+
 provider "helm" {
   kubernetes {
     host                   = aws_eks_cluster.kubernetes_cluster.endpoint
     cluster_ca_certificate = base64decode(aws_eks_cluster.kubernetes_cluster.certificate_authority[0].data)
-    exec {
-      api_version = "client.authentication.k8s.io/v1beta1"
-      args        = ["eks", "get-token", "--cluster-name", var.cluster_name]
-      command     = "aws"
-    }
+    token = data.aws_eks_cluster_auth.kubernetes_cluster.token
   }
-}
+}
diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/oidc.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/oidc.tf
@@ -0,0 +1,9 @@
+data "tls_certificate" "cluster_oidc_provider" {
+  url = aws_eks_cluster.kubernetes_cluster.identity[0].oidc[0].issuer
+}
+
+resource "aws_iam_openid_connect_provider" "cluster_provider" {
+  client_id_list  = ["sts.amazonaws.com"]
+  thumbprint_list = data.tls_certificate.cluster_oidc_provider.certificates[*].sha1_fingerprint
+  url             = data.tls_certificate.cluster_oidc_provider.url
+}
diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/output.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/output.tf
@@ -52,4 +52,8 @@ output "gateway_address" {
 
 output "workload_subnet" {
   value = data.aws_subnet.main_subnet.id
+}
+
+output "iam_role_node_group_arn" {
+  value = aws_iam_role.dss-cluster-node-group.arn
 }
diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.tf
@@ -33,6 +33,16 @@ variable "aws_route53_zone_id" {
   EOT
 }
 
+variable "aws_iam_permissions_boundary" {
+  type        = string
+  description = <<-EOT
+    AWS IAM Policy ARN to be used for permissions boundaries on created roles.
+
+    Example: `arn:aws:iam::123456789012:policy/GithubCIPermissionBoundaries`
+  EOT
+}
+
+
 variable "app_hostname" {
   type        = string
   description = <<-EOT

diff --git a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.md b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.md
@@ -40,6 +40,15 @@ Leave empty to disable record creation.
 Example: `Z0123456789ABCDEFGHIJ`
 
 
+### aws_iam_permissions_boundary
+
+*Type: `string`*
+
+AWS IAM Policy ARN to be used for permissions boundaries on created roles.
+
+Example: `arn:aws:iam::123456789012:policy/GithubCIPermissionBoundaries`
+
+
 ### app_hostname
 
 *Type: `string`*

diff --git a/deploy/infrastructure/modules/terraform-aws-dss/main.tf b/deploy/infrastructure/modules/terraform-aws-dss/main.tf
@@ -1,12 +1,14 @@
 module "terraform-aws-kubernetes" {
   # See variables.tf for variables description.
-  cluster_name         = var.cluster_name
-  aws_region           = var.aws_region
-  app_hostname         = var.app_hostname
-  crdb_hostname_suffix = var.crdb_hostname_suffix
-  aws_instance_type    = var.aws_instance_type
-  aws_route53_zone_id  = var.aws_route53_zone_id
-  node_count           = var.node_count
+  cluster_name                 = var.cluster_name
+  aws_region                   = var.aws_region
+  app_hostname                 = var.app_hostname
+  crdb_hostname_suffix         = var.crdb_hostname_suffix
+  aws_instance_type            = var.aws_instance_type
+  aws_route53_zone_id          = var.aws_route53_zone_id
+  aws_iam_path                 = var.aws_iam_path
+  aws_iam_permissions_boundary = var.aws_iam_permissions_boundary
+  node_count                   = var.node_count
 
   source = "../../dependencies/terraform-aws-kubernetes"
 }

diff --git a/deploy/infrastructure/modules/terraform-aws-dss/variables.tf b/deploy/infrastructure/modules/terraform-aws-dss/variables.tf
@@ -33,6 +33,16 @@ variable "aws_route53_zone_id" {
   EOT
 }
 
+variable "aws_iam_permissions_boundary" {
+  type        = string
+  description = <<-EOT
+    AWS IAM Policy ARN to be used for permissions boundaries on created roles.
+
+    Example: `arn:aws:iam::123456789012:policy/GithubCIPermissionBoundaries`
+  EOT
+}
+
+
 variable "app_hostname" {
   type        = string
   description = <<-EOT

diff --git a/deploy/infrastructure/utils/README.md b/deploy/infrastructure/utils/README.md
@@ -4,4 +4,4 @@ This directory contains the following tools to simplify the management of the te
 
 1. `generate_terraform_variables.sh`: Terraform variables can't be shared between modules without repeating their definition at every level of encapsulation.
    To prevent repeating ourselves and to maintain a consistent level of quality for every module and dependencies, this script takes variables 
-   in the `definitions` directory and creates a `variables.tf` file in each modules with the appropriate content.
+   in the `definitions` directory and creates a `variables.tf` file in each modules with the appropriate content.
diff --git a/deploy/infrastructure/utils/definitions/aws_iam_permissions_boundary.tf b/deploy/infrastructure/utils/definitions/aws_iam_permissions_boundary.tf
@@ -0,0 +1,8 @@
+variable "aws_iam_permissions_boundary" {
+  type        = string
+  description = <<-EOT
+    AWS IAM Policy ARN to be used for permissions boundaries on created roles.
+
+    Example: `arn:aws:iam::123456789012:policy/GithubCIPermissionBoundaries`
+  EOT
+}