From 456d0cc8e762baefd4c10fed79e32689a26cf58d Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Mon, 11 Dec 2023 11:55:28 -0500 Subject: [PATCH] [ci] Test deployment to AWS (#974) * 874: Automate creation of the infrastructure for the first ci environment * Fix login with session token * Add aws_iam_path variable to scope resources created * Add support for permissions boundary * Add AWS_SESSION_TOKEN to docker container * Only plan * Remove path from iam resources * Expose node group arn as output * Grant administrator access to kubernetes * Add README with debugging instructions * Trigger on PR * Update workflow * Update workflow * Update workflow * Update workflow * Update workflow * Update test script * exit code from container * Add kubernetes provider configuration * Grant explicitely GithubCI role to K8s * Keep only generated files output * Add title * Remove unnecessary version * Clean up * Manage operations/ci/aws-1/variables.tf with utility * new line end of file * Update documentation * Improve README --- .github/workflows/dss-deploy.yml | 49 ++++ .gitignore | 5 +- .../terraform-aws-kubernetes/cluster.tf | 6 +- .../terraform-aws-kubernetes/ebs.tf | 9 - .../terraform-aws-kubernetes/iam.tf | 98 ++++--- .../terraform-aws-kubernetes/main.tf | 15 +- .../terraform-aws-kubernetes/oidc.tf | 9 + .../terraform-aws-kubernetes/output.tf | 4 + .../terraform-aws-kubernetes/variables.tf | 10 + .../modules/terraform-aws-dss/TFVARS.md | 9 + .../modules/terraform-aws-dss/main.tf | 16 +- .../modules/terraform-aws-dss/variables.tf | 10 + deploy/infrastructure/utils/README.md | 2 +- .../aws_iam_permissions_boundary.tf | 8 + deploy/infrastructure/utils/variables.py | 9 +- deploy/operations/Dockerfile | 22 ++ deploy/operations/ci/aws-1/README.md | 39 +++ deploy/operations/ci/aws-1/data.tf | 1 + .../ci/aws-1/kubernetes_admin_access.tf | 38 +++ deploy/operations/ci/aws-1/local_variables.tf | 21 ++ deploy/operations/ci/aws-1/main.tf | 44 +++ deploy/operations/ci/aws-1/output.tf | 4 + deploy/operations/ci/aws-1/providers.tf | 19 ++ deploy/operations/ci/aws-1/terraform.tfvars | 30 ++ deploy/operations/ci/aws-1/test.sh | 26 ++ deploy/operations/ci/aws-1/variables.tf | 270 ++++++++++++++++++ deploy/operations/docker-compose.yaml | 15 + 27 files changed, 717 insertions(+), 71 deletions(-) create mode 100644 .github/workflows/dss-deploy.yml create mode 100644 deploy/infrastructure/dependencies/terraform-aws-kubernetes/oidc.tf create mode 100644 deploy/infrastructure/utils/definitions/aws_iam_permissions_boundary.tf create mode 100644 deploy/operations/Dockerfile create mode 100644 deploy/operations/ci/aws-1/README.md create mode 100644 deploy/operations/ci/aws-1/data.tf create mode 100644 deploy/operations/ci/aws-1/kubernetes_admin_access.tf create mode 100644 deploy/operations/ci/aws-1/local_variables.tf create mode 100644 deploy/operations/ci/aws-1/main.tf create mode 100644 deploy/operations/ci/aws-1/output.tf create mode 100644 deploy/operations/ci/aws-1/providers.tf create mode 100644 deploy/operations/ci/aws-1/terraform.tfvars create mode 100755 deploy/operations/ci/aws-1/test.sh create mode 100644 deploy/operations/ci/aws-1/variables.tf create mode 100644 deploy/operations/docker-compose.yaml diff --git a/.github/workflows/dss-deploy.yml b/.github/workflows/dss-deploy.yml new file mode 100644 index 000000000..7b20dc265 --- /dev/null +++ b/.github/workflows/dss-deploy.yml @@ -0,0 +1,49 @@ +name: Deploy DSS +on: + workflow_dispatch: {} + pull_request: {} +jobs: + deploy: + name: Deploy DSS to AWS + runs-on: ubuntu-latest + if: github.repository == 'interuss/dss' || github.repository == 'Orbitalize/dss' + concurrency: + group: dss-deploy-aws + cancel-in-progress: false + permissions: + id-token: write + contents: read + steps: + - name: Job information + run: | + echo "Job information" + echo "Trigger: ${{ github.event_name }}" + echo "Host: ${{ runner.os }}" + echo "Repository: ${{ github.repository }}" + echo "Branch: ${{ github.ref }}" + docker images + + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::301042233698:role/InterUSSGithubCI + aws-region: us-east-1 + mask-aws-account-id: true + role-duration-seconds: 1800 + + - name: Caller Id + run: | + aws sts get-caller-identity + + - name: Test Deployment Scenario AWS-1 + shell: bash + working-directory: ./deploy/operations/ + env: + COMPOSE_PROFILES: aws-1 + run: | + docker compose up --exit-code-from ci-aws-1 diff --git a/.gitignore b/.gitignore index 9268591f8..a24e88084 100644 --- a/.gitignore +++ b/.gitignore @@ -128,4 +128,7 @@ test/e2e_test_result go # vscode files -.vscode \ No newline at end of file +.vscode + +# terraform +.terraform* \ No newline at end of file diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/cluster.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/cluster.tf index dfca0743f..1c5153026 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/cluster.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/cluster.tf @@ -1,11 +1,11 @@ resource "aws_eks_cluster" "kubernetes_cluster" { name = var.cluster_name role_arn = aws_iam_role.dss-cluster.arn - + vpc_config { subnet_ids = aws_subnet.dss[*].id endpoint_public_access = true - public_access_cidrs = [ + public_access_cidrs = [ "0.0.0.0/0" ] } @@ -26,7 +26,7 @@ resource "aws_eks_node_group" "eks_node_group" { node_role_arn = aws_iam_role.dss-cluster-node-group.arn disk_size = 100 node_group_name_prefix = aws_eks_cluster.kubernetes_cluster.name - instance_types = [ + instance_types = [ var.aws_instance_type ] diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/ebs.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/ebs.tf index eedf02822..dc7eefd8b 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/ebs.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/ebs.tf @@ -1,12 +1,3 @@ -data "tls_certificate" "cluster_oidc_provider" { - url = aws_eks_cluster.kubernetes_cluster.identity[0].oidc[0].issuer -} - -resource "aws_iam_openid_connect_provider" "cluster_provider" { - client_id_list = ["sts.amazonaws.com"] - thumbprint_list = data.tls_certificate.cluster_oidc_provider.certificates[*].sha1_fingerprint - url = data.tls_certificate.cluster_oidc_provider.url -} resource "aws_eks_addon" "aws-ebs-csi-driver" { addon_name = "aws-ebs-csi-driver" diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf index 6eabc6ead..00131e28c 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/iam.tf @@ -7,22 +7,24 @@ locals { } resource "aws_iam_role" "dss-cluster" { + // EKS does not support a path in the role arn name = "${var.cluster_name}-dss-cluster" - assume_role_policy = < kubectl create secret -n VAR_NAMESPACE docker-registry VAR_DOCKER_IMAGE_PULL_SECRET \ + --docker-server=DOCKER_REGISTRY_SERVER \ + --docker-username=DOCKER_USER \ + --docker-password=DOCKER_PASSWORD \ + --docker-email=DOCKER_EMAIL + + Replace `VAR_DOCKER_IMAGE_PULL_SECRET` with the secret name (for instance: `private-registry-credentials`). + For docker hub private repository, use `docker.io` as `DOCKER_REGISTRY_SERVER` and an + [access token](https://hub.docker.com/settings/security) as `DOCKER_PASSWORD`. + + Example: docker-registry + EOT + default = "" +} + +variable "authorization" { + type = object({ + public_key_pem_path = optional(string) + jwks = optional(object({ + endpoint = string + key_id = string + })) + }) + description = <_", + and the = character is not allowed. However, any unique (among all other participating + DSS instances) value is acceptable. + + Example: + EOT +} + +variable "crdb_external_nodes" { + type = list(string) + description = <<-EOT + Fully-qualified domain name of existing CRDB nodes outside of the cluster if you are joining an existing pool. + Example: ["0.db.dss.example.com", "1.db.dss.example.com", "2.db.dss.example.com"] + EOT + default = [] +} + +variable "kubernetes_namespace" { + type = string + description = <<-EOT + Namespace where to deploy Kubernetes resources. Only default is supported at the moment. + + Example: `default` + EOT + + default = "default" + + # TODO: Adapt current deployment scripts in /build/deploy to support default is supported for the moment. + validation { + condition = var.kubernetes_namespace == "default" + error_message = "Only default namespace is supported at the moment" + } +} + diff --git a/deploy/operations/docker-compose.yaml b/deploy/operations/docker-compose.yaml new file mode 100644 index 000000000..4618089db --- /dev/null +++ b/deploy/operations/docker-compose.yaml @@ -0,0 +1,15 @@ +services: + ci-aws-1: + build: . + image: interuss-deploy + profiles: ["aws-1"] + command: operations/ci/aws-1/test.sh + working_dir: /opt/dss + environment: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + volumes: + - type: bind + source: ../ + target: /opt/dss/