From ea795dca7dda4420cdb4e31c4732143ebad8d0f5 Mon Sep 17 00:00:00 2001 From: Andy Roth Date: Tue, 19 Oct 2021 12:47:52 -0700 Subject: [PATCH] Prep the GitHub Actions setup for handling more than one E2E test (#106) Signed-off-by: Jeff McCoy --- .github/workflows/test-command.yml | 129 ++++++++++++++++--- Makefile | 15 ++- test/e2e/common.go | 139 ++++++++++++++++++++ test/e2e/e2e_example_game_test.go | 82 ++++++++++++ test/e2e/terraform_ssh_e2e_test.go | 196 ----------------------------- 5 files changed, 342 insertions(+), 219 deletions(-) create mode 100644 test/e2e/common.go create mode 100644 test/e2e/e2e_example_game_test.go delete mode 100644 test/e2e/terraform_ssh_e2e_test.go diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 07441f64ea..da4c22a71c 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -8,7 +8,7 @@ on: defaults: run: - # We need -e -o pipefail for consistency with GitHub Actions's default behavior + # We need -e -o pipefail for consistency with GitHub Actions' default behavior shell: bash -e -o pipefail {0} jobs: @@ -18,6 +18,7 @@ jobs: runs-on: ubuntu-latest outputs: run-ping: ${{ steps.parse.outputs.ping }} + run-build: ${{ steps.parse.outputs.build }} run-e2e: ${{ steps.parse.outputs.e2e }} steps: - name: Parse Args @@ -31,7 +32,7 @@ jobs: ARGS="${ARGS_V1}${ARGS_V2}" printf "Args are %s\n" "$ARGS" printf "\n\nslash_command is %s\n\n" "$DEBUG" - COMMANDS=(PING E2E) + COMMANDS=(PING BUILD E2E) if printf "%s" "${ARGS^^}" | grep -qE '\bALL\b'; then # "all" explicitly does not include "ping" for cmd in "${COMMANDS[@]}"; do @@ -70,10 +71,107 @@ jobs: GITHUB_REF: ${{ github.event.client_payload.pull_request.head.ref }} GITHUB_OWNER: ${{ github.event.client_payload.github.payload.repository.owner.login }} - # Run E2E tests - e2e: + # Build and upload the artifacts so they can be used later in the pipeline + build: runs-on: ubuntu-latest needs: parse + # Run if they explicitly want it, or run if they want a different stage that depends on this + if: needs.parse.outputs.run-build == 'true' || needs.parse.outputs.run-e2e == 'true' + container: cloudposse/test-harness:latest + steps: + # Update GitHub status for pending pipeline run + - name: "Update GitHub Status for pending" + uses: docker://cloudposse/github-status-updater + with: + args: "-action update_state -ref ${{ github.event.client_payload.pull_request.head.sha }} -repo ${{ github.event.client_payload.github.payload.repository.name }}" + env: + GITHUB_TOKEN: ${{ secrets.PAT }} + GITHUB_STATE: pending + GITHUB_CONTEXT: "/test build" + GITHUB_DESCRIPTION: "started by @${{ github.event.client_payload.github.actor }}" + GITHUB_TARGET_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + GITHUB_REF: ${{ github.event.client_payload.pull_request.head.ref }} + GITHUB_OWNER: ${{ github.event.client_payload.github.payload.repository.owner.login }} + + # Checkout the code from GitHub Pull Request + - name: "Checkout the code" + uses: actions/checkout@v2 + with: + token: ${{ secrets.PAT }} + repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }} + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: "Build the artifacts" + shell: bash -x -e -o pipefail {0} + run: | + # cloudposse/test-harness has golang 1.15, we need 1.16. This is the easiest way I know to do it. This should definitely be revisited and cleaned up. + git clone --branch v0.8.0 --depth 1 https://github.com/asdf-vm/asdf.git $HOME/.asdf + source ~/.asdf/asdf.sh + export PATH="$HOME/.asdf/bin:$PATH" + asdf plugin-add golang https://github.com/kennyp/asdf-golang.git + asdf install golang 1.16.7 + asdf global golang 1.16.7 + export GOPATH="$HOME/go" + export PATH="$PATH:$GOPATH/bin" + make build-cli-linux + ./build/zarf tools registry login registry1.dso.mil --username "${{ secrets.REGISTRY1_USERNAME_ROTHANDREW2 }}" --password "${{ secrets.REGISTRY1_PASSWORD_ROTHANDREW2 }}" + make init-package + + - name: "Upload the artifacts" + uses: actions/upload-artifact@v2 + with: + name: build + path: build + if-no-files-found: error + + # Update GitHub status for failing pipeline run + - name: "Update GitHub Status for failure" + if: ${{ failure() }} + uses: docker://cloudposse/github-status-updater + with: + args: "-action update_state -ref ${{ github.event.client_payload.pull_request.head.sha }} -repo ${{ github.event.client_payload.github.payload.repository.name }}" + env: + GITHUB_TOKEN: ${{ secrets.PAT }} + GITHUB_STATE: failure + GITHUB_CONTEXT: "/test build" + GITHUB_DESCRIPTION: "run failed" + GITHUB_TARGET_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + GITHUB_REF: ${{ github.event.client_payload.pull_request.head.ref }} + GITHUB_OWNER: ${{ github.event.client_payload.github.payload.repository.owner.login }} + + # Update GitHub status for successful pipeline run + - name: "Update GitHub Status for success" + uses: docker://cloudposse/github-status-updater + with: + args: "-action update_state -ref ${{ github.event.client_payload.pull_request.head.sha }} -repo ${{ github.event.client_payload.github.payload.repository.name }}" + env: + GITHUB_TOKEN: ${{ secrets.PAT }} + GITHUB_STATE: success + GITHUB_CONTEXT: "/test build" + GITHUB_DESCRIPTION: "run passed" + GITHUB_TARGET_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + GITHUB_REF: ${{ github.event.client_payload.pull_request.head.ref }} + GITHUB_OWNER: ${{ github.event.client_payload.github.payload.repository.owner.login }} + + # Update GitHub status for cancelled pipeline run + - name: "Update GitHub Status for cancelled" + if: ${{ cancelled() }} + uses: docker://cloudposse/github-status-updater + with: + args: "-action update_state -ref ${{ github.event.client_payload.pull_request.head.sha }} -repo ${{ github.event.client_payload.github.payload.repository.name }}" + env: + GITHUB_TOKEN: ${{ secrets.PAT }} + GITHUB_STATE: error + GITHUB_CONTEXT: "/test build" + GITHUB_DESCRIPTION: "run cancelled" + GITHUB_TARGET_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + GITHUB_REF: ${{ github.event.client_payload.pull_request.head.ref }} + GITHUB_OWNER: ${{ github.event.client_payload.github.payload.repository.owner.login }} + + # Run the Game E2E test + e2e-game: + runs-on: ubuntu-latest + needs: [parse, build] if: needs.parse.outputs.run-e2e == 'true' container: cloudposse/test-harness:latest steps: @@ -85,7 +183,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.PAT }} GITHUB_STATE: pending - GITHUB_CONTEXT: "/test e2e" + GITHUB_CONTEXT: "/test e2e - Game Example" GITHUB_DESCRIPTION: "started by @${{ github.event.client_payload.github.actor }}" GITHUB_TARGET_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} GITHUB_REF: ${{ github.event.client_payload.pull_request.head.ref }} @@ -99,13 +197,9 @@ jobs: repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }} ref: ${{ github.event.client_payload.pull_request.head.ref }} -# # Checkout the code from GitHub Pull Request -# - name: "Checkout the code" -# uses: actions/checkout@v2 -# with: -# token: ${{ secrets.PAT }} -# repository: defenseunicorns/zarf -# ref: feature/add-terratest-e2e-to-pipeline + # Download the built artifacts + - name: "Download the built artifacts" + uses: actions/download-artifact@v2 - name: "Run E2E tests" shell: bash -x -e -o pipefail {0} @@ -123,9 +217,8 @@ jobs: asdf global golang 1.16.7 export GOPATH="$HOME/go" export PATH="$PATH:$GOPATH/bin" - make build-cli-linux - ./build/zarf tools registry login registry1.dso.mil --username "${{ secrets.REGISTRY1_USERNAME_ROTHANDREW2 }}" --password "${{ secrets.REGISTRY1_PASSWORD_ROTHANDREW2 }}" - make init-package test-e2e + chmod +x build/zarf + make package-example-game test-cloud-e2e-example-game # Update GitHub status for failing pipeline run - name: "Update GitHub Status for failure" @@ -136,7 +229,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.PAT }} GITHUB_STATE: failure - GITHUB_CONTEXT: "/test e2e" + GITHUB_CONTEXT: "/test e2e - Game Example" GITHUB_DESCRIPTION: "run failed" GITHUB_TARGET_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} GITHUB_REF: ${{ github.event.client_payload.pull_request.head.ref }} @@ -150,7 +243,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.PAT }} GITHUB_STATE: success - GITHUB_CONTEXT: "/test e2e" + GITHUB_CONTEXT: "/test e2e - Game Example" GITHUB_DESCRIPTION: "run passed" GITHUB_TARGET_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} GITHUB_REF: ${{ github.event.client_payload.pull_request.head.ref }} @@ -165,7 +258,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.PAT }} GITHUB_STATE: error - GITHUB_CONTEXT: "/test e2e" + GITHUB_CONTEXT: "/test e2e - Game Example" GITHUB_DESCRIPTION: "run cancelled" GITHUB_TARGET_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} GITHUB_REF: ${{ github.event.client_payload.pull_request.head.ref }} diff --git a/Makefile b/Makefile index be81cfb88e..334d45cdaa 100644 --- a/Makefile +++ b/Makefile @@ -33,9 +33,6 @@ vm-init: ## usage -> make vm-init OS=ubuntu vm-destroy: ## Destroy the VM vagrant destroy -f -test-e2e: ## Run E2E tests. Requires access to an AWS account. Costs money. Make sure you ran the `build-cli` and `init-package` targets first - cd test/e2e && go test ./... -v -timeout 1200s - e2e-ssh: ## Run this if you set SKIP_teardown=1 and want to SSH into the still-running test server. Don't forget to unset SKIP_teardown when you're done cd test/tf/public-ec2-instance/.test-data && cat Ec2KeyPair.json | jq -r .PrivateKey > privatekey.pem && chmod 600 privatekey.pem cd test/tf/public-ec2-instance && ssh -i .test-data/privatekey.pem ubuntu@$$(terraform output public_instance_ip) @@ -61,5 +58,13 @@ build-test: build-cli init-package ## Build the CLI and create the init package ci-release: init-package ## Create the init package -package-examples: ## automatically package all example directories and add the tarballs to the examples/sync directory - cd examples && $(MAKE) package-examples +.PHONY: package-example-game +package-example-game: ## Create the Doom example + cd examples/game && ../../$(ZARF_BIN) package create --confirm && mv zarf-package-* ../../build/ + +.PHONY: test-cloud-e2e-example-game +test-cloud-e2e-example-game: ## Runs the Doom game as an E2E test in the cloud. Requires access to an AWS account. Costs money. Make sure you ran the `build-cli`, `init-package`, and `package-example-game` targets first + cd test/e2e && go test ./... -run TestE2eExampleGame -v -timeout 1200s + +.PHONY: test-e2e +test-e2e: package-example-game test-cloud-e2e-example-game ## DEPRECATED - to be replaced by individual e2e test targets diff --git a/test/e2e/common.go b/test/e2e/common.go new file mode 100644 index 0000000000..04ae0be542 --- /dev/null +++ b/test/e2e/common.go @@ -0,0 +1,139 @@ +package test + +import ( + "bufio" + "encoding/base64" + "fmt" + "github.com/gruntwork-io/terratest/modules/random" + "github.com/gruntwork-io/terratest/modules/retry" + teststructure "github.com/gruntwork-io/terratest/modules/test-structure" + "github.com/stretchr/testify/require" + "io/ioutil" + "os" + "testing" + "time" + + "github.com/gruntwork-io/terratest/modules/aws" + "github.com/gruntwork-io/terratest/modules/ssh" + "github.com/gruntwork-io/terratest/modules/terraform" +) + +func teardown(t *testing.T, tmpFolder string) { + keyPair := teststructure.LoadEc2KeyPair(t, tmpFolder) + aws.DeleteEC2KeyPair(t, keyPair) + + terraformOptions := teststructure.LoadTerraformOptions(t, tmpFolder) + terraform.Destroy(t, terraformOptions) +} + +func setup(t *testing.T, tmpFolder string) { + terraformOptions, keyPair, err := configureTerraformOptions(t, tmpFolder) + require.NoError(t, err) + + // Save the options and key pair so later test stages can use them + teststructure.SaveTerraformOptions(t, tmpFolder, terraformOptions) + teststructure.SaveEc2KeyPair(t, tmpFolder, keyPair) + + // This will run `terraform init` and `terraform apply` and fail the test if there are any errors + terraform.InitAndApply(t, terraformOptions) +} + +func configureTerraformOptions(t *testing.T, tmpFolder string) (*terraform.Options, *aws.Ec2Keypair, error) { + // A unique ID we can use to namespace resources so we don't clash with anything already in the AWS account or + // tests running in parallel + uniqueID := random.UniqueId() + namespace := "zarf" + stage := "terratest" + name := fmt.Sprintf("e2e-%s", uniqueID) + + // Get the region to use from the system's environment + awsRegion, err := getAwsRegion() + if err != nil { + return nil, nil, err + } + + instanceType := "t3a.large" + + // Create an EC2 KeyPair that we can use for SSH access + keyPairName := fmt.Sprintf("%s-%s-%s", namespace, stage, name) + keyPair := aws.CreateAndImportEC2KeyPair(t, awsRegion, keyPairName) + + // Construct the terraform options with default retryable errors to handle the most common retryable errors in + // terraform testing. + terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ + // The path to where our Terraform code is located + TerraformDir: tmpFolder, + + // Variables to pass to our Terraform code using -var options + Vars: map[string]interface{}{ + "aws_region": awsRegion, + "namespace": namespace, + "stage": stage, + "name": name, + "instance_type": instanceType, + "key_pair_name": keyPairName, + }, + }) + + return terraformOptions, keyPair, nil +} + +// syncFileToRemoteServer uses SCP to sync a file from source to destination. `destPath` can be absolute or relative to +// the SSH user's home directory. It has to be in a directory that the SSH user is allowed to write to. +func syncFileToRemoteServer(t *testing.T, terraformOptions *terraform.Options, keyPair *aws.Ec2Keypair, sshUsername string, srcPath string, destPath string, chmod string) { + // Run `terraform output` to get the value of an output variable + publicInstanceIP := terraform.Output(t, terraformOptions, "public_instance_ip") + + // We're going to try to SSH to the instance IP, using the Key Pair we created earlier, and the user "ubuntu", + // as we know the Instance is running an Ubuntu AMI that has such a user + host := ssh.Host{ + Hostname: publicInstanceIP, + SshKeyPair: keyPair.KeyPair, + SshUserName: sshUsername, + } + + // It can take a minute or so for the Instance to boot up, so retry a few times + maxRetries := 15 + timeBetweenRetries, err := time.ParseDuration("5s") + require.NoError(t, err) + + // Wait for the instance to be ready + _, err = retry.DoWithRetryE(t, "Wait for the instance to be ready", maxRetries, timeBetweenRetries, func() (string, error){ + _, err := ssh.CheckSshCommandE(t, host, "whoami") + if err != nil { + return "", err + } + return "", nil + }) + require.NoError(t, err) + + // Create the folder structure + output, err := ssh.CheckSshCommandE(t, host,fmt.Sprintf("bash -c 'install -m 644 -D /dev/null \"%s\"'", destPath)) + require.NoError(t, err, output) + + // The ssh lib only supports sending strings so we'll base64encode it first + f, err := os.Open(srcPath) + require.NoError(t, err) + reader := bufio.NewReader(f) + content, err := ioutil.ReadAll(reader) + require.NoError(t, err) + encodedContent := base64.StdEncoding.EncodeToString(content) + err = ssh.ScpFileToE(t, host, 0600, fmt.Sprintf("%s.b64", destPath), encodedContent) + require.NoError(t, err) + output, err = ssh.CheckSshCommandE(t, host, fmt.Sprintf("base64 -d \"%s.b64\" > \"%s\" && chmod \"%s\" \"%s\"", destPath, destPath, chmod, destPath)) + require.NoError(t, err, output) +} + +// getAwsRegion returns the desired AWS region to use by first checking the env var AWS_REGION, then checking +// AWS_DEFAULT_REGION if AWS_REGION isn't set. If neither is set it returns an error +func getAwsRegion() (string, error) { + val, present := os.LookupEnv("AWS_REGION") + if !present { + val, present = os.LookupEnv("AWS_DEFAULT_REGION") + } + if !present { + return "", fmt.Errorf("expected either AWS_REGION or AWS_DEFAULT_REGION env var to be set, but they were not") + } else { + return val, nil + } +} diff --git a/test/e2e/e2e_example_game_test.go b/test/e2e/e2e_example_game_test.go new file mode 100644 index 0000000000..d517a2055e --- /dev/null +++ b/test/e2e/e2e_example_game_test.go @@ -0,0 +1,82 @@ +package test + +import ( + "fmt" + "github.com/gruntwork-io/terratest/modules/aws" + "github.com/gruntwork-io/terratest/modules/ssh" + "github.com/gruntwork-io/terratest/modules/terraform" + teststructure "github.com/gruntwork-io/terratest/modules/test-structure" + "github.com/stretchr/testify/require" + "testing" +) + +func TestE2eExampleGame(t *testing.T) { + t.Parallel() + + // Our SSH username, will change based on which AMI we use + username := "ubuntu" + + // Copy the terraform folder to a temp directory so we can run multiple tests in parallel + tmpFolder := teststructure.CopyTerraformFolderToTemp(t, "..", "tf/public-ec2-instance") + + // At the end of the test, run `terraform destroy` to clean up any resources that were created + defer teststructure.RunTestStage(t, "TEARDOWN", func() { + teardown(t, tmpFolder) + }) + + // Deploy the terraform infra + teststructure.RunTestStage(t, "SETUP", func() { + setup(t, tmpFolder) + }) + + // Upload the Zarf artifacts + teststructure.RunTestStage(t, "UPLOAD", func() { + terraformOptions := teststructure.LoadTerraformOptions(t, tmpFolder) + keyPair := teststructure.LoadEc2KeyPair(t, tmpFolder) + + syncFileToRemoteServer(t, terraformOptions, keyPair, username, "../../build/zarf", fmt.Sprintf("/home/%s/build/zarf", username), "0700") + syncFileToRemoteServer(t, terraformOptions, keyPair, username, "../../build/zarf-init.tar.zst", fmt.Sprintf("/home/%s/build/zarf-init.tar.zst", username), "0600") + syncFileToRemoteServer(t, terraformOptions, keyPair, username, "../../build/zarf-package-appliance-demo-doom.tar.zst", fmt.Sprintf("/home/%s/build/zarf-package-appliance-demo-doom.tar.zst", username), "0600") + }) + + teststructure.RunTestStage(t, "TEST", func() { + terraformOptions := teststructure.LoadTerraformOptions(t, tmpFolder) + keyPair := teststructure.LoadEc2KeyPair(t, tmpFolder) + + // Finally run the actual test + test(t, terraformOptions, keyPair,username) + }) +} + +func test(t *testing.T, terraformOptions *terraform.Options, keyPair *aws.Ec2Keypair, username string) { + // Run `terraform output` to get the value of an output variable + publicInstanceIP := terraform.Output(t, terraformOptions, "public_instance_ip") + + // We're going to try to SSH to the instance IP, using the Key Pair we created earlier, and the user "ubuntu", + // as we know the Instance is running an Ubuntu AMI that has such a user + publicHost := ssh.Host{ + Hostname: publicInstanceIP, + SshKeyPair: keyPair.KeyPair, + SshUserName: username, + } + + // Make sure `zarf --help` doesn't error + output, err := ssh.CheckSshCommandE(t, publicHost, fmt.Sprintf("sudo /home/%s/build/zarf --help", username)) + require.NoError(t, err, output) + + // run `zarf init` + output, err = ssh.CheckSshCommandE(t, publicHost, fmt.Sprintf("cd /home/%s/build && sudo ./zarf init --confirm --components management --host localhost", username)) + require.NoError(t, err, output) + + // Wait until the Docker registry is ready + output, err = ssh.CheckSshCommandE(t, publicHost, "curl -sfSL --retry 15 --retry-connrefused --retry-delay 10 -o /dev/null -w \"%{http_code}\" \"https://localhost/v2/\"") + require.NoError(t, err, output) + + // Deploy the game + output, err = ssh.CheckSshCommandE(t, publicHost, fmt.Sprintf("cd /home/%s/build && sudo ./zarf package deploy zarf-package-appliance-demo-doom.tar.zst --confirm", username)) + require.NoError(t, err, output) + + // Wait for the game to be live. Right now we're just checking that `curl` returns 0. It can be enhanced by scraping the HTML that gets returned or something. + output, err = ssh.CheckSshCommandE(t, publicHost, "timeout 60 bash -c 'while [[ \"$(curl -sfSL --retry 15 --retry-connrefused --retry-delay 5 -o /dev/null -w \"%{http_code}\" \"https://localhost\")\" != \"200\" ]]; do sleep 1; done' || false") + require.NoError(t, err, output) +} diff --git a/test/e2e/terraform_ssh_e2e_test.go b/test/e2e/terraform_ssh_e2e_test.go deleted file mode 100644 index 40e7603b76..0000000000 --- a/test/e2e/terraform_ssh_e2e_test.go +++ /dev/null @@ -1,196 +0,0 @@ -package test - -import ( - "bufio" - "encoding/base64" - "fmt" - "github.com/gruntwork-io/terratest/modules/random" - "github.com/gruntwork-io/terratest/modules/retry" - "github.com/stretchr/testify/require" - "io/ioutil" - "os" - "testing" - "time" - - "github.com/gruntwork-io/terratest/modules/aws" - "github.com/gruntwork-io/terratest/modules/ssh" - "github.com/gruntwork-io/terratest/modules/terraform" - teststructure "github.com/gruntwork-io/terratest/modules/test-structure" -) - -func TestTerraformSshExample(t *testing.T) { - t.Parallel() - - // Copy the terraform folder to a temp directory so we can run multiple tests in parallel - tmpFolder := teststructure.CopyTerraformFolderToTemp(t, "..", "tf/public-ec2-instance") - - // At the end of the test, run `terraform destroy` to clean up any resources that were created - defer teststructure.RunTestStage(t, "TEARDOWN", func() { - keyPair := teststructure.LoadEc2KeyPair(t, tmpFolder) - aws.DeleteEC2KeyPair(t, keyPair) - - terraformOptions := teststructure.LoadTerraformOptions(t, tmpFolder) - terraform.Destroy(t, terraformOptions) - }) - - // Deploy the terraform infra - teststructure.RunTestStage(t, "SETUP", func() { - terraformOptions, keyPair, err := configureTerraformOptions(t, tmpFolder) - require.NoError(t, err) - - // Save the options and key pair so later test stages can use them - teststructure.SaveTerraformOptions(t, tmpFolder, terraformOptions) - teststructure.SaveEc2KeyPair(t, tmpFolder, keyPair) - - // This will run `terraform init` and `terraform apply` and fail the test if there are any errors - terraform.InitAndApply(t, terraformOptions) - }) - - // Upload the Zarf artifacts - teststructure.RunTestStage(t, "UPLOAD", func() { - terraformOptions := teststructure.LoadTerraformOptions(t, tmpFolder) - keyPair := teststructure.LoadEc2KeyPair(t, tmpFolder) - - // This will upload the Zarf binary, init package, and other necessary files to the server so we can use them for - // tests - syncFilesToRemoteServer(t, terraformOptions, keyPair) - }) - - // Make sure we can SSH to the public Instance directly from the public Internet - teststructure.RunTestStage(t, "TEST", func() { - terraformOptions := teststructure.LoadTerraformOptions(t, tmpFolder) - keyPair := teststructure.LoadEc2KeyPair(t, tmpFolder) - - // Finally run the actual test - test(t, terraformOptions, keyPair) - }) -} - -func configureTerraformOptions(t *testing.T, tmpFolder string) (*terraform.Options, *aws.Ec2Keypair, error) { - // A unique ID we can use to namespace resources so we don't clash with anything already in the AWS account or - // tests running in parallel - uniqueID := random.UniqueId() - namespace := "zarf" - stage := "terratest" - name := fmt.Sprintf("e2e-%s", uniqueID) - - // Get the region to use from the system's environment - awsRegion, err := getAwsRegion() - if err != nil { - return nil, nil, err - } - - // Some AWS regions are missing certain instance types, so pick an available type based on the region we picked - instanceType := aws.GetRecommendedInstanceType(t, awsRegion, []string{"t3a.large", "t3.large", "t2.large"}) - - // Create an EC2 KeyPair that we can use for SSH access - keyPairName := fmt.Sprintf("%s-%s-%s", namespace, stage, name) - keyPair := aws.CreateAndImportEC2KeyPair(t, awsRegion, keyPairName) - - // Construct the terraform options with default retryable errors to handle the most common retryable errors in - // terraform testing. - terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ - // The path to where our Terraform code is located - TerraformDir: tmpFolder, - - // Variables to pass to our Terraform code using -var options - Vars: map[string]interface{}{ - "aws_region": awsRegion, - "namespace": namespace, - "stage": stage, - "name": name, - "instance_type": instanceType, - "key_pair_name": keyPairName, - }, - }) - - return terraformOptions, keyPair, nil -} - -func syncFilesToRemoteServer(t *testing.T, terraformOptions *terraform.Options, keyPair *aws.Ec2Keypair) { - // Run `terraform output` to get the value of an output variable - publicInstanceIP := terraform.Output(t, terraformOptions, "public_instance_ip") - - // We're going to try to SSH to the instance IP, using the Key Pair we created earlier, and the user "ubuntu", - // as we know the Instance is running an Ubuntu AMI that has such a user - publicHost := ssh.Host{ - Hostname: publicInstanceIP, - SshKeyPair: keyPair.KeyPair, - SshUserName: "ubuntu", - } - - // It can take a minute or so for the Instance to boot up, so retry a few times - maxRetries := 15 - timeBetweenRetries, err := time.ParseDuration("5s") - require.NoError(t, err) - - // Wait for the instance to be ready - _, err = retry.DoWithRetryE(t, "Wait for the instance to be ready", maxRetries, timeBetweenRetries, func() (string, error){ - _, err := ssh.CheckSshCommandE(t, publicHost, "whoami") - if err != nil { - return "", err - } - return "", nil - }) - require.NoError(t, err) - - // Upload the compiled Zarf binary to the server. The ssh lib only supports sending strings so we'll base64encode it - // first - f, err := os.Open("../../build/zarf") - require.NoError(t, err) - reader := bufio.NewReader(f) - content, err := ioutil.ReadAll(reader) - require.NoError(t, err) - encodedZarfBinary := base64.StdEncoding.EncodeToString(content) - err = ssh.ScpFileToE(t, publicHost, 0644, "$HOME/zarf.b64", encodedZarfBinary) - require.NoError(t, err) - output, err := ssh.CheckSshCommandE(t, publicHost, "cd $HOME && sudo bash -c 'base64 -d zarf.b64 > /usr/local/bin/zarf && chmod 0777 /usr/local/bin/zarf'") - require.NoError(t, err, output) - - // Upload zarf-init.tar.zst - f, err = os.Open("../../build/zarf-init.tar.zst") - require.NoError(t, err) - reader = bufio.NewReader(f) - content, err = ioutil.ReadAll(reader) - require.NoError(t, err) - encodedZarfInit := base64.StdEncoding.EncodeToString(content) - err = ssh.ScpFileToE(t, publicHost, 0644, "$HOME/zarf-init.tar.zst.b64", encodedZarfInit) - require.NoError(t, err) - output, err = ssh.CheckSshCommandE(t, publicHost, "cd $HOME && base64 -d zarf-init.tar.zst.b64 > zarf-init.tar.zst") - require.NoError(t, err, output) -} - -func test(t *testing.T, terraformOptions *terraform.Options, keyPair *aws.Ec2Keypair) { - // Run `terraform output` to get the value of an output variable - publicInstanceIP := terraform.Output(t, terraformOptions, "public_instance_ip") - - // We're going to try to SSH to the instance IP, using the Key Pair we created earlier, and the user "ubuntu", - // as we know the Instance is running an Ubuntu AMI that has such a user - publicHost := ssh.Host{ - Hostname: publicInstanceIP, - SshKeyPair: keyPair.KeyPair, - SshUserName: "ubuntu", - } - - // Make sure `zarf --help` doesn't error - output, err := ssh.CheckSshCommandE(t, publicHost, "zarf --help") - require.NoError(t, err, output) - - // Test `zarf init just to make sure it returns a zero exit code.` - output, err = ssh.CheckSshCommandE(t, publicHost, "sudo bash -c 'zarf init --confirm --components management,logging,gitops-service --host localhost'") - require.NoError(t, err, output) -} - -// getAwsRegion returns the desired AWS region to use by first checking the env var AWS_REGION, then checking -//AWS_DEFAULT_REGION if AWS_REGION isn't set. If neither is set it returns an error -func getAwsRegion() (string, error) { - val, present := os.LookupEnv("AWS_REGION") - if !present { - val, present = os.LookupEnv("AWS_DEFAULT_REGION") - } - if !present { - return "", fmt.Errorf("expected either AWS_REGION or AWS_DEFAULT_REGION env var to be set, but they were not") - } else { - return val, nil - } -}