Run Integration Tests #1460
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | |
| # SPDX-License-Identifier: MIT | |
| name: Run Integration Tests | |
| env: | |
| PRIVATE_KEY: ${{ secrets.AWS_PRIVATE_KEY }} | |
| TERRAFORM_AWS_ASSUME_ROLE: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }} | |
| TERRAFORM_AWS_ASSUME_ROLE_DURATION: 14400 # 4 hours | |
| S3_INTEGRATION_BUCKET: ${{ vars.S3_INTEGRATION_BUCKET }} | |
| KEY_NAME: ${{ secrets.KEY_NAME }} | |
| CF_IAM_ROLE: ${{ secrets.CF_IAM_ROLE }} | |
| CF_KEY_NAME: ${{ secrets.CF_KEY_NAME }} | |
| ECR_INTEGRATION_TEST_REPO: "cwagent-integration-test" | |
| CWA_GITHUB_TEST_REPO_NAME: "aws/amazon-cloudwatch-agent-test" | |
| CWA_GITHUB_TEST_REPO_URL: "https://github.com/aws/amazon-cloudwatch-agent-test.git" | |
| CWA_GITHUB_TEST_REPO_BRANCH: "add-logs-agent-integ-test" | |
| TERRAFORM_AWS_ASSUME_ROLE_ITAR: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }} | |
| S3_INTEGRATION_BUCKET_ITAR: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }} | |
| TERRAFORM_AWS_ASSUME_ROLE_CN: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }} | |
| S3_INTEGRATION_BUCKET_CN: ${{ vars.S3_INTEGRATION_BUCKET_CN }} | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| build_run_id: | |
| description: 'The ID of the build-test-artifacts workflow run' | |
| type: number | |
| required: true | |
| build_sha: | |
| description: 'The SHA of the build-test-artifacts workflow run' | |
| type: string | |
| required: true | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref_name }} | |
| cancel-in-progress: true | |
| jobs: | |
| CheckBuildTestArtifacts: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: | | |
| if [[ ${{ inputs.build_sha }} == ${{ github.sha }} ]]; then | |
| echo "Build SHA matches test SHA" | |
| else | |
| echo "Build SHA does not match test SHA" | |
| exit 1 | |
| fi | |
| - run: | | |
| conclusion=$(gh run view ${{ inputs.build_run_id }} --repo $GITHUB_REPOSITORY --json conclusion -q '.conclusion') | |
| if [[ $conclusion == "success" ]]; then | |
| echo "Run succeeded" | |
| else | |
| echo "Run failed" | |
| exit 1 | |
| fi | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GenerateTestMatrix: | |
| needs: [ CheckBuildTestArtifacts ] | |
| name: 'GenerateTestMatrix' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| ec2_gpu_matrix: ${{ steps.set-matrix.outputs.ec2_gpu_matrix }} | |
| ec2_linux_matrix: ${{ steps.set-matrix.outputs.ec2_linux_matrix }} | |
| ec2_windows_matrix: ${{ steps.set-matrix.outputs.ec2_windows_matrix }} | |
| ec2_mac_matrix: ${{ steps.set-matrix.outputs.ec2_mac_matrix }} | |
| ec2_performance_matrix: ${{steps.set-matrix.outputs.ec2_performance_matrix}} | |
| ec2_windows_performance_matrix: ${{steps.set-matrix.outputs.ec2_windows_performance_matrix}} | |
| ec2_stress_matrix: ${{steps.set-matrix.outputs.ec2_stress_matrix}} | |
| ec2_windows_stress_matrix: ${{steps.set-matrix.outputs.ec2_windows_stress_matrix}} | |
| ecs_ec2_launch_daemon_matrix: ${{ steps.set-matrix.outputs.ecs_ec2_launch_daemon_matrix }} | |
| ecs_fargate_matrix: ${{ steps.set-matrix.outputs.ecs_fargate_matrix }} | |
| eks_daemon_matrix: ${{ steps.set-matrix.outputs.eks_daemon_matrix }} | |
| eks_deployment_matrix: ${{ steps.set-matrix.outputs.eks_deployment_matrix }} | |
| ec2_linux_itar_matrix: ${{ steps.set-matrix.outputs.ec2_linux_itar_matrix }} | |
| ec2_linux_china_matrix: ${{ steps.set-matrix.outputs.ec2_linux_china_matrix }} | |
| eks_addon_matrix: ${{ steps.set-matrix.outputs.eks_addon_matrix }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} | |
| ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} | |
| - name: Set up Go 1.x | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: ~1.22.2 | |
| - name: Generate matrix | |
| id: set-matrix | |
| run: | | |
| go run --tags=generator generator/test_case_generator.go | |
| echo "::set-output name=ec2_gpu_matrix::$(echo $(cat generator/resources/ec2_gpu_complete_test_matrix.json))" | |
| echo "::set-output name=eks_addon_matrix::$(echo $(cat generator/resources/eks_addon_complete_test_matrix.json))" | |
| echo "::set-output name=ec2_linux_matrix::$(echo $(cat generator/resources/ec2_linux_complete_test_matrix.json))" | |
| echo "::set-output name=ec2_windows_matrix::$(echo $(cat generator/resources/ec2_windows_complete_test_matrix.json))" | |
| echo "::set-output name=ec2_mac_matrix::$(echo $(cat generator/resources/ec2_mac_complete_test_matrix.json))" | |
| echo "::set-output name=ec2_performance_matrix::$(echo $(cat generator/resources/ec2_performance_complete_test_matrix.json))" | |
| echo "::set-output name=ec2_windows_performance_matrix::$(echo $(cat generator/resources/ec2_windows_performance_complete_test_matrix.json))" | |
| echo "::set-output name=ec2_stress_matrix::$(echo $(cat generator/resources/ec2_stress_complete_test_matrix.json))" | |
| echo "::set-output name=ec2_windows_stress_matrix::$(echo $(cat generator/resources/ec2_windows_stress_complete_test_matrix.json))" | |
| echo "::set-output name=ecs_ec2_launch_daemon_matrix::$(echo $(cat generator/resources/ecs_ec2_daemon_complete_test_matrix.json))" | |
| echo "::set-output name=ecs_fargate_matrix::$(echo $(cat generator/resources/ecs_fargate_complete_test_matrix.json))" | |
| echo "::set-output name=eks_daemon_matrix::$(echo $(cat generator/resources/eks_daemon_complete_test_matrix.json))" | |
| echo "::set-output name=eks_deployment_matrix::$(echo $(cat generator/resources/eks_deployment_complete_test_matrix.json))" | |
| echo "::set-output name=ec2_linux_itar_matrix::$(echo $(cat generator/resources/ec2_linux_itar_complete_test_matrix.json))" | |
| echo "::set-output name=ec2_linux_china_matrix::$(echo $(cat generator/resources/ec2_linux_china_complete_test_matrix.json))" | |
| - name: Echo test plan matrix | |
| run: | | |
| echo "ec2_gpu_matrix: ${{ steps.set-matrix.outputs.ec2_gpu_matrix }}" | |
| echo "eks_addon_matrix: ${{ steps.set-matrix.outputs.eks_addon_matrix }}" | |
| echo "ec2_linux_matrix: ${{ steps.set-matrix.outputs.ec2_linux_matrix }}" | |
| echo "ec2_windows_matrix: ${{ steps.set-matrix.outputs.ec2_windows_matrix }}" | |
| echo "ec2_mac_matrix: ${{ steps.set-matrix.outputs.ec2_mac_matrix }}" | |
| echo "ec2_performance_matrix: ${{ steps.set-matrix.outputs.ec2_performance_matrix}}" | |
| echo "ec2_windows_performance_matrix: ${{ steps.set-matrix.outputs.ec2_windows_performance_matrix}}" | |
| echo "ec2_stress_matrix: ${{ steps.set-matrix.outputs.ec2_stress_matrix}}" | |
| echo "ec2_windows_stress_matrix: ${{ steps.set-matrix.outputs.ec2_windows_stress_matrix}}" | |
| echo "ecs_ec2_launch_daemon_matrix: ${{ steps.set-matrix.outputs.ecs_ec2_launch_daemon_matrix }}" | |
| echo "ecs_fargate_matrix: ${{ steps.set-matrix.outputs.ecs_fargate_matrix }}" | |
| echo "eks_daemon_matrix: ${{ steps.set-matrix.outputs.eks_daemon_matrix }}" | |
| echo "eks_deployment_matrix: ${{ steps.set-matrix.outputs.eks_deployment_matrix }}" | |
| echo "ec2_linux_itar_matrix: ${{ steps.set-matrix.outputs.ec2_linux_itar_matrix }}" | |
| echo "ec2_linux_china_matrix: ${{ steps.set-matrix.outputs.ec2_linux_china_matrix }}" | |
| CloudformationTest: | |
| needs: [GenerateTestMatrix] | |
| name: 'CFTest' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} | |
| ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} | |
| path: test | |
| - name: Set up Go 1.x | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: ~1.22.2 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} | |
| aws-region: us-west-2 | |
| role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} | |
| - name: Cache if success | |
| id: cf-integration-test | |
| uses: actions/cache@v2 | |
| with: | |
| path: go.mod | |
| key: "cf-integration-${{ github.sha }}-test" | |
| - name: Test cf | |
| if: steps.ec2-linux-integration-test.outputs.cache-hit != 'true' | |
| run: | | |
| cd test/test/cloudformation | |
| go test -timeout 1h -package_path=s3://${S3_INTEGRATION_BUCKET}/integration-test/binary/${{ github.sha }}/linux/amd64/amazon-cloudwatch-agent.rpm -iam_role=${CF_IAM_ROLE} -key_name=${CF_KEY_NAME} -metric_name=mem_used_percent | |
| StartLocalStack: | |
| name: 'StartLocalStack' | |
| needs: [OutputEnvVariables] | |
| uses: ./.github/workflows/start-localstack.yml | |
| secrets: inherit | |
| permissions: | |
| id-token: write | |
| contents: read | |
| with: | |
| region: us-west-2 | |
| test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} | |
| test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} | |
| terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }} | |
| test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} | |
| github_sha: ${{github.sha}} | |
| s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }} | |
| StartLocalStackITAR: | |
| name: 'StartLocalStackITAR' | |
| needs: [OutputEnvVariables] | |
| uses: ./.github/workflows/start-localstack.yml | |
| secrets: inherit | |
| permissions: | |
| id-token: write | |
| contents: read | |
| with: | |
| region: us-gov-east-1 | |
| test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} | |
| test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} | |
| terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }} | |
| test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} | |
| github_sha: ${{github.sha}} | |
| s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }} | |
| StartLocalStackCN: | |
| name: 'StartLocalStackCN' | |
| needs: [ OutputEnvVariables ] | |
| uses: ./.github/workflows/start-localstack.yml | |
| secrets: inherit | |
| permissions: | |
| id-token: write | |
| contents: read | |
| with: | |
| region: cn-north-1 | |
| test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} | |
| test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} | |
| terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }} | |
| test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} | |
| github_sha: ${{github.sha}} | |
| s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_CN }} | |
| EC2NvidiaGPUIntegrationTest: | |
| needs: [ StartLocalStack, GenerateTestMatrix ] | |
| name: 'EC2NVIDIAGPUIntegrationTest' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_gpu_matrix) }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} | |
| ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} | |
| aws-region: us-west-2 | |
| role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} | |
| - name: Cache if success | |
| id: ec2-linux-integration-test | |
| uses: actions/cache@v3 | |
| with: | |
| path: go.mod | |
| key: ec2-nvidia-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} | |
| - name: Echo Test Info | |
| run: echo run on ec2 instance os ${{ matrix.arrays.os }} arc ${{ matrix.arrays.arc }} test dir ${{ matrix.arrays.test_dir }} | |
| - name: Verify Terraform version | |
| run: terraform --version | |
| # nick-fields/retry@v2 starts at base dir | |
| - name: Terraform apply | |
| if: ${{ matrix.arrays.family == 'linux' && steps.ec2-nvidia-integration-test.outputs.cache-hit != 'true' }} | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 30 | |
| retry_wait_seconds: 5 | |
| command: | | |
| if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then | |
| cd "${{ matrix.arrays.terraform_dir }}" | |
| else | |
| cd terraform/ec2/linux | |
| fi | |
| terraform init | |
| if terraform apply --auto-approve \ | |
| -var="ssh_key_value=${PRIVATE_KEY}" -var="github_test_repo=${{env.CWA_GITHUB_TEST_REPO_URL}}" \ | |
| -var="test_name=${{ matrix.arrays.os }}" \ | |
| -var="cwa_github_sha=${GITHUB_SHA}" -var="install_agent=${{ matrix.arrays.installAgentCommand }}" \ | |
| -var="github_test_repo_branch=${{env.CWA_GITHUB_TEST_REPO_BRANCH}}" \ | |
| -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ | |
| -var="user=${{ matrix.arrays.username }}" \ | |
| -var="ami=${{ matrix.arrays.ami }}" \ | |
| -var="ca_cert_path=${{ matrix.arrays.caCertPath }}" \ | |
| -var="arc=${{ matrix.arrays.arc }}" \ | |
| -var="binary_name=${{ matrix.arrays.binaryName }}" \ | |
| -var="local_stack_host_name=${{ needs.StartLocalStack.outputs.local_stack_host_name }}" \ | |
| -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ | |
| -var="ssh_key_name=${KEY_NAME}" \ | |
| -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve | |
| else | |
| terraform destroy -auto-approve && exit 1 | |
| fi | |
| - name: Terraform apply | |
| if: ${{ matrix.arrays.family == 'window' && steps.ec2-nvidia-integration-test.outputs.cache-hit != 'true' }} | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 30 | |
| retry_wait_seconds: 5 | |
| command: | | |
| if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then | |
| cd "${{ matrix.arrays.terraform_dir }}" | |
| else | |
| cd terraform/ec2/win | |
| fi | |
| terraform init | |
| if terraform apply --auto-approve \ | |
| -var="ssh_key_value=${PRIVATE_KEY}" -var="ssh_key_name=${KEY_NAME}" \ | |
| -var="github_repo=${{env.CWA_GITHUB_TEST_REPO_URL}}" \ | |
| -var="cwa_github_sha=${GITHUB_SHA}" -var="ami=${{ matrix.arrays.ami }}" \ | |
| -var="test_dir=${{ matrix.arrays.test_dir }}" \ | |
| -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ | |
| -var="github_test_repo=${{env.CWA_GITHUB_TEST_REPO_URL}}" \ | |
| -var="github_test_repo_branch=${{env.CWA_GITHUB_TEST_REPO_BRANCH}}" \ | |
| -var="s3_bucket=${S3_INTEGRATION_BUCKET}" ; then terraform destroy -auto-approve | |
| else | |
| terraform destroy -auto-approve && exit 1 | |
| fi | |
| #This is here just in case workflow cancel | |
| - name: Terraform destroy | |
| if: ${{ cancelled() || failure() }} | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 8 | |
| retry_wait_seconds: 5 | |
| command: | | |
| if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then | |
| cd "${{ matrix.arrays.terraform_dir }}" | |
| elif if "${{ matrix.arrays.os }}" == window; then | |
| cd terraform/ec2/win | |
| else | |
| cd terraform/ec2/linux | |
| fi | |
| terraform destroy --auto-approve | |
| OutputEnvVariables: | |
| needs: [CheckBuildTestArtifacts] | |
| name: 'OutputEnvVariables' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| CWA_GITHUB_TEST_REPO_NAME: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_NAME }} | |
| CWA_GITHUB_TEST_REPO_URL: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_URL }} | |
| CWA_GITHUB_TEST_REPO_BRANCH: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} | |
| ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} | |
| - name: Set up Go 1.x | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: ~1.22.2 | |
| - name: SetOutputs | |
| id: set-outputs | |
| run: | | |
| echo "::set-output name=CWA_GITHUB_TEST_REPO_NAME::${{ env.CWA_GITHUB_TEST_REPO_NAME }}" | |
| echo "::set-output name=CWA_GITHUB_TEST_REPO_URL::${{ env.CWA_GITHUB_TEST_REPO_URL }}" | |
| echo "::set-output name=CWA_GITHUB_TEST_REPO_BRANCH::${{ env.CWA_GITHUB_TEST_REPO_BRANCH }}" | |
| - name: Echo test variables | |
| run: | | |
| echo "CWA_GITHUB_TEST_REPO_NAME: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_NAME }}" | |
| echo "CWA_GITHUB_TEST_REPO_URL: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_URL }}" | |
| echo "CWA_GITHUB_TEST_REPO_BRANCH: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}" | |
| EC2LinuxIntegrationTest: | |
| needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables ] | |
| name: 'EC2Linux' | |
| uses: ./.github/workflows/ec2-integration-test.yml | |
| with: | |
| github_sha: ${{github.sha}} | |
| test_dir: terraform/ec2/linux | |
| job_id: ec2-linux-integration-test | |
| test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_matrix}} | |
| test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} | |
| test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} | |
| test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} | |
| localstack_host: ${{needs.StartLocalStack.outputs.local_stack_host_name}} | |
| region: us-west-2 | |
| terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }} | |
| s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }} | |
| secrets: inherit | |
| EC2LinuxIntegrationTestITAR: | |
| needs: [ StartLocalStackITAR, GenerateTestMatrix, OutputEnvVariables ] | |
| name: 'EC2LinuxITAR' | |
| uses: ./.github/workflows/ec2-integration-test.yml | |
| with: | |
| github_sha: ${{github.sha}} | |
| test_dir: terraform/ec2/linux | |
| job_id: ec2-linux-integration-test | |
| test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_itar_matrix}} | |
| test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} | |
| test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} | |
| test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} | |
| localstack_host: ${{needs.StartLocalStackITAR.outputs.local_stack_host_name}} | |
| region: us-gov-east-1 | |
| terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }} | |
| s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }} | |
| secrets: inherit | |
| EC2LinuxIntegrationTestCN: | |
| needs: [ StartLocalStackCN, GenerateTestMatrix, OutputEnvVariables ] | |
| name: 'EC2LinuxCN' | |
| uses: ./.github/workflows/ec2-integration-test.yml | |
| with: | |
| github_sha: ${{github.sha}} | |
| test_dir: terraform/ec2/linux | |
| job_id: ec2-linux-integration-test | |
| test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_china_matrix}} | |
| test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} | |
| test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} | |
| test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} | |
| localstack_host: ${{needs.StartLocalStackCN.outputs.local_stack_host_name}} | |
| region: cn-north-1 | |
| terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }} | |
| s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_CN }} | |
| secrets: inherit | |
| LinuxOnPremIntegrationTest: | |
| needs: [StartLocalStack, GenerateTestMatrix, OutputEnvVariables] | |
| name: 'OnpremLinux' | |
| uses: ./.github/workflows/ec2-integration-test.yml | |
| with: | |
| github_sha: ${{github.sha}} | |
| test_dir: terraform/ec2/linux_onprem | |
| job_id: linux-onprem-integration-test | |
| test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_onprem_matrix}} | |
| test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} | |
| test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} | |
| test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} | |
| localstack_host: ${{needs.StartLocalStack.outputs.local_stack_host_name}} | |
| region: us-west-2 | |
| secrets: inherit | |
| EC2WinIntegrationTest: | |
| needs: [GenerateTestMatrix] | |
| name: 'EC2WinIntegrationTest' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_windows_matrix) }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} | |
| ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} | |
| aws-region: us-west-2 | |
| role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} | |
| - name: Cache if success | |
| id: ec2-win-integration-test | |
| uses: actions/cache@v3 | |
| with: | |
| path: go.mod | |
| key: ec2-win-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} | |
| - name: Echo Test Info | |
| run: echo run on ec2 instance os ${{ matrix.arrays.os }} use ssm ${{ matrix.arrays.useSSM }} test ${{ matrix.arrays.test_dir }} | |
| - name: Verify Terraform version | |
| run: terraform --version | |
| # nick-fields/retry@v2 starts at base dir | |
| - name: Terraform apply | |
| if: steps.ec2-win-integration-test.outputs.cache-hit != 'true' | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 30 | |
| retry_wait_seconds: 5 | |
| command: | | |
| if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then | |
| cd "${{ matrix.arrays.terraform_dir }}" | |
| else | |
| cd terraform/ec2/win | |
| fi | |
| terraform init | |
| if terraform apply --auto-approve \ | |
| -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ | |
| -var="ssh_key_value=${PRIVATE_KEY}" -var="ssh_key_name=${KEY_NAME}" \ | |
| -var="test_name=${{ matrix.arrays.os }}" \ | |
| -var="cwa_github_sha=${GITHUB_SHA}" \ | |
| -var="test_dir=${{ matrix.arrays.test_dir }}" \ | |
| -var="ami=${{ matrix.arrays.ami }}" \ | |
| -var="use_ssm=${{ matrix.arrays.useSSM }}" \ | |
| -var="s3_bucket=${S3_INTEGRATION_BUCKET}" ; then | |
| terraform destroy -auto-approve | |
| else | |
| terraform destroy -auto-approve && exit 1 | |
| fi | |
| #This is here just in case workflow cancel | |
| - name: Terraform destroy | |
| if: ${{ cancelled() || failure() }} | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 8 | |
| retry_wait_seconds: 5 | |
| command: | | |
| if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then | |
| cd "${{ matrix.arrays.terraform_dir }}" | |
| else | |
| cd terraform/ec2/win | |
| fi | |
| terraform destroy --auto-approve | |
| EC2DarwinIntegrationTest: | |
| needs: [GenerateTestMatrix] | |
| name: 'EC2DarwinIntegrationTest' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_mac_matrix) }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} | |
| ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} | |
| aws-region: us-west-2 | |
| role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} | |
| - name: Cache if success | |
| id: ec2-mac-integration-test | |
| uses: actions/cache@v3 | |
| with: | |
| path: go.mod | |
| key: ec2-mac-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} | |
| - name: Echo OS | |
| run: echo run on ec2 instance os ${{ matrix.arrays.os }} | |
| - name: Verify Terraform version | |
| run: terraform --version | |
| # nick-fields/retry@v2 starts at base dir | |
| - name: Terraform apply | |
| if: steps.ec2-mac-integration-test.outputs.cache-hit != 'true' | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 30 | |
| retry_wait_seconds: 5 | |
| command: | | |
| if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then | |
| cd "${{ matrix.arrays.terraform_dir }}" | |
| else | |
| cd terraform/ec2/mac | |
| fi | |
| terraform init | |
| if terraform apply --auto-approve \ | |
| -var="ssh_key_value=${PRIVATE_KEY}" -var="ssh_key_name=${KEY_NAME}" \ | |
| -var="arc=${{ matrix.arrays.arc }}" \ | |
| -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ | |
| -var="cwa_github_sha=${GITHUB_SHA}" -var="ami=${{ matrix.arrays.ami }}" \ | |
| -var="test_dir=${{ matrix.arrays.test_dir }}" \ | |
| -var="license_manager_arn=${{ env.LICENSE_MANAGER_ARN }}" \ | |
| -var="s3_bucket=${S3_INTEGRATION_BUCKET}" ; then | |
| terraform destroy -auto-approve | |
| else | |
| terraform destroy -auto-approve && exit 1 | |
| fi | |
| #This is here just in case workflow cancel | |
| - name: Terraform destroy | |
| if: ${{ cancelled() || failure() }} | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 8 | |
| retry_wait_seconds: 5 | |
| command: | | |
| if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then | |
| cd "${{ matrix.arrays.terraform_dir }}" | |
| else | |
| cd terraform/ec2/mac | |
| fi | |
| terraform destroy --auto-approve | |
| StopLocalStack: | |
| name: 'StopLocalStack' | |
| if: ${{ always() && needs.StartLocalStack.result == 'success' }} | |
| needs: [ StartLocalStack, EC2LinuxIntegrationTest, LinuxOnPremIntegrationTest, OutputEnvVariables ] | |
| uses: ./.github/workflows/stop-localstack.yml | |
| secrets: inherit | |
| permissions: | |
| id-token: write | |
| contents: read | |
| with: | |
| region: us-west-2 | |
| test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} | |
| test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} | |
| terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }} | |
| github_sha: ${{github.sha}} | |
| s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }} | |
| StopLocalStackITAR: | |
| name: 'StopLocalStackITAR' | |
| if: ${{ always() && needs.StartLocalStackITAR.result == 'success' }} | |
| needs: [ StartLocalStackITAR, EC2LinuxIntegrationTestITAR, OutputEnvVariables ] | |
| uses: ./.github/workflows/stop-localstack.yml | |
| secrets: inherit | |
| permissions: | |
| id-token: write | |
| contents: read | |
| with: | |
| region: us-gov-east-1 | |
| test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} | |
| test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} | |
| terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }} | |
| github_sha: ${{github.sha}} | |
| s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }} | |
| StopLocalStackCN: | |
| name: 'StopLocalStackCN' | |
| if: ${{ always() && needs.StartLocalStackCN.result == 'success' }} | |
| needs: [ StartLocalStackCN, EC2LinuxIntegrationTestCN ] | |
| uses: ./.github/workflows/stop-localstack.yml | |
| secrets: inherit | |
| permissions: | |
| id-token: write | |
| contents: read | |
| with: | |
| region: cn-north-1 | |
| test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} | |
| test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} | |
| terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }} | |
| github_sha: ${{github.sha}} | |
| s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_CN }} | |
| ECSEC2IntegrationTest: | |
| name: 'ECSEC2IntegrationTest' | |
| runs-on: ubuntu-latest | |
| needs: [ GenerateTestMatrix ] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ecs_ec2_launch_daemon_matrix) }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} | |
| ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} | |
| aws-region: us-west-2 | |
| role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} | |
| - name: Cache if success | |
| id: ecs-ec2-integration-test | |
| uses: actions/cache@v3 | |
| with: | |
| path: go.mod | |
| key: ecs-ec2-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.test_dir }} | |
| - name: Login ECR | |
| id: login-ecr | |
| if: steps.ecs-ec2-integration-test.outputs.cache-hit != 'true' | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| - name: Verify Terraform version | |
| if: steps.ecs-ec2-integration-test.outputs.cache-hit != 'true' | |
| run: terraform --version | |
| - name: Terraform apply | |
| if: steps.ecs-ec2-integration-test.outputs.cache-hit != 'true' | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 15 | |
| retry_wait_seconds: 5 | |
| command: | | |
| if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then | |
| cd "${{ matrix.arrays.terraform_dir }}" | |
| else | |
| cd terraform/ecs_ec2/daemon | |
| fi | |
| terraform init | |
| if terraform apply --auto-approve\ | |
| -var="test_dir=${{ matrix.arrays.test_dir }}"\ | |
| -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ | |
| -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}"\ | |
| -var="cwagent_image_tag=${{ github.sha }}"\ | |
| -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ | |
| -var="metadataEnabled=${{ matrix.arrays.metadataEnabled }}" \ | |
| -var="ami=${{ matrix.arrays.ami }}" ; then | |
| terraform destroy -auto-approve | |
| else | |
| terraform destroy -auto-approve && exit 1 | |
| fi | |
| - name: Terraform destroy | |
| if: ${{ cancelled() || failure() }} | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 8 | |
| retry_wait_seconds: 5 | |
| command: | | |
| if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then | |
| cd "${{ matrix.arrays.terraform_dir }}" | |
| else | |
| cd terraform/ecs_ec2/daemon | |
| fi | |
| terraform destroy --auto-approve | |
| ECSFargateIntegrationTest: | |
| name: 'ECSFargateIntegrationTest' | |
| runs-on: ubuntu-latest | |
| needs: [GenerateTestMatrix] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ecs_fargate_matrix) }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} | |
| ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} | |
| aws-region: us-west-2 | |
| role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} | |
| - name: Cache if success | |
| id: ecs-fargate-integration-test | |
| uses: actions/cache@v3 | |
| with: | |
| path: go.mod | |
| key: ecs-fargate-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.test_dir }} | |
| - name: Login ECR | |
| id: login-ecr | |
| if: steps.ecs-fargate-integration-test.outputs.cache-hit != 'true' | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| - name: Verify Terraform version | |
| if: steps.ecs-fargate-integration-test.outputs.cache-hit != 'true' | |
| run: terraform --version | |
| - name: Terraform apply | |
| if: steps.ecs-fargate-integration-test.outputs.cache-hit != 'true' | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 15 | |
| retry_wait_seconds: 5 | |
| command: | | |
| if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then | |
| cd "${{ matrix.arrays.terraform_dir }}" | |
| else | |
| cd terraform/ecs_fargate/linux | |
| fi | |
| terraform init | |
| if terraform apply --auto-approve\ | |
| -var="test_dir=${{ matrix.arrays.test_dir }}"\ | |
| -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}"\ | |
| -var="cwagent_image_tag=${{ github.sha }}"; then | |
| terraform destroy -auto-approve | |
| else | |
| terraform destroy -auto-approve && exit 1 | |
| fi | |
| - name: Terraform destroy | |
| if: ${{ cancelled() || failure() }} | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 8 | |
| retry_wait_seconds: 5 | |
| command: | | |
| if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then | |
| cd "${{ matrix.arrays.terraform_dir }}" | |
| else | |
| cd terraform/ecs_fargate/linux | |
| fi | |
| terraform destroy --auto-approve | |
| EKSIntegrationTest: | |
| name: 'EKSIntegrationTest' | |
| runs-on: ubuntu-latest | |
| needs: [ GenerateTestMatrix ] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.eks_daemon_matrix) }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} | |
| ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} | |
| aws-region: us-west-2 | |
| role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} | |
| - name: Cache if success | |
| id: eks-ec2-integration-test | |
| uses: actions/cache@v3 | |
| with: | |
| path: go.mod | |
| key: ${{ matrix.arrays.terraform_dir }}-${{ matrix.arrays.k8s_version }}-${{ matrix.arrays.instanceType }}-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.test_dir }} | |
| - name: Login ECR | |
| id: login-ecr | |
| if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| - name: Verify Terraform version | |
| if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' | |
| run: terraform --version | |
| - name: Terraform apply | |
| if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 2 | |
| timeout_minutes: 90 # EKS takes about 20 minutes to spin up a cluster and service on the cluster | |
| retry_wait_seconds: 5 | |
| command: | | |
| if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then | |
| cd "${{ matrix.arrays.terraform_dir }}" | |
| else | |
| cd terraform/eks/daemon | |
| fi | |
| terraform init | |
| if terraform apply --auto-approve \ | |
| -var="test_dir=${{ matrix.arrays.test_dir }}"\ | |
| -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}" \ | |
| -var="cwagent_image_tag=${{ github.sha }}" \ | |
| -var="ami_type=${{ matrix.arrays.ami }}" \ | |
| -var="instance_type=${{ matrix.arrays.instanceType }}" \ | |
| -var="k8s_version=${{ matrix.arrays.k8s_version }}"; then | |
| terraform destroy -auto-approve | |
| else | |
| terraform destroy -auto-approve && exit 1 | |
| fi | |
| - name: Terraform destroy | |
| if: ${{ cancelled() || failure() }} | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 8 | |
| retry_wait_seconds: 5 | |
| command: | | |
| if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then | |
| cd "${{ matrix.arrays.terraform_dir }}" | |
| else | |
| cd terraform/eks/daemon | |
| fi | |
| terraform destroy --auto-approve | |
| EKSPrometheusIntegrationTest: | |
| name: 'EKSPrometheusIntegrationTest' | |
| runs-on: ubuntu-latest | |
| needs: [ GenerateTestMatrix ] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.eks_deployment_matrix) }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} | |
| ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} | |
| aws-region: us-west-2 | |
| role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} | |
| - name: Cache if success | |
| id: eks-ec2-integration-test | |
| uses: actions/cache@v3 | |
| with: | |
| path: go.mod | |
| key: eks-ec2-integration-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.test_dir }} | |
| - name: Login ECR | |
| id: login-ecr | |
| if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| - name: Verify Terraform version | |
| if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' | |
| run: terraform --version | |
| - name: Terraform apply | |
| if: steps.eks-ec2-integration-test.outputs.cache-hit != 'true' | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 60 # EKS takes about 20 minutes to spin up a cluster and service on the cluster | |
| retry_wait_seconds: 5 | |
| command: | | |
| if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then | |
| cd "${{ matrix.arrays.terraform_dir }}" | |
| else | |
| cd terraform/eks/deployment | |
| fi | |
| terraform init | |
| if terraform apply --auto-approve \ | |
| -var="test_dir=${{ matrix.arrays.test_dir }}"\ | |
| -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}" \ | |
| -var="cwagent_image_tag=${{ github.sha }}" \ | |
| -var="k8s_version=${{ matrix.arrays.k8s_version }}"; then | |
| terraform destroy -auto-approve | |
| else | |
| terraform destroy -auto-approve && exit 1 | |
| fi | |
| - name: Terraform destroy | |
| if: ${{ cancelled() || failure() }} | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 8 | |
| retry_wait_seconds: 5 | |
| command: | | |
| if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then | |
| cd "${{ matrix.arrays.terraform_dir }}" | |
| else | |
| cd terraform/eks/deployment | |
| fi | |
| terraform destroy --auto-approve | |
| PerformanceTrackingTest: | |
| name: "PerformanceTrackingTest" | |
| needs: [GenerateTestMatrix] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_performance_matrix) }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} | |
| ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} | |
| aws-region: us-west-2 | |
| role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} | |
| - name: Cache if success | |
| id: performance-tracking | |
| uses: actions/cache@v3 | |
| with: | |
| path: go.mod | |
| key: performance-tracking-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} | |
| - name: Verify Terraform version | |
| if: steps.performance-tracking.outputs.cache-hit != 'true' | |
| run: terraform --version | |
| - name: Terraform apply | |
| if: steps.performance-tracking.outputs.cache-hit != 'true' | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 1 | |
| timeout_minutes: 60 | |
| retry_wait_seconds: 5 | |
| command: | | |
| cd terraform/performance | |
| terraform init | |
| if terraform apply --auto-approve \ | |
| -var="ssh_key_value=${PRIVATE_KEY}" \ | |
| -var="cwa_github_sha=${GITHUB_SHA}" \ | |
| -var="ami=${{ matrix.arrays.ami }}" \ | |
| -var="arc=${{ matrix.arrays.arc }}" \ | |
| -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ | |
| -var="ssh_key_name=${KEY_NAME}" \ | |
| -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ | |
| -var="family=${{ matrix.arrays.family}}"\ | |
| -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve | |
| else | |
| terraform destroy -auto-approve && exit 1 | |
| fi | |
| - name: Terraform destroy | |
| if: ${{ cancelled() || failure() }} | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 8 | |
| retry_wait_seconds: 5 | |
| command: cd terraform/performance && terraform destroy --auto-approve | |
| EC2WinPerformanceTest: | |
| name: "EC2WinPerformanceTest" | |
| needs: [ GenerateTestMatrix ] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_windows_performance_matrix) }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} | |
| ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} | |
| aws-region: us-west-2 | |
| role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} | |
| - name: Cache if success | |
| id: performance-tracking | |
| uses: actions/cache@v3 | |
| with: | |
| path: go.mod | |
| key: performance-tracking-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} | |
| - name: Verify Terraform version | |
| if: steps.performance-tracking.outputs.cache-hit != 'true' | |
| run: terraform --version | |
| - name: Terraform apply | |
| if: steps.performance-tracking.outputs.cache-hit != 'true' | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 1 | |
| timeout_minutes: 60 | |
| retry_wait_seconds: 5 | |
| command: | | |
| cd terraform/performance | |
| terraform init | |
| if terraform apply --auto-approve \ | |
| -var="ssh_key_value=${PRIVATE_KEY}" \ | |
| -var="cwa_github_sha=${GITHUB_SHA}" \ | |
| -var="ami=${{ matrix.arrays.ami }}" \ | |
| -var="arc=${{ matrix.arrays.arc }}" \ | |
| -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ | |
| -var="ssh_key_name=${KEY_NAME}" \ | |
| -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ | |
| -var="family=${{ matrix.arrays.family}}"\ | |
| -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve | |
| else | |
| terraform destroy -auto-approve && exit 1 | |
| fi | |
| - name: Terraform destroy | |
| if: ${{ cancelled() || failure() }} | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 8 | |
| retry_wait_seconds: 5 | |
| command: cd terraform/performance && terraform destroy --auto-approve | |
| StressTrackingTest: | |
| name: "StressTrackingTest" | |
| needs: [GenerateTestMatrix] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_stress_matrix) }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} | |
| ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} | |
| aws-region: us-west-2 | |
| role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} | |
| - name: Cache if success | |
| id: stress-tracking | |
| uses: actions/cache@v3 | |
| with: | |
| path: go.mod | |
| key: stress-tracking-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} | |
| - name: Verify Terraform version | |
| if: steps.stress-tracking.outputs.cache-hit != 'true' | |
| run: terraform --version | |
| - name: Echo Test Info | |
| run: echo run on ec2 instance os ${{ matrix.arrays.os }} arc ${{ matrix.arrays.arc }} test dir ${{ matrix.arrays.test_dir }} values per minute ${{ matrix.arrays.values_per_minute }} | |
| - name: Terraform apply | |
| if: steps.stress-tracking.outputs.cache-hit != 'true' | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 1 | |
| timeout_minutes: 60 | |
| retry_wait_seconds: 5 | |
| command: | | |
| cd terraform/stress | |
| terraform init | |
| if terraform apply --auto-approve \ | |
| -var="ssh_key_value=${PRIVATE_KEY}" \ | |
| -var="cwa_github_sha=${GITHUB_SHA}" \ | |
| -var="ami=${{ matrix.arrays.ami }}" \ | |
| -var="arc=${{ matrix.arrays.arc }}" \ | |
| -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ | |
| -var="ssh_key_name=${KEY_NAME}" \ | |
| -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ | |
| -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve | |
| else | |
| terraform destroy -auto-approve && exit 1 | |
| fi | |
| - name: Terraform destroy | |
| if: ${{ cancelled() || failure() }} | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 8 | |
| retry_wait_seconds: 5 | |
| command: cd terraform/stress && terraform destroy --auto-approve | |
| EC2WinStressTrackingTest: | |
| name: "EC2WinStressTrackingTest" | |
| needs: [GenerateTestMatrix] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_windows_stress_matrix) }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} | |
| ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} | |
| aws-region: us-west-2 | |
| role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} | |
| - name: Cache if success | |
| id: ec2-win-stress-tracking-test | |
| uses: actions/cache@v3 | |
| with: | |
| path: go.mod | |
| key: ec2-win-stress-tracking-test-${{ github.sha }}-${{ matrix.arrays.os }}-${{ matrix.arrays.arc }}-${{ matrix.arrays.test_dir }} | |
| - name: Verify Terraform version | |
| if: steps.ec2-win-stress-tracking-test.outputs.cache-hit != 'true' | |
| run: terraform --version | |
| - name: Echo Test Info | |
| run: echo run on ec2 instance os ${{ matrix.arrays.os }} arc ${{ matrix.arrays.arc }} test dir ${{ matrix.arrays.test_dir }} values per minute ${{ matrix.arrays.values_per_minute }} | |
| - name: Terraform apply | |
| if: steps.ec2-win-stress-tracking-test.outputs.cache-hit != 'true' | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 1 | |
| timeout_minutes: 60 | |
| retry_wait_seconds: 5 | |
| command: | | |
| cd terraform/stress | |
| terraform init | |
| if terraform apply --auto-approve \ | |
| -var="ssh_key_value=${PRIVATE_KEY}" \ | |
| -var="cwa_github_sha=${GITHUB_SHA}" \ | |
| -var="ami=${{ matrix.arrays.ami }}" \ | |
| -var="arc=${{ matrix.arrays.arc }}" \ | |
| -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ | |
| -var="ssh_key_name=${KEY_NAME}" \ | |
| -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ | |
| -var="family=${{ matrix.arrays.family}}"\ | |
| -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve | |
| else | |
| terraform destroy -auto-approve && exit 1 | |
| fi | |
| - name: Terraform destroy | |
| if: ${{ cancelled() || failure() }} | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 8 | |
| retry_wait_seconds: 5 | |
| command: cd terraform/stress && terraform destroy --auto-approve | |
| GPUEndToEndTest: | |
| name: "GPU E2E Test" | |
| needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables ] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.eks_addon_matrix) }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: ${{env.CWA_GITHUB_TEST_REPO_NAME}} | |
| ref: ${{env.CWA_GITHUB_TEST_REPO_BRANCH}} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} | |
| aws-region: us-west-2 | |
| role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} | |
| - name: Verify Terraform version | |
| run: terraform --version | |
| - name: Terraform apply and setup | |
| run: | | |
| if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then | |
| cd "${{ matrix.arrays.terraform_dir }}" | |
| else | |
| cd terraform/eks/addon/gpu | |
| fi | |
| terraform init | |
| if terraform apply --auto-approve \ | |
| -var="beta=true" \ | |
| -var="addon_name=amazon-cloudwatch-observability" \ | |
| -var="addon_version=v1.6.0-eksbuild.1" \ | |
| -var="k8s_version=1.29" ; then | |
| echo "Terraform apply successful." | |
| # Capture the output | |
| echo "Getting EKS cluster name" | |
| EKS_CLUSTER_NAME=$(terraform output -raw eks_cluster_name) | |
| echo "Cluster name is ${EKS_CLUSTER_NAME}" | |
| kubectl apply -f ./gpuBurner.yaml | |
| kubectl create -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.15.0/deployments/static/nvidia-device-plugin.yml | |
| kubectl patch amazoncloudwatchagents -n amazon-cloudwatch cloudwatch-agent --type='json' -p='[{"op": "replace", "path": "/spec/image", "value": ${{ secrets.AWS_ECR_PRIVATE_REGISTRY }}/${{ env.ECR_INTEGRATION_TEST_REPO }}:${{ github.sha }}}]' | |
| else | |
| terraform destroy -var="beta=${{ github.event.inputs.run_in_beta }}" -auto-approve && exit 1 | |
| fi | |
| - name: Run Go tests with retry | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 10 | |
| timeout_minutes: 60 | |
| retry_wait_seconds: 60 | |
| command: | | |
| if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then | |
| cd "${{ matrix.arrays.terraform_dir }}" | |
| else | |
| cd terraform/eks/addon/gpu | |
| fi | |
| echo "Getting EKS cluster name" | |
| EKS_CLUSTER_NAME=$(terraform output -raw eks_cluster_name) | |
| echo "Cluster name is ${EKS_CLUSTER_NAME}" | |
| if go test ${{ matrix.arrays.test_dir }} -eksClusterName ${EKS_CLUSTER_NAME} -computeType=EKS -v -eksDeploymentStrategy=DAEMON -eksGpuType=nvidia -useE2EMetrics; then | |
| echo "Tests passed" | |
| else | |
| echo "Tests failed" | |
| exit 1 | |
| fi | |
| - name: Terraform destroy | |
| if: always() | |
| uses: nick-fields/retry@v2 | |
| with: | |
| max_attempts: 3 | |
| timeout_minutes: 8 | |
| retry_wait_seconds: 5 | |
| command: | | |
| if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then | |
| cd "${{ matrix.arrays.terraform_dir }}" | |
| else | |
| cd terraform/eks/addon/gpu | |
| fi | |
| terraform destroy --auto-approve |