Deploy a staging environment #1806
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy a staging environment | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| manifest: | |
| description: 'The manifest file to deploy (base64 encoded)' | |
| required: true | |
| type: string | |
| sandbox-id: | |
| description: 'The sandbox ID to deploy under' | |
| required: true | |
| type: string | |
| with-rds: | |
| description: 'Stage a new RDS instance' | |
| required: false | |
| default: 'False' | |
| type: string | |
| pipeline-pr: | |
| description: Pipeline PR number to stage | |
| required: false | |
| default: "" | |
| type: string | |
| secrets: | |
| description: 'Encrypted secrets to use for this task' | |
| required: true | |
| type: string | |
| env: | |
| region: ${{ secrets.AWS_REGION }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| load-config: | |
| uses: ./.github/workflows/load-config.yaml | |
| deploy-staging: | |
| name: Deploy staging environment | |
| runs-on: ubuntu-20.04 | |
| needs: load-config | |
| environment: ${{needs.load-config.outputs.domain_name}} | |
| steps: | |
| - id: setup-aws | |
| name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/ci-iac-role | |
| aws-region: ${{ env.region }} | |
| - id: install-aws-cli | |
| name: Install AWS CLI | |
| uses: unfor19/install-aws-cli-action@v1 | |
| with: | |
| version: 2 | |
| - id: checkout | |
| name: Check out source code | |
| uses: actions/checkout@v3 | |
| with: | |
| token: ${{ secrets.API_TOKEN_GITHUB }} | |
| - id: install-eksctl | |
| name: Install eksctl | |
| run: |- | |
| curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp | |
| sudo mv /tmp/eksctl /usr/local/bin | |
| - id: set-stack-name | |
| name: Set name of the CloudFormation stacks | |
| run: |- | |
| BASE_NAME=biomage-rds-staging-$SANDBOX_ID | |
| echo "rds-name=$BASE_NAME" >> $GITHUB_OUTPUT | |
| BASE_NAME=biomage-sns-staging-$SANDBOX_ID | |
| echo "sns-name=$BASE_NAME" >> $GITHUB_OUTPUT | |
| BASE_NAME=biomage-userpoolclient-staging-$SANDBOX_ID | |
| echo "userpoolclient-name=$BASE_NAME" >> $GITHUB_OUTPUT | |
| BASE_NAME=biomage-batch-job-definition-staging-$SANDBOX_ID | |
| echo "batch-job-definition-name=$BASE_NAME" >> $GITHUB_OUTPUT | |
| env: | |
| SANDBOX_ID: ${{ github.event.inputs.sandbox-id }} | |
| - id: create-worker-fargate-profile | |
| name: Attempt to create worker Fargate profile | |
| uses: nick-invision/retry@v2 | |
| with: | |
| timeout_seconds: 900 | |
| max_attempts: 30 | |
| retry_on: error | |
| command: | | |
| output=$(eksctl create fargateprofile --cluster biomage-staging --region ${{ secrets.AWS_REGION }} --name worker-${SANDBOX_ID} --labels type=worker --namespace worker-${SANDBOX_ID} 2>&1) | |
| echo $output | |
| echo $output | grep "created Fargate profile" | |
| # Add jitter to break up correlated events. | |
| on_retry_command: sleep $((20 + RANDOM % 10)); | |
| env: | |
| SANDBOX_ID: ${{ github.event.inputs.sandbox-id }} | |
| - id: create-pipeline-fargate-profile | |
| name: Attempt to create pipeline Fargate profile | |
| uses: nick-invision/retry@v2 | |
| with: | |
| timeout_seconds: 900 | |
| max_attempts: 30 | |
| retry_on: error | |
| command: | | |
| output=$(eksctl create fargateprofile --cluster biomage-staging --region ${{ secrets.AWS_REGION }} --name pipeline-${SANDBOX_ID} --labels type=pipeline --namespace pipeline-${SANDBOX_ID} 2>&1) | |
| echo $output | |
| echo $output | grep "created Fargate profile" | |
| # Add jitter to break up correlated events. | |
| on_retry_command: sleep $((20 + RANDOM % 10)) | |
| env: | |
| SANDBOX_ID: ${{ github.event.inputs.sandbox-id }} | |
| - id: deploy-template-rds | |
| # if: ${{ github.ref == 'refs/heads/master' && github.event.inputs.with-rds == 'True' }} | |
| if: ${{ github.event.inputs.with-rds == 'True' }} | |
| name: Deploy CloudFormation stack for RDS | |
| uses: aws-actions/aws-cloudformation-github-deploy@v1 | |
| with: | |
| parameter-overrides: "Environment=staging,SandboxID=${{ github.event.inputs.sandbox-id }}" | |
| name: ${{ steps.set-stack-name.outputs.rds-name }} | |
| template: cf/rds.yaml | |
| no-fail-on-empty-changeset: "1" | |
| capabilities: "CAPABILITY_IAM,CAPABILITY_NAMED_IAM,CAPABILITY_AUTO_EXPAND" | |
| - id: rds-secrets | |
| if: ${{ github.event.inputs.with-rds == 'True' }} | |
| uses: t-botz/aws-secrets-manager-read-action@v2 | |
| name: Outputs RDS secrets to get into the db | |
| with: | |
| secret-id: aurora-staging-${{ github.event.inputs.sandbox-id }} | |
| mask-value: true | |
| mask-json-values: true | |
| keys-as-outputs: true | |
| - id: setup-rds-roles | |
| # if: ${{ github.ref == 'refs/heads/master' && github.event.inputs.with-rds == 'True' }} | |
| if: ${{ github.event.inputs.with-rds == 'True' }} | |
| name: Setup RDS roles | |
| run: |- | |
| INSTANCE_ID=$(aws ec2 describe-instances \ | |
| --filters 'Name=tag:Name,Values=rds-staging-ssm-agent' \ | |
| --output text \ | |
| --query 'Reservations[*].Instances[*].InstanceId') | |
| if [ -z $INSTANCE_ID ]; then | |
| echo "Can not connect to RDS agent: No instances found for staging" | |
| exit 1 | |
| fi | |
| CLUSTER_NAME=aurora-cluster-staging-${SANDBOX_ID} | |
| RDSHOST=$(aws rds describe-db-cluster-endpoints \ | |
| --region $REGION \ | |
| --db-cluster-identifier $CLUSTER_NAME \ | |
| --filter Name=db-cluster-endpoint-type,Values='writer' \ | |
| --query 'DBClusterEndpoints[0].Endpoint' \ | |
| --output text) | |
| if [ -z $RDSHOST ]; then | |
| echo "Failed getting RDS host with name $CLUSTER_NAME" | |
| exit 1 | |
| fi | |
| ENSURE_PSQL_INSTALLED_COMMAND="sudo yum -y install postgresql" | |
| aws ssm send-command --instance-ids "$INSTANCE_ID" \ | |
| --document-name AWS-RunShellScript \ | |
| --parameters "commands='$ENSURE_PSQL_INSTALLED_COMMAND'" | |
| SETUP_ROLES_CMD=" | |
| PGPASSWORD=\'${AURORA_STAGING_PASSWORD}\' psql \ | |
| --host=${RDSHOST} \ | |
| --port=5432 \ | |
| --username=${AURORA_STAGING_USERNAME} \ | |
| --dbname=aurora_db <<EOF | |
| CREATE ROLE api_role WITH LOGIN; | |
| CREATE ROLE dev_role WITH LOGIN; | |
| GRANT USAGE ON SCHEMA public TO api_role; | |
| GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public to api_role; | |
| GRANT dev_role TO ${AURORA_STAGING_USERNAME}; | |
| ALTER DEFAULT PRIVILEGES FOR USER dev_role IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO api_role; | |
| ALTER DEFAULT PRIVILEGES FOR USER dev_role IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO api_role; | |
| REVOKE dev_role FROM ${AURORA_STAGING_USERNAME}; | |
| GRANT rds_iam TO api_role; | |
| GRANT rds_iam, ${AURORA_STAGING_USERNAME} TO dev_role; | |
| EOF" | |
| aws ssm send-command --instance-ids "$INSTANCE_ID" \ | |
| --document-name AWS-RunShellScript \ | |
| --parameters "commands='$SETUP_ROLES_CMD'" | |
| env: | |
| REGION: ${{ env.region }} | |
| SANDBOX_ID: ${{ github.event.inputs.sandbox-id }} | |
| AURORA_STAGING_USERNAME: ${{ steps.rds-secrets.outputs.username }} | |
| AURORA_STAGING_PASSWORD: ${{ steps.rds-secrets.outputs.password }} | |
| - id: deploy-template-sns | |
| name: Deploy CloudFormation stack for SNS topic | |
| uses: aws-actions/aws-cloudformation-github-deploy@v1 | |
| with: | |
| parameter-overrides: "Environment=staging,SandboxID=${{ github.event.inputs.sandbox-id }}" | |
| name: ${{ steps.set-stack-name.outputs.sns-name }} | |
| template: cf/sns.yaml | |
| no-fail-on-empty-changeset: "1" | |
| capabilities: "CAPABILITY_IAM,CAPABILITY_NAMED_IAM" | |
| - id: deploy-template-userpool-client | |
| name: Deploy CloudFormation stack for Cognito userpool clients | |
| uses: aws-actions/aws-cloudformation-github-deploy@v1 | |
| with: | |
| parameter-overrides: "Environment=staging,SandboxID=${{ github.event.inputs.sandbox-id }}" | |
| name: ${{ steps.set-stack-name.outputs.userpoolclient-name }} | |
| template: cf/userpoolclient.yaml | |
| no-fail-on-empty-changeset: "1" | |
| capabilities: "CAPABILITY_IAM,CAPABILITY_NAMED_IAM" | |
| - id: deploy-template-batch-job-definition | |
| name: Deploy CloudFormation stack for Batch job definition | |
| uses: aws-actions/aws-cloudformation-github-deploy@v1 | |
| with: | |
| parameter-overrides: >- | |
| Environment=staging, | |
| SandboxID=${{ github.event.inputs.sandbox-id }}, | |
| PipelinePRNum=${{ github.event.inputs.pipeline-pr }}, | |
| ImageAccountId=${{ needs.load-config.outputs.image_account_id }}, | |
| ImageAccountRegion=${{ needs.load-config.outputs.image_account_region }} | |
| name: ${{ steps.set-stack-name.outputs.batch-job-definition-name }} | |
| template: cf/batch-job-definition.yaml | |
| no-fail-on-empty-changeset: "1" | |
| capabilities: "CAPABILITY_IAM,CAPABILITY_NAMED_IAM" | |
| - id: create-manifest-file | |
| name: Create manifest file to repository. | |
| run: |- | |
| echo "$MANIFEST" | base64 -d > $SANDBOX_ID.yaml | |
| env: | |
| MANIFEST: ${{ github.event.inputs.manifest }} | |
| SANDBOX_ID: ${{ github.event.inputs.sandbox-id }} | |
| - id: push-deployment-to-releases | |
| name: Push staging deployment template to releases | |
| uses: dmnemec/[email protected] | |
| env: | |
| API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }} | |
| with: | |
| source_file: ${{ github.event.inputs.sandbox-id }}.yaml | |
| destination_repo: ${{ github.repository_owner }}/releases | |
| destination_folder: staging | |
| user_email: [email protected] | |
| user_name: 'Biomage CI/CD' | |
| - id: add-kubeconfig | |
| name: Add k8s config to enable removal of Fargate profile on failure | |
| if: ${{ failure() }} | |
| run: |- | |
| aws eks update-kubeconfig --name biomage-staging | |
| - id: cleanup-pipeline-fargate-profile-if-failure | |
| name: Remove Fargate profile for the pipelines | |
| if: ${{ failure() }} | |
| uses: nick-invision/retry@v2 | |
| with: | |
| timeout_seconds: 900 | |
| max_attempts: 30 | |
| retry_on: error | |
| command: | | |
| output=$(eksctl delete fargateprofile --cluster biomage-staging --region ${{ secrets.AWS_REGION }} --name pipeline-${SANDBOX_ID} 2>&1) | |
| echo $output | |
| echo $output | egrep "deleted Fargate profile|No Fargate Profile found" | |
| # Add jitter to break up correlated events. | |
| on_retry_command: sleep $((20 + RANDOM % 10)); | |
| env: | |
| SANDBOX_ID: ${{ github.event.inputs.sandbox-id }} | |
| - id: cleanup-worker-fargate-profile-if-failure | |
| name: Remove Fargate profile for the workers | |
| if: ${{ failure() }} | |
| uses: nick-invision/retry@v2 | |
| with: | |
| timeout_seconds: 900 | |
| max_attempts: 30 | |
| retry_on: error | |
| command: | | |
| output=$(eksctl delete fargateprofile --cluster biomage-staging --region ${{ secrets.AWS_REGION }} --name worker-${SANDBOX_ID} 2>&1) | |
| echo $output | |
| echo $output | egrep "deleted Fargate profile|No Fargate Profile found" | |
| # Add jitter to break up correlated events. | |
| on_retry_command: sleep $((20 + RANDOM % 10)); | |
| env: | |
| SANDBOX_ID: ${{ github.event.inputs.sandbox-id }} |