Merge pull request #520 from biomage-org/fix-submir-work-hook #325

Workflow file for this run

	name: Build and Deploy the API
	on:
	push:
	branches:
	- master
	release:
	types:
	- released
	pull_request_target:
	branches:
	- master
	types:
	- labeled
	- unlabeled
	- opened
	- synchronize
	- reopened

	permissions:
	id-token: write
	contents: read
	pull-requests: write

	jobs:
	is-safe-to-run:
	name: Sensitive jobs are safe to be run
	runs-on: ubuntu-20.04
	if: (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to run')) \|\| github.event_name == 'release' \|\| github.event_name == 'push'
	steps:
	- id: is-safe-to-run
	name: Is safe to run
	run: \|-
	echo "Safe to run, checks will proceed"

	is-not-safe-to-run:
	name: Sensitive jobs aren't labeled safe to be run
	runs-on: ubuntu-20.04
	if: (github.event_name == 'pull_request_target' && !contains(github.event.pull_request.labels.*.name, 'safe to run'))
	steps:
	- id: is-not-safe-to-run
	name: The pull request hasn't been labeled safe to run
	run: \|-
	echo "Pull request not labeled safe to run"
	exit 1

	uncheck-integration-test:
	name: Mark integration test as not run
	runs-on: ubuntu-20.04
	needs: is-safe-to-run
	if: github.event_name == 'pull_request_target'
	steps:
	- id: get-pr-body
	name: Get the current PR body
	uses: jwalton/gh-find-current-pr@v1
	with:
	state: open

	- id: create-unchecked-pr-body
	name: Create unchecked PR body
	run: \|-
	UNCHECKED_BODY=$(sed 's/- \[[Xx]\] Started end-to-end tests on the latest commit./- \[ \] Started end-to-end tests on the latest commit./' <<\EOF
	${{ steps.get-pr-body.outputs.body }}
	EOF
	)

	echo "Unchecked PR body"
	echo $UNCHECKED_BODY

	# This sets multiline strings into the output variable
	# See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#example-of-a-multiline-string
	echo "body<<EOF" >> "$GITHUB_OUTPUT"
	echo "$UNCHECKED_BODY" >> "$GITHUB_OUTPUT"
	echo "EOF" >> "$GITHUB_OUTPUT"

	- id: uncheck-integration-checkbox
	if: steps.create-unchecked-pr-body.outputs.body != ''
	name: Uncheck the integration checkbox
	uses: tzkhan/pr-update-action@v2
	with:
	repo-token: "${{ secrets.API_TOKEN_GITHUB }}"
	head-branch-regex: "${{ github.head_ref }}"
	lowercase-branch: false
	body-template: ${{ steps.create-unchecked-pr-body.outputs.body }}
	body-update-action: 'replace'

	test:
	name: Run tests
	permissions: write-all
	runs-on: ubuntu-20.04
	needs: is-safe-to-run
	env:
	CI: "true"
	steps:

	- id: setup-node
	uses: actions/setup-node@v2
	with:
	node-version: '14'

	- id: checkout
	name: Check out source code
	uses: actions/checkout@v3
	with:
	ref: ${{github.head_ref}}
	repository: ${{github.event.pull_request.head.repo.full_name}}

	- id: cache-seek
	name: Check for npm cache hit
	uses: c-hive/gha-npm-cache@v1

	- id: install
	name: Install dependencies
	run: \|-
	echo "Running CI with "
	echo "Node version: $(node --version)"
	echo "NPM version: $(npm --version)"

	git config --global url."https://".insteadOf ssh://
	npm ci

	- id: test-codecov
	name: Run unit tests with coverage
	uses: mattallty/jest-github-action@v1
	env:
	AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	with:
	test-command: "npm run coverage"
	coverage-comment: false

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v1

	- id: check-licenses
	name: Check licenses
	env:
	ALLOWED_LICENSES: "MIT;ISC;Apache-2.0;BSD-3-Clause;BSD-2-Clause;(MIT OR Apache-2.0);Unlicense;Python-2.0;BSD;(AFL-2.1 OR BSD-3-Clause);0BSD"
	EXCLUDE_PACKAGES: "[email protected]"
	run: \|-
	npm install -g license-checker
	license-checker --production --json --onlyAllow="${ALLOWED_LICENSES}" --excludePackages="${EXCLUDE_PACKAGES}"

	- id: send-to-slack
	name: Send failure notification to Slack
	if: failure() && github.event_name == 'push'
	env:
	SLACK_BOT_TOKEN: ${{ secrets.BUILD_STATUS_BOT_TOKEN }}
	uses: voxmedia/github-action-slack-notify-build@v1
	with:
	channel: pipelines
	status: FAILED
	color: danger

	build-docker:
	name: Build Docker container
	runs-on: ubuntu-20.04
	needs: [is-safe-to-run]
	outputs:
	ref-id: ${{ steps.ref.outputs.ref-id }}
	image-tag: ${{ steps.ref.outputs.image-tag }}
	timestamp: ${{ steps.ref.outputs.timestamp }}
	steps:
	- id: checkout
	name: Check out source code
	uses: actions/checkout@v3
	with:
	ref: ${{github.head_ref}}
	repository: ${{github.event.pull_request.head.repo.full_name}}

	- id: ref
	name: Format docker tag and repository name.
	run: \|-
	# This will take a ref like `refs/heads/master`
	# and turn it into `refs-heads-master`

	if [ "${{ github.event_name }}" = "pull_request_target" ]; then
	# Construct the PR_GITHUB_REF for our pull request copying GITHUB_REF's format
	# We can't use GITHUB_REF because it is set to master in this event
	# More info: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target
	PR_GITHUB_REF="refs/pull/${{ github.event.pull_request.number }}/merge"
	REF_ID=$(echo $PR_GITHUB_REF \| sed 's/\//-/g')
	else
	REF_ID=$(echo $GITHUB_REF \| sed 's/\//-/g')
	fi

	echo "ref-id=$REF_ID" >> $GITHUB_OUTPUT

	# the final tag is something like:
	# refs-heads-master-a4f8bc313dae
	# this is what we push to ECR
	# we will also take semver'd tags like `1.0.0` and use them for releases

	# In push & PR events we want the tag to contain the latest commit on the branch:
	# in push events, the latest commit of the master branch is GITHUB_SHA
	# in PR synch the latest commit of the branch is found in github.sha instead

	TIMESTAMP=$(date +%s)

	if [ "${{ github.event_name }}" = "release" ] && [ "${{ github.event.action }}" = "released" ]; then
	COMMIT_SHA=""
	IMAGE_TAG="${REF_ID/refs-tags-/}"
	elif [ "${{ github.event_name }}" = "pull_request_target" ]; then
	COMMIT_SHA="${{ github.sha }}"
	IMAGE_TAG="$REF_ID-$COMMIT_SHA-$TIMESTAMP"
	else
	COMMIT_SHA=$GITHUB_SHA
	IMAGE_TAG="$REF_ID-$COMMIT_SHA-$TIMESTAMP"
	fi

	# IMAGE_TAG is used in the Build Docker Image step.
	# We can easily build the image-tag from REF_ID and COMMIT_SHA for non-production releases.
	# But we can not easily create the image tag for production releases, so we're bulding it here
	echo "image-tag=$IMAGE_TAG" >> $GITHUB_OUTPUT

	# TIMESTAMP is used to postfix images in the "push docker images to ECR" step.
	# The timestamp is used by Flux to auto update images for staging environments.
	# Images for production uses semantic versioning to determine the latest image.
	echo "timestamp=$TIMESTAMP" >> $GITHUB_OUTPUT

	# This will take a GitHub repo name like `hms-dbmi-cellenics/iac`
	# and turns it into `iac`. This will be the name of the
	# ECR repository.
	IMAGE_REPO_NAME=$(echo $GITHUB_REPOSITORY \| awk -F '/' '{print $2}')
	echo "repo-name=$IMAGE_REPO_NAME" >> $GITHUB_OUTPUT

	- id: setup-aws
	name: Configure AWS credentials
	uses: aws-actions/configure-aws-credentials@v4
	with:
	role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/ci-role-api
	aws-region: ${{ secrets.AWS_REGION }}

	- id: login-ecr
	name: Login to Amazon ECR
	uses: aws-actions/amazon-ecr-login@v1

	- id: create-ecr-registry
	name: Create an ECR repository (if needed)
	# This will fail if the registry already exists, which is fine. If there is some other
	# error, the `push` step will fail instead.
	continue-on-error: true
	run: \|-
	aws ecr create-repository --repository-name $REPO_NAME --image-tag-mutability IMMUTABLE
	env:
	REPO_NAME: ${{ steps.ref.outputs.repo-name }}

	- id: build
	name: Build Docker image
	uses: docker/build-push-action@v2
	with:
	context: .
	file: ./Dockerfile
	platforms: linux/amd64
	tags: ${{ format('{0}/{1}:{2}', steps.login-ecr.outputs.registry, steps.ref.outputs.repo-name, steps.ref.outputs.image-tag) }}
	push: false

	- id: push
	name: Push docker image to ECR
	run: \|-
	echo Pushing image $IMAGE_NAME to ECR.
	docker push $IMAGE_NAME
	env:
	IMAGE_NAME: ${{ format('{0}/{1}:{2}', steps.login-ecr.outputs.registry, steps.ref.outputs.repo-name, steps.ref.outputs.image-tag) }}

	deploy:
	name: Deploy to Kubernetes
	runs-on: ubuntu-20.04
	needs: [build-docker, is-safe-to-run]
	strategy:
	max-parallel: 1
	matrix:
	environment: ['production', 'staging', 'develop']
	steps:
	- id: checkout
	name: Check out source code
	uses: actions/checkout@v3
	with:
	ref: ${{github.head_ref}}
	repository: ${{github.event.pull_request.head.repo.full_name}}

	- id: setup-aws
	name: Configure AWS credentials
	uses: aws-actions/configure-aws-credentials@v4
	with:
	role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/ci-role-api
	aws-region: ${{ secrets.AWS_REGION }}

	- id: login-ecr
	name: Login to Amazon ECR
	uses: aws-actions/amazon-ecr-login@v1

	- id: fill-metadata
	name: Fill out a new HelmRelease resource
	run: \|-
	export DEPLOYMENT_NAME=$(echo $GITHUB_REPOSITORY \| awk -F '/' '{print $2}')
	echo "deployment-name=$DEPLOYMENT_NAME" >> $GITHUB_OUTPUT

	if [ "${{ matrix.environment }}" = "production" ]; then
	export SANDBOX_ID="default"
	export RDS_SANDBOX_ID="default"
	export KUBERNETES_ENV="production"
	export CHART_REF="master"
	export IMAGE_PATTERN="^(?P<version>[0-9]+\.[0-9]+\.[0-9]+)$"
	export IMAGE_EXTRACT='$version'
	export IMAGE_POLICY_TYPE="semver"
	export IMAGE_POLICY_KEY="range"
	export IMAGE_POLICY_VALUE=">=0.0.0"
	export REPLICA_COUNT="4"
	export MANIFEST_REPOSITORY="./production"
	fi

	if [ "${{ matrix.environment }}" = "develop" ]; then
	export SANDBOX_ID="default"
	export RDS_SANDBOX_ID="default"
	export KUBERNETES_ENV="staging"
	export CHART_REF="master"
	export IMAGE_PATTERN="^$REF_ID-[a-f0-9]+-(?P<timestamp>[0-9]+)$"
	export IMAGE_EXTRACT='$timestamp'
	export IMAGE_POLICY_TYPE="numerical"
	export IMAGE_POLICY_KEY="order"
	export IMAGE_POLICY_VALUE="asc"
	export REPLICA_COUNT="2"
	export MANIFEST_REPOSITORY="./staging"
	fi

	if [ "${{ matrix.environment }}" = "staging" ]; then

	# The chart template for the API repo is stored in IAC branch master
	# If you are working on the chart for UI/API, you can modify this value manualy in your PR's release
	export CHART_REF="master"
	export SANDBOX_ID="STAGING_SANDBOX_ID"
	export RDS_SANDBOX_ID="STAGING_RDS_SANDBOX_ID"
	export KUBERNETES_ENV="staging"
	export IMAGE_PATTERN="^$REF_ID-[a-f0-9]+-(?P<timestamp>[0-9]+)$"
	export IMAGE_EXTRACT='$timestamp'
	export IMAGE_POLICY_TYPE="numerical"
	export IMAGE_POLICY_KEY="order"
	export IMAGE_POLICY_VALUE="asc"
	export REPLICA_COUNT="1"
	export MANIFEST_REPOSITORY="./staging"
	fi

	echo "kubernetes-env=$KUBERNETES_ENV" >> $GITHUB_OUTPUT

	export NAMESPACE="$DEPLOYMENT_NAME-$SANDBOX_ID"
	export CHART_SOURCE_NAME="$DEPLOYMENT_NAME-chart"
	export IMAGE_POLICY_TAG="{\"\$imagepolicy\": \"$NAMESPACE:$DEPLOYMENT_NAME:tag\"}"

	cp .flux.yaml $DEPLOYMENT_NAME.yaml

	yq '
	select(di == 0).metadata.name = strenv(NAMESPACE) \|
	select(di == 0).metadata.labels.sandboxId = strenv(SANDBOX_ID) \|
	select(di == 1).metadata.name = strenv(CHART_SOURCE_NAME) \|
	select(di == 1).metadata.namespace = strenv(NAMESPACE) \|
	select(di == 1).spec.url = "https://github.com/" + strenv(GITHUB_OWNER) + "/iac" \|
	select(di == 1).spec.ref.branch = strenv(CHART_REF) \|
	select(di == 2).metadata.name = strenv(DEPLOYMENT_NAME) \|
	select(di == 2).metadata.namespace = strenv(NAMESPACE) \|
	select(di == 2).spec.image = strenv(REGISTRY) + "/" + strenv(DEPLOYMENT_NAME) \|
	select(di == 3).metadata.name = strenv(DEPLOYMENT_NAME) \|
	select(di == 3).metadata.namespace = strenv(NAMESPACE) \|
	select(di == 3).spec.imageRepositoryRef.name = strenv(DEPLOYMENT_NAME) \|
	select(di == 3).spec.filterTags.pattern = strenv(IMAGE_PATTERN) \|
	select(di == 3).spec.filterTags.extract = strenv(IMAGE_EXTRACT) \|
	select(di == 3).spec.policy.[strenv(IMAGE_POLICY_TYPE)].[strenv(IMAGE_POLICY_KEY)] = strenv(IMAGE_POLICY_VALUE) \|
	select(di == 4).metadata.name = strenv(DEPLOYMENT_NAME) + "-image-update" \|
	select(di == 4).metadata.namespace = strenv(NAMESPACE) \|
	select(di == 4).spec.update.path = strenv(MANIFEST_REPOSITORY) \|
	select(di == 5).metadata.name = strenv(DEPLOYMENT_NAME) \|
	select(di == 5).metadata.namespace = strenv(NAMESPACE) \|
	select(di == 5).metadata.labels.sandboxId = strenv(SANDBOX_ID) \|
	select(di == 5).spec.releaseName = strenv(DEPLOYMENT_NAME) \|
	select(di == 5).spec.chart.spec.sourceRef.name = strenv(CHART_SOURCE_NAME) \|
	select(di == 5).spec.chart.spec.sourceRef.namespace = strenv(NAMESPACE) \|
	select(di == 5).spec.values.kubernetes.env = strenv(KUBERNETES_ENV) \|
	select(di == 5).spec.values.biomageCi.repo = strenv(GITHUB_REPOSITORY) \|
	select(di == 5).spec.values.biomageCi.sandboxId = strenv(SANDBOX_ID) \|
	select(di == 5).spec.values.biomageCi.rdsSandboxId = strenv(RDS_SANDBOX_ID) \|
	select(di == 5).spec.values.image.registry = strenv(REGISTRY) \|
	select(di == 5).spec.values.image.repository = strenv(DEPLOYMENT_NAME) \|
	select(di == 5).spec.values.image.tag = strenv(IMAGE_TAG) \|
	select(di == 5).spec.values.image.tag line_comment = strenv(IMAGE_POLICY_TAG) \|
	select(di == 5).spec.values.replicaCount = env(REPLICA_COUNT) \|
	select(di == 5).spec.values.serviceAccount.iamRole = "api-role-" + strenv(KUBERNETES_ENV)
	' .flux.yaml > $DEPLOYMENT_NAME.yaml

	cat $DEPLOYMENT_NAME.yaml
	env:
	GITHUB_OWNER: ${{ github.repository_owner }}
	REF_ID: ${{ needs.build-docker.outputs.ref-id }}
	IMAGE_TAG: ${{ needs.build-docker.outputs.image-tag }}
	REGISTRY: ${{ steps.login-ecr.outputs.registry }}
	- name: Push production/develop template to releases
	if:
	(matrix.environment == 'production' && github.event_name == 'release' && github.event.action == 'released') \|\| (matrix.environment == 'develop' && github.event_name == 'push')
	uses: dmnemec/[email protected]
	env:
	API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }}
	with:
	source_file: '${{ steps.fill-metadata.outputs.deployment-name }}.yaml'
	destination_repo: '${{ github.repository_owner }}/releases'
	destination_folder: '${{ steps.fill-metadata.outputs.kubernetes-env }}'
	user_email: '[email protected]'
	user_name: 'Biomage CI/CD'

	- name: Change name of deployment file for staging deployment
	if:
	(github.event_name == 'pull_request_target' \|\| github.event_name == 'push') && matrix.environment == 'staging'
	env:
	DEPLOYMENT_NAME: ${{ steps.fill-metadata.outputs.deployment-name }}
	REF_ID: ${{ needs.build-docker.outputs.ref-id }}
	run: \|-
	mv $DEPLOYMENT_NAME.yaml $REF_ID.yaml

	- name: Push staging deployment template to releases
	if:
	(github.event_name == 'pull_request_target' \|\| github.event_name == 'push') && matrix.environment == 'staging'
	uses: dmnemec/[email protected]
	env:
	API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }}
	with:
	source_file: ${{ needs.build-docker.outputs.ref-id }}.yaml
	destination_repo: '${{ github.repository_owner }}/releases'
	destination_folder: 'staging-candidates/${{ steps.fill-metadata.outputs.deployment-name }}'
	user_email: '[email protected]'
	user_name: 'Biomage CI/CD'
	- id: disable-admin-enforcement
	if:
	(matrix.environment == 'production' && github.event_name == 'release' && github.event.action == 'released') \|\| (matrix.environment == 'develop' && github.event_name == 'push')
	name: Temporarily disable admin enforcement
	uses: benjefferies/branch-protection-bot@master
	with:
	access_token: ${{ secrets.API_TOKEN_GITHUB }}
	owner: ${{ github.repository_owner }}
	repo: iac
	enforce_admins: false
	retries: 8
	- name: Push migrations into iac
	if:
	github.repository_owner == 'hms-dbmi-cellenics' && matrix.environment == 'develop' && github.event_name == 'push'
	uses: cpina/github-action-push-to-another-repository@main
	env:
	API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }}
	with:
	source-directory: 'src/sql/migrations'
	destination-github-username: '${{ github.repository_owner }}'
	destination-repository-name: 'iac'
	user-email: '[email protected]'
	target-branch: master
	target-directory: 'migrations/sql'

	ready-to-merge:
	name: Ready for merging
	runs-on: ubuntu-20.04
	needs: [deploy, is-safe-to-run]
	steps:
	- id: ready-to-merge
	name: Signal readiness to merge
	run: \|-
	exit 0

	report-if-failed:
	name: Report if workflow failed
	runs-on: ubuntu-20.04
	needs: [test, build-docker, deploy, is-safe-to-run]
	if: failure() && github.ref == 'refs/heads/master'
	steps:
	- id: send-to-slack
	name: Send failure notification to Slack on failure
	env:
	SLACK_BOT_TOKEN: ${{ secrets.WORKFLOW_STATUS_BOT_TOKEN }}
	uses: voxmedia/github-action-slack-notify-build@v1
	with:
	channel: workflow-failures
	status: FAILED
	color: danger

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Merge pull request #520 from biomage-org/fix-submir-work-hook #325

Workflow file

Merge pull request #520 from biomage-org/fix-submir-work-hook #325

Jobs

Run details

Workflow file for this run