Skip to content

update flux api version #1859

update flux api version

update flux api version #1859

Workflow file for this run

name: CI
on:
push:
branches:
- master
release:
types:
- released
pull_request_target:
branches:
- master
types:
- labeled
- unlabeled
- opened
- synchronize
- reopened
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
permissions:
id-token: write
contents: read
jobs:
is-safe-to-run:
name: Sensitive jobs are safe to be run
runs-on: ubuntu-20.04
if: (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to run')) || github.event_name == 'release' || github.event_name == 'push'
steps:
- id: is-safe-to-run
name: Is safe to run
run: |-
echo "Safe to run, checks will proceed"
is-not-safe-to-run:
name: Sensitive jobs aren't labeled safe to be run
runs-on: ubuntu-20.04
if: (github.event_name == 'pull_request_target' && !contains(github.event.pull_request.labels.*.name, 'safe to run'))
steps:
- id: is-not-safe-to-run
name: The pull request hasn't been labeled safe to run
run: |-
echo "Pull request not labeled safe to run"
exit 1
uncheck-integration-test:
name: Mark integration test as not run
runs-on: ubuntu-20.04
needs: is-safe-to-run
if: github.event_name == 'pull_request_target'
steps:
- id: get-pr-body
name: Get the current PR body
uses: jwalton/gh-find-current-pr@v1
with:
state: open
- id: create-unchecked-pr-body
name: Create unchecked PR body
run: |-
UNCHECKED_BODY=$(sed 's/- \[[Xx]\] Started end-to-end tests on the latest commit./- \[ \] Started end-to-end tests on the latest commit./' <<\EOF
${{ steps.get-pr-body.outputs.body }}
EOF
)
echo "Unchecked PR body"
echo $UNCHECKED_BODY
# This sets multiline strings into the output variable
# See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#example-of-a-multiline-string
echo "body<<EOF" >> "$GITHUB_OUTPUT"
echo "$UNCHECKED_BODY" >> "$GITHUB_OUTPUT"
echo "EOF" >> "$GITHUB_OUTPUT"
- id: uncheck-integration-checkbox
if: steps.create-unchecked-pr-body.outputs.body != ''
name: Uncheck the integration checkbox
uses: tzkhan/pr-update-action@v2
with:
repo-token: "${{ secrets.API_TOKEN_GITHUB }}"
head-branch-regex: "${{ github.head_ref }}"
lowercase-branch: false
body-template: "${{ steps.create-unchecked-pr-body.outputs.body }}"
body-update-action: "replace"
build-docker:
name: Build Docker container
runs-on: ubuntu-20.04
needs: is-safe-to-run
strategy:
matrix:
project: ['pipeline-runner']
outputs:
repo-name: ${{ steps.ref.outputs.repo-name }}
image-tag: ${{ steps.ref.outputs.image-tag }}
ref-id: ${{ steps.ref.outputs.ref-id }}
timestamp: ${{ steps.ref.outputs.timestamp }}
defaults:
run:
working-directory: ${{ matrix.project }}
steps:
- id: checkout
name: Check out source code
uses: actions/checkout@v3
with:
ref: ${{github.head_ref}}
repository: ${{github.event.pull_request.head.repo.full_name}}
- id: ref
name: Format docker tag and repository name for current image.
run: |-
if [ "${{ github.event_name }}" = "pull_request_target" ]; then
# Construct the PR_GITHUB_REF for our pull request copying GITHUB_REF's format
# We can't use GITHUB_REF because it is set to master in this event
# More info: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target
PR_GITHUB_REF="refs/pull/${{ github.event.pull_request.number }}/merge"
REF_ID=$(echo $PR_GITHUB_REF | sed 's/\//-/g')
else
# This will take a ref like `refs/heads/master`
# and turn it into `refs-heads-master`
REF_ID=$(echo $GITHUB_REF | sed 's/\//-/g')
fi
echo "ref-id=$REF_ID" >> $GITHUB_OUTPUT
# the final tag is something like:
# refs-heads-master-a4f8bc313dae
# this is what we push to ECR
# we will also take semver'd tags like `1.0.0` and use them for releases
# In push & PR events we want the tag to contain the latest commit on the branch:
# in push events, the latest commit of the master branch is GITHUB_SHA
# in PR synch the latest commit of the branch is found in github.sha instead
TIMESTAMP=$(date +%s)
if [ "${{ github.event_name }}" = "release" ] && [ "${{ github.event.action }}" = "released" ]; then
COMMIT_SHA=""
IMAGE_TAG=$REF_ID
TIMESTAMP=""
BATCH_IMAGE_TAG=production
elif [ "${{ github.event_name }}" = "pull_request_target" ]; then
COMMIT_SHA="${{ github.sha }}"
IMAGE_TAG="$REF_ID-$COMMIT_SHA"
TIMESTAMP=$TIMESTAMP
BATCH_IMAGE_TAG=$REF_ID
else
COMMIT_SHA=$GITHUB_SHA
IMAGE_TAG="$REF_ID-$COMMIT_SHA"
TIMESTAMP=$TIMESTAMP
BATCH_IMAGE_TAG=staging
fi
# IMAGE_TAG is used in the Build Docker Image step.
# We can easily build the image-tag from REF_ID and COMMIT_SHA for non-production releases.
# But we can not easily create the image tag for production releases, so we're bulding it here
echo "image-tag=$IMAGE_TAG" >> $GITHUB_OUTPUT
echo "batch-image-tag=$BATCH_IMAGE_TAG" >> $GITHUB_OUTPUT
# TIMESTAMP is used to postfix images in the "push docker images to ECR" step.
# The timestamp is used by Flux to auto update images for staging environments.
# Images for production uses semantic versioning to determine the latest image.
echo "timestamp=$TIMESTAMP" >> $GITHUB_OUTPUT
# This will take a GitHub repo name like `hms-dbmi-cellenics/releases`
# and turns it into `releases`. This will be the name of the
# ECR repository.
IMAGE_REPO_NAME=$(echo $GITHUB_REPOSITORY | awk -F '/' '{print $2}')
echo "repo-name=$IMAGE_REPO_NAME" >> $GITHUB_OUTPUT
- id: ref-previous
name: Format docker tag and repository name for the previous pushed image.
run: |-
if [ "${{ github.event_name }}" = "pull_request_target" ]; then
echo "This is a pull request, base ref and sha set to the target branch."
BASE_REF="refs-heads-${{ github.event.pull_request.base.ref }}"
BASE_SHA=${{ github.event.pull_request.base.sha }}
echo "tag=$BASE_REF-$BASE_SHA" >> $GITHUB_OUTPUT
fi
if [ "${{ github.event_name }}" = "push" ]; then
echo "This is a push, base ref and sha set to the previous commit."
BASE_REF=$(echo $GITHUB_REF | sed 's/\//-/g')
BASE_SHA="${{ github.event.before }}"
echo "tag=$BASE_REF-$BASE_SHA" >> $GITHUB_OUTPUT
fi
if [ "${{ github.event_name }}" = "release" ]; then
echo "This is a release, base ref and sha set to the current commit."
BASE_SHA="$GITHUB_SHA"
echo "tag=refs-heads-master-$BASE_SHA" >> $GITHUB_OUTPUT
fi
- id: setup-aws
name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/ci-role-pipeline
aws-region: ${{ secrets.AWS_REGION }}
- id: login-ecr
name: Login to Amazon ECR
uses: aws-actions/amazon-ecr-login@v1
- id: create-ecr-registry
name: Create an ECR repository (if needed)
# This will fail if the registry already exists, which is fine. If there is some other
# error, the `push` step will fail instead.
continue-on-error: true
run: |-
aws ecr create-repository --repository-name $REPO_NAME
env:
REPO_NAME: ${{ steps.ref.outputs.repo-name }}
- id: setup-docker-buildx
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- id: build
name: Build Docker images
run: |-
docker buildx build --target common \
--output=type=docker \
--cache-from type=registry,ref=$PREVIOUS_IMAGE_NAME-common \
--build-arg GITHUB_PAT=$GITHUB_PAT \
--tag $IMAGE_NAME-common .
docker buildx build --target prod \
--output=type=docker \
--cache-from=$IMAGE_NAME-common \
--build-arg GITHUB_PAT=$GITHUB_PAT \
--tag $IMAGE_NAME .
docker buildx build --target batch \
--output=type=docker \
--cache-from=$IMAGE_NAME-common \
--build-arg GITHUB_PAT=$GITHUB_PAT \
--tag $BATCH_IMAGE_NAME .
env:
PREVIOUS_IMAGE_NAME: ${{ format('{0}/{1}:{2}-{3}', steps.login-ecr.outputs.registry, steps.ref.outputs.repo-name, steps.ref-previous.outputs.tag, matrix.project) }}
IMAGE_NAME: ${{ format('{0}/{1}:{2}-{3}', steps.login-ecr.outputs.registry, steps.ref.outputs.repo-name, steps.ref.outputs.image-tag, matrix.project) }}
BATCH_IMAGE_NAME: ${{ format('{0}/{1}:batch-{2}', steps.login-ecr.outputs.registry, steps.ref.outputs.repo-name, steps.ref.outputs.batch-image-tag) }}
GITHUB_PAT: ${{ secrets.API_TOKEN_GITHUB }}
- id: codecov
name: Generate codecov report
run: |-
docker run \
--entrypoint /bin/bash \
-v $(pwd)/covr:/covr $IMAGE_NAME \
-c "R -e 'tryCatch({cov <- covr::package_coverage(); covr::to_cobertura(cov, \"/covr/coverage.xml\")}, error = function(e) {testthat::test_local(); stop()})'"
env:
IMAGE_NAME: ${{ format('{0}/{1}:{2}-{3}', steps.login-ecr.outputs.registry, steps.ref.outputs.repo-name, steps.ref.outputs.image-tag, matrix.project) }}
- id: upload-coverage
name: Upload coverage to Codecov
uses: codecov/codecov-action@v2
with:
files: ./pipeline-runner/covr/coverage.xml
- id: push
name: Push docker images to ECR
run: |-
echo Pushing image $IMAGE_NAME-common to ECR.
docker buildx build --push --target common \
--cache-to type=registry,ref=$IMAGE_NAME-common,mode=max,image-manifest=true \
--build-arg GITHUB_PAT=$GITHUB_PAT \
--tag $IMAGE_NAME-common .
echo Pushing image $IMAGE_NAME to ECR.
docker push $IMAGE_NAME
echo Pushing batch image $BATCH_IMAGE_NAME to ECR.
docker push $BATCH_IMAGE_NAME
if [ ! -z "$TIMESTAMP" ]; then
echo Pushing timestamped image $IMAGE_NAME-$TIMESTAMP to ECR
docker tag $IMAGE_NAME $IMAGE_NAME-$TIMESTAMP
docker push $IMAGE_NAME-$TIMESTAMP
fi
env:
IMAGE_NAME: ${{ format('{0}/{1}:{2}-{3}', steps.login-ecr.outputs.registry, steps.ref.outputs.repo-name, steps.ref.outputs.image-tag, matrix.project) }}
BATCH_IMAGE_NAME: ${{ format('{0}/{1}:batch-{2}', steps.login-ecr.outputs.registry, steps.ref.outputs.repo-name, steps.ref.outputs.batch-image-tag) }}
TIMESTAMP: ${{ steps.ref.outputs.timestamp }}
GITHUB_PAT: ${{ secrets.API_TOKEN_GITHUB }}
deploy:
name: Deploy to Kubernetes
runs-on: ubuntu-20.04
needs: [build-docker, is-safe-to-run]
strategy:
max-parallel: 1
matrix:
environment: ['production', 'staging', 'develop']
steps:
- id: checkout
name: Check out source code
uses: actions/checkout@v3
with:
ref: ${{github.head_ref}}
repository: ${{github.event.pull_request.head.repo.full_name}}
- id: setup-aws
name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/ci-role-pipeline
aws-region: ${{ secrets.AWS_REGION }}
- id: login-ecr
name: Login to Amazon ECR
uses: aws-actions/amazon-ecr-login@v1
- id: fill-metadata
name: Fill out a new HelmRelease resource
run: |-
export DEPLOYMENT_NAME=$(echo $GITHUB_REPOSITORY | awk -F '/' '{print $2}')
echo "deployment-name=$DEPLOYMENT_NAME" >> $GITHUB_OUTPUT
# Deployment config for `production-default`
if [ "${{ matrix.environment }}" = "production" ]; then
export SANDBOX_ID="default"
export CHART_REF_TYPE="commit"
export CHART_REF="$GITHUB_SHA"
export KUBERNETES_ENV="production"
export IMAGE_NAME=$IMAGE_TAG-pipeline-runner
export REPLICA_COUNT="1"
export VERSION_NUMBER=${REF_ID/refs-tags-/}
export IMAGE_PATTERN="^refs-tags-(?P<version>[0-9]+\.[0-9]+\.[0-9]+)-pipeline-runner$"
export IMAGE_EXTRACT='$version'
export IMAGE_POLICY_TYPE="semver"
export IMAGE_POLICY_KEY="range"
export IMAGE_POLICY_VALUE=">=0.0.0"
export MEMORY_REQUEST="40Gi"
export MANIFEST_PATH="./production"
fi
# Deployment config for `staging-default`
if [ "${{ matrix.environment }}" = "develop" ]; then
export SANDBOX_ID="default"
export CHART_REF_TYPE="branch"
export CHART_REF="master"
export KUBERNETES_ENV="staging"
export IMAGE_NAME="$IMAGE_TAG-pipeline-runner-$TIMESTAMP"
export REPLICA_COUNT="0"
export IMAGE_PATTERN="^$REF_ID-[a-f0-9]+-pipeline-runner-(?P<timestamp>[0-9]+)"
export IMAGE_EXTRACT='$timestamp'
export IMAGE_POLICY_TYPE="numerical"
export IMAGE_POLICY_KEY="order"
export IMAGE_POLICY_VALUE="asc"
export MEMORY_REQUEST="16Gi"
export MANIFEST_PATH="./staging"
fi
# Deployment config for other staging env i.e. non `default`
if [ "${{ matrix.environment }}" = "staging" ]; then
export CHART_REF="$GITHUB_HEAD_REF"
# $GITHUB_HEAD_REF references the source branch of the pull request in a workflow run
# $GITHUB_HEAD_REF does not have a value when we merge, because merging is counted as a "push" event to master branch
# Therefore we set this value to "master" manually
if [ -z "$CHART_REF" ]; then
export CHART_REF="master"
fi
export SANDBOX_ID="STAGING_SANDBOX_ID"
export CHART_REF_TYPE="branch"
export KUBERNETES_ENV="staging"
export IMAGE_NAME="$IMAGE_TAG-pipeline-runner-$TIMESTAMP"
export REPLICA_COUNT="0"
export IMAGE_PATTERN="^$REF_ID-[a-f0-9]+-pipeline-runner-(?P<timestamp>[0-9]+)"
export IMAGE_EXTRACT='$timestamp'
export IMAGE_POLICY_TYPE="numerical"
export IMAGE_POLICY_KEY="order"
export IMAGE_POLICY_VALUE="asc"
export MEMORY_REQUEST="4Gi"
export MANIFEST_PATH="./staging"
fi
echo "sandbox-id=$SANDBOX_ID" >> $GITHUB_OUTPUT
echo "kubernetes-env=$KUBERNETES_ENV" >> $GITHUB_OUTPUT
export GITHUB_OWNER=${{ github.repository_owner }}
export NAMESPACE="$DEPLOYMENT_NAME-$SANDBOX_ID"
export CHART_CRD_NAME="$DEPLOYMENT_NAME-chart"
export IMAGE_POLICY_TAG="{\"\$imagepolicy\": \"$NAMESPACE:$DEPLOYMENT_NAME:tag\"}"
yq '
select(di == 0).metadata.name = strenv(NAMESPACE) |
select(di == 0).metadata.labels.sandboxId = strenv(SANDBOX_ID) |
select(di == 1).metadata.name = strenv(CHART_CRD_NAME) |
select(di == 1).metadata.namespace = strenv(NAMESPACE) |
select(di == 1).spec.ref.[strenv(CHART_REF_TYPE)] = strenv(CHART_REF) |
select(di == 1).spec.url = "https://github.com/" + strenv(GITHUB_OWNER) + "/pipeline" |
select(di == 2).metadata.name = strenv(DEPLOYMENT_NAME) |
select(di == 2).metadata.namespace = strenv(NAMESPACE) |
select(di == 2).spec.image = strenv(REGISTRY) + "/" + strenv(DEPLOYMENT_NAME) |
select(di == 3).metadata.name = strenv(DEPLOYMENT_NAME) |
select(di == 3).metadata.namespace = strenv(NAMESPACE) |
select(di == 3).spec.imageRepositoryRef.name = strenv(DEPLOYMENT_NAME) |
select(di == 3).spec.imageRepositoryRef.namespace = strenv(NAMESPACE) |
select(di == 3).spec.filterTags.pattern = strenv(IMAGE_PATTERN) |
select(di == 3).spec.filterTags.extract = strenv(IMAGE_EXTRACT) |
select(di == 3).spec.policy.[strenv(IMAGE_POLICY_TYPE)].[strenv(IMAGE_POLICY_KEY)] = strenv(IMAGE_POLICY_VALUE) |
select(di == 4).metadata.name = strenv(DEPLOYMENT_NAME) + "-image-update" |
select(di == 4).metadata.namespace = strenv(NAMESPACE) |
select(di == 4).spec.update.path = strenv(MANIFEST_PATH) |
select(di == 5).metadata.name = strenv(DEPLOYMENT_NAME) |
select(di == 5).metadata.namespace = strenv(NAMESPACE) |
select(di == 5).metadata.labels.sandboxId = strenv(SANDBOX_ID) |
select(di == 5).spec.releaseName = strenv(DEPLOYMENT_NAME) |
select(di == 5).spec.chart.spec.sourceRef.name = strenv(CHART_CRD_NAME) |
select(di == 5).spec.chart.spec.sourceRef.namespace = strenv(NAMESPACE) |
select(di == 5).spec.values.clusterEnv = strenv(KUBERNETES_ENV) |
select(di == 5).spec.values.image.registry = strenv(REGISTRY) |
select(di == 5).spec.values.image.repository = strenv(REPOSITORY) |
select(di == 5).spec.values.image.tag = strenv(IMAGE_NAME) |
select(di == 5).spec.values.image.tag line_comment = strenv(IMAGE_POLICY_TAG) |
select(di == 5).spec.values.sandboxId = strenv(SANDBOX_ID) |
select(di == 5).spec.values.memoryRequest = strenv(MEMORY_REQUEST) |
select(di == 5).spec.values.replicaCount = env(REPLICA_COUNT) |
select(di == 5).spec.values.serviceAccount.iamRole = "pipeline-role-" + strenv(KUBERNETES_ENV) |
select(di == 5).spec.values.datadogTags = "kube_namespace:" + strenv(NAMESPACE)
' .flux.yaml > $DEPLOYMENT_NAME.yaml
cat $DEPLOYMENT_NAME.yaml
env:
IMAGE_TAG: ${{ needs.build-docker.outputs.image-tag }}
REF_ID: ${{ needs.build-docker.outputs.ref-id }}
REGISTRY: ${{ steps.login-ecr.outputs.registry }}
REPOSITORY: ${{ needs.build-docker.outputs.repo-name }}
TIMESTAMP: ${{ needs.build-docker.outputs.timestamp }}
- name: Push production/develop template to releases
if:
(matrix.environment == 'production' && github.event_name == 'release' && github.event.action == 'released') || (matrix.environment == 'develop' && github.event_name == 'push')
uses: dmnemec/[email protected]
env:
API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }}
with:
source_file: '${{ steps.fill-metadata.outputs.deployment-name }}.yaml'
destination_repo: '${{ github.repository_owner }}/releases'
destination_folder: '${{ steps.fill-metadata.outputs.kubernetes-env }}'
user_email: '[email protected]'
user_name: 'Biomage CI/CD'
- name: Change name of deployment file for staging deployment
if:
(github.event_name == 'pull_request_target' || github.event_name == 'push') && matrix.environment == 'staging'
env:
DEPLOYMENT_NAME: ${{ steps.fill-metadata.outputs.deployment-name }}
REF_ID: ${{ needs.build-docker.outputs.ref-id }}
run: |-
mv $DEPLOYMENT_NAME.yaml $REF_ID.yaml
- name: Push staging deployment template to releases
if:
(github.event_name == 'pull_request_target' || github.event_name == 'push') && matrix.environment == 'staging'
uses: dmnemec/[email protected]
env:
API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }}
with:
source_file: ${{ needs.build-docker.outputs.ref-id }}.yaml
destination_repo: '${{ github.repository_owner }}/releases'
destination_folder: 'staging-candidates/${{ steps.fill-metadata.outputs.deployment-name }}'
user_email: '[email protected]'
user_name: 'Biomage CI/CD'
ready-to-merge:
name: Ready for merging
runs-on: ubuntu-20.04
needs: [deploy, is-safe-to-run]
steps:
- id: ready-to-merge
name: Signal readiness to merge
run: |-
exit 0
report-if-failed:
name: Report if workflow failed
runs-on: ubuntu-20.04
needs: [build-docker, deploy, is-safe-to-run]
if: failure() && github.ref == 'refs/heads/master'
steps:
- id: send-to-slack
name: Send failure notification to Slack on failure
env:
SLACK_BOT_TOKEN: ${{ secrets.WORKFLOW_STATUS_BOT_TOKEN }}
uses: voxmedia/github-action-slack-notify-build@v1
with:
channel: workflow-failures
status: FAILED
color: danger