ci.yaml

name: CI
on:
  push:
    branches:
      - master
  release:
    types:
      - released
  pull_request:
    branches:
      - master

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  uncheck-integration-test:
    name: Mark integration test as not run
    runs-on: ubuntu-20.04
    if: github.event_name == 'pull_request'
    steps:
      - id: get-pr-body
        name: Get the current PR body
        uses: jwalton/gh-find-current-pr@v1
        with:
          state: open

      - id: create-unchecked-pr-body
        name: Create unchecked PR body
        run: |-
          UNCHECKED_BODY=$(sed 's/- \[[Xx]\] Started end-to-end tests on the latest commit./- \[ \] Started end-to-end tests on the latest commit./' <<\EOF
          ${{ steps.get-pr-body.outputs.body }}
          EOF
          )

          UNCHECKED_BODY="${UNCHECKED_BODY//'%'/'%25'}"
          UNCHECKED_BODY="${UNCHECKED_BODY//$'\n'/'%0A'}"
          UNCHECKED_BODY="${UNCHECKED_BODY//$'\r'/'%0D'}"

          echo "Unchecked PR body"
          echo $UNCHECKED_BODY

          echo "::set-output name=body::$UNCHECKED_BODY"

      - id: uncheck-integration-checkbox
        name: Uncheck the integration checkbox
        uses: tzkhan/pr-update-action@v2
        with:
          repo-token: "${{ secrets.API_TOKEN_GITHUB }}"
          head-branch-regex: "${{ github.head_ref }}"
          lowercase-branch: false
          body-template: "${{  steps.create-unchecked-pr-body.outputs.body }}"
          body-update-action: "replace"

  build-docker:
    name: Build Docker container
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        project: ['pipeline-runner']
    outputs:
      repo-name: ${{ steps.ref.outputs.repo-name }}
      image-tag: ${{ steps.ref.outputs.image-tag }}
      ref-id: ${{ steps.ref.outputs.ref-id }}
      timestamp: ${{ steps.ref.outputs.timestamp }}
    defaults:
      run:
        working-directory: ${{ matrix.project }}
    steps:
      - id: checkout
        name: Check out source code
        uses: actions/checkout@v2

      - id: ref
        name: Format docker tag and repository name for current image.
        run: |-
          # This will take a ref like `refs/heads/master`
          # and turn it into `refs-heads-master`
          REF_ID=$(echo $GITHUB_REF | sed 's/\//-/g')
          echo "::set-output name=ref-id::$REF_ID"

          # the final tag is something like:
          # refs-heads-master-a4f8bc313dae
          # this is what we push to ECR
          # we will also take semver'd tags like `1.0.0` and use them for releases

          # In push & PR events we want the tag to contain the latest commit on the branch:
          # in push events, the latest commit of the master branch is GITHUB_SHA
          # in PR synch the latest commit of the branch is found in github.event.pull_request.head.sha instead

          TIMESTAMP=$(date +%s)

          if [ "${{ github.event_name }}" = "release" ] && [ "${{ github.event.action }}" = "released" ]; then
            COMMIT_SHA=""
            IMAGE_TAG=$REF_ID
            TIMESTAMP=""
            BATCH_IMAGE_TAG=production
          elif [ "${{ github.event_name }}" = "pull_request" ]; then
            COMMIT_SHA="${{ github.event.pull_request.head.sha }}"
            IMAGE_TAG="$REF_ID-$COMMIT_SHA"
            TIMESTAMP=$TIMESTAMP
            BATCH_IMAGE_TAG=$REF_ID
          else
            COMMIT_SHA=$GITHUB_SHA
            IMAGE_TAG="$REF_ID-$COMMIT_SHA"
            TIMESTAMP=$TIMESTAMP
            BATCH_IMAGE_TAG=staging
          fi

          # IMAGE_TAG is used in the Build Docker Image step.
          # We can easily build the image-tag from REF_ID and COMMIT_SHA for non-production releases.
          # But we can not easily create the image tag for production releases, so we're bulding it here
          echo "::set-output name=image-tag::$IMAGE_TAG"
          echo "::set-output name=batch-image-tag::$BATCH_IMAGE_TAG"

          # TIMESTAMP is used to postfix images in the "push docker images to ECR" step.
          # The timestamp is used by Flux to auto update images for staging environments.
          # Images for production uses semantic versioning to determine the latest image.
          echo "::set-output name=timestamp::$TIMESTAMP"

          # This will take a GitHub repo name like `biomage-org/releases`
          # and turns it into `releases`. This will be the name of the
          # ECR repository.
          IMAGE_REPO_NAME=$(echo $GITHUB_REPOSITORY | awk -F '/' '{print $2}')
          echo "::set-output name=repo-name::$IMAGE_REPO_NAME"

      - id: ref-previous
        name: Format docker tag and repository name for the previous pushed image.
        run: |-
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            echo "This is a pull request, base ref and sha set to the target branch."
            BASE_REF="refs-heads-${{ github.event.pull_request.base.ref }}"
            BASE_SHA=${{ github.event.pull_request.base.sha }}
            echo "::set-output name=tag::$BASE_REF-$BASE_SHA"
          fi

          if [ "${{ github.event_name }}" = "push" ]; then
            echo "This is a push, base ref and sha set to the previous commit."
            BASE_REF=$(echo $GITHUB_REF | sed 's/\//-/g')
            BASE_SHA="${{ github.event.before }}"
            echo "::set-output name=tag::$BASE_REF-$BASE_SHA"
          fi

          if [ "${{ github.event_name }}" = "release" ]; then
            echo "This is a release, base ref and sha set to the current commit."
            BASE_SHA="$GITHUB_SHA"
            echo "::set-output name=tag::refs-heads-master-$BASE_SHA"
          fi

      - id: set-up-creds
        name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-1

      - id: login-ecr
        name: Login to Amazon ECR
        uses: aws-actions/amazon-ecr-login@v1

      - id: create-ecr-registry
        name: Create an ECR repository (if needed)
        # This will fail if the registry already exists, which is fine. If there is some other
        # error, the `push` step will fail instead.
        continue-on-error: true
        run: |-
          aws ecr create-repository --repository-name $REPO_NAME
        env:
          REPO_NAME: ${{ steps.ref.outputs.repo-name }}

      - id: pull-latest-image
        name: Pull latest Docker images from mainline
        run: |-
          echo Pulling $IMAGE_NAME from ECR.
          docker pull $IMAGE_NAME || true

          echo Pulling $IMAGE_NAME-builder from ECR.
          docker pull $IMAGE_NAME-builder || true
        env:
          IMAGE_NAME: ${{ format('{0}/{1}:{2}-{3}', steps.login-ecr.outputs.registry, steps.ref.outputs.repo-name, steps.ref-previous.outputs.tag, matrix.project) }}

      - id: build
        name: Build Docker images
        run: |-
          docker build --target builder \
            --cache-from=$PREVIOUS_IMAGE_NAME-builder \
            --build-arg GITHUB_PAT=$GITHUB_PAT \
            --tag $IMAGE_NAME-builder .

          docker build --target prod \
            --cache-from=$IMAGE_NAME-builder \
            --cache-from=$PREVIOUS_IMAGE_NAME \
            --build-arg GITHUB_PAT=$GITHUB_PAT \
            --tag $IMAGE_NAME .

          docker build --target batch \
            --cache-from=$IMAGE_NAME-builder \
            --cache-from=$PREVIOUS_IMAGE_NAME \
            --build-arg GITHUB_PAT=$GITHUB_PAT \
            --tag $BATCH_IMAGE_NAME .
        env:
          PREVIOUS_IMAGE_NAME: ${{ format('{0}/{1}:{2}-{3}', steps.login-ecr.outputs.registry, steps.ref.outputs.repo-name, steps.ref-previous.outputs.tag, matrix.project) }}
          IMAGE_NAME: ${{ format('{0}/{1}:{2}-{3}', steps.login-ecr.outputs.registry, steps.ref.outputs.repo-name, steps.ref.outputs.image-tag, matrix.project) }}
          BATCH_IMAGE_NAME: ${{ format('{0}/{1}:batch-{2}', steps.login-ecr.outputs.registry, steps.ref.outputs.repo-name, steps.ref.outputs.batch-image-tag) }}
          GITHUB_PAT: ${{ secrets.API_TOKEN_GITHUB }}

      - id: codecov
        name: Generate codecov report
        run: |-
          docker run \
            --entrypoint /bin/bash \
            -v $(pwd)/covr:/covr $IMAGE_NAME \
            -c "R -e 'cov <- covr::package_coverage(); covr::to_cobertura(cov, \"/covr/coverage.xml\")'"

        env:
          IMAGE_NAME: ${{ format('{0}/{1}:{2}-{3}', steps.login-ecr.outputs.registry, steps.ref.outputs.repo-name, steps.ref.outputs.image-tag, matrix.project) }}

      - id: upload-coverage
        name: Upload coverage to Codecov
        uses: codecov/codecov-action@v2
        with:
          files: ./pipeline-runner/covr/coverage.xml

      - id: push
        name: Push docker images to ECR
        run: |-
          echo Pushing image $IMAGE_NAME-builder to ECR.
          docker push $IMAGE_NAME-builder

          echo Pushing image $IMAGE_NAME to ECR.
          docker push $IMAGE_NAME

          echo Pushing batch image $BATCH_IMAGE_NAME to ECR.
          docker push $BATCH_IMAGE_NAME

          if [ ! -z "$TIMESTAMP" ]; then
            echo Pushing timestamped image $IMAGE_NAME-$TIMESTAMP to ECR

            docker tag $IMAGE_NAME $IMAGE_NAME-$TIMESTAMP
            docker push $IMAGE_NAME-$TIMESTAMP
          fi
        env:
          IMAGE_NAME: ${{ format('{0}/{1}:{2}-{3}', steps.login-ecr.outputs.registry, steps.ref.outputs.repo-name, steps.ref.outputs.image-tag, matrix.project) }}
          BATCH_IMAGE_NAME: ${{ format('{0}/{1}:batch-{2}', steps.login-ecr.outputs.registry, steps.ref.outputs.repo-name, steps.ref.outputs.batch-image-tag) }}
          TIMESTAMP: ${{ steps.ref.outputs.timestamp }}

  deploy:
    name: Deploy to Kubernetes
    runs-on: ubuntu-20.04
    needs: build-docker
    strategy:
      max-parallel: 1
      matrix:
        environment: ['production', 'staging', 'develop']
    steps:
      - id: checkout
        name: Check out source code
        uses: actions/checkout@v2

      - id: setup-aws
        name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-1

      - id: login-ecr
        name: Login to Amazon ECR
        uses: aws-actions/amazon-ecr-login@v1

      - id: fill-metadata
        name: Fill out a new HelmRelease resource
        run: |-
          export DEPLOYMENT_NAME=$(echo $GITHUB_REPOSITORY | awk -F '/' '{print $2}')
          echo "::set-output name=deployment-name::$DEPLOYMENT_NAME"

          # Deployment config for `production-default`
          if [ "${{ matrix.environment }}" = "production" ]; then
            export SANDBOX_ID="default"
            export CHART_REF_TYPE="commit"
            export CHART_REF="$GITHUB_SHA"
            export KUBERNETES_ENV="production"
            export IMAGE_NAME=$IMAGE_TAG-pipeline-runner
            export REPLICA_COUNT="3"
            export VERSION_NUMBER=${REF_ID/refs-tags-/}
            export IMAGE_PATTERN="^refs-tags-(?P<version>[0-9]+\.[0-9]+\.[0-9]+)-pipeline-runner$"
            export IMAGE_EXTRACT='$version'
            export IMAGE_POLICY_TYPE="semver"
            export IMAGE_POLICY_KEY="range"
            export IMAGE_POLICY_VALUE=">=0.0.0"
            export MEMORY_REQUEST="29Gi"
            export MANIFEST_PATH="./production"
          fi

          # Deployment config for `staging-default`
          if [ "${{ matrix.environment }}" = "develop" ]; then
            export SANDBOX_ID="default"
            export CHART_REF_TYPE="branch"
            export CHART_REF="master"
            export KUBERNETES_ENV="staging"
            export IMAGE_NAME="$IMAGE_TAG-pipeline-runner-$TIMESTAMP"
            export REPLICA_COUNT="0"
            export IMAGE_PATTERN="^$REF_ID-[a-f0-9]+-pipeline-runner-(?P<timestamp>[0-9]+)"
            export IMAGE_EXTRACT='$timestamp'
            export IMAGE_POLICY_TYPE="numerical"
            export IMAGE_POLICY_KEY="order"
            export IMAGE_POLICY_VALUE="asc"
            export MEMORY_REQUEST="29Gi"
            export MANIFEST_PATH="./staging"
          fi

          # Deployment config for other staging env i.e. non `default`
          if [ "${{ matrix.environment }}" = "staging" ]; then

            export CHART_REF="$GITHUB_HEAD_REF"

            # $GITHUB_HEAD_REF references the source branch of the pull request in a workflow run
            # $GITHUB_HEAD_REF does not have a value when we merge, because merging is counted as a "push" event to master branch
            # Therefore we set this value to "master" manually
            if [ -z "$CHART_REF" ]; then
              export CHART_REF="master"
            fi

            export SANDBOX_ID="STAGING_SANDBOX_ID"
            export CHART_REF_TYPE="branch"
            export KUBERNETES_ENV="staging"
            export IMAGE_NAME="$IMAGE_TAG-pipeline-runner-$TIMESTAMP"
            export REPLICA_COUNT="0"
            export IMAGE_PATTERN="^$REF_ID-[a-f0-9]+-pipeline-runner-(?P<timestamp>[0-9]+)"
            export IMAGE_EXTRACT='$timestamp'
            export IMAGE_POLICY_TYPE="numerical"
            export IMAGE_POLICY_KEY="order"
            export IMAGE_POLICY_VALUE="asc"
            export MEMORY_REQUEST="4Gi"
            export MANIFEST_PATH="./staging"
          fi

          echo "::set-output name=sandbox-id::$SANDBOX_ID"
          echo "::set-output name=kubernetes-env::$KUBERNETES_ENV"

          export NAMESPACE="$DEPLOYMENT_NAME-$SANDBOX_ID"
          export CHART_CRD_NAME="$DEPLOYMENT_NAME-chart"
          export IMAGE_POLICY_TAG="{\"\$imagepolicy\": \"$NAMESPACE:$DEPLOYMENT_NAME:tag\"}"

          yq '
            select(di == 0).metadata.name = strenv(NAMESPACE) |
            select(di == 0).metadata.labels.sandboxId = strenv(SANDBOX_ID) |
            select(di == 1).metadata.name = strenv(CHART_CRD_NAME) |
            select(di == 1).metadata.namespace = strenv(NAMESPACE) |
            select(di == 1).spec.ref.[strenv(CHART_REF_TYPE)] = strenv(CHART_REF) |
            select(di == 2).metadata.name = strenv(DEPLOYMENT_NAME) |
            select(di == 2).metadata.namespace = strenv(NAMESPACE) |
            select(di == 2).spec.image = strenv(REGISTRY) + "/" + strenv(DEPLOYMENT_NAME) |
            select(di == 3).metadata.name = strenv(DEPLOYMENT_NAME) |
            select(di == 3).metadata.namespace = strenv(NAMESPACE) |
            select(di == 3).spec.imageRepositoryRef.name = strenv(DEPLOYMENT_NAME) |
            select(di == 3).spec.imageRepositoryRef.namespace = strenv(NAMESPACE) |
            select(di == 3).spec.filterTags.pattern = strenv(IMAGE_PATTERN) |
            select(di == 3).spec.filterTags.extract = strenv(IMAGE_EXTRACT) |
            select(di == 3).spec.policy.[strenv(IMAGE_POLICY_TYPE)].[strenv(IMAGE_POLICY_KEY)] = strenv(IMAGE_POLICY_VALUE) |
            select(di == 4).metadata.name = strenv(DEPLOYMENT_NAME)  + "-image-update" |
            select(di == 4).metadata.namespace = strenv(NAMESPACE) |
            select(di == 4).spec.update.path = strenv(MANIFEST_PATH) |
            select(di == 5).metadata.name = strenv(DEPLOYMENT_NAME) |
            select(di == 5).metadata.namespace = strenv(NAMESPACE) |
            select(di == 5).metadata.labels.sandboxId = strenv(SANDBOX_ID) |
            select(di == 5).spec.releaseName = strenv(DEPLOYMENT_NAME) |
            select(di == 5).spec.chart.spec.sourceRef.name = strenv(CHART_CRD_NAME) |
            select(di == 5).spec.chart.spec.sourceRef.namespace = strenv(NAMESPACE) |
            select(di == 5).spec.values.clusterEnv = strenv(KUBERNETES_ENV) |
            select(di == 5).spec.values.image.registry = strenv(REGISTRY) |
            select(di == 5).spec.values.image.repository = strenv(REPOSITORY) |
            select(di == 5).spec.values.image.tag = strenv(IMAGE_NAME) |
            select(di == 5).spec.values.image.tag line_comment = strenv(IMAGE_POLICY_TAG) |
            select(di == 5).spec.values.sandboxId = strenv(SANDBOX_ID) |
            select(di == 5).spec.values.memoryRequest = strenv(MEMORY_REQUEST) |
            select(di == 5).spec.values.replicaCount = env(REPLICA_COUNT) |
            select(di == 5).spec.values.serviceAccount.iamRole = "pipeline-role-" + strenv(KUBERNETES_ENV) |
            select(di == 5).spec.values.datadogTags = "kube_namespace:" + strenv(NAMESPACE)
          ' .flux.yaml > $DEPLOYMENT_NAME.yaml

          cat $DEPLOYMENT_NAME.yaml
        env:
          IMAGE_TAG: ${{ needs.build-docker.outputs.image-tag }}
          REF_ID: ${{ needs.build-docker.outputs.ref-id }}
          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
          REPOSITORY: ${{ needs.build-docker.outputs.repo-name  }}
          TIMESTAMP: ${{ needs.build-docker.outputs.timestamp }}

      - name: Push production/develop template to releases
        if:
          (matrix.environment == 'production' && github.event_name == 'release' && github.event.action == 'released') || (matrix.environment == 'develop' && github.event_name == 'push')
        uses: dmnemec/copy_file_to_another_repo_action@v1.0.4
        env:
          API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }}
        with:
          source_file: '${{ steps.fill-metadata.outputs.deployment-name }}.yaml'
          destination_repo: '${{ github.repository_owner }}/releases'
          destination_folder: '${{ steps.fill-metadata.outputs.kubernetes-env }}'
          user_email: 'ci@biomage.net'
          user_name: 'Biomage CI/CD'

      - name: Change name of deployment file for staging deployment
        if:
          (github.event_name == 'pull_request' || github.event_name == 'push') && matrix.environment == 'staging'
        env:
          DEPLOYMENT_NAME: ${{ steps.fill-metadata.outputs.deployment-name }}
          REF_ID: ${{ needs.build-docker.outputs.ref-id }}
        run: |-
          mv $DEPLOYMENT_NAME.yaml $REF_ID.yaml

      - name: Push staging deployment template to releases
        if:
          (github.event_name == 'pull_request' || github.event_name == 'push') && matrix.environment == 'staging'
        uses: dmnemec/copy_file_to_another_repo_action@v1.0.4
        env:
          API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }}
        with:
          source_file: ${{ needs.build-docker.outputs.ref-id }}.yaml
          destination_repo: '${{ github.repository_owner }}/releases'
          destination_folder: 'staging-candidates/${{ steps.fill-metadata.outputs.deployment-name }}'
          user_email: 'ci@biomage.net'
          user_name: 'Biomage CI/CD'

  ready-to-merge:
    name: Ready for merging
    runs-on: ubuntu-20.04
    needs: deploy
    steps:
      - id: ready-to-merge
        name: Signal readiness to merge
        run: |-
          exit 0

  report-if-failed:
    name: Report if workflow failed
    runs-on: ubuntu-20.04
    needs: [build-docker, deploy]
    if: failure() && github.ref == 'refs/heads/master'
    steps:
      - id: send-to-slack
        name: Send failure notification to Slack on failure
        env:
          SLACK_BOT_TOKEN: ${{ secrets.WORKFLOW_STATUS_BOT_TOKEN }}
        uses: voxmedia/github-action-slack-notify-build@v1
        with:
          channel: workflow-failures
          status: FAILED
          color: danger