ci-pipeline.yml

name: CaraML CI Pipeline

on:
  # Automatically run CI on Release and Pre-Release tags and main branch
  push:
    tags:
      - 'v[0-9]+.[0-9]+.[0-9]+*'
    branches:
      - main
    paths-ignore:
      - "docs/**"

  # Automatically run CI on branches, that have active PR opened
  pull_request:
    branches:
      - main
    paths-ignore:
      - "docs/**"

  # To make it possible to trigger e2e CI workflow for any arbitrary git ref
  workflow_dispatch:

env:
  GO_VERSION: "1.20"
  GO_LINT_VERSION: v1.51.2
  NODE_VERSION: 16
  ARTIFACT_RETENTION_DAYS: 7
  CONTAINER_REGISTRY: ghcr.io
  LOCAL_REGISTRY: registry.localhost:5000

  CARAML_CHARTS_REPOSITORY: caraml-dev/helm-charts
  CARAML_CHARTS_REF: main

jobs:
  build-ui:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-node@v3
        with:
          node-version: ${{ env.NODE_VERSION }}

      - name: Cache Dependencies
        uses: actions/cache@v3
        with:
          path: ui/node_modules
          key: |
            ${{ runner.os }}-modules-${{ hashFiles('ui/yarn.lock') }}
          restore-keys: ${{ runner.os }}-modules-

      - name: Install
        working-directory: ui
        run: yarn install

      - name: Lint code
        run: make lint-ui

      - name: Build UI
        env:
          NODE_OPTIONS: "--max_old_space_size=4096"
        run: make build-ui

      - name: Publish Artifact
        uses: actions/upload-artifact@v3
        with:
          name: mlp-ui-dist
          path: ui/build/
          retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}

  build-api:
    runs-on: ubuntu-latest
    outputs:
      api-version: ${{ steps.build-image.outputs.api-version }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Build Docker image
        id: build-image
        run: |
          set -o pipefail
          make build-api-image | tee output.log
          echo "api-version=$(sed -n 's%version: \(.*\)%\1%p' output.log)" >> $GITHUB_OUTPUT

      - name: Save Docker image
        run: |
          docker image save \
            --output mlp-api.${{ steps.build-image.outputs.api-version }}.tar \
            mlp-api:${{ steps.build-image.outputs.api-version }}

      - name: Publish Artifact
        uses: actions/upload-artifact@v3
        with:
          name: mlp-api.${{ steps.build-image.outputs.api-version }}.tar
          path: mlp-api.${{ steps.build-image.outputs.api-version }}.tar
          retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}

  test-api:
    runs-on: ubuntu-latest
    env:
      GOPATH: ${{ github.workspace }}/.go
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v3
        with:
          go-version: ${{ env.GO_VERSION }}

      - name: Cache Dependencies
        uses: actions/cache@v3
        with:
          path: .go/pkg/mod/
          key: |
            gomod-${{ hashFiles('go.mod') }}
          restore-keys: gomod-

      - name: Lint code
        uses: golangci/golangci-lint-action@v2
        with:
          version: ${{ env.GO_LINT_VERSION }}
          skip-go-installation: true
          args: --timeout 3m --verbose api/...

      - name: Run Integration Test
        run: make it-test-api

      - uses: codecov/codecov-action@v4
        with:
          flags: api-test
          name: api-test
          token: ${{ secrets.CODECOV_TOKEN }}
          working-directory: ./api
          codecov_yml_path: ../.github/workflows/codecov-config/codecov.yml

  e2e-test:
    runs-on: ubuntu-latest
    needs:
      - build-api
      - test-api
    steps:
      - uses: actions/checkout@v3

      - uses: actions/checkout@v3
        with:
          repository: ${{ env.CARAML_CHARTS_REPOSITORY }}
          ref: ${{ env.CARAML_CHARTS_REF }}
          path: 'helm-charts'

      - uses: actions/setup-go@v3
        with:
          go-version: ${{ env.GO_VERSION }}

      - name: Set up Helm
        uses: azure/setup-helm@v3
        with:
          version: v3.9.3

      - name: Setup local k8s cluster
        uses: AbsaOSS/k3d-action@v2.1.0
        with:
          cluster-name: mlp-e2e
          args: >-
            --agents 1
            --port 80:80@agent:*
            --registry-create ${{ env.LOCAL_REGISTRY }}
            --k3s-arg "--no-deploy=metrics-server@server:*"

      - name: Download API image tar
        uses: actions/download-artifact@v3
        with:
          name: mlp-api.${{ needs.build-api.outputs.api-version }}.tar

      - name: Publish image to the local registry
        env:
          DOCKER_REPOSITORY: ${{ env.LOCAL_REGISTRY }}/${{ github.repository }}
        shell: bash
        run: |
          docker image load --input mlp-api.${{ needs.build-api.outputs.api-version }}.tar
          docker tag \
            mlp-api:${{ needs.build-api.outputs.api-version }} \
            ${{ env.DOCKER_REPOSITORY }}/mlp-api:${{ needs.build-api.outputs.api-version }}
          docker push ${{ env.DOCKER_REPOSITORY }}/mlp-api:${{ needs.build-api.outputs.api-version }}

      - name: Deploy MLP to the local cluster
        id: deploy
        run: |
          helm install mlp ./helm-charts/charts/mlp \
            --dependency-update \
            --wait --timeout=3m \
            --values=.github/e2e/values.yaml \
            --set deployment.image.registry=${{ env.LOCAL_REGISTRY }} \
            --set deployment.image.repository=${{ github.repository }}/mlp-api \
            --set deployment.image.tag=${{ needs.build-api.outputs.api-version }}

      - if: steps.deploy.outcome == 'failure' && always()
        name: "Debug Deployment Failure"
        run: |
          echo "::group::describe deployment/mlp"
          kubectl describe deployment/mlp
          echo "::endgroup::"
          
          echo "::group::configmap/mlp-config"
          kubectl get cm/mlp-config -o jsonpath='{.data.mlp-config\.yaml}'
          echo "::endgroup::"
          
          echo "::group::logs deployment/mlp"
          kubectl logs deployment/mlp
          echo "::endgroup::"
          
          echo "::group::kubernetes events"
          kubectl get events
          echo "::endgroup::"

          echo "::group::kubernetes pod describe"
          kubectl describe pod
          echo "::endgroup::"      

      - name: Setup MLP project
        shell: bash
        run: |
          sleep 2
          
          tee payload.json <<EOF
          {
            "name": "e2e-test",
            "team": "myteam",
            "stream": "mystream"
          }
          EOF

          curl -v \
            --header 'Content-Type: application/json' \
            --request POST \
            --data @payload.json \
            http://mlp.127.0.0.1.nip.io/v1/projects

      - name: Run E2E tests
        env:
          MLP_API_BASEPATH: http://mlp.127.0.0.1.nip.io
        shell: bash
        run: |
          for example in api/client/examples/*; do
            [[ -e $example ]] || continue
            echo $example
            go run $example/main.go
          done

  release-rules:
    runs-on: ubuntu-latest
    outputs:
      release-type: ${{ steps.release-rules.outputs.release-type }}
    steps:
      - id: release-rules
        uses: caraml-dev/turing/.github/actions/release-rules@main

  publish-docker:
    # Automatically publish release and pre-release artifacts.
    #
    # As for dev releases, make it possible to publish artifacts
    # manually by approving 'deployment' in the 'manual' environment.
    #
    # Dev build can be released either from the 'main' branch or
    # by running this workflow manually with `workflow_dispatch` event.
    if: >-
      contains('release,pre-release', needs.release-rules.outputs.release-type)
        || ( github.event_name != 'pull_request' )
        || ( github.event.pull_request.head.repo.full_name == github.repository )
    needs:
      - build-ui
      - build-api
      - test-api
      - e2e-test
      - release-rules
    runs-on: ubuntu-latest
    environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }}
    steps:
      - uses: actions/checkout@v3

      - name: Log in to the Container registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.CONTAINER_REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Download API image tar
        uses: actions/download-artifact@v3
        with:
          name: mlp-api.${{ needs.build-api.outputs.api-version }}.tar

      - name: Download MLP UI Dist
        uses: actions/download-artifact@v3
        with:
          name: mlp-ui-dist
          path: ui/build

      - name: Build and Publish Turing Docker image
        env:
          DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }}/${{ github.repository_owner }}
          MLP_API_IMAGE: mlp-api:${{ needs.build-api.outputs.api-version }}
          OVERWRITE_VERSION: ${{ needs.build-api.outputs.api-version }}
        run: |
          docker image load --input mlp-api.${{ needs.build-api.outputs.api-version }}.tar
          make build-image
          docker push ${{ env.DOCKER_REGISTRY }}/mlp:${{ env.OVERWRITE_VERSION }}

  detect-ui-lib-changes:
    outputs:
      changed: ${{ steps.changes.outputs.ui-lib }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: dorny/paths-filter@v2
        id: changes
        with:
          filters: |
            ui-lib:
              - 'ui/packages/lib/**'

  publish-ui-lib:
    # Automatically publish release and pre-release UI lib
    # (if any library component has been changed).
    #
    # As for dev releases, make it possible to publish the UI lib bundle
    # manually by approving 'deployment' in the 'manual' environment.
    #
    # Dev build can be released either from the 'main' branch or
    # from the PR opened to the 'main' branch from the same (not forked) repo.
    if: >-
      needs.detect-ui-lib-changes.outputs.changed == 'true'
        && (
          contains('release,pre-release', needs.release-rules.outputs.release-type)
            || ( github.event_name == 'push' )
            || ( github.event.pull_request.head.repo.full_name == github.repository )
        )
    needs:
      - build-ui
      - release-rules
      - detect-ui-lib-changes
    runs-on: ubuntu-latest
    environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - uses: actions/setup-node@v3
        with:
          node-version: ${{ env.NODE_VERSION }}
          registry-url: https://registry.npmjs.org

      - name: Cache Dependencies
        uses: actions/cache@v3
        with:
          path: ui/node_modules
          key: |
            ${{ runner.os }}-modules-${{ hashFiles('ui/yarn.lock') }}
          restore-keys: ${{ runner.os }}-modules-

      - name: Install dependencies
        working-directory: ui
        run: yarn install

      - name: Publish @caraml-dev/ui-lib library
        working-directory: ui
        env:
          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
        run: |
          yarn set-version-from-git
          yarn lib publish