cd.yml

name: CD

on:
  push:
    branches:
      - master

#on:
#  push:
#    branches-ignore:
#      - '**'  # temporally ignore all


jobs:
  update-schema:
    if: |
      !startsWith(github.event.head_commit.message, 'chore') &&
      !startsWith(github.event.head_commit.message, 'build: hotfix') &&
      !endsWith(github.event.head_commit.message, 'reformatted by jina-dev-bot')
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2.5.0
        with:
#          submodules: true
          fetch-depth: 100
      - run: |
          truncate --size=24KB README.md > README-trunc.md
      - uses: peter-evans/dockerhub-description@v3
        with:
          username: ${{ secrets.DOCKERHUB_DEVBOT_USER  }}
          password: ${{ secrets.DOCKERHUB_DEVBOT_PWD }}
          repository: jinaai/jina
          readme-filepath: ./README-trunc.md
      - uses: actions/checkout@v2.5.0
        with:
          repository: jina-ai/api
          path: schema
          token: ${{ secrets.JINA_DEV_BOT }}
      - uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - uses: actions/setup-node@v2
        with:
          node-version: '14'
      - run: |
          pip install . --no-cache-dir
          pip install docarray==0.21.0 # only valid for this version. I think we should remove this schema loading
          JINA_VERSION=$(sed -n '/^__version__/p' ./jina/__init__.py | cut -d \' -f2)-master
          echo "JINA_VERSION=${JINA_VERSION}" >> $GITHUB_ENV
          cd schema
          mkdir -p schemas
          jina export schema --schema-path schemas/"$JINA_VERSION.json" schemas/master.json schemas/master --yaml-path "$JINA_VERSION.yml" master.yml --json-path "$JINA_VERSION.json" master.json master
          python ../scripts/get-openapi-schemas.py
          npm install --prefix ~ snippet-enricher-cli
          ~/node_modules/.bin/snippet-enricher-cli --input=gateway.json --targets=shell_curl > gateway-with-code.json
          cd -
      - name: redoc-cli-gateway
        uses: seeebiii/redoc-cli-github-action@v10
        with:
          args: 'bundle schema/gateway-with-code.json -o rest.html'
      - name: push-to-api-repo
        run: |
          mkdir -p schema/rest/
          cp rest.html schema/rest/master.html
          cp schema/gateway.json schema/rest/master.json
          rm schema/gateway.json schema/gateway-with-code.json
          cd schema
          git config --local user.email "dev-bot@jina.ai"
          git config --local user.name "Jina Dev Bot"
          if [[ `git status --porcelain` ]]; then
            git pull && git add . && git commit -m "update ${{env.JINA_VERSION}} due to ${{github.event_name}} on ${{github.repository}}" && git push
          fi

  update-docker:
    needs: update-schema
    runs-on: ubuntu-latest
    steps:
      - uses: benc-uk/workflow-dispatch@v1
        with:
          workflow: Manual Docker Build
          inputs: '{ "release_token": "${{ env.release_token }}", "triggered_by": "CD"}'
          token: ${{ secrets.JINA_DEV_BOT }}
        env:
          release_token: ${{ secrets.JINA_CORE_RELEASE_TOKEN }}

  prep-testbed:
    runs-on: ubuntu-latest
    needs: update-schema
    env:
      release_token: ${{ secrets.JINA_CORE_RELEASE_TOKEN }}
    steps:
      - uses: actions/checkout@v2.5.0
      - id: set-matrix
        run: |
          sudo apt-get install jq
          export value=$(bash scripts/get-all-test-paths.sh)
          echo "matrix=$value" >> $GITHUB_OUTPUT
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}

  docarray-v-two-test:
    needs: prep-testbed
    runs-on: ubuntu-latest
    env:
      JINA_RANDOM_PORT_MIN: 16384
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.8]
    steps:
      - uses: actions/checkout@v2.5.0
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Set up Golang
        uses: actions/setup-go@v2
        with:
          go-version: 1.19.5
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install setuptools wheel
          pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code
          pip install cibuildwheel

      - name: Build wheels with setuptools-golang-build-manylinux-wheel
        run: |
          setuptools-golang-build-manylinux-wheels --pythons cp38-cp38
      - name: Prepare environment
        run: |
          docker build -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
          python -m pip install --upgrade pip
          python -m pip install wheel
          WHEEL_FILE=$(ls dist/*whl)
          pip install "$WHEEL_FILE[common,devel,test]" --no-cache-dir
          pip install -U protobuf${{ matrix.protobuf-version }}
          jina
          export JINA_LOG_LEVEL="ERROR"
      - name: Test
        id: test
        run: |
          pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/unit/serve/runtimes/test_helper.py
          pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_v2.py
          pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/deployment_http_composite
          pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_singleton.py
          pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_parameters_as_pydantic.py
          pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_streaming.py
          pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/sagemaker/test_sagemaker.py
          pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/docker
          echo "flag it as jina for codeoverage"
          echo "codecov_flag=jina" >> $GITHUB_OUTPUT
        timeout-minutes: 45
        env:
          JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}"
      - name: Test stateful
        id: test_stateful
        run: |
          JINA_LOG_LEVEL=DEBUG JINA_RANDOM_PORT_MAX="60535" pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/stateful
          echo "flag it as jina for codeoverage"
          echo "codecov_flag=jina" >> $GITHUB_OUTPUT
        timeout-minutes: 30
        env:
          JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}"
      - name: Check codecov file
        id: check_files
        uses: andstor/file-existence-action@v1
        with:
          files: "coverage.xml"
      - name: Upload coverage from test to Codecov
        uses: codecov/codecov-action@v3.1.1
        if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
        with:
          file: coverage.xml
          name: ${{ matrix.test-path }}-codecov
          flags: ${{ steps.test.outputs.codecov_flag }}
          fail_ci_if_error: false


  mark-dev-N:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository
        uses: actions/checkout@v2
        with:
          fetch-depth: 200

      - name: Set up Python ${{ matrix.python }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python }}

      - id: computedevdelta
        run: |
          git fetch --depth=200
          LAST_VER=$(git tag -l | sort -V | tail -n1)
          COMMITS_SINCE_LAST_VER=$(git rev-list $LAST_VER..HEAD --count)
          echo "devdelta=$COMMITS_SINCE_LAST_VER" >> $GITHUB_OUTPUT
          INIT_FILE='jina/__init__.py'
          RELEASE_VER=$(sed -n '/^__version__/p' $INIT_FILE | cut -d \' -f2)
          echo "LAST_VER $LAST_VER"
          echo "COMMITS_SINCE_LAST_VER $COMMITS_SINCE_LAST_VER"
          echo "RELEASE_VER $RELEASE_VER"
          echo "devdelta=$COMMITS_SINCE_LAST_VER" >> $GITHUB_OUTPUT
          echo "releasever=$RELEASE_VER" >> $GITHUB_OUTPUT
    outputs:
      devdelta: ${{ steps.computedevdelta.outputs.devdelta }}
      releasever: ${{ steps.computedevdelta.outputs.releasever }}

  # Build the wheels for Linux and macOS for Python 3.8 and newer
  build-wheels:
    runs-on: ${{ matrix.os }}
    needs: [mark-dev-N]
    strategy:
      # Ensure that a wheel builder finishes even if another fails
      fail-fast: false
      matrix:
        include:
          # linux
          - os: ubuntu-latest
            python: '3.7'
            python-manylinux-tag: "cp37-cp37m"
          - os: ubuntu-latest
            python: '3.8'
            python-manylinux-tag: "cp38-cp38"
          - os: ubuntu-latest
            python: '3.9'
            python-manylinux-tag: "cp39-cp39"
          - os: ubuntu-latest
            python: '3.10'
            python-manylinux-tag: "cp310-cp310"
          - os: ubuntu-latest
            python: '3.11'
            python-manylinux-tag: "cp311-cp311"

          # MacOS emulated
          - os: macos-latest
            python: '3.8'
            python-cibuildwheels: '38'
            platform_id: macosx_x86_64
            arch: x86_64
            goarch: amd64
          - os: macos-latest
            python: '3.9'
            python-cibuildwheels: '39'
            platform_id: macosx_x86_64
            arch: x86_64
            goarch: amd64
          - os: macos-latest
            python: '3.10'
            python-cibuildwheels: '310'
            platform_id: macosx_x86_64
            arch: x86_64
            goarch: amd64
          - os: macos-latest
            python: '3.11'
            python-cibuildwheels: '311'
            platform_id: macosx_x86_64
            arch: x86_64
            goarch: amd64

          # MacOS native
          - os: macos-latest
            python: '3.8'
            python-cibuildwheels: '38'
            platform_id: macosx_arm64
            arch: arm64
            goarch: arm64
          - os: macos-latest
            python: '3.9'
            python-cibuildwheels: '39'
            platform_id: macosx_arm64
            arch: arm64
            goarch: arm64
          - os: macos-latest
            python: '3.10'
            python-cibuildwheels: '310'
            platform_id: macosx_arm64
            arch: arm64
            goarch: arm64
          - os: macos-latest
            python: '3.11'
            python-cibuildwheels: '311'
            platform_id: macosx_arm64
            arch: arm64
            goarch: arm64
    steps:
      - name: Check out repository
        uses: actions/checkout@v2

      - name: Set up Python ${{ matrix.python }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python }}

      - name: Set up Golang
        uses: actions/setup-go@v2
        with:
          go-version: 1.19.5

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install setuptools wheel
          pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code
          pip install cibuildwheel

      - name: Update version
        run: |
          DEVDELTA=${{needs.mark-dev-N.outputs.devdelta}}
          RELEASE_VER=${{needs.mark-dev-N.outputs.releasever}}
          INIT_FILE='jina/__init__.py'
          DEV_VER=$RELEASE_VER".dev"$DEVDELTA
          echo "DEVDELTA $DEVDELTA"
          echo "RELEASE_VER $RELEASE_VER"
          echo "DEV_VER $DEV_VER"
          if [[ "${{ matrix.os }}" == "macos-latest" ]]; then
            sed -i '' '/'"${RELEASE_VER}"'/s/.*/'"__version__ = '${DEV_VER}'"'/' "${INIT_FILE}"
          else
            sed -i '/'"${RELEASE_VER}"'/s/.*/'"__version__ = '${DEV_VER}'"'/' "${INIT_FILE}"
          fi

      - name: Build wheels with setuptools-golang-build-manylinux-wheel
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: |
          setuptools-golang-build-manylinux-wheels --pythons ${{ matrix.python-manylinux-tag }}

      - name: Build wheels with cibuildwheels on macos
        env:
          CIBW_BUILD: cp${{ matrix.python-cibuildwheels }}-${{ matrix.platform_id }}
          CIBW_ARCHS: ${{ matrix.arch }}
          CIBW_TEST_COMMAND: python -c "import jina; import jraft"
          CIBW_BUILD_VERBOSITY: 1
          GOARCH: ${{ matrix.goarch }}
          CGO_ENABLED: 1
        if: ${{ matrix.os == 'macos-latest' }}
        run: |
          python -m cibuildwheel --output-dir dist

      - name: Test wheels
        run: |
          WHEEL_FILE=$(ls dist/*.whl)
          python -m pip install $WHEEL_FILE
          python -c "import jraft"
        if: ${{ matrix.os != 'macos-latest' || matrix.platform_id != 'macosx_arm64' }} # runners do not necessarily have macos ARM, so cannot run this test for it

      - name: Upload wheels as artifacts
        uses: actions/upload-artifact@v2
        with:
          path: dist/*.whl

# comment for now, do it manually if needed
#  pre-release:
#   if: |
#     !startsWith(github.event.head_commit.message, 'chore') &&
#     !startsWith(github.event.head_commit.message, 'build: hotfix') &&
#     !endsWith(github.event.head_commit.message, 'reformatted by jina-dev-bot')
#   needs: [build-wheels, mark-dev-N]
#   runs-on: ubuntu-latest
#   steps:
#     - uses: actions/checkout@v2.5.0
#       with:
#         #          submodules: true
#         fetch-depth: 100
#     - run: |
#         truncate --size=24KB README.md > README-trunc.md
#     - uses: actions/download-artifact@v3
#       with:
#         name: artifact
#         path: dist
#
#     - name: Update version
#       run: |
#         DEVDELTA=${{needs.mark-dev-N.outputs.devdelta}}
#         RELEASE_VER=${{needs.mark-dev-N.outputs.releasever}}
#         INIT_FILE='jina/__init__.py'
#         DEV_VER=$RELEASE_VER".dev"$DEVDELTA
#         echo "DEVDELTA $DEVDELTA"
#         echo "RELEASE_VER $RELEASE_VER"
#         echo "DEV_VER $DEV_VER"
#         sed -i '/'"${RELEASE_VER}"'/s/.*/'"__version__ = '${DEV_VER}'"'/' "${INIT_FILE}"
#     - name: Pre-release (.devN)
#       run: |
#         pip install twine wheel
#         ./scripts/release.sh
#       env:
#         TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
#         TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
#         JINA_SLACK_WEBHOOK: ${{ secrets.JINA_SLACK_WEBHOOK }}

  core-test:
    needs: prep-testbed
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.8]
        test-path: ${{fromJson(needs.prep-testbed.outputs.matrix)}}
    steps:
      - uses: actions/checkout@v2.5.0
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Set up Golang
        uses: actions/setup-go@v2
        with:
          go-version: 1.19.5

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install setuptools wheel
          pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code
          pip install cibuildwheel

      - name: Build wheels with setuptools-golang-build-manylinux-wheel
        run: |
          setuptools-golang-build-manylinux-wheels --pythons cp38-cp38
      - name: Prepare environment
        run: |
          docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
          python -m pip install --upgrade pip
          python -m pip install wheel
          WHEEL_FILE=$(ls dist/*whl)
          pip install "$WHEEL_FILE[all]" --no-cache-dir
          pip install docarray==0.21.0
          jina
          export JINA_LOG_LEVEL="ERROR"
      - name: Test
        id: test
        run: |
          pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/docarray_v2/*' --ignore-glob='tests/integration/stateful/*' --ignore-glob='tests/integration/hub_usage/dummyhub*' ${{ matrix.test-path }}
          echo "flag it as jina for codeoverage"
          echo "codecov_flag=jina" >> $GITHUB_OUTPUT
        timeout-minutes: 30
        env:
          JINAHUB_USERNAME: ${{ secrets.JINAHUB_USERNAME }}
          JINAHUB_PASSWORD: ${{ secrets.JINAHUB_PASSWORD }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}"
      - name: Check file existence
        id: check_files
        uses: andstor/file-existence-action@v1
        with:
          files: "coverage.xml"
      - name: Upload coverage from test to Codecov
        uses: codecov/codecov-action@v3.1.1
        if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
        with:
          file: coverage.xml
          name: ${{ matrix.test-path }}-codecov
          flags: ${{ steps.test.outputs.codecov_flag }}
          fail_ci_if_error: false

  import-test:
    runs-on: ubuntu-latest
    needs: update-schema
    strategy:
      fail-fast: false
      matrix:
        core: ['', 'true']
        perf: ['', 'true']
        python-env: ['3.7', '3.8', '3.9', '3.10', '3.11']
        exclude: 
          - core: 'true'
            perf: 'true'
    steps:
      - uses: actions/checkout@v2.5.0
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-env }}
      - name: Prepare enviroment
        run: |
          python -m pip install --upgrade pip
          python -m pip install wheel
          pip install --no-cache-dir .
        env:
          JINA_PIP_INSTALL_CORE: ${{ matrix.core }}
          JINA_PIP_INSTALL_PERF: ${{ matrix.perf }}
      - name: Test basic import
        run: python -c 'from jina import Executor,requests,Client,Flow,dynamic_batching,Document,DocumentArray'
  docker-image-test:
    needs: update-schema
    runs-on: ubuntu-latest
    services:
      registry:
        image: registry:2
        ports:
          - 5000:5000
    strategy:
      fail-fast: false
      matrix:
        test-arch: ["linux/amd64", "linux/arm64"]
    steps:
      #      - name: Cancel Previous Runs
      #        uses: styfle/cancel-workflow-action@0.9.0
      #        with:
      #          access_token: ${{ github.token }}
      - uses: actions/checkout@v2.5.0
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v1
        with:
          install: true
          driver-opts: network=host
      - run: |
          docker run --privileged --rm tonistiigi/binfmt --uninstall qemu-aarch64
          docker run --rm --privileged tonistiigi/binfmt --install all
      - name: Build and test
        uses: docker/build-push-action@v2
        with:
          context: .
          file: Dockerfiles/debianx.Dockerfile
          platforms: ${{ matrix.test-arch }}
          push: true
          tags: localhost:5000/jina/multiarch:latest
          target: jina
      - run: |
          docker run --platform ${{ matrix.test-arch }} localhost:5000/jina/multiarch:latest -v

  hub-test:
    needs: update-schema
    runs-on: ubuntu-latest
    if: ${{ !github.event.pull_request.head.repo.fork }}
    steps:
      #      - name: Cancel Previous Runs
      #        uses: styfle/cancel-workflow-action@0.9.0
      #        with:
      #          access_token: ${{ github.token }}
      - uses: actions/checkout@v2.5.0
      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Test hubapp with hubpods
        run: |
          ./tests/jinahub/test_integration.sh
        timeout-minutes: 30
        env:
          JINAHUB_USERNAME: ${{ secrets.JINAHUB_USERNAME }}
          JINAHUB_PASSWORD: ${{ secrets.JINAHUB_PASSWORD }}

  k8s-flow-test:
    needs: update-schema
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2.5.0
      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Prepare enviroment
        run: |
          docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
          python -m pip install --upgrade pip
          python -m pip install wheel
          pip install ".[all]" --no-cache-dir
          pip install docarray==0.21.0
          jina
          export JINA_LOG_LEVEL="ERROR"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Test k8s Flow
        run: |
          export LINKERD2_VERSION=stable-2.11.4
          curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install | sh
          pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_flow.py
        timeout-minutes: 30
      - name: Check codecov file
        id: check_files
        uses: andstor/file-existence-action@v1
        with:
          files: "coverage.xml"
      - name: Upload coverage from test to Codecov
        uses: codecov/codecov-action@v3.1.1
        if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
        with:
          file: coverage.xml
          name: ${{ matrix.test-path }}-codecov
          flags: ${{ steps.test.outputs.codecov_flag }}
          fail_ci_if_error: false

  k8s-deployment-test:
    needs: update-schema
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2.5.0
      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Prepare enviroment
        run: |
          docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
          python -m pip install --upgrade pip
          python -m pip install wheel
          pip install ".[all]" --no-cache-dir
          pip install docarray==0.21.0
          jina
          export JINA_LOG_LEVEL="ERROR"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Test k8s Deployment
        run: |
          export LINKERD2_VERSION=stable-2.11.4
          curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install | sh
          pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_deployment.py
        timeout-minutes: 30
      - name: Check codecov file
        id: check_files
        uses: andstor/file-existence-action@v1
        with:
          files: "coverage.xml"
      - name: Upload coverage from test to Codecov
        uses: codecov/codecov-action@v3.1.1
        if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
        with:
          file: coverage.xml
          name: ${{ matrix.test-path }}-codecov
          flags: ${{ steps.test.outputs.codecov_flag }}
          fail_ci_if_error: false

  k8s-graceful-test:
    needs: update-schema
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2.5.0
      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Prepare enviroment
        run: |
          docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
          python -m pip install --upgrade pip
          python -m pip install wheel
          pip install ".[all]" --no-cache-dir
          pip install docarray==0.21.0
          jina
          export JINA_LOG_LEVEL="ERROR"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Test graceful request handling on k8s
        run: |
          export LINKERD2_VERSION=stable-2.11.4
          curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install | sh
          pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_graceful_request_handling.py
        timeout-minutes: 45
      - name: Check codecov file
        id: check_files
        uses: andstor/file-existence-action@v1
        with:
          files: "coverage.xml"
      - name: Upload coverage from test to Codecov
        uses: codecov/codecov-action@v3.1.1
        if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
        with:
          file: coverage.xml
          name: ${{ matrix.test-path }}-codecov
          flags: ${{ steps.test.outputs.codecov_flag }}
          fail_ci_if_error: false

  k8s-failures-test:
    needs: update-schema
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2.5.0
      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Prepare enviroment
        run: |
          docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
          python -m pip install --upgrade pip
          python -m pip install wheel
          pip install ".[all]" --no-cache-dir
          pip install docarray==0.21.0
          jina
          export JINA_LOG_LEVEL="ERROR"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Test k8s failures
        run: |
          export LINKERD2_VERSION=stable-2.11.4
          curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install | sh
          curl --proto '=https' --tlsv1.2 -sSfL https://linkerd.github.io/linkerd-smi/install | sh
          pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_failures.py
        timeout-minutes: 45
      - name: Check codecov file
        id: check_files
        uses: andstor/file-existence-action@v1
        with:
          files: "coverage.xml"
      - name: Upload coverage from test to Codecov
        uses: codecov/codecov-action@v3.1.1
        if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
        with:
          file: coverage.xml
          name: ${{ matrix.test-path }}-codecov
          flags: ${{ steps.test.outputs.codecov_flag }}
          fail_ci_if_error: false

  k8s-otel-test:
    needs: update-schema
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2.5.0
      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Prepare enviroment
        run: |
          docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
          python -m pip install --upgrade pip
          python -m pip install wheel
          pip install ".[all]" --no-cache-dir
          pip install docarray==0.21.0
          jina
          export JINA_LOG_LEVEL="ERROR"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Test K8s with OpenTelemetry
        run: |
          pytest -v -s --cov=jina --cov-report=xml ./tests/k8s_otel
        timeout-minutes: 30
      - name: Check codecov file
        id: check_files
        uses: andstor/file-existence-action@v1
        with:
          files: "coverage.xml"
      - name: Upload coverage from test to Codecov
        uses: codecov/codecov-action@v3.1.1
        if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
        with:
          file: coverage.xml
          name: ${{ matrix.test-path }}-codecov
          flags: ${{ steps.test.outputs.codecov_flag }}
          fail_ci_if_error: false

  docker-compose-test:
    needs: update-schema
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2.5.0
      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Prepare enviroment
        run: |
          docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
          python -m pip install --upgrade pip
          python -m pip install wheel
          pip install ".[all]" --no-cache-dir
          pip install docarray==0.21.0
          jina
          export JINA_LOG_LEVEL="ERROR"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Test docker compose
        run: |
          pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/docker_compose/test_*.py
        timeout-minutes: 30
      - name: Check codecov file
        id: check_files
        uses: andstor/file-existence-action@v1
        with:
          files: "coverage.xml"
      - name: Upload coverage from test to Codecov
        uses: codecov/codecov-action@v3.1.1
        if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
        with:
          file: coverage.xml
          name: ${{ matrix.test-path }}-codecov
          flags: ${{ steps.test.outputs.codecov_flag }}
          fail_ci_if_error: false

  benchmark-pre-release:
    runs-on: ubuntu-latest
    needs: [update-schema, mark-dev-N]
    steps:
      - uses: actions/checkout@v2.5.0
        with:
          fetch-depth: 200
      - run: |
          DEVDELTA=${{needs.mark-dev-N.outputs.devdelta}}
          RELEASE_VER=${{needs.mark-dev-N.outputs.releasever}}
          NEXT_VER=$RELEASE_VER".dev"$COMMITS_SINCE_LAST_VER
          echo "NEXT_VER=$NEXT_VER" >> $GITHUB_ENV
      - uses: benc-uk/workflow-dispatch@v1
        with:
          token: ${{ secrets.JINA_DEV_BOT }}
          workflow: Benchmark Jina
          repo: jina-ai/jina-terraform
          ref: "main"
          inputs: '{ "pypi_releases": "[\"${{ env.NEXT_VER }}\"]"}'
  # just for blocking the merge until all parallel core-test are successful
  success-all-steps:
    runs-on: ubuntu-latest
    needs: [core-test, docarray-v-two-test, import-test, hub-test, k8s-flow-test, k8s-deployment-test, k8s-graceful-test, k8s-failures-test, k8s-otel-test, docker-compose-test, docker-image-test, benchmark-pre-release, update-schema, update-docker] #, pre-release]
    if: always()
    steps:
      - uses: technote-space/workflow-conclusion-action@v2
      - name: Check Failure
        if: env.WORKFLOW_CONCLUSION == 'failure'
        run: exit 1
      - name: Success
        if: ${{ success() }}
        run: echo "All Done"