ci-testing-deploy.yml

name: CI
# NOTE: this CI workflow is running a number of unit/integration/system tests jobs
# and also builds a set of "test" docker images (required for integration/system tests)
# if all tests are passing, and the CI runs on the master branch, then it will also deploy
# the "test" docker images to dockerhub registry
#
# in PRs: only the jobs that needs to run will be run by detecting file changes
#
# NOTE2: to keep the github branch protection simpler, the required jobs are the ones named:
# unit-tests, integration-tests, and system-tests

on:
  push:
    branches:
      - "*"
    tags-ignore:
      - "*"
  pull_request:
    branches:
      - "*"
  # https://github.blog/changelog/2023-02-08-pull-request-merge-queue-public-beta/
  merge_group:
    branches:
      - "master"

  workflow_dispatch:
    inputs:
      tests:
        description: tests to run
        required: true
        default: "all"
        type: choice
        options:
          - unit-tests
          - integration-tests
          - system-tests
          - all

env:
  DEFAULT_MAX_NANO_CPUS: 10000000
  DEFAULT_MAX_MEMORY: 268435456

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  changes:
    name: detect file changes since last push
    runs-on: ubuntu-latest
    # Set job outputs to values from filter step
    outputs:
      aws-library: ${{ steps.filter.outputs.aws-library }}
      dask-task-models-library: ${{ steps.filter.outputs.dask-task-models-library }}
      models-library: ${{ steps.filter.outputs.models-library }}
      notifications-library: ${{ steps.filter.outputs.notifications-library }}
      postgres-database: ${{ steps.filter.outputs.postgres-database }}
      service-integration: ${{ steps.filter.outputs.service-integration }}
      service-library: ${{ steps.filter.outputs.service-library }}
      settings-library: ${{ steps.filter.outputs.settings-library }}
      simcore-sdk: ${{ steps.filter.outputs.simcore-sdk }}
      agent: ${{ steps.filter.outputs.agent }}
      api: ${{ steps.filter.outputs.api }}
      api-server: ${{ steps.filter.outputs.api-server }}
      autoscaling: ${{ steps.filter.outputs.autoscaling }}
      catalog: ${{ steps.filter.outputs.catalog }}
      clusters-keeper: ${{ steps.filter.outputs.clusters-keeper }}
      dask-sidecar: ${{ steps.filter.outputs.dask-sidecar }}
      datcore-adapter: ${{ steps.filter.outputs.datcore-adapter }}
      director: ${{ steps.filter.outputs.director }}
      director-v2: ${{ steps.filter.outputs.director-v2 }}
      dynamic-sidecar: ${{ steps.filter.outputs.dynamic-sidecar }}
      invitations: ${{ steps.filter.outputs.invitations }}
      migration: ${{ steps.filter.outputs.migration }}
      osparc-gateway-server: ${{ steps.filter.outputs.osparc-gateway-server }}
      payments: ${{ steps.filter.outputs.payments }}
      dynamic-scheduler: ${{ steps.filter.outputs.dynamic-scheduler }}
      resource-usage-tracker: ${{ steps.filter.outputs.resource-usage-tracker }}
      static-webserver: ${{ steps.filter.outputs.static-webserver }}
      storage: ${{ steps.filter.outputs.storage }}
      webserver: ${{ steps.filter.outputs.webserver }}
      anything: ${{ steps.filter.outputs.anything }}
      anything-py: ${{ steps.filter.outputs.anything-py }}
      anything-js: ${{ steps.filter.outputs.anything-js }}
    steps:
      - uses: actions/checkout@v4
      # For pull requests it's not necessary to checkout the code
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            aws-library:
              - 'packages/aws-library/**'
              - 'packages/pytest-simcore/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            dask-task-models-library:
              - 'packages/dask-task-models-library/**'
              - 'packages/pytest-simcore/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            models-library:
              - 'packages/models-library/**'
              - 'packages/postgres-database/**'
              - 'packages/pytest-simcore/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            notifications-library:
              - 'packages/notifications-library/**'
              - 'packages/postgres-database/**'
              - 'packages/pytest-simcore/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            postgres-database:
              - 'packages/postgres-database/**'
              - 'packages/pytest-simcore/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            service-integration:
              - 'packages/models-library/**'
              - 'packages/pytest-simcore/**'
              - 'packages/service-integration/**'
              - 'services/docker-compose*'
            service-library:
              - 'packages/pytest-simcore/**'
              - 'packages/service-library/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            settings-library:
              - 'packages/pytest-simcore/**'
              - 'packages/settings-library/**'
              - 'services/docker-compose*'
            simcore-sdk:
              - 'packages/**'
              - 'services/docker-compose*'
            agent:
              - 'packages/**'
              - 'services/agent/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            api:
              - 'api/**'
            api-server:
              - 'packages/**'
              - 'services/api-server/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            autoscaling:
              - 'packages/**'
              - 'services/autoscaling/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            catalog:
              - 'packages/**'
              - 'services/catalog/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            clusters-keeper:
              - 'packages/**'
              - 'services/clusters-keeper/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            dask-sidecar:
              - 'packages/**'
              - 'services/dask-sidecar/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            datcore-adapter:
              - 'packages/**'
              - 'services/datcore-adapter/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            director:
              - 'packages/**'
              - 'services/director/**'
              - 'services/docker-compose*'
            director-v2:
              - 'packages/**'
              - 'services/director-v2/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            dynamic-sidecar:
              - 'packages/**'
              - 'services/dynamic-sidecar/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            invitations:
              - 'packages/**'
              - 'services/invitations/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            migration:
              - 'packages/**'
              - 'services/migration/**'
              - 'services/docker-compose*'
            osparc-gateway-server:
              - 'packages/**'
              - 'services/osparc-gateway-server/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            payments:
                - 'packages/**'
                - 'services/payments/**'
                - 'services/docker-compose*'
                - 'scripts/mypy/*'
                - 'mypy.ini'
            dynamic-scheduler:
                - 'packages/**'
                - 'services/dynamic-scheduler/**'
                - 'services/docker-compose*'
                - 'scripts/mypy/*'
                - 'mypy.ini'
            resource-usage-tracker:
              - 'packages/**'
              - 'services/resource-usage-tracker/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            static-webserver:
              - 'services/static-webserver/**'
              - 'services/docker-compose*'
            storage:
              - 'packages/**'
              - 'services/storage/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            webserver:
              - 'packages/**'
              - 'services/web/**'
              - 'services/docker-compose*'
            anything:
              - 'packages/**'
              - 'services/**'
              - 'tests/**'
            anything-py:
              - '**/*.py'
            anything-js:
              - '**/*.js'
  build-test-images:
    # this step comes first, so that it is executed as first job in push calls
    # in PR calls this step is anyway skipped
    needs: changes
    if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    name: "[build] docker images"
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      - name: show system environs
        run: ./ci/helpers/show_system_versions.bash
      - name: build images
        run: |
          export DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash)
          mkdir --parents /${{ runner.temp }}/build
          make build local-dest=/${{ runner.temp }}/build
      - name: upload build artifacts
        uses: actions/upload-artifact@v4
        with:
          name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
          path: /${{ runner.temp }}/build

  unit-test-webserver-01:
    needs: changes
    if: ${{ needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
    timeout-minutes: 25 # if this timeout gets too small, then split the tests
    name: "[unit] webserver 01"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install webserver
        run: ./ci/github/unit-testing/webserver.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/webserver.bash typecheck
      - name: test isolated
        if: always()
        run: ./ci/github/unit-testing/webserver.bash test_isolated
      - name: test
        if: always()
        run: ./ci/github/unit-testing/webserver.bash test_with_db 01
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-webserver-02:
    needs: changes
    if: ${{ needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
    timeout-minutes: 25 # if this timeout gets too small, then split the tests
    name: "[unit] webserver 02"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install webserver
        run: ./ci/github/unit-testing/webserver.bash install
      - name: test
        run: ./ci/github/unit-testing/webserver.bash test_with_db 02
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-webserver-03:
    needs: changes
    if: ${{ needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
    timeout-minutes: 25 # if this timeout gets too small, then split the tests
    name: "[unit] webserver 03"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install webserver
        run: ./ci/github/unit-testing/webserver.bash install
      - name: test
        run: ./ci/github/unit-testing/webserver.bash test_with_db 03
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-storage:
    needs: changes
    if: ${{ needs.changes.outputs.storage == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] storage"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/storage.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/storage.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/storage.bash test
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-agent:
    needs: changes
    if: ${{ needs.changes.outputs.agent == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] agent"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install rclone
        run: sudo ./ci/github/helpers/install_rclone.bash
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/agent.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/agent.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/agent.bash test
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-api:
    needs: changes
    if: ${{ needs.changes.outputs.api == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] api"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install api
        run: ./ci/github/unit-testing/api.bash install
      - name: test
        run: ./ci/github/unit-testing/api.bash test

  unit-test-api-server:
    needs: changes
    if: ${{ needs.changes.outputs.api-server == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] api-server"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/api-server.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/api-server.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/api-server.bash test
      - name: OAS backwards compatibility check
        if: always()
        run: ./ci/github/unit-testing/api-server.bash openapi-diff
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-autoscaling:
    needs: changes
    if: ${{ needs.changes.outputs.autoscaling == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] autoscaling"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/autoscaling.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/autoscaling.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/autoscaling.bash test
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-catalog:
    needs: changes
    if: ${{ needs.changes.outputs.catalog == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] catalog"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/catalog.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/catalog.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/catalog.bash test
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/catalog/test_failures
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-clusters-keeper:
    needs: changes
    if: ${{ needs.changes.outputs.clusters-keeper == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] clusters-keeper"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: |
          make devenv
          source .venv/bin/activate && \
          pushd services/clusters-keeper && \
          make install-ci
      - name: typecheck
        run: |
          source .venv/bin/activate && \
          pushd services/clusters-keeper && \
          make mypy
      - name: test
        if: always()
        run: |
          source .venv/bin/activate && \
          pushd services/clusters-keeper && \
          make test-ci-unit
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-datcore-adapter:
    needs: changes
    if: ${{ needs.changes.outputs.datcore-adapter == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] datcore-adapter"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/datcore-adapter.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/datcore-adapter.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/datcore-adapter.bash test
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/datcore-adapter/test_failures
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-director:
    needs: changes
    if: ${{ needs.changes.outputs.director == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] director"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        # KEEP 3.6 Development of this service is frozen
        # KEEP ubuntu 20.04, else no python 3.6
        python: [3.6]
        os: [ubuntu-20.04]
        docker_buildx: [v0.10.4]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/director.bash install
      - name: test
        run: ./ci/github/unit-testing/director.bash test
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-director-v2:
    needs: changes
    if: ${{ needs.changes.outputs.director-v2 == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] director-v2"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/director-v2.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/director-v2.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/director-v2.bash test
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/director-v2/test_failures
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-aws-library:
    needs: changes
    if: ${{ needs.changes.outputs.aws-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] aws-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/aws-library.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/aws-library.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/aws-library.bash test
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-dask-task-models-library:
    needs: changes
    if: ${{ needs.changes.outputs.dask-task-models-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] dask-task-models-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/dask-task-models-library.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/dask-task-models-library.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/dask-task-models-library.bash test
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-dask-sidecar:
    needs: changes
    if: ${{ needs.changes.outputs.dask-sidecar == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] dask-sidecar"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/dask-sidecar.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/dask-sidecar.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/dask-sidecar.bash test
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-osparc-gateway-server:
    needs: changes
    if: ${{ needs.changes.outputs.osparc-gateway-server == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] osparc-gateway-server"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: |
          make devenv
          source .venv/bin/activate && \
          pushd services/osparc-gateway-server && \
          make install-ci
      - name: typecheck
        run: |
          source .venv/bin/activate && \
          pushd services/osparc-gateway-server && \
          make mypy
      - name: test
        if: always()
        run: |
          source .venv/bin/activate && \
          pushd services/osparc-gateway-server && \
          make test-ci-unit
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-payments:
    needs: changes
    if: ${{ needs.changes.outputs.payments == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] payments"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/payments.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/payments.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/payments.bash test
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-dynamic-scheduler:
    needs: changes
    if: ${{ needs.changes.outputs.dynamic-scheduler == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] dynamic-scheduler"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/dynamic-scheduler.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/dynamic-scheduler.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/dynamic-scheduler.bash test
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-resource-usage-tracker:
    needs: changes
    if: ${{ needs.changes.outputs.resource-usage-tracker == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] resource-usage-tracker"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: |
          make devenv
          source .venv/bin/activate && \
          pushd services/resource-usage-tracker && \
          make install-ci
      - name: typecheck
        run: |
          source .venv/bin/activate && \
          pushd services/resource-usage-tracker && \
          make mypy
      - name: test
        if: always()
        run: |
          source .venv/bin/activate && \
          pushd services/resource-usage-tracker && \
          make test-ci-unit
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-dynamic-sidecar:
    needs: changes
    if: ${{ needs.changes.outputs.dynamic-sidecar == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] dynamic-sidecar"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/dynamic-sidecar.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/dynamic-sidecar.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/dynamic-sidecar.bash test
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-frontend:
    needs: changes
    if: ${{ needs.changes.outputs.static-webserver == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] frontend"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        node: [14]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - uses: actions/setup-node@v4.0.2
        with:
          node-version: ${{ matrix.node }}
          cache: "npm"
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/frontend.bash install
      - name: test
        run: ./ci/github/unit-testing/frontend.bash test

  unit-test-python-linting:
    needs: changes
    if: ${{ needs.changes.outputs.anything-py == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] python-linting"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10", "3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/python-linting.bash install
      - name: test
        run: ./ci/github/unit-testing/python-linting.bash test

  unit-test-postgres-database:
    needs: changes
    if: ${{ needs.changes.outputs.postgres-database == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] postgres-database"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/postgres-database.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/postgres-database.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/postgres-database.bash test
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-invitations:
    needs: changes
    if: ${{ needs.changes.outputs.invitations == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] invitations"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/invitations.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/invitations.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/invitations.bash test
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-service-integration:
    needs: changes
    if: ${{ needs.changes.outputs.service-integration == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] service-integration"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/service-integration.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/service-integration.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/service-integration.bash test
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-service-library:
    needs: changes
    if: ${{ needs.changes.outputs.service-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] service-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/service-library.bash install_all
      - name: typecheck
        run: ./ci/github/unit-testing/service-library.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/service-library.bash test_all
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-settings-library:
    needs: changes
    if: ${{ needs.changes.outputs.settings-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] settings-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/settings-library.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/settings-library.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/settings-library.bash test
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-models-library:
    needs: changes
    if: ${{ needs.changes.outputs.models-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] models-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/models-library.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/models-library.bash typecheck
      - name: test
        run: ./ci/github/unit-testing/models-library.bash test
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-notifications-library:
    needs: changes
    if: ${{ needs.changes.outputs.notifications-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit]notifications-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
        docker_buildx: [v0.10.4]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "packages/notifications-library/requirements/ci.txt"
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/notifications-library.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/notifications-library.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/notifications-library.bash test
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-test-simcore-sdk:
    needs: changes
    if: ${{ needs.changes.outputs.simcore-sdk == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] simcore-sdk"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: install rclone
        run: sudo ./ci/github/helpers/install_rclone.bash
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/simcore-sdk.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/simcore-sdk.bash typecheck
      - name: test
        if: always()
        run: ./ci/github/unit-testing/simcore-sdk.bash test
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: unittests #optional

  unit-tests:
    # NOTE: this is a github required status check!
    if: ${{ always() }}
    needs:
      [
        unit-test-agent,
        unit-test-api-server,
        unit-test-api,
        unit-test-autoscaling,
        unit-test-catalog,
        unit-test-clusters-keeper,
        unit-test-dask-sidecar,
        unit-test-aws-library,
        unit-test-dask-task-models-library,
        unit-test-datcore-adapter,
        unit-test-director-v2,
        unit-test-director,
        unit-test-dynamic-sidecar,
        unit-test-frontend,
        unit-test-models-library,
        unit-test-notifications-library,
        unit-test-osparc-gateway-server,
        unit-test-payments,
        unit-test-dynamic-scheduler,
        unit-test-postgres-database,
        unit-test-python-linting,
        unit-test-resource-usage-tracker,
        unit-test-service-integration,
        unit-test-service-library,
        unit-test-settings-library,
        unit-test-simcore-sdk,
        unit-test-storage,
        unit-test-webserver-01,
        unit-test-webserver-02,
        unit-test-webserver-03,
      ]
    runs-on: ubuntu-latest
    steps:
      - name: a previous unit-test job failed
        if: ${{ contains(join(needs.*.result, ','), 'failure') }}
        run: |
          echo "::error title=ERROR::one of the unit-tests failed!"
          echo "${{ join(needs.*.result, ',') }}"
          exit 1
      - name: all the previous unit-tests were run successfully or skipped
        if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
        run: echo "::notice All good!"

  integration-test-webserver-01:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] webserver 01"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/webserver.bash install
      - name: test
        run: ./ci/github/integration-testing/webserver.bash test 01
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/web/server/test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/integration-testing/webserver.bash clean_up
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: integrationtests #optional

  integration-test-webserver-02:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] webserver 02"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/webserver.bash install
      - name: test
        run: ./ci/github/integration-testing/webserver.bash test 02
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/web/server/test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/integration-testing/webserver.bash clean_up
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: integrationtests #optional

  integration-test-director-v2-01:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.director-v2 == 'true' || github.event_name == 'push'}}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] director-v2 01"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/director-v2.bash install
      - name: test
        run: ./ci/github/integration-testing/director-v2.bash test 01
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/director-v2/test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/integration-testing/director-v2.bash clean_up
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: integrationtests #optional

  integration-test-director-v2-02:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.director-v2 == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] director-v2 02"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup rclone docker volume plugin
        run: sudo ./ci/github/helpers/install_rclone_docker_volume_plugin.bash
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install rclone
        run: sudo ./ci/github/helpers/install_rclone.bash
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/director-v2.bash install
      - name: test
        run: ./ci/github/integration-testing/director-v2.bash test 02
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/director-v2/test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/integration-testing/director-v2.bash clean_up
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: integrationtests #optional

  integration-test-dynamic-sidecar:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.dynamic-sidecar == 'true' || github.event_name == 'push'}}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] dynamic-sidecar"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install rclone
        run: sudo ./ci/github/helpers/install_rclone.bash
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/dynamic-sidecar.bash install
      - name: test
        run: ./ci/github/integration-testing/dynamic-sidecar.bash test 01
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/dynamic-sidecar/test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/integration-testing/dynamic-sidecar.bash clean_up
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: integrationtests #optional

  integration-test-osparc-gateway-server:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.osparc-gateway-server == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] osparc-gateway-server"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container

      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: |
          make devenv && \
          source .venv/bin/activate && \
          pushd services/osparc-gateway-server && \
          make install-ci
      - name: integration-test
        run: |
          source .venv/bin/activate && \
          pushd services/osparc-gateway-server && \
          make test-ci-integration
      - name: system-test
        run: |
          source .venv/bin/activate && \
          pushd services/osparc-gateway-server && \
          make test-system
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/director-v2/test_failures
      - name: cleanup
        if: always()
        run: |
          pushd services/osparc-gateway-server && \
          make down
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: integrationtests #optional

  integration-test-simcore-sdk:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.simcore-sdk == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] simcore-sdk"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: install rclone
        run: sudo ./ci/github/helpers/install_rclone.bash
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/simcore-sdk.bash install
      - name: test
        run: ./ci/github/integration-testing/simcore-sdk.bash test
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./packages/simcore-sdk/test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/integration-testing/simcore-sdk.bash clean_up
      - uses: codecov/codecov-action@v4.3.0
        with:
          flags: integrationtests

  integration-tests:
    # NOTE: this is a github required status check!
    if: always()
    needs:
      [
        integration-test-director-v2-01,
        integration-test-director-v2-02,
        integration-test-dynamic-sidecar,
        integration-test-osparc-gateway-server,
        integration-test-simcore-sdk,
        integration-test-webserver-01,
        integration-test-webserver-02,
      ]
    runs-on: ubuntu-latest
    steps:
      - name: a previous integration-test job failed
        if: ${{ contains(join(needs.*.result, ','), 'failure') }}
        run: |
          echo "::error title=ERROR::one of the integration-tests failed!"
          exit 1
      - name: all the previous integration-tests were run successfully or skipped
        if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
        run: echo "::notice All good!"

  system-test-public-api:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
    timeout-minutes: 25 # if this timeout gets too small, then split the tests
    name: "[sys] public api"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: ./ci/github/system-testing/public-api.bash install
      - name: test
        run: ./ci/github/system-testing/public-api.bash test
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/system-testing/public-api.bash clean_up

  system-test-swarm-deploy:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[sys] deploy simcore"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/system-testing/swarm-deploy.bash install
      - name: test
        run: ./ci/github/system-testing/swarm-deploy.bash test
      - name: dump services setting schemas
        run: export DOCKER_REGISTRY=local; export DOCKER_IMAGE_TAG=production; make settings-schema.json
      - name: upload services settings schemas
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_services_settings_schemas
          path: ./services/**/settings-schema.json
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/system-testing/swarm-deploy.bash clean_up

  system-test-e2e:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[sys] e2e"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        node: [14]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - uses: actions/setup-node@v4.0.2
        with:
          node-version: ${{ matrix.node }}
          cache: "npm"
          cache-dependency-path: "tests/e2e/package-lock.json"
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: setup
        run: ./ci/github/system-testing/e2e.bash install
      - name: test
        run: ./ci/github/system-testing/e2e.bash test
      - name: dump docker logs
        id: docker_logs_dump
        if: failure()
        run: ./ci/github/system-testing/e2e.bash dump_docker_logs
      - name: upload docker logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./tests/e2e/test_failures
      - name: upload screenshots
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_screenshots
          path: tests/e2e/screenshots
      - name: upload e2e logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_logs
          path: tests/e2e/logs
      - name: cleanup
        if: always()
        run: ./ci/github/system-testing/e2e.bash clean_up

  system-test-e2e-playwright:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[sys] e2e-playwright"
    runs-on: ${{ matrix.os }}
    # NOTE: this is an interesting way, but generate a load of issues like not having docker installed, etc etc.
    # container:
    #   image: mcr.microsoft.com/playwright/python:v1.39.0-jammy
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      - name: download docker images
        uses: actions/download-artifact@v4
        with:
          name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
          path: /${{ runner.temp }}/build
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: prepare devenv
        run: make devenv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: setup
        run: |
          ./ci/github/system-testing/e2e-playwright.bash install
      - name: test
        run: |
          ./ci/github/system-testing/e2e-playwright.bash test
      - name: dump docker logs
        id: docker_logs_dump
        if: failure()
        run: ./ci/github/system-testing/e2e-playwright.bash dump_docker_logs
      - name: upload docker logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./tests/e2e-playwright/test_failures
      - name: upload tracing if failed
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_tracing
          path: tests/e2e-playwright/test-results


  system-test-environment-setup:
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[sys] environment setup"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/system-testing/environment-setup.bash install
      - name: test
        run: ./ci/github/system-testing/environment-setup.bash test
      - name: cleanup
        if: always()
        run: ./ci/github/system-testing/environment-setup.bash clean_up

  system-tests:
    # NOTE: this is a github required status check!
    if: always()
    needs:
      [
        system-test-e2e,
        system-test-e2e-playwright,
        system-test-environment-setup,
        system-test-public-api,
        system-test-swarm-deploy,
      ]
    runs-on: ubuntu-latest
    steps:
      - name: a previous system-test job failed
        if: ${{ contains(join(needs.*.result, ','), 'failure') }}
        run: |
          echo "::error title=ERROR::one of the system-tests failed!"
          exit 1
      - name: all the previous system-tests were run successfully or skipped
        if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
        run: echo "::notice All good!"

  deploy:
    name: deploy to dockerhub
    if: github.event_name == 'push'
    needs: [unit-tests, integration-tests, system-tests]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.10"]
        os: [ubuntu-22.04]
      fail-fast: false
    env:
      # secrets can be set in settings/secrets on github
      DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: |
          make load-images local-src=/${{ runner.temp }}/build
      - name: set owner variable
        run: echo "OWNER=${GITHUB_REPOSITORY%/*}" >> $GITHUB_ENV
      - if: github.ref == 'refs/heads/master'
        name: deploy master image
        env:
          TAG_PREFIX: master-github
        run: ./ci/deploy/dockerhub-deploy.bash -n
      - if: contains(github.ref, 'refs/heads/hotfix_v')
        name: deploy release hotfix image
        env:
          TAG_PREFIX: hotfix-github
        run: ./ci/deploy/dockerhub-deploy.bash -n
      - if: contains(github.ref, 'refs/heads/hotfix_staging_')
        name: deploy staging hotfix image
        env:
          TAG_PREFIX: hotfix-staging-github
        run: ./ci/deploy/dockerhub-deploy.bash -n