notebooks-cloud.yml

name: Notebooks - cloud

on:
  workflow_dispatch:
  pull_request:
    branches:
      - master
  push:
    branches:
      - master
  schedule:
    - cron: "0 11 * * 0"

concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: false

jobs:
  get-notebooks-cloud:
    runs-on: ubuntu-latest
    outputs:
      notebooks: ${{ steps.set_output.outputs.notebooks }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set output variable (Make sure it is this quote format - "[path/to/notebook1.ipynb", "path/to/notebook2.ipynb]")
        id: set_output
        run: |
          notebooks=$(find docs/sphinx/source -name '*cloud.ipynb' ! -name 'pdf-retrieval-with-ColQwen2-vlm_Vespa-cloud.ipynb' ! -name 'mother-of-all-embedding-models-cloud.ipynb' ! -name 'scaling-personal-ai-assistants-with-streaming-mode-cloud.ipynb' ! -name 'colpali-benchmark-vqa-vlm_Vespa-cloud.ipynb' | jq -R -s -c 'split("\n")[:-1]')
          # Print all notebooks echo
          echo $notebooks
          echo "notebooks=$notebooks" >> $GITHUB_OUTPUT

  notebooks-cloud:
    runs-on: ubuntu-latest
    needs: get-notebooks-cloud
    strategy:
      matrix:
        notebook: ${{fromJson(needs.get-notebooks-cloud.outputs.notebooks)}}

    steps:
      - name: Free disk space
        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
        with:
          large-packages: false
          docker-images: false
          swap-storage: false

      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Set up uv
        # Install a specific uv version using the installer
        run: curl -LsSf https://astral.sh/uv/0.4.5/install.sh | sh

      - name: Install dependencies
        run: |
          uv sync --extra dev --extra notebooks

      - name: Run notebooks tests
        env:
          VESPA_TEAM_API_KEY: ${{ secrets.VESPA_TEAM_API_KEY }}
          VESPA_CLOUD_SECRET_TOKEN: ${{ secrets.VESPA_CLOUD_SECRET_TOKEN }}
          CO_API_KEY: ${{ secrets.CO_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
        run: |
          echo "Running ${{ matrix.notebook }}"

          # Run bash script to extract, modify, and comment out the pip install command in the notebook
          # Will save all packages to be installed (excluding pyvespa and vespacli) to additional_requirements.txt
          echo "Running replace_pip_install_notebooks.sh"
          sh .github/scripts/replace_pip_install_notebooks.sh ${{ matrix.notebook }}

          # Install additional requirements with uv
          uv pip install -r additional_requirements.txt

          # Run the modified notebook
          uv run papermill --log-output ${{ matrix.notebook }} ${{ matrix.notebook }}-out.ipynb