diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbb271681f..470b222ba7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -243,7 +243,7 @@ jobs: uses: arduino/setup-protoc@v3 - name: Install venv - run: pdm sync -d --clean --no-editable + run: pdm sync -d --clean --no-editable -G nidx - name: Upload virtualenv to cache uses: actions/cache/save@v4 with: @@ -332,70 +332,147 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - nucliadb-search-tests-nidx: - name: NucliaDB search tests (nidx) + node-rust-tests: + name: Node Rust tests + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + - name: test + run: cargo test --workspace --all-features + + nucliadb-tests: + name: NucliaDB tests runs-on: ubuntu-24.04 needs: - - build-node-binaries - - build-nidx-binary - build-virtual-env + strategy: + fail-fast: false + matrix: + maindb_driver: ["pg"] + shard: [0, 1, 2] + steps: - - uses: actions/checkout@v4 + - name: Checkout the repository + uses: actions/checkout@v4 + - name: Setup Python uses: actions/setup-python@v5 with: python-version: "3.12.5" + - name: Restore venv uses: actions/cache/restore@v4 with: path: .venv key: venv-${{ github.sha }} fail-on-cache-miss: true - - name: Restore binaries - uses: actions/cache/restore@v4 + + - name: Run NucliaDB tests + # These tests can be flaky, let's retry them... + uses: nick-fields/retry@v3 + env: + TESTING_MAINDB_DRIVERS: ${{ matrix.maindb_driver }} with: - path: "target/release/node_*er" - key: node-bin-${{ github.sha }} - fail-on-cache-miss: true + max_attempts: 2 + retry_on: error + timeout_minutes: 10 + command: source .venv/bin/activate && make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=${{ matrix.shard }} --num-shards=3 -n 4" || make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=${{ matrix.shard }} --num-shards=3 --lf" - - name: Build docker images for node - run: mkdir builds && mv target/release/node_* builds && docker build -t europe-west4-docker.pkg.dev/nuclia-internal/nuclia/node:latest -f Dockerfile.node_prebuilt . + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + file: ./nucliadb/coverage.xml + flags: nucliadb + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - - name: Build sidecar image - run: make build-sidecar + nucliadb-components-tests: + name: NucliaDB components tests + runs-on: ubuntu-24.04 + needs: + - build-virtual-env + strategy: + fail-fast: false + matrix: + component: + - ingest + - reader + - standalone + - train + - writer - - name: Restore nidx binary + steps: + - name: Checkout the repository + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12.5" + + - name: Restore venv uses: actions/cache/restore@v4 with: - path: "nidx/target/release/nidx" - key: nidx-bin-${{ github.sha }} + path: .venv + key: venv-${{ github.sha }} fail-on-cache-miss: true - - name: Build docker images for nidx - run: mv nidx/target/release/nidx builds && docker build -t nidx -f Dockerfile.nidx_prebuilt . - - - name: Run tests - env: - NIDX_ENABLED: 1 - run: source .venv/bin/activate && make -C nucliadb test-cov-search + - name: Run package tests + run: source .venv/bin/activate && make -C nucliadb test-cov-${{ matrix.component }} - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 with: file: ./nucliadb/coverage.xml - flags: nucliadb-search + flags: nucliadb-${{ matrix.component }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - node-rust-tests: - name: Node Rust tests + python-packages-tests: + name: Python packages tests runs-on: ubuntu-24.04 + needs: + - build-virtual-env + strategy: + fail-fast: false + matrix: + package: + - nucliadb_dataset + - nucliadb_models + - nucliadb_node_binding + - nucliadb_sdk + - nucliadb_telemetry + - nucliadb_utils + steps: - - uses: actions/checkout@v4 - - uses: Swatinem/rust-cache@v2 - - name: test - run: cargo test --workspace --all-features + - name: Checkout the repository + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12.5" + + - name: Restore venv + uses: actions/cache/restore@v4 + with: + path: .venv + key: venv-${{ github.sha }} + fail-on-cache-miss: true + + - name: Run package tests + run: source .venv/bin/activate && make -C ${{ matrix.package }} test-cov + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + file: ./${{ matrix.package }}/coverage.xml + flags: ${{ matrix.package }} + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + # Tests with nidx nidx-tests: name: nidx tests runs-on: ubuntu-24.04 @@ -424,66 +501,71 @@ jobs: SQLX_OFFLINE: true run: cd nidx && cargo test --workspace --all-features - nucliadb-tests: - name: NucliaDB tests + nucliadb-search-tests-nidx: + name: NucliaDB search tests (nidx) runs-on: ubuntu-24.04 needs: + - build-node-binaries + - build-nidx-binary - build-virtual-env - strategy: - fail-fast: false - matrix: - maindb_driver: ["pg"] - shard: [0, 1, 2] - steps: - - name: Checkout the repository - uses: actions/checkout@v4 - + - uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v5 with: python-version: "3.12.5" - - name: Restore venv uses: actions/cache/restore@v4 with: path: .venv key: venv-${{ github.sha }} fail-on-cache-miss: true + - name: Restore binaries + uses: actions/cache/restore@v4 + with: + path: "target/release/node_*er" + key: node-bin-${{ github.sha }} + fail-on-cache-miss: true - - name: Run NucliaDB tests - # These tests can be flaky, let's retry them... - uses: nick-fields/retry@v3 - env: - TESTING_MAINDB_DRIVERS: ${{ matrix.maindb_driver }} + - name: Build docker images for node + run: mkdir builds && mv target/release/node_* builds && docker build -t europe-west4-docker.pkg.dev/nuclia-internal/nuclia/node:latest -f Dockerfile.node_prebuilt . + + - name: Build sidecar image + run: make build-sidecar + + - name: Restore nidx binary + uses: actions/cache/restore@v4 with: - max_attempts: 2 - retry_on: error - timeout_minutes: 10 - command: source .venv/bin/activate && make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=${{ matrix.shard }} --num-shards=3 -n 4" || make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=${{ matrix.shard }} --num-shards=3 --lf" + path: "nidx/target/release/nidx" + key: nidx-bin-${{ github.sha }} + fail-on-cache-miss: true + + - name: Build docker images for nidx + run: mv nidx/target/release/nidx builds && docker build -t nidx -f Dockerfile.nidx_prebuilt . + + - name: Run tests + env: + NIDX_ENABLED: 1 + run: source .venv/bin/activate && make -C nucliadb test-cov-search - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 with: file: ./nucliadb/coverage.xml - flags: nucliadb + flags: nucliadb-search env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - nucliadb-components-tests: - name: NucliaDB components tests + nucliadb-tests-nidx: + name: NucliaDB tests (nidx) runs-on: ubuntu-24.04 needs: - build-virtual-env strategy: fail-fast: false matrix: - component: - - ingest - - reader - - standalone - - train - - writer + maindb_driver: ["pg"] + shard: [0, 1, 2] steps: - name: Checkout the repository @@ -501,32 +583,40 @@ jobs: key: venv-${{ github.sha }} fail-on-cache-miss: true - - name: Run package tests - run: source .venv/bin/activate && make -C nucliadb test-cov-${{ matrix.component }} + - name: Run NucliaDB tests + # These tests can be flaky, let's retry them... + uses: nick-fields/retry@v3 + env: + TESTING_MAINDB_DRIVERS: ${{ matrix.maindb_driver }} + NIDX_ENABLED: 1 + with: + max_attempts: 2 + retry_on: error + timeout_minutes: 10 + command: source .venv/bin/activate && make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=${{ matrix.shard }} --num-shards=3 -n 4" || make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=${{ matrix.shard }} --num-shards=3 --lf" - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 with: file: ./nucliadb/coverage.xml - flags: nucliadb-${{ matrix.component }} + flags: nucliadb env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - python-packages-tests: - name: Python packages tests + nucliadb-components-tests-nidx: + name: NucliaDB components tests (nidx) runs-on: ubuntu-24.04 needs: - build-virtual-env strategy: fail-fast: false matrix: - package: - - nucliadb_dataset - - nucliadb_models - - nucliadb_node_binding - - nucliadb_sdk - - nucliadb_telemetry - - nucliadb_utils + component: + - ingest + - reader + - standalone + - train + - writer steps: - name: Checkout the repository @@ -545,13 +635,15 @@ jobs: fail-on-cache-miss: true - name: Run package tests - run: source .venv/bin/activate && make -C ${{ matrix.package }} test-cov + run: source .venv/bin/activate && make -C nucliadb test-cov-${{ matrix.component }} + env: + NIDX_ENABLED: 1 - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 with: - file: ./${{ matrix.package }}/coverage.xml - flags: ${{ matrix.package }} + file: ./nucliadb/coverage.xml + flags: nucliadb-${{ matrix.component }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}