CI: enable nidx search tests (#2628)

.github/workflows/ci.yml

@@ -112,13 +112,21 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: |
+            .
+            nidx
       - name: Set up PDM
         uses: pdm-project/setup-pdm@v4
         with:
           python-version: "3.9.20"
           cache: true
+      # Needed for nidx_protos (rust)
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+
       - name: Install venv
-        run: pdm sync -d --clean --no-editable --without nidx
+        run: pdm sync -d --clean --no-editable
       - name: Clean build
         run: find . -name build | xargs rm -r || true
       - name: Lint source code
@@ -197,19 +205,45 @@ jobs:
           path: "target/release/node_*er"
           key: node-bin-${{ github.sha }}
 
+  build-nidx-binary:
+    name: Build nidx binary
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: nidx
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+      - name: Compile
+        run: cd nidx && cargo build --release
+      - name: Upload binaries to cache
+        uses: actions/cache/save@v4
+        with:
+          path: "nidx/target/release/nidx"
+          key: nidx-bin-${{ github.sha }}
+
   build-virtual-env:
     name: Build virtual environment
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: |
+            .
+            nidx
       - name: Set up PDM
         uses: pdm-project/setup-pdm@v4
         with:
           python-version: "3.12.5"
           cache: true
+      # Needed for nidx_protos (rust)
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+
       - name: Install venv
-        run: pdm sync -d --clean --no-editable --without nidx
+        run: pdm sync -d --clean --no-editable
       - name: Upload virtualenv to cache
         uses: actions/cache/save@v4
         with:
@@ -298,6 +332,61 @@ jobs:
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
+  nucliadb-search-tests-nidx:
+    name: NucliaDB search tests (nidx)
+    runs-on: ubuntu-24.04
+    needs:
+      - build-node-binaries
+      - build-nidx-binary
+      - build-virtual-env
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12.5"
+      - name: Restore venv
+        uses: actions/cache/restore@v4
+        with:
+          path: .venv
+          key: venv-${{ github.sha }}
+          fail-on-cache-miss: true
+      - name: Restore binaries
+        uses: actions/cache/restore@v4
+        with:
+          path: "target/release/node_*er"
+          key: node-bin-${{ github.sha }}
+          fail-on-cache-miss: true
+
+      - name: Build docker images for node
+        run: mkdir builds && mv target/release/node_* builds && docker build -t europe-west4-docker.pkg.dev/nuclia-internal/nuclia/node:latest -f Dockerfile.node_prebuilt .
+
+      - name: Build sidecar image
+        run: make build-sidecar
+
+      - name: Restore nidx binary
+        uses: actions/cache/restore@v4
+        with:
+          path: "nidx/target/release/nidx"
+          key: nidx-bin-${{ github.sha }}
+          fail-on-cache-miss: true
+
+      - name: Build docker images for nidx
+        run: mv nidx/target/release/nidx builds && docker build -t nidx -f Dockerfile.nidx_prebuilt .
+
+      - name: Run tests
+        env:
+          NIDX_ENABLED: 1
+        run: source .venv/bin/activate && make -C nucliadb test-cov-search
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          file: ./nucliadb/coverage.xml
+          flags: nucliadb-search
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
   node-rust-tests:
     name: Node Rust tests
     runs-on: ubuntu-24.04

Dockerfile.nidx_prebuilt

@@ -0,0 +1,11 @@
+#
+# Dockerfile for nucliadb's index node, using the provided binaries
+# NOTE: Binaries should be built in the same image as this Dockerfile or older (for glibc compatibility)
+# For a version that compiles the binaries inside docker, see `Dockerfile.nidx`
+#
+
+FROM ubuntu:24.04
+
+RUN apt update && apt install -y ca-certificates && rm -rf /var/lib/apt/lists/*
+COPY ./builds/nidx /usr/local/bin/nidx
+RUN chmod +x /usr/local/bin/nidx

nidx/nidx_protos/build.py

@@ -1,5 +1,5 @@
 import os
-
+from importlib import resources
 from grpc_tools import protoc
 
 
@@ -11,18 +11,22 @@ def pdm_build_initialize(context):
     except FileExistsError:
         pass
 
+    well_known_path = resources.files("grpc_tools") / "_proto"
+
     # Compile protos
     for proto in [
         "src/nidx.proto",
     ]:
         command = [
             "grpc_tools.protoc",
-            "--proto_path={}".format("src"),
-            "--proto_path={}".format("../../"),
-            "--python_out={}".format(python_dir),
-            "--pyi_out={}".format(python_dir),
-            "--grpc_python_out={}".format(python_dir),
-        ] + [proto]
+            "--proto_path=src",
+            "--proto_path=../../",
+            f"--proto_path={well_known_path}",
+            f"--python_out={python_dir}",
+            f"--pyi_out={python_dir}",
+            f"--grpc_python_out={python_dir}",
+            proto
+        ]
         if protoc.main(command) != 0:
             raise Exception("error: {} failed".format(command))
 

nucliadb/tests/search/fixtures.py

@@ -238,4 +238,6 @@ async def wait_for_shard(knowledgebox_ingest: str, count: int) -> str:
             await asyncio.sleep(1)
 
     assert all(checks.values())
+    # Wait an extra couple of seconds for reader/searcher to catch up
+    await asyncio.sleep(2)
     return knowledgebox_ingest

nucliadb/tests/search/integration/api/v1/test_search.py

@@ -38,6 +38,7 @@
 RUNNING_IN_GH_ACTIONS = os.environ.get("CI", "").lower() == "true"
 
 
+@pytest.mark.flaky(reruns=5)
 @pytest.mark.asyncio
 async def test_multiple_fuzzy_search_resource_all(
     search_api: Callable[..., AsyncClient], multiple_search_resource: str
@@ -62,6 +63,7 @@ async def test_multiple_fuzzy_search_resource_all(
         )
 
 
+@pytest.mark.flaky(reruns=5)
 @pytest.mark.asyncio
 async def test_multiple_search_resource_all(
     search_api: Callable[..., AsyncClient], multiple_search_resource: str
@@ -117,6 +119,7 @@ async def test_multiple_search_resource_all(
 
 
 @pytest.mark.asyncio
+@pytest.mark.flaky(reruns=3)
 async def test_search_resource_all(
     search_api: Callable[..., AsyncClient],
     test_search_resource: str,