From 4bf1729d486adeb5cecb7044633c2433a16feb43 Mon Sep 17 00:00:00 2001
From: Rodrigo Lopez <rodrigo.lopez@weaviate.io>
Date: Fri, 13 Dec 2024 17:32:46 +0100
Subject: [PATCH] Modify filter_memory_leak to use async indexing

---
 .github/workflows/matrix.yaml                 | 20 +++++++++++++++++++
 .github/workflows/tests.yaml                  |  1 +
 .../docker-compose-with-memlimit.yml          |  1 +
 .../docker-compose.yml                        |  1 +
 filter_memory_leak.sh                         |  1 +
 5 files changed, 24 insertions(+)

diff --git a/.github/workflows/matrix.yaml b/.github/workflows/matrix.yaml
index 6d09de5d..8577e68e 100644
--- a/.github/workflows/matrix.yaml
+++ b/.github/workflows/matrix.yaml
@@ -44,6 +44,7 @@ jobs:
     with:
       lsm_access_strategy: ${{matrix.lsm_access_strategy}}
       weaviate_version: ${{ github.event_name == 'schedule' && 'latest' || inputs.weaviate_version || '1.27.2' }}
+      async_indexing: "false"
     secrets:
       AWS_ACCESS_KEY: ${{secrets.AWS_ACCESS_KEY}}
       AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
@@ -51,6 +52,25 @@ jobs:
       DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
       GCP_SERVICE_ACCOUNT_BENCHMARKS: ${{secrets.GCP_SERVICE_ACCOUNT_BENCHMARKS}}
       POLARSIGNALS_TOKEN: ${{secrets.POLARSIGNALS_TOKEN}}
+
+  run-with-async-indexing:
+    strategy:
+      fail-fast: false
+      matrix:
+        lsm_access_strategy: ["mmap", "pread"]
+    uses: ./.github/workflows/tests.yaml
+    with:
+      lsm_access_strategy: ${{matrix.lsm_access_strategy}}
+      weaviate_version: ${{ github.event_name == 'schedule' && 'latest' || inputs.weaviate_version || '1.27.2' }}
+      async_indexing: "true"
+    secrets:
+      AWS_ACCESS_KEY: ${{secrets.AWS_ACCESS_KEY}}
+      AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
+      DOCKER_USERNAME: ${{secrets.DOCKER_USERNAME}}
+      DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
+      GCP_SERVICE_ACCOUNT_BENCHMARKS: ${{secrets.GCP_SERVICE_ACCOUNT_BENCHMARKS}}
+      POLARSIGNALS_TOKEN: ${{secrets.POLARSIGNALS_TOKEN}}
+
   send-slack-message-on-failure:
     needs: [weaviate-version-information, run-with-sync-indexing]
     if: failure() && github.event_name == 'schedule'
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 48f3133d..db17a6b2 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -26,6 +26,7 @@ on:
 env:
   WEAVIATE_VERSION: ${{ inputs.weaviate_version }}
   DISABLE_RECOVERY_ON_PANIC: true
+  ASYNC_INDEXING: ${{ inputs.async_indexing }}
 
 jobs:
   real-version-in-tag:
diff --git a/apps/weaviate-no-restart-on-crash/docker-compose-with-memlimit.yml b/apps/weaviate-no-restart-on-crash/docker-compose-with-memlimit.yml
index 6d6a38aa..dcafe92f 100644
--- a/apps/weaviate-no-restart-on-crash/docker-compose-with-memlimit.yml
+++ b/apps/weaviate-no-restart-on-crash/docker-compose-with-memlimit.yml
@@ -16,6 +16,7 @@ services:
     volumes:
     - "$PWD/apps/weaviate/data:/var/lib/weaviate"
     environment:
+      ASYNC_INDEXING: ${ASYNC_INDEXING}
       PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
       PERSISTENCE_FLUSH_IDLE_MEMTABLES_AFTER: 2
       QUERY_DEFAULTS_LIMIT: 25
diff --git a/apps/weaviate-no-restart-on-crash/docker-compose.yml b/apps/weaviate-no-restart-on-crash/docker-compose.yml
index aa9604aa..869b01e1 100644
--- a/apps/weaviate-no-restart-on-crash/docker-compose.yml
+++ b/apps/weaviate-no-restart-on-crash/docker-compose.yml
@@ -18,6 +18,7 @@ services:
     volumes:
     - "$PWD/apps/weaviate/data:/var/lib/weaviate"
     environment:
+      - ASYNC_INDEXING
       - PERSISTENCE_DATA_PATH
       - PERSISTENCE_MEMTABLES_FLUSH_IDLE_AFTER_SECONDS
       - QUERY_DEFAULTS_LIMIT
diff --git a/filter_memory_leak.sh b/filter_memory_leak.sh
index e60ed455..d4a815bf 100755
--- a/filter_memory_leak.sh
+++ b/filter_memory_leak.sh
@@ -10,6 +10,7 @@ echo "Building all required containers"
 export COMPOSE="apps/weaviate-no-restart-on-crash/docker-compose-with-memlimit.yml"
 
 echo "Starting Weaviate..."
+echo "ASYNC_INDEXING: $ASYNC_INDEXING"
 docker compose -f $COMPOSE up -d
 
 wait_weaviate