Modify filter_memory_leak to use async indexing

weaviate · Dec 13, 2024 · 4bf1729 · 4bf1729
1 parent 2247b2b
commit 4bf1729
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 0 deletions.
diff --git a/.github/workflows/matrix.yaml b/.github/workflows/matrix.yaml
@@ -44,13 +44,33 @@ jobs:
     with:
       lsm_access_strategy: ${{matrix.lsm_access_strategy}}
       weaviate_version: ${{ github.event_name == 'schedule' && 'latest' || inputs.weaviate_version || '1.27.2' }}
+      async_indexing: "false"
     secrets:
       AWS_ACCESS_KEY: ${{secrets.AWS_ACCESS_KEY}}
       AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
       DOCKER_USERNAME: ${{secrets.DOCKER_USERNAME}}
       DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
       GCP_SERVICE_ACCOUNT_BENCHMARKS: ${{secrets.GCP_SERVICE_ACCOUNT_BENCHMARKS}}
       POLARSIGNALS_TOKEN: ${{secrets.POLARSIGNALS_TOKEN}}
+
+  run-with-async-indexing:
+    strategy:
+      fail-fast: false
+      matrix:
+        lsm_access_strategy: ["mmap", "pread"]
+    uses: ./.github/workflows/tests.yaml
+    with:
+      lsm_access_strategy: ${{matrix.lsm_access_strategy}}
+      weaviate_version: ${{ github.event_name == 'schedule' && 'latest' || inputs.weaviate_version || '1.27.2' }}
+      async_indexing: "true"
+    secrets:
+      AWS_ACCESS_KEY: ${{secrets.AWS_ACCESS_KEY}}
+      AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
+      DOCKER_USERNAME: ${{secrets.DOCKER_USERNAME}}
+      DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
+      GCP_SERVICE_ACCOUNT_BENCHMARKS: ${{secrets.GCP_SERVICE_ACCOUNT_BENCHMARKS}}
+      POLARSIGNALS_TOKEN: ${{secrets.POLARSIGNALS_TOKEN}}
+
   send-slack-message-on-failure:
     needs: [weaviate-version-information, run-with-sync-indexing]
     if: failure() && github.event_name == 'schedule'

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -26,6 +26,7 @@ on:
 env:
   WEAVIATE_VERSION: ${{ inputs.weaviate_version }}
   DISABLE_RECOVERY_ON_PANIC: true
+  ASYNC_INDEXING: ${{ inputs.async_indexing }}
 
 jobs:
   real-version-in-tag:

diff --git a/apps/weaviate-no-restart-on-crash/docker-compose-with-memlimit.yml b/apps/weaviate-no-restart-on-crash/docker-compose-with-memlimit.yml
@@ -16,6 +16,7 @@ services:
     volumes:
     - "$PWD/apps/weaviate/data:/var/lib/weaviate"
     environment:
+      ASYNC_INDEXING: ${ASYNC_INDEXING}
       PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
       PERSISTENCE_FLUSH_IDLE_MEMTABLES_AFTER: 2
       QUERY_DEFAULTS_LIMIT: 25

diff --git a/apps/weaviate-no-restart-on-crash/docker-compose.yml b/apps/weaviate-no-restart-on-crash/docker-compose.yml
@@ -18,6 +18,7 @@ services:
     volumes:
     - "$PWD/apps/weaviate/data:/var/lib/weaviate"
     environment:
+      - ASYNC_INDEXING
       - PERSISTENCE_DATA_PATH
       - PERSISTENCE_MEMTABLES_FLUSH_IDLE_AFTER_SECONDS
       - QUERY_DEFAULTS_LIMIT

diff --git a/filter_memory_leak.sh b/filter_memory_leak.sh
@@ -10,6 +10,7 @@ echo "Building all required containers"
 export COMPOSE="apps/weaviate-no-restart-on-crash/docker-compose-with-memlimit.yml"
 
 echo "Starting Weaviate..."
+echo "ASYNC_INDEXING: $ASYNC_INDEXING"
 docker compose -f $COMPOSE up -d
 
 wait_weaviate