Bump faiss commit to 33c0ba5 (opensearch-project#1796)

* Bump faiss commit to 33c0ba5 and update patches

Signed-off-by: Naveen Tatikonda <[email protected]>

* Add Changelog

Signed-off-by: Naveen Tatikonda <[email protected]>

* Set cmake minimum requirement to 3.24.0

Signed-off-by: Naveen Tatikonda <[email protected]>

---------

Signed-off-by: Naveen Tatikonda <[email protected]>

CHANGELOG.md

@@ -24,4 +24,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Documentation
 * Update dev guide to fix clang linking issue on arm [#1746](https://github.com/opensearch-project/k-NN/pull/1746)
 ### Maintenance
+* Bump faiss commit to 33c0ba5 [#1796](https://github.com/opensearch-project/k-NN/pull/1796)
 ### Refactoring

DEVELOPER_GUIDE.md

@@ -57,11 +57,11 @@ In addition to this, the plugin has been tested with JDK 17, and this JDK versio
 
 #### CMake
 
-The plugin requires that cmake >= 3.23.3 is installed in order to build the JNI libraries.
+The plugin requires that cmake >= 3.24.0 is installed in order to build the JNI libraries.
 
 One easy way to install on mac or linux is to use pip:
 ```bash
-pip install cmake==3.23.3
+pip install cmake==3.24.0
 ```
 
 On Mac M series machines, install cmake using:

jni/CMakeLists.txt

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-cmake_minimum_required(VERSION 3.23.1)
+cmake_minimum_required(VERSION 3.24.0)
 
 project(KNNPlugin_JNI)
 

jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch

@@ -1,4 +1,4 @@
-From 35ef01f59b8903dfbd4d08ff874b085e851e4228 Mon Sep 17 00:00:00 2001
+From 9e5affabe2caacf38f5585a0b906620dd35deef5 Mon Sep 17 00:00:00 2001
 From: Heemin Kim <[email protected]>
 Date: Tue, 30 Jan 2024 14:43:56 -0800
 Subject: [PATCH] Add IDGrouper for HNSW
@@ -13,20 +13,20 @@ Signed-off-by: Heemin Kim <[email protected]>
  faiss/impl/HNSW.cpp        |   6 +
  faiss/impl/IDGrouper.cpp   |  51 ++++++++
  faiss/impl/IDGrouper.h     |  51 ++++++++
- faiss/impl/ResultHandler.h | 190 +++++++++++++++++++++++++++++
+ faiss/impl/ResultHandler.h | 189 +++++++++++++++++++++++++++++
  faiss/utils/GroupHeap.h    | 182 ++++++++++++++++++++++++++++
  tests/CMakeLists.txt       |   2 +
  tests/test_group_heap.cpp  |  98 +++++++++++++++
  tests/test_id_grouper.cpp  | 241 +++++++++++++++++++++++++++++++++++++
- 13 files changed, 891 insertions(+), 5 deletions(-)
+ 13 files changed, 890 insertions(+), 5 deletions(-)
  create mode 100644 faiss/impl/IDGrouper.cpp
  create mode 100644 faiss/impl/IDGrouper.h
  create mode 100644 faiss/utils/GroupHeap.h
  create mode 100644 tests/test_group_heap.cpp
  create mode 100644 tests/test_id_grouper.cpp
 
 diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt
-index a890a46f..137e68d4 100644
+index 1b0860f3..f3d72df3 100644
 --- a/faiss/CMakeLists.txt
 +++ b/faiss/CMakeLists.txt
 @@ -54,6 +54,7 @@ set(FAISS_SRC
@@ -45,10 +45,10 @@ index a890a46f..137e68d4 100644
    impl/DistanceComputer.h
    impl/FaissAssert.h
    impl/FaissException.h
-@@ -183,6 +185,7 @@ set(FAISS_HEADERS
-   invlists/InvertedLists.h
+@@ -184,6 +186,7 @@ set(FAISS_HEADERS
    invlists/InvertedListsIOHook.h
    utils/AlignedTable.h
+   utils/bf16.h
 +  utils/GroupHeap.h
    utils/Heap.h
    utils/WorkerThread.h
@@ -81,10 +81,10 @@ index 3d1bdb99..a8622858 100644
      virtual ~SearchParameters() {}
  };
 diff --git a/faiss/IndexHNSW.cpp b/faiss/IndexHNSW.cpp
-index 9a67332d..a5e0fea0 100644
+index 8e5c654f..d473b6ad 100644
 --- a/faiss/IndexHNSW.cpp
 +++ b/faiss/IndexHNSW.cpp
-@@ -354,10 +354,17 @@ void IndexHNSW::search(
+@@ -320,10 +320,17 @@ void IndexHNSW::search(
          const SearchParameters* params_in) const {
      FAISS_THROW_IF_NOT(k > 0);
 
@@ -198,10 +198,10 @@ index 2d164123..a68887bd 100644
 +
  } // namespace faiss
 diff --git a/faiss/impl/HNSW.cpp b/faiss/impl/HNSW.cpp
-index a9fb9daf..33b56638 100644
+index 3ba5f72f..c574ce39 100644
 --- a/faiss/impl/HNSW.cpp
 +++ b/faiss/impl/HNSW.cpp
-@@ -804,6 +804,12 @@ int extract_k_from_ResultHandler(ResultHandler<C>& res) {
+@@ -831,6 +831,12 @@ int extract_k_from_ResultHandler(ResultHandler<C>& res) {
      if (auto hres = dynamic_cast<RH::SingleResultHandler*>(&res)) {
          return hres->k;
      }
@@ -329,20 +329,19 @@ index 00000000..d56113d9
 +
 +} // namespace faiss
 diff --git a/faiss/impl/ResultHandler.h b/faiss/impl/ResultHandler.h
-index 270de8dc..3199634f 100644
+index 713fe8e4..d307fd70 100644
 --- a/faiss/impl/ResultHandler.h
 +++ b/faiss/impl/ResultHandler.h
-@@ -12,6 +12,9 @@
- #pragma once
+@@ -13,6 +13,8 @@
 
  #include <faiss/impl/AuxIndexStructures.h>
-+#include <faiss/impl/FaissException.h>
+ #include <faiss/impl/FaissException.h>
 +#include <faiss/impl/IDGrouper.h>
 +#include <faiss/utils/GroupHeap.h>
  #include <faiss/utils/Heap.h>
  #include <faiss/utils/partitioning.h>
-
-@@ -265,6 +268,193 @@ struct HeapBlockResultHandler : BlockResultHandler<C> {
+ #include <iostream>
+@@ -267,6 +269,193 @@ struct HeapBlockResultHandler : BlockResultHandler<C> {
      }
  };
 
@@ -726,7 +725,7 @@ index 00000000..3b7078da
 +} // namespace faiss
 \ No newline at end of file
 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
-index 9017edc5..a8e9d30c 100644
+index 3980d7dd..c888a5a6 100644
 --- a/tests/CMakeLists.txt
 +++ b/tests/CMakeLists.txt
 @@ -27,6 +27,8 @@ set(FAISS_TEST_SRC
@@ -1090,5 +1089,5 @@ index 00000000..6601795b
 +    delete[] xb;
 +}
 -- 
-2.39.3 (Apple Git-145)
+2.37.0
 

jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch

@@ -1,4 +1,4 @@
-From c5ca07299b427dedafc738b98bd20f8f286f6783 Mon Sep 17 00:00:00 2001
+From a33e6ef35385009f24200586294f96235cb95d61 Mon Sep 17 00:00:00 2001
 From: John Mazanec <[email protected]>
 Date: Wed, 21 Feb 2024 15:34:15 -0800
 Subject: [PATCH] Enable precomp table to be shared ivfpq
@@ -167,7 +167,7 @@ index d5d21da4..850bbe44 100644
  };
 
 diff --git a/faiss/IndexIVFPQFastScan.cpp b/faiss/IndexIVFPQFastScan.cpp
-index d069db13..09a335ff 100644
+index 2844ae49..895df342 100644
 --- a/faiss/IndexIVFPQFastScan.cpp
 +++ b/faiss/IndexIVFPQFastScan.cpp
 @@ -46,6 +46,8 @@ IndexIVFPQFastScan::IndexIVFPQFastScan(
@@ -302,13 +302,13 @@ index 00dd2f11..91f35a6e 100644
      /// same as the regular IVFPQ encoder. The codes are not reorganized by
      /// blocks a that point
 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
-index 9017edc5..0889bf72 100644
+index c888a5a6..83ecedfd 100644
 --- a/tests/CMakeLists.txt
 +++ b/tests/CMakeLists.txt
-@@ -33,6 +33,7 @@ set(FAISS_TEST_SRC
-   test_partitioning.cpp
-   test_fastscan_perf.cpp
+@@ -37,6 +37,7 @@ set(FAISS_TEST_SRC
    test_disable_pq_sdc_tables.cpp
+   test_common_ivf_empty_index.cpp
+   test_callback.cpp
 +  test_ivfpq_share_table.cpp
  )
 
@@ -508,5 +508,5 @@ index 00000000..f827315d
 +            "IVF16,PQ8x4fsr", "/tmp/ivfpqfsip", faiss::METRIC_INNER_PRODUCT);
 +}
 -- 
-2.39.3 (Apple Git-145)
+2.37.0