diff --git a/CHANGELOG.md b/CHANGELOG.md index 52e15f5e8..5cb1665f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,4 +23,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Documentation * Update dev guide to fix clang linking issue on arm [#1746](https://github.com/opensearch-project/k-NN/pull/1746) ### Maintenance +* Bump faiss commit to 33c0ba5 [#1796](https://github.com/opensearch-project/k-NN/pull/1796) ### Refactoring diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 0ff5a0bdb..daab5b936 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -57,11 +57,11 @@ In addition to this, the plugin has been tested with JDK 17, and this JDK versio #### CMake -The plugin requires that cmake >= 3.23.3 is installed in order to build the JNI libraries. +The plugin requires that cmake >= 3.24.0 is installed in order to build the JNI libraries. One easy way to install on mac or linux is to use pip: ```bash -pip install cmake==3.23.3 +pip install cmake==3.24.0 ``` On Mac M series machines, install cmake using: diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 2fe26875d..3c071fc1f 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # -cmake_minimum_required(VERSION 3.23.1) +cmake_minimum_required(VERSION 3.24.0) project(KNNPlugin_JNI) diff --git a/jni/external/faiss b/jni/external/faiss index 12b92e9fa..33c0ba5d0 160000 --- a/jni/external/faiss +++ b/jni/external/faiss @@ -1 +1 @@ -Subproject commit 12b92e9fa5d8e8fb3da53c57af9ff007c826b1ee +Subproject commit 33c0ba5d002a7cd9761513f06ecc9822079d4a2f diff --git a/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch b/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch index 227630c63..de5806ef0 100644 --- a/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch +++ b/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch @@ -1,4 +1,4 @@ -From 35ef01f59b8903dfbd4d08ff874b085e851e4228 Mon Sep 17 00:00:00 2001 +From 9e5affabe2caacf38f5585a0b906620dd35deef5 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Tue, 30 Jan 2024 14:43:56 -0800 Subject: [PATCH] Add IDGrouper for HNSW @@ -13,12 +13,12 @@ Signed-off-by: Heemin Kim faiss/impl/HNSW.cpp | 6 + faiss/impl/IDGrouper.cpp | 51 ++++++++ faiss/impl/IDGrouper.h | 51 ++++++++ - faiss/impl/ResultHandler.h | 190 +++++++++++++++++++++++++++++ + faiss/impl/ResultHandler.h | 189 +++++++++++++++++++++++++++++ faiss/utils/GroupHeap.h | 182 ++++++++++++++++++++++++++++ tests/CMakeLists.txt | 2 + tests/test_group_heap.cpp | 98 +++++++++++++++ tests/test_id_grouper.cpp | 241 +++++++++++++++++++++++++++++++++++++ - 13 files changed, 891 insertions(+), 5 deletions(-) + 13 files changed, 890 insertions(+), 5 deletions(-) create mode 100644 faiss/impl/IDGrouper.cpp create mode 100644 faiss/impl/IDGrouper.h create mode 100644 faiss/utils/GroupHeap.h @@ -26,7 +26,7 @@ Signed-off-by: Heemin Kim create mode 100644 tests/test_id_grouper.cpp diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt -index a890a46f..137e68d4 100644 +index 1b0860f3..f3d72df3 100644 --- a/faiss/CMakeLists.txt +++ b/faiss/CMakeLists.txt @@ -54,6 +54,7 @@ set(FAISS_SRC @@ -45,10 +45,10 @@ index a890a46f..137e68d4 100644 impl/DistanceComputer.h impl/FaissAssert.h impl/FaissException.h -@@ -183,6 +185,7 @@ set(FAISS_HEADERS - invlists/InvertedLists.h +@@ -184,6 +186,7 @@ set(FAISS_HEADERS invlists/InvertedListsIOHook.h utils/AlignedTable.h + utils/bf16.h + utils/GroupHeap.h utils/Heap.h utils/WorkerThread.h @@ -81,10 +81,10 @@ index 3d1bdb99..a8622858 100644 virtual ~SearchParameters() {} }; diff --git a/faiss/IndexHNSW.cpp b/faiss/IndexHNSW.cpp -index 9a67332d..a5e0fea0 100644 +index 8e5c654f..d473b6ad 100644 --- a/faiss/IndexHNSW.cpp +++ b/faiss/IndexHNSW.cpp -@@ -354,10 +354,17 @@ void IndexHNSW::search( +@@ -320,10 +320,17 @@ void IndexHNSW::search( const SearchParameters* params_in) const { FAISS_THROW_IF_NOT(k > 0); @@ -198,10 +198,10 @@ index 2d164123..a68887bd 100644 + } // namespace faiss diff --git a/faiss/impl/HNSW.cpp b/faiss/impl/HNSW.cpp -index a9fb9daf..33b56638 100644 +index 3ba5f72f..c574ce39 100644 --- a/faiss/impl/HNSW.cpp +++ b/faiss/impl/HNSW.cpp -@@ -804,6 +804,12 @@ int extract_k_from_ResultHandler(ResultHandler& res) { +@@ -831,6 +831,12 @@ int extract_k_from_ResultHandler(ResultHandler& res) { if (auto hres = dynamic_cast(&res)) { return hres->k; } @@ -329,20 +329,19 @@ index 00000000..d56113d9 + +} // namespace faiss diff --git a/faiss/impl/ResultHandler.h b/faiss/impl/ResultHandler.h -index 270de8dc..3199634f 100644 +index 713fe8e4..d307fd70 100644 --- a/faiss/impl/ResultHandler.h +++ b/faiss/impl/ResultHandler.h -@@ -12,6 +12,9 @@ - #pragma once +@@ -13,6 +13,8 @@ #include -+#include + #include +#include +#include #include #include - -@@ -265,6 +268,193 @@ struct HeapBlockResultHandler : BlockResultHandler { + #include +@@ -267,6 +269,193 @@ struct HeapBlockResultHandler : BlockResultHandler { } }; @@ -726,7 +725,7 @@ index 00000000..3b7078da +} // namespace faiss \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt -index 9017edc5..a8e9d30c 100644 +index 3980d7dd..c888a5a6 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -27,6 +27,8 @@ set(FAISS_TEST_SRC @@ -1090,5 +1089,5 @@ index 00000000..6601795b + delete[] xb; +} -- -2.39.3 (Apple Git-145) +2.37.0 diff --git a/jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch b/jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch index dfc5099aa..619832f62 100644 --- a/jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch +++ b/jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch @@ -1,4 +1,4 @@ -From c5ca07299b427dedafc738b98bd20f8f286f6783 Mon Sep 17 00:00:00 2001 +From a33e6ef35385009f24200586294f96235cb95d61 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Wed, 21 Feb 2024 15:34:15 -0800 Subject: [PATCH] Enable precomp table to be shared ivfpq @@ -167,7 +167,7 @@ index d5d21da4..850bbe44 100644 }; diff --git a/faiss/IndexIVFPQFastScan.cpp b/faiss/IndexIVFPQFastScan.cpp -index d069db13..09a335ff 100644 +index 2844ae49..895df342 100644 --- a/faiss/IndexIVFPQFastScan.cpp +++ b/faiss/IndexIVFPQFastScan.cpp @@ -46,6 +46,8 @@ IndexIVFPQFastScan::IndexIVFPQFastScan( @@ -302,13 +302,13 @@ index 00dd2f11..91f35a6e 100644 /// same as the regular IVFPQ encoder. The codes are not reorganized by /// blocks a that point diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt -index 9017edc5..0889bf72 100644 +index c888a5a6..83ecedfd 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt -@@ -33,6 +33,7 @@ set(FAISS_TEST_SRC - test_partitioning.cpp - test_fastscan_perf.cpp +@@ -37,6 +37,7 @@ set(FAISS_TEST_SRC test_disable_pq_sdc_tables.cpp + test_common_ivf_empty_index.cpp + test_callback.cpp + test_ivfpq_share_table.cpp ) @@ -508,5 +508,5 @@ index 00000000..f827315d + "IVF16,PQ8x4fsr", "/tmp/ivfpqfsip", faiss::METRIC_INNER_PRODUCT); +} -- -2.39.3 (Apple Git-145) +2.37.0