From b58b950a9b726e689932c8d03fbe0a58ffa6dcb4 Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Fri, 11 Oct 2024 12:11:32 -0500 Subject: [PATCH] Bump Faiss commit from 33c0ba5 to 4eecd91 (#2194) * Bump Faiss commit from 33c0ba5 to 4eecd91 Signed-off-by: Naveen Tatikonda * Update Faiss patches after commit bump Signed-off-by: Naveen Tatikonda --------- Signed-off-by: Naveen Tatikonda (cherry picked from commit d9c7ba5e1b25856cd52e33aa54b37f1c3eb6a882) --- jni/external/faiss | 2 +- ...Custom-patch-to-support-multi-vector.patch | 46 +++++++++---------- ...ble-precomp-table-to-be-shared-ivfpq.patch | 22 ++++----- 3 files changed, 34 insertions(+), 36 deletions(-) diff --git a/jni/external/faiss b/jni/external/faiss index 33c0ba5d0..4eecd9165 160000 --- a/jni/external/faiss +++ b/jni/external/faiss @@ -1 +1 @@ -Subproject commit 33c0ba5d002a7cd9761513f06ecc9822079d4a2f +Subproject commit 4eecd9165ae56bc8ff4afbf14cc8b359fdfe4004 diff --git a/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch b/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch index 7afdabc1f..26b143346 100644 --- a/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch +++ b/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch @@ -1,4 +1,4 @@ -From 9e5affabe2caacf38f5585a0b906620dd35deef5 Mon Sep 17 00:00:00 2001 +From e775a8e65da96232822d5aed77f538592fccffda Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Tue, 30 Jan 2024 14:43:56 -0800 Subject: [PATCH] Add IDGrouper for HNSW @@ -6,7 +6,7 @@ Subject: [PATCH] Add IDGrouper for HNSW Signed-off-by: Heemin Kim --- faiss/CMakeLists.txt | 3 + - faiss/Index.h | 8 +- + faiss/Index.h | 6 +- faiss/IndexHNSW.cpp | 13 +- faiss/IndexIDMap.cpp | 29 +++++ faiss/IndexIDMap.h | 22 ++++ @@ -18,7 +18,7 @@ Signed-off-by: Heemin Kim tests/CMakeLists.txt | 2 + tests/test_group_heap.cpp | 98 +++++++++++++++ tests/test_id_grouper.cpp | 241 +++++++++++++++++++++++++++++++++++++ - 13 files changed, 890 insertions(+), 5 deletions(-) + 13 files changed, 889 insertions(+), 4 deletions(-) create mode 100644 faiss/impl/IDGrouper.cpp create mode 100644 faiss/impl/IDGrouper.h create mode 100644 faiss/utils/GroupHeap.h @@ -26,10 +26,10 @@ Signed-off-by: Heemin Kim create mode 100644 tests/test_id_grouper.cpp diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt -index 1b0860f3..f3d72df3 100644 +index 2871d974..d0bcec6a 100644 --- a/faiss/CMakeLists.txt +++ b/faiss/CMakeLists.txt -@@ -54,6 +54,7 @@ set(FAISS_SRC +@@ -55,6 +55,7 @@ set(FAISS_SRC impl/AuxIndexStructures.cpp impl/CodePacker.cpp impl/IDSelector.cpp @@ -37,7 +37,7 @@ index 1b0860f3..f3d72df3 100644 impl/FaissException.cpp impl/HNSW.cpp impl/NSG.cpp -@@ -149,6 +150,7 @@ set(FAISS_HEADERS +@@ -151,6 +152,7 @@ set(FAISS_HEADERS impl/AuxIndexStructures.h impl/CodePacker.h impl/IDSelector.h @@ -45,7 +45,7 @@ index 1b0860f3..f3d72df3 100644 impl/DistanceComputer.h impl/FaissAssert.h impl/FaissException.h -@@ -184,6 +186,7 @@ set(FAISS_HEADERS +@@ -186,6 +188,7 @@ set(FAISS_HEADERS invlists/InvertedListsIOHook.h utils/AlignedTable.h utils/bf16.h @@ -54,23 +54,21 @@ index 1b0860f3..f3d72df3 100644 utils/WorkerThread.h utils/distances.h diff --git a/faiss/Index.h b/faiss/Index.h -index 3d1bdb99..a8622858 100644 +index f57140ec..8f511e5d 100644 --- a/faiss/Index.h +++ b/faiss/Index.h -@@ -38,9 +38,10 @@ - +@@ -51,8 +51,9 @@ namespace faiss { --/// Forward declarations see impl/AuxIndexStructures.h, impl/IDSelector.h and --/// impl/DistanceComputer.h -+/// Forward declarations see impl/AuxIndexStructures.h, impl/IDSelector.h + /// Forward declarations see impl/AuxIndexStructures.h, impl/IDSelector.h +-/// and impl/DistanceComputer.h +/// ,impl/IDGrouper.h and impl/DistanceComputer.h struct IDSelector; +struct IDGrouper; struct RangeSearchResult; struct DistanceComputer; -@@ -52,6 +53,9 @@ struct DistanceComputer; +@@ -64,6 +65,9 @@ struct DistanceComputer; struct SearchParameters { /// if non-null, only these IDs will be considered during search. IDSelector* sel = nullptr; @@ -81,10 +79,10 @@ index 3d1bdb99..a8622858 100644 virtual ~SearchParameters() {} }; diff --git a/faiss/IndexHNSW.cpp b/faiss/IndexHNSW.cpp -index 8e5c654f..d473b6ad 100644 +index 6a1186ca..9c8a8255 100644 --- a/faiss/IndexHNSW.cpp +++ b/faiss/IndexHNSW.cpp -@@ -320,10 +320,17 @@ void IndexHNSW::search( +@@ -301,10 +301,17 @@ void IndexHNSW::search( const SearchParameters* params_in) const { FAISS_THROW_IF_NOT(k > 0); @@ -198,10 +196,10 @@ index 2d164123..a68887bd 100644 + } // namespace faiss diff --git a/faiss/impl/HNSW.cpp b/faiss/impl/HNSW.cpp -index 3ba5f72f..c574ce39 100644 +index c3693fd9..7ae28062 100644 --- a/faiss/impl/HNSW.cpp +++ b/faiss/impl/HNSW.cpp -@@ -831,6 +831,12 @@ int extract_k_from_ResultHandler(ResultHandler& res) { +@@ -906,6 +906,12 @@ int extract_k_from_ResultHandler(ResultHandler& res) { if (auto hres = dynamic_cast(&res)) { return hres->k; } @@ -329,19 +327,19 @@ index 00000000..d56113d9 + +} // namespace faiss diff --git a/faiss/impl/ResultHandler.h b/faiss/impl/ResultHandler.h -index 713fe8e4..d307fd70 100644 +index 3116eb24..126ed015 100644 --- a/faiss/impl/ResultHandler.h +++ b/faiss/impl/ResultHandler.h -@@ -13,6 +13,8 @@ - +@@ -14,6 +14,8 @@ #include #include + #include +#include +#include #include #include - #include -@@ -267,6 +269,193 @@ struct HeapBlockResultHandler : BlockResultHandler { + +@@ -286,6 +288,193 @@ struct HeapBlockResultHandler : BlockResultHandler { } }; @@ -725,7 +723,7 @@ index 00000000..3b7078da +} // namespace faiss \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt -index 3980d7dd..c888a5a6 100644 +index c41edf0c..87ab2020 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -27,6 +27,8 @@ set(FAISS_TEST_SRC diff --git a/jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch b/jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch index 619832f62..88e6a8106 100644 --- a/jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch +++ b/jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch @@ -1,4 +1,4 @@ -From a33e6ef35385009f24200586294f96235cb95d61 Mon Sep 17 00:00:00 2001 +From 9b33874562c9e62abf4a863657c54f0d349b0f67 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Wed, 21 Feb 2024 15:34:15 -0800 Subject: [PATCH] Enable precomp table to be shared ivfpq @@ -22,7 +22,7 @@ Signed-off-by: John Mazanec create mode 100644 tests/test_ivfpq_share_table.cpp diff --git a/faiss/IndexIVFPQ.cpp b/faiss/IndexIVFPQ.cpp -index 0b7f4d05..07bc7e83 100644 +index 100f499c..09508890 100644 --- a/faiss/IndexIVFPQ.cpp +++ b/faiss/IndexIVFPQ.cpp @@ -59,6 +59,29 @@ IndexIVFPQ::IndexIVFPQ( @@ -55,7 +55,7 @@ index 0b7f4d05..07bc7e83 100644 } /**************************************************************** -@@ -466,11 +489,23 @@ void IndexIVFPQ::precompute_table() { +@@ -464,11 +487,23 @@ void IndexIVFPQ::precompute_table() { use_precomputed_table, quantizer, pq, @@ -80,7 +80,7 @@ index 0b7f4d05..07bc7e83 100644 namespace { #define TIC t0 = get_cycles() -@@ -650,7 +685,7 @@ struct QueryTables { +@@ -648,7 +683,7 @@ struct QueryTables { fvec_madd( pq.M * pq.ksub, @@ -89,7 +89,7 @@ index 0b7f4d05..07bc7e83 100644 -2.0, sim_table_2, sim_table); -@@ -679,7 +714,7 @@ struct QueryTables { +@@ -677,7 +712,7 @@ struct QueryTables { k >>= cpq.nbits; // get corresponding table @@ -98,7 +98,7 @@ index 0b7f4d05..07bc7e83 100644 (ki * pq.M + cm * Mf) * pq.ksub; if (polysemous_ht == 0) { -@@ -709,7 +744,7 @@ struct QueryTables { +@@ -707,7 +742,7 @@ struct QueryTables { dis0 = coarse_dis; const float* s = @@ -107,7 +107,7 @@ index 0b7f4d05..07bc7e83 100644 for (int m = 0; m < pq.M; m++) { sim_table_ptrs[m] = s; s += pq.ksub; -@@ -729,7 +764,7 @@ struct QueryTables { +@@ -727,7 +762,7 @@ struct QueryTables { int ki = k & ((uint64_t(1) << cpq.nbits) - 1); k >>= cpq.nbits; @@ -116,7 +116,7 @@ index 0b7f4d05..07bc7e83 100644 (ki * pq.M + cm * Mf) * pq.ksub; for (int m = m0; m < m0 + Mf; m++) { -@@ -1346,6 +1381,8 @@ IndexIVFPQ::IndexIVFPQ() { +@@ -1344,6 +1379,8 @@ IndexIVFPQ::IndexIVFPQ() { do_polysemous_training = false; polysemous_ht = 0; polysemous_training = nullptr; @@ -302,13 +302,13 @@ index 00dd2f11..91f35a6e 100644 /// same as the regular IVFPQ encoder. The codes are not reorganized by /// blocks a that point diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt -index c888a5a6..83ecedfd 100644 +index 87ab2020..a859516c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt -@@ -37,6 +37,7 @@ set(FAISS_TEST_SRC - test_disable_pq_sdc_tables.cpp +@@ -38,6 +38,7 @@ set(FAISS_TEST_SRC test_common_ivf_empty_index.cpp test_callback.cpp + test_utils.cpp + test_ivfpq_share_table.cpp )