forked from opensearch-project/k-NN
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use Lucene HNSW searcher to search FAISS
- Loading branch information
Dooyong Kim
committed
Jan 1, 2025
1 parent
f13e9f4
commit ddeb38d
Showing
5 changed files
with
269 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
59 changes: 59 additions & 0 deletions
59
src/main/java/org/opensearch/knn/index/store/partial_loading/KdyFaissHnswGraph.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.knn.index.store.partial_loading; | ||
|
|
||
| import java.io.IOException; | ||
| import org.apache.lucene.util.hnsw.HnswGraph; | ||
| import org.apache.lucene.store.IndexInput; | ||
| import org.apache.lucene.util.hnsw.HnswGraph; | ||
|
|
||
| public class KdyFaissHnswGraph extends HnswGraph { | ||
| public FaissHNSW hnsw; | ||
| private long begin; | ||
| private long end; | ||
| private long currOffset; | ||
| private IndexInput indexInput; | ||
|
|
||
| public KdyFaissHnswGraph(KdyHNSW kdyHNSW, IndexInput indexInput) { | ||
| this.hnsw = kdyHNSW.hnswFlatIndex.hnsw; | ||
| this.indexInput = indexInput; | ||
| } | ||
|
|
||
| @Override public void seek(int level, int target) throws IOException { | ||
| long o = hnsw.offsets[target]; | ||
| begin = o + hnsw.cumNumberNeighborPerLevel[level]; | ||
| end = o + hnsw.cumNumberNeighborPerLevel[level + 1]; | ||
| currOffset = begin; | ||
| } | ||
|
|
||
| @Override public int size() { | ||
| return 1000_0000; | ||
| } | ||
|
|
||
| @Override public int nextNeighbor() throws IOException { | ||
| if (currOffset < end) { | ||
| final int id = hnsw.neighbors.readInt(indexInput, currOffset); | ||
| ++currOffset; | ||
| if (id >= 0) { | ||
| return id; | ||
| } | ||
| } | ||
|
|
||
| return Integer.MAX_VALUE; | ||
| } | ||
|
|
||
| @Override public int numLevels() throws IOException { | ||
| return hnsw.maxLevel; | ||
| } | ||
|
|
||
| @Override public int entryNode() throws IOException { | ||
| return hnsw.entryPoint; | ||
| } | ||
|
|
||
| @Override public HnswGraph.NodesIterator getNodesOnLevel(int level) throws IOException { | ||
| throw new UnsupportedOperationException(); | ||
| } | ||
| } |