-
Notifications
You must be signed in to change notification settings - Fork 3.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
13 changed files
with
838 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| /** | ||
| * Copyright (c) Facebook, Inc. and its affiliates. | ||
| * | ||
| * This source code is licensed under the MIT license found in the | ||
| * LICENSE file in the root directory of this source tree. | ||
| */ | ||
|
|
||
| #include <assert.h> | ||
| #include <faiss/impl/FaissAssert.h> | ||
| #include <faiss/impl/IDGrouper.h> | ||
|
|
||
| namespace faiss { | ||
|
|
||
| /*********************************************************************** | ||
| * IDGrouperBitmap | ||
| ***********************************************************************/ | ||
|
|
||
| IDGrouperBitmap::IDGrouperBitmap(size_t n, uint64_t* bitmap) | ||
| : n(n), bitmap(bitmap) {} | ||
|
|
||
| idx_t IDGrouperBitmap::get_group(idx_t id) const { | ||
| assert(id >= 0 && "id shouldn't be less than zero"); | ||
| assert(id < this->n * 64 && "is should be less than total number of bits"); | ||
|
|
||
| idx_t index = id >> 6; // div by 64 | ||
| uint64_t block = this->bitmap[index] >> | ||
| (id & 63); // Equivalent of words[i] >> (index % 64) | ||
| // block is non zero after right shift, it means, next set bit is in current | ||
| // block The index of set bit is "given index" + "trailing zero in the right | ||
| // shifted word" | ||
| if (block != 0) { | ||
| return id + __builtin_ctzll(block); | ||
| } | ||
|
|
||
| while (++index < this->n) { | ||
| block = this->bitmap[index]; | ||
| if (block != 0) { | ||
| return (index << 6) + __builtin_ctzll(block); | ||
| } | ||
| } | ||
|
|
||
| return NO_MORE_DOCS; | ||
| } | ||
|
|
||
| void IDGrouperBitmap::set_group(idx_t group_id) { | ||
| idx_t index = group_id >> 6; | ||
| this->bitmap[index] |= 1ULL | ||
| << (group_id & 63); // Equivalent of 1ULL << (value % 64) | ||
| } | ||
|
|
||
| } // namespace faiss |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| /** | ||
| * Copyright (c) Facebook, Inc. and its affiliates. | ||
| * | ||
| * This source code is licensed under the MIT license found in the | ||
| * LICENSE file in the root directory of this source tree. | ||
| */ | ||
|
|
||
| #pragma once | ||
|
|
||
| #include <limits> | ||
| #include <unordered_set> | ||
| #include <vector> | ||
|
|
||
| #include <faiss/MetricType.h> | ||
|
|
||
| /** IDGrouper is intended to define a group of vectors to include only | ||
| * the nearest vector of each group during search */ | ||
|
|
||
| namespace faiss { | ||
|
|
||
| /** Encapsulates a group id of ids */ | ||
| struct IDGrouper { | ||
| const idx_t NO_MORE_DOCS = std::numeric_limits<idx_t>::max(); | ||
| virtual idx_t get_group(idx_t id) const = 0; | ||
| virtual ~IDGrouper() {} | ||
| }; | ||
|
|
||
| /** One bit per element. Constructed with a bitmap, size ceil(n / 8). | ||
| */ | ||
| struct IDGrouperBitmap : IDGrouper { | ||
| // length of the bitmap array | ||
| size_t n; | ||
|
|
||
| // Array of uint64_t holding the bits | ||
| // Using uint64_t to leverage function __builtin_ctzll which is defined in | ||
| // faiss/impl/platform_macros.h Group id of a given id is next set bit in | ||
| // the bitmap | ||
| uint64_t* bitmap; | ||
|
|
||
| /** Construct with a binary mask | ||
| * | ||
| * @param n size of the bitmap array | ||
| * @param bitmap group id of a given id is next set bit in the bitmap | ||
| */ | ||
| IDGrouperBitmap(size_t n, uint64_t* bitmap); | ||
| idx_t get_group(idx_t id) const final; | ||
| void set_group(idx_t group_id); | ||
| ~IDGrouperBitmap() override {} | ||
| }; | ||
|
|
||
| } // namespace faiss |
Oops, something went wrong.