-
Notifications
You must be signed in to change notification settings - Fork 3.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add example of how to build, link, and test an external SWIG module (#…
…3922) Summary: Pull Request resolved: #3922 We need to be able to build external modules into FAISS, but don't have an example yet. This diff shows what CMakeLists.txt changes need to happen to incorporate an external module. Reference: #3699 Reviewed By: mdouze Differential Revision: D63991471 fbshipit-source-id: 0c1cd25eabbffb01d2a7170d6725a0c4a13c5bf0
- Loading branch information
1 parent
a99dbcd
commit 847cde8
Showing
9 changed files
with
301 additions
and
29 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,133 @@ | ||
|
|
||
| %module faiss_example_external_module; | ||
|
|
||
|
|
||
| // Put C++ includes here | ||
| %{ | ||
|
|
||
| #include <faiss/impl/FaissException.h> | ||
| #include <faiss/impl/IDSelector.h> | ||
|
|
||
| %} | ||
|
|
||
| #pragma SWIG nowarn=322 | ||
|
|
||
| typedef unsigned char uint8_t; | ||
| typedef unsigned short uint16_t; | ||
| typedef unsigned int uint32_t; | ||
|
|
||
| typedef signed char int8_t; | ||
| typedef short int16_t; | ||
| typedef int int32_t; | ||
|
|
||
| #ifdef SWIGWORDSIZE64 | ||
| typedef unsigned long uint64_t; | ||
| typedef long int64_t; | ||
| #else | ||
| typedef unsigned long long uint64_t; | ||
| typedef long long int64_t; | ||
| #endif | ||
|
|
||
| typedef uint64_t size_t; | ||
|
|
||
| // This means: assume what's declared in these .h files is provided | ||
| // by the Faiss module. | ||
| %import(module="faiss") "faiss/MetricType.h" | ||
| %import(module="faiss") "faiss/impl/IDSelector.h" | ||
|
|
||
| // functions to be parsed here | ||
|
|
||
| // This is important to release GIL and do Faiss exception handing | ||
| %exception { | ||
| Py_BEGIN_ALLOW_THREADS | ||
| try { | ||
| $action | ||
| } catch(faiss::FaissException & e) { | ||
| PyEval_RestoreThread(_save); | ||
|
|
||
| if (PyErr_Occurred()) { | ||
| // some previous code already set the error type. | ||
| } else { | ||
| PyErr_SetString(PyExc_RuntimeError, e.what()); | ||
| } | ||
| SWIG_fail; | ||
| } catch(std::bad_alloc & ba) { | ||
| PyEval_RestoreThread(_save); | ||
| PyErr_SetString(PyExc_MemoryError, "std::bad_alloc"); | ||
| SWIG_fail; | ||
| } | ||
| Py_END_ALLOW_THREADS | ||
| } | ||
|
|
||
|
|
||
| // any class or function declared below will be made available | ||
| // in the module. | ||
| %inline %{ | ||
|
|
||
| struct IDSelectorModulo : faiss::IDSelector { | ||
| int mod; | ||
|
|
||
| IDSelectorModulo(int mod): mod(mod) {} | ||
|
|
||
| bool is_member(faiss::idx_t id) const { | ||
| return id % mod == 0; | ||
| } | ||
|
|
||
| ~IDSelectorModulo() override {} | ||
| }; | ||
|
|
||
| faiss::idx_t sum_of_idx(size_t n, const faiss::idx_t *tab) { | ||
| faiss::idx_t sum = 0; | ||
| for(size_t i = 0; i < n; i++) { | ||
| sum += tab[i]; | ||
| } | ||
| return sum; | ||
| } | ||
|
|
||
| float sum_of_float32(size_t n, const float *tab) { | ||
| float sum = 0; | ||
| for(size_t i = 0; i < n; i++) { | ||
| sum += tab[i]; | ||
| } | ||
| return sum; | ||
| } | ||
|
|
||
| double sum_of_float64(size_t n, const double *tab) { | ||
| double sum = 0; | ||
| for(size_t i = 0; i < n; i++) { | ||
| sum += tab[i]; | ||
| } | ||
| return sum; | ||
| } | ||
|
|
||
| %} | ||
|
|
||
| /********************************************** | ||
| * To test if passing a swig_ptr on all array types works | ||
| **********************************************/ | ||
|
|
||
| %define SUM_OF_TYPE(ty) | ||
|
|
||
| %inline %{ | ||
|
|
||
| ty##_t sum_of_##ty (size_t n, const ty##_t * tab) { | ||
| ty##_t sum = 0; | ||
| for(size_t i = 0; i < n; i++) { | ||
| sum += tab[i]; | ||
| } | ||
| return sum; | ||
| } | ||
|
|
||
| %} | ||
|
|
||
| %enddef | ||
|
|
||
| SUM_OF_TYPE(uint8); | ||
| SUM_OF_TYPE(uint16); | ||
| SUM_OF_TYPE(uint32); | ||
| SUM_OF_TYPE(uint64); | ||
|
|
||
| SUM_OF_TYPE(int8); | ||
| SUM_OF_TYPE(int16); | ||
| SUM_OF_TYPE(int32); | ||
| SUM_OF_TYPE(int64); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,10 +4,12 @@ | |
| # LICENSE file in the root directory of this source tree. | ||
|
|
||
| from __future__ import print_function | ||
| from setuptools import setup, find_packages | ||
|
|
||
| import os | ||
| import shutil | ||
| import platform | ||
| import shutil | ||
|
|
||
| from setuptools import find_packages, setup | ||
|
|
||
| # make the faiss python package dir | ||
| shutil.rmtree("faiss", ignore_errors=True) | ||
|
|
@@ -20,25 +22,32 @@ | |
| shutil.copyfile("extra_wrappers.py", "faiss/extra_wrappers.py") | ||
| shutil.copyfile("array_conversions.py", "faiss/array_conversions.py") | ||
|
|
||
| ext = ".pyd" if platform.system() == 'Windows' else ".so" | ||
| prefix = "Release/" * (platform.system() == 'Windows') | ||
| ext = ".pyd" if platform.system() == "Windows" else ".so" | ||
| prefix = "Release/" * (platform.system() == "Windows") | ||
|
|
||
| swigfaiss_generic_lib = f"{prefix}_swigfaiss{ext}" | ||
| swigfaiss_avx2_lib = f"{prefix}_swigfaiss_avx2{ext}" | ||
| swigfaiss_avx512_lib = f"{prefix}_swigfaiss_avx512{ext}" | ||
| callbacks_lib = f"{prefix}libfaiss_python_callbacks{ext}" | ||
| swigfaiss_sve_lib = f"{prefix}_swigfaiss_sve{ext}" | ||
| faiss_example_external_module_lib = f"_faiss_example_external_module{ext}" | ||
|
|
||
| found_swigfaiss_generic = os.path.exists(swigfaiss_generic_lib) | ||
| found_swigfaiss_avx2 = os.path.exists(swigfaiss_avx2_lib) | ||
| found_swigfaiss_avx512 = os.path.exists(swigfaiss_avx512_lib) | ||
| found_callbacks = os.path.exists(callbacks_lib) | ||
| found_swigfaiss_sve = os.path.exists(swigfaiss_sve_lib) | ||
| found_faiss_example_external_module_lib = os.path.exists( | ||
| faiss_example_external_module_lib | ||
| ) | ||
|
|
||
| assert (found_swigfaiss_generic or found_swigfaiss_avx2 or found_swigfaiss_avx512 or found_swigfaiss_sve), \ | ||
| f"Could not find {swigfaiss_generic_lib} or " \ | ||
| f"{swigfaiss_avx2_lib} or {swigfaiss_avx512_lib} or {swigfaiss_sve_lib}. " \ | ||
| assert ( | ||
| found_swigfaiss_generic or found_swigfaiss_avx2 or found_swigfaiss_avx512 or found_swigfaiss_sve or found_faiss_example_external_module_lib | ||
| ), ( | ||
| f"Could not find {swigfaiss_generic_lib} or " | ||
| f"{swigfaiss_avx2_lib} or {swigfaiss_avx512_lib} or {swigfaiss_sve_lib} or {faiss_example_external_module_lib}. " | ||
| f"Faiss may not be compiled yet." | ||
| ) | ||
|
|
||
| if found_swigfaiss_generic: | ||
| print(f"Copying {swigfaiss_generic_lib}") | ||
|
|
@@ -64,7 +73,17 @@ | |
| shutil.copyfile("swigfaiss_sve.py", "faiss/swigfaiss_sve.py") | ||
| shutil.copyfile(swigfaiss_sve_lib, f"faiss/_swigfaiss_sve{ext}") | ||
|
|
||
| long_description=""" | ||
| if found_faiss_example_external_module_lib: | ||
| print(f"Copying {faiss_example_external_module_lib}") | ||
| shutil.copyfile( | ||
| "faiss_example_external_module.py", "faiss/faiss_example_external_module.py" | ||
| ) | ||
| shutil.copyfile( | ||
| faiss_example_external_module_lib, | ||
| f"faiss/_faiss_example_external_module{ext}", | ||
| ) | ||
|
|
||
| long_description = """ | ||
| Faiss is a library for efficient similarity search and clustering of dense | ||
| vectors. It contains algorithms that search in sets of vectors of any size, | ||
| up to ones that possibly do not fit in RAM. It also contains supporting | ||
|
|
@@ -73,20 +92,19 @@ | |
| are implemented on the GPU. It is developed by Facebook AI Research. | ||
| """ | ||
| setup( | ||
| name='faiss', | ||
| version='1.9.0', | ||
| description='A library for efficient similarity search and clustering of dense vectors', | ||
| name="faiss", | ||
| version="1.9.0", | ||
| description="A library for efficient similarity search and clustering of dense vectors", | ||
| long_description=long_description, | ||
| url='https://github.com/facebookresearch/faiss', | ||
| author='Matthijs Douze, Jeff Johnson, Herve Jegou, Lucas Hosseini', | ||
| author_email='[email protected]', | ||
| license='MIT', | ||
| keywords='search nearest neighbors', | ||
|
|
||
| install_requires=['numpy', 'packaging'], | ||
| packages=['faiss', 'faiss.contrib', 'faiss.contrib.torch'], | ||
| url="https://github.com/facebookresearch/faiss", | ||
| author="Matthijs Douze, Jeff Johnson, Herve Jegou, Lucas Hosseini", | ||
| author_email="[email protected]", | ||
| license="MIT", | ||
| keywords="search nearest neighbors", | ||
| install_requires=["numpy", "packaging"], | ||
| packages=["faiss", "faiss.contrib", "faiss.contrib.torch"], | ||
| package_data={ | ||
| 'faiss': ['*.so', '*.pyd'], | ||
| "faiss": ["*.so", "*.pyd"], | ||
| }, | ||
| zip_safe=False, | ||
| ) | ||
Oops, something went wrong.