From 7c2d48a38a190cb241921fe89e8823de4ce71082 Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 24 Feb 2022 22:16:07 -0500 Subject: [PATCH 1/3] 49 - allow read_index_with_flags for memory mapping some index types --- Cargo.toml | 2 +- src/index/io.rs | 31 +++++++++++++++++++++++++++++++ src/index/io_flags.rs | 24 ++++++++++++++++++++++++ src/index/mod.rs | 1 + 4 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 src/index/io_flags.rs diff --git a/Cargo.toml b/Cargo.toml index 445ff33..acf9b17 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "faiss" description = "High-level bindings for Faiss, the vector similarity search engine" -version = "0.10.1-alpha.0" +version = "0.11.1-alpha.0" authors = ["Eduardo Pinho "] license = "MIT/Apache-2.0" repository = "https://github.com/Enet4/faiss-rs" diff --git a/src/index/io.rs b/src/index/io.rs index d8e55a2..2134f4a 100644 --- a/src/index/io.rs +++ b/src/index/io.rs @@ -47,10 +47,34 @@ where } } +/// Read an index from a file with io flags. You can memory map some index types with this. +/// +/// # Error +/// +/// This function returns an error if the description contains any byte with the value `\0` (since +/// it cannot be converted to a C string), or if the internal index reading operation fails. +pub fn read_index_with_flags

(file_name: P, io_flags: u8) -> Result +where + P: AsRef, +{ + unsafe { + let f = file_name.as_ref(); + let f = CString::new(f).map_err(|_| Error::BadFilePath)?; + let mut inner = ptr::null_mut(); + faiss_try(faiss_read_index_fname( + f.as_ptr(), + io_flags as i32, + &mut inner, + ))?; + Ok(IndexImpl::from_inner_ptr(inner)) + } +} + #[cfg(test)] mod tests { use super::*; use crate::index::flat::FlatIndex; + use crate::index::io_flags::io_flag; use crate::index::Index; const D: u32 = 8; @@ -74,4 +98,11 @@ mod tests { assert_eq!(index.ntotal(), 5); ::std::fs::remove_file(&filepath).unwrap(); } + + #[test] + fn test_read_with_flags() { + let index = read_index_with_flags("file_name", io_flag::MEM_MAP | io_flag::READ_ONLY); + // we just want to ensure the method signature is right here + assert!(index.is_err()); + } } diff --git a/src/index/io_flags.rs b/src/index/io_flags.rs new file mode 100644 index 0000000..67f3f58 --- /dev/null +++ b/src/index/io_flags.rs @@ -0,0 +1,24 @@ +//! Module containing the io flags. + +/// Io Flags used during index reading, not all flags applicable to all indices +// This is a set of constants rather than enum so that bitwise operations exist +pub mod io_flag { + /// Load entire index into memory (default behavior) + pub const MEM_RESIDENT: u8 = 0x00; + /// Memory-map index + pub const MEM_MAP: u8 = 0x01; + /// Index is read-only + pub const READ_ONLY: u8 = 0x02; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn can_do_bitor() { + let mmap = io_flag::MEM_MAP; + let ro = io_flag::READ_ONLY; + assert_eq!(3, mmap | ro); + } +} diff --git a/src/index/mod.rs b/src/index/mod.rs index 39a97a5..a1ea8ce 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -25,6 +25,7 @@ pub mod autotune; pub mod flat; pub mod id_map; pub mod io; +pub mod io_flags; pub mod ivf_flat; pub mod lsh; pub mod pretransform; From dfab04e9bf127d50695afd1acf3e7df8e2a3e430 Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 24 Feb 2022 22:16:07 -0500 Subject: [PATCH 2/3] 49 - allow read_index_with_flags for memory mapping some index types --- Cargo.toml | 2 +- src/index/io.rs | 39 ++++++++++++++++++++++++++++++++++++++- src/index/io_flags.rs | 24 ++++++++++++++++++++++++ src/index/mod.rs | 1 + 4 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 src/index/io_flags.rs diff --git a/Cargo.toml b/Cargo.toml index 445ff33..acf9b17 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "faiss" description = "High-level bindings for Faiss, the vector similarity search engine" -version = "0.10.1-alpha.0" +version = "0.11.1-alpha.0" authors = ["Eduardo Pinho "] license = "MIT/Apache-2.0" repository = "https://github.com/Enet4/faiss-rs" diff --git a/src/index/io.rs b/src/index/io.rs index d8e55a2..651c7fd 100644 --- a/src/index/io.rs +++ b/src/index/io.rs @@ -7,6 +7,8 @@ use faiss_sys::*; use std::ffi::CString; use std::ptr; +use super::io_flags::io_flag; + /// Write an index to a file. /// /// # Error @@ -42,7 +44,34 @@ where let f = file_name.as_ref(); let f = CString::new(f).map_err(|_| Error::BadFilePath)?; let mut inner = ptr::null_mut(); - faiss_try(faiss_read_index_fname(f.as_ptr(), 0, &mut inner))?; + faiss_try(faiss_read_index_fname( + f.as_ptr(), + io_flag::MEM_RESIDENT as i32, + &mut inner, + ))?; + Ok(IndexImpl::from_inner_ptr(inner)) + } +} + +/// Read an index from a file with io flags. You can memory map some index types with this. +/// +/// # Error +/// +/// This function returns an error if the description contains any byte with the value `\0` (since +/// it cannot be converted to a C string), or if the internal index reading operation fails. +pub fn read_index_with_flags

(file_name: P, io_flags: u8) -> Result +where + P: AsRef, +{ + unsafe { + let f = file_name.as_ref(); + let f = CString::new(f).map_err(|_| Error::BadFilePath)?; + let mut inner = ptr::null_mut(); + faiss_try(faiss_read_index_fname( + f.as_ptr(), + io_flags as i32, + &mut inner, + ))?; Ok(IndexImpl::from_inner_ptr(inner)) } } @@ -51,6 +80,7 @@ where mod tests { use super::*; use crate::index::flat::FlatIndex; + use crate::index::io_flags::io_flag; use crate::index::Index; const D: u32 = 8; @@ -74,4 +104,11 @@ mod tests { assert_eq!(index.ntotal(), 5); ::std::fs::remove_file(&filepath).unwrap(); } + + #[test] + fn test_read_with_flags() { + let index = read_index_with_flags("file_name", io_flag::MEM_MAP | io_flag::READ_ONLY); + // we just want to ensure the method signature is right here + assert!(index.is_err()); + } } diff --git a/src/index/io_flags.rs b/src/index/io_flags.rs new file mode 100644 index 0000000..67f3f58 --- /dev/null +++ b/src/index/io_flags.rs @@ -0,0 +1,24 @@ +//! Module containing the io flags. + +/// Io Flags used during index reading, not all flags applicable to all indices +// This is a set of constants rather than enum so that bitwise operations exist +pub mod io_flag { + /// Load entire index into memory (default behavior) + pub const MEM_RESIDENT: u8 = 0x00; + /// Memory-map index + pub const MEM_MAP: u8 = 0x01; + /// Index is read-only + pub const READ_ONLY: u8 = 0x02; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn can_do_bitor() { + let mmap = io_flag::MEM_MAP; + let ro = io_flag::READ_ONLY; + assert_eq!(3, mmap | ro); + } +} diff --git a/src/index/mod.rs b/src/index/mod.rs index 39a97a5..a1ea8ce 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -25,6 +25,7 @@ pub mod autotune; pub mod flat; pub mod id_map; pub mod io; +pub mod io_flags; pub mod ivf_flat; pub mod lsh; pub mod pretransform; From 4481cf41f4a94bc15b80918620e757314581fff3 Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 24 Mar 2022 20:09:29 -0400 Subject: [PATCH 3/3] IoFlags Struct impl BitOr --- src/index/io.rs | 34 +++++----------------------------- src/index/io_flags.rs | 43 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 41 insertions(+), 36 deletions(-) diff --git a/src/index/io.rs b/src/index/io.rs index 36d4510..1561652 100644 --- a/src/index/io.rs +++ b/src/index/io.rs @@ -7,7 +7,7 @@ use faiss_sys::*; use std::ffi::CString; use std::ptr; -use super::io_flags::io_flag; +use super::io_flags::IoFlags; /// Write an index to a file. /// @@ -46,7 +46,7 @@ where let mut inner = ptr::null_mut(); faiss_try(faiss_read_index_fname( f.as_ptr(), - io_flag::MEM_RESIDENT as i32, + IoFlags::MEM_RESIDENT.into(), &mut inner, ))?; Ok(IndexImpl::from_inner_ptr(inner)) @@ -59,7 +59,7 @@ where /// /// This function returns an error if the description contains any byte with the value `\0` (since /// it cannot be converted to a C string), or if the internal index reading operation fails. -pub fn read_index_with_flags

(file_name: P, io_flags: u8) -> Result +pub fn read_index_with_flags

(file_name: P, io_flags: IoFlags) -> Result where P: AsRef, { @@ -69,30 +69,7 @@ where let mut inner = ptr::null_mut(); faiss_try(faiss_read_index_fname( f.as_ptr(), - io_flags as i32, - &mut inner, - ))?; - Ok(IndexImpl::from_inner_ptr(inner)) - } -} - -/// Read an index from a file with io flags. You can memory map some index types with this. -/// -/// # Error -/// -/// This function returns an error if the description contains any byte with the value `\0` (since -/// it cannot be converted to a C string), or if the internal index reading operation fails. -pub fn read_index_with_flags

(file_name: P, io_flags: u8) -> Result -where - P: AsRef, -{ - unsafe { - let f = file_name.as_ref(); - let f = CString::new(f).map_err(|_| Error::BadFilePath)?; - let mut inner = ptr::null_mut(); - faiss_try(faiss_read_index_fname( - f.as_ptr(), - io_flags as i32, + io_flags.into(), &mut inner, ))?; Ok(IndexImpl::from_inner_ptr(inner)) @@ -103,7 +80,6 @@ where mod tests { use super::*; use crate::index::flat::FlatIndex; - use crate::index::io_flags::io_flag; use crate::index::Index; const D: u32 = 8; @@ -130,7 +106,7 @@ mod tests { #[test] fn test_read_with_flags() { - let index = read_index_with_flags("file_name", io_flag::MEM_MAP | io_flag::READ_ONLY); + let index = read_index_with_flags("file_name", IoFlags::MEM_MAP | IoFlags::READ_ONLY); // we just want to ensure the method signature is right here assert!(index.is_err()); } diff --git a/src/index/io_flags.rs b/src/index/io_flags.rs index 67f3f58..d65a2b9 100644 --- a/src/index/io_flags.rs +++ b/src/index/io_flags.rs @@ -2,13 +2,36 @@ /// Io Flags used during index reading, not all flags applicable to all indices // This is a set of constants rather than enum so that bitwise operations exist -pub mod io_flag { +#[derive(Debug, Copy, Clone, Eq, Hash, PartialEq)] +pub struct IoFlags(u16); + +impl IoFlags { /// Load entire index into memory (default behavior) - pub const MEM_RESIDENT: u8 = 0x00; + pub const MEM_RESIDENT: Self = IoFlags(0x00); /// Memory-map index - pub const MEM_MAP: u8 = 0x01; + pub const MEM_MAP: Self = IoFlags(0x01); /// Index is read-only - pub const READ_ONLY: u8 = 0x02; + pub const READ_ONLY: Self = IoFlags(0x02); +} + +impl std::ops::BitOr for IoFlags { + type Output = Self; + + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl From for IoFlags { + fn from(n: i32) -> IoFlags { + IoFlags(n as u16) + } +} + +impl From for i32 { + fn from(io_flag: IoFlags) -> i32 { + io_flag.0 as i32 + } } #[cfg(test)] @@ -17,8 +40,14 @@ mod tests { #[test] fn can_do_bitor() { - let mmap = io_flag::MEM_MAP; - let ro = io_flag::READ_ONLY; - assert_eq!(3, mmap | ro); + let mmap = IoFlags::MEM_MAP; + let ro = IoFlags::READ_ONLY; + assert_eq!(IoFlags(0x03), mmap | ro); + } + + #[test] + fn can_coerce_to_i32() { + let mmap = IoFlags::MEM_MAP; + assert_eq!(1, mmap.into()); } }