reorganize traits to allow proper wasm compilation

nix/sources.json

@@ -17,10 +17,10 @@
         "homepage": "",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "46821ea01c8f54d2a20f5a503809abfc605269d7",
-        "sha256": "093daxh4yib44ddq2xgbnvk0kxgk754sfagqiplgd5rh7zmb1bpb",
+        "rev": "8ca77a63599ed951d6a2d244c1d62092776a3fe1",
+        "sha256": "11yc4lnp24x7mkn495impsx6rcmmc5m4ym44yjqihvpdz9yb0w9d",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/46821ea01c8f54d2a20f5a503809abfc605269d7.tar.gz",
+        "url": "https://github.com/NixOS/nixpkgs/archive/8ca77a63599ed951d6a2d244c1d62092776a3fe1.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "rust-overlay": {
@@ -29,10 +29,10 @@
         "homepage": null,
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "27fb59f3f4c687d599ec63a6c328e8432cd61101",
-        "sha256": "0mxqj0zrm4qyrjdhgyx243x2wknrp3032x91a6nff2zbfnc64wn2",
+        "rev": "9fb49daf1bbe1d91e6c837706c481f9ebb3d8097",
+        "sha256": "1h8v9346kw70glmsg58dz3fa260iy38p9kdf73nxphnnf6dy2yd4",
         "type": "tarball",
-        "url": "https://github.com/oxalica/rust-overlay/archive/27fb59f3f4c687d599ec63a6c328e8432cd61101.tar.gz",
+        "url": "https://github.com/oxalica/rust-overlay/archive/9fb49daf1bbe1d91e6c837706c481f9ebb3d8097.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     }
 }

shell.nix

@@ -6,10 +6,16 @@ in
   with pkgs;
 
   pkgs.mkShell {
+    nativeBuildInputs = [
+      clang_13
+    ];
+
     buildInputs = [
       rustPlatform.rust.cargo
+      openssl
+      pkg-config
+
       git
-      stdenv.cc.cc.lib
       (python38.withPackages(ps: with ps; [ virtualenv tox setuptools ]))
       (python39.withPackages(ps: with ps; [ virtualenv setuptools ]))
       (python37.withPackages(ps: with ps; [ virtualenv setuptools ]))
@@ -25,11 +31,14 @@ in
       cargo-watch
       cargo-limit
       cargo-udeps
+
+      llvmPackages_13.libclang
+      llvmPackages_13.libcxxClang
     ];
 
-    shellHook = ''
-       # workaround for https://github.com/NixOS/nixpkgs/blob/48dfc9fa97d762bce28cc8372a2dd3805d14c633/doc/languages-frameworks/python.section.md#python-setuppy-bdist_wheel-cannot-create-whl
-       export SOURCE_DATE_EPOCH=315532800 # 1980
-       export LD_LIBRARY_PATH="${stdenv.cc.cc.lib}/lib64:$LD_LIBRARY_PATH";
-    '';
+    BINDGEN_EXTRA_CLANG_ARGS = "-isystem ${llvmPackages_13.libclang.lib}/lib/clang/${lib.getVersion clang}/include";
+    LIBCLANG_PATH = "${llvmPackages_13.libclang.lib}/lib";
+
+    # workaround for https://github.com/NixOS/nixpkgs/blob/48dfc9fa97d762bce28cc8372a2dd3805d14c633/doc/languages-frameworks/python.section.md#python-setuppy-bdist_wheel-cannot-create-whl
+    SOURCE_DATE_EPOCH = 315532800; # 1980
   }

src/core/Cargo.toml

@@ -80,3 +80,6 @@ features = ["console", "File"]
 
 [target.'cfg(all(target_arch = "wasm32", target_vendor="unknown"))'.dev-dependencies]
 wasm-bindgen-test = "0.3.0"
+
+### These crates don't compile on wasm
+[target.'cfg(not(all(target_arch = "wasm32", target_vendor="unknown")))'.dependencies]

src/core/src/cmd.rs

@@ -2,22 +2,9 @@ use getset::{CopyGetters, Getters, Setters};
 use typed_builder::TypedBuilder;
 
 use crate::encodings::HashFunctions;
-use crate::index::MHBT;
 use crate::signature::Signature;
 use crate::sketch::minhash::{max_hash_for_scaled, KmerMinHashBTree};
 use crate::sketch::Sketch;
-use crate::Error;
-
-pub fn prepare(index_path: &str) -> Result<(), Error> {
-    let mut index = MHBT::from_path(index_path)?;
-
-    // TODO equivalent to fill_internal in python
-    //unimplemented!();
-
-    index.save_file(index_path, None)?;
-
-    Ok(())
-}
 
 impl Signature {
     pub fn from_params(params: &ComputeParameters) -> Signature {

src/core/src/errors.rs

@@ -43,10 +43,10 @@ pub enum SourmashError {
     HLLPrecisionBounds,
 
     #[error(transparent)]
-    ReadDataError(#[from] crate::index::storage::ReadDataError),
+    ReadDataError(#[from] ReadDataError),
 
     #[error(transparent)]
-    StorageError(#[from] crate::index::storage::StorageError),
+    StorageError(#[from] crate::storage::StorageError),
 
     #[error(transparent)]
     SerdeError(#[from] serde_json::error::Error),
@@ -65,6 +65,12 @@ pub enum SourmashError {
     Panic(#[from] crate::ffi::utils::Panic),
 }
 
+#[derive(Debug, Error)]
+pub enum ReadDataError {
+    #[error("Could not load data")]
+    LoadError,
+}
+
 #[repr(u32)]
 pub enum SourmashErrorCode {
     // no error

src/core/src/ffi/hyperloglog.rs

@@ -2,7 +2,7 @@ use std::ffi::CStr;
 use std::os::raw::c_char;
 use std::slice;
 
-use crate::index::sbt::Update;
+use crate::prelude::*;
 use crate::signature::SigsTrait;
 use crate::sketch::hyperloglog::HyperLogLog;
 

src/core/src/ffi/nodegraph.rs

@@ -2,7 +2,7 @@ use std::ffi::CStr;
 use std::os::raw::c_char;
 use std::slice;
 
-use crate::index::sbt::Update;
+use crate::prelude::*;
 use crate::sketch::nodegraph::Nodegraph;
 
 use crate::ffi::minhash::SourmashKmerMinHash;

src/core/src/index/linear.rs

@@ -7,8 +7,9 @@ use std::rc::Rc;
 use serde::{Deserialize, Serialize};
 use typed_builder::TypedBuilder;
 
-use crate::index::storage::{FSStorage, ReadData, Storage, StorageInfo, ToWriter};
 use crate::index::{Comparable, DatasetInfo, Index, SigStore};
+use crate::prelude::*;
+use crate::storage::{FSStorage, Storage, StorageInfo};
 use crate::Error;
 
 #[derive(TypedBuilder)]

src/core/src/index/mod.rs

@@ -7,8 +7,6 @@ pub mod bigsi;
 pub mod linear;
 pub mod sbt;
 
-pub mod storage;
-
 pub mod search;
 
 use std::ops::Deref;
@@ -19,12 +17,14 @@ use once_cell::sync::OnceCell;
 use serde::{Deserialize, Serialize};
 use typed_builder::TypedBuilder;
 
+use crate::errors::ReadDataError;
 use crate::index::sbt::{Node, SBT};
 use crate::index::search::{search_minhashes, search_minhashes_containment};
-use crate::index::storage::{ReadData, ReadDataError, Storage};
-use crate::signature::{Signature, SigsTrait};
+use crate::prelude::*;
+use crate::signature::SigsTrait;
 use crate::sketch::nodegraph::Nodegraph;
 use crate::sketch::Sketch;
+use crate::storage::Storage;
 use crate::Error;
 
 pub type MHBT = SBT<Node<Nodegraph>, Signature>;
@@ -103,12 +103,6 @@ pub trait Index<'a> {
     */
 }
 
-// TODO: split into two traits, Similarity and Containment?
-pub trait Comparable<O> {
-    fn similarity(&self, other: &O) -> f64;
-    fn containment(&self, other: &O) -> f64;
-}
-
 impl<'a, N, L> Comparable<L> for &'a N
 where
     N: Comparable<L>,

src/core/src/index/sbt/mhbt.rs

@@ -1,10 +1,10 @@
 use std::collections::HashMap;
 use std::io::Write;
 
-use crate::index::sbt::{Factory, FromFactory, Node, Update, SBT};
-use crate::index::storage::{ReadData, ReadDataError, ToWriter};
-use crate::index::Comparable;
-use crate::signature::{Signature, SigsTrait};
+use crate::errors::ReadDataError;
+use crate::index::sbt::{Factory, FromFactory, Node, SBT};
+use crate::prelude::*;
+use crate::signature::SigsTrait;
 use crate::sketch::nodegraph::Nodegraph;
 use crate::sketch::Sketch;
 use crate::Error;
@@ -157,9 +157,8 @@ mod test {
     use crate::index::linear::LinearIndex;
     use crate::index::sbt::scaffold;
     use crate::index::search::{search_minhashes, search_minhashes_containment};
-    use crate::index::storage::ReadData;
     use crate::index::{Index, SigStore, MHBT};
-    use crate::signature::Signature;
+    use crate::prelude::*;
 
     #[test]
     fn save_sbt() {

src/core/src/index/sbt/mod.rs

@@ -24,19 +24,11 @@ use once_cell::sync::OnceCell;
 use serde::{Deserialize, Serialize};
 use typed_builder::TypedBuilder;
 
-use crate::index::storage::{FSStorage, ReadData, Storage, StorageInfo, ToWriter};
 use crate::index::{Comparable, DatasetInfo, Index, SigStore};
-use crate::signature::Signature;
+use crate::prelude::*;
+use crate::storage::{FSStorage, StorageInfo};
 use crate::Error;
 
-pub trait Update<O> {
-    fn update(&self, other: &mut O) -> Result<(), Error>;
-}
-
-pub trait FromFactory<N> {
-    fn factory(&self, name: &str) -> Result<N, Error>;
-}
-
 #[derive(TypedBuilder)]
 pub struct SBT<N, L> {
     #[builder(default = 2)]

src/core/src/lib.rs

@@ -22,12 +22,13 @@
 pub mod errors;
 pub use errors::SourmashError as Error;
 
-pub mod cmd;
+pub mod prelude;
 
-pub mod index;
+pub mod cmd;
 
 pub mod signature;
 pub mod sketch;
+pub mod storage;
 
 pub mod encodings;
 
@@ -42,6 +43,7 @@ cfg_if! {
         pub mod wasm;
     } else {
         pub mod ffi;
+        pub mod index;
     }
 }
 

src/core/src/prelude.rs

@@ -0,0 +1,31 @@
+use std::io::Write;
+
+use crate::Error;
+
+pub use crate::signature::Signature;
+pub use crate::storage::Storage;
+
+pub trait ToWriter {
+    fn to_writer<W>(&self, writer: &mut W) -> Result<(), Error>
+    where
+        W: Write;
+}
+
+pub trait Update<O> {
+    fn update(&self, other: &mut O) -> Result<(), Error>;
+}
+
+pub trait FromFactory<N> {
+    fn factory(&self, name: &str) -> Result<N, Error>;
+}
+
+/// Implemented by anything that wants to read specific data from a storage.
+pub trait ReadData<D> {
+    fn data(&self) -> Result<&D, Error>;
+}
+
+// TODO: split into two traits, Similarity and Containment?
+pub trait Comparable<O> {
+    fn similarity(&self, other: &O) -> f64;
+    fn containment(&self, other: &O) -> f64;
+}

src/core/src/signature.rs

@@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize};
 use typed_builder::TypedBuilder;
 
 use crate::encodings::{aa_to_dayhoff, aa_to_hp, revcomp, to_aa, HashFunctions, VALID};
-use crate::index::storage::ToWriter;
+use crate::prelude::*;
 use crate::sketch::Sketch;
 use crate::Error;
 use crate::HashIntoType;

src/core/src/sketch/hyperloglog/mod.rs

@@ -16,7 +16,7 @@ use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
 use serde::{Deserialize, Serialize};
 
 use crate::encodings::HashFunctions;
-use crate::index::sbt::Update;
+use crate::prelude::*;
 use crate::signature::SigsTrait;
 use crate::sketch::KmerMinHash;
 use crate::Error;

src/core/src/sketch/mod.rs

@@ -1,5 +1,6 @@
 pub mod hyperloglog;
 pub mod minhash;
+
 pub mod nodegraph;
 
 use serde::{Deserialize, Serialize};

src/core/src/sketch/nodegraph.rs

@@ -6,7 +6,7 @@ use std::slice;
 use byteorder::{BigEndian, ByteOrder, LittleEndian, ReadBytesExt, WriteBytesExt};
 use fixedbitset::FixedBitSet;
 
-use crate::index::sbt::Update;
+use crate::prelude::*;
 use crate::sketch::minhash::KmerMinHash;
 use crate::Error;
 use crate::HashIntoType;

src/core/src/index/storage.rs → src/core/src/storage.rs

@@ -14,17 +14,6 @@ pub enum StorageError {
     EmptyPathError,
 }
 
-#[derive(Debug, Error)]
-pub enum ReadDataError {
-    #[error("Could not load data")]
-    LoadError,
-}
-
-/// Implemented by anything that wants to read specific data from a storage.
-pub trait ReadData<D> {
-    fn data(&self) -> Result<&D, Error>;
-}
-
 #[derive(Serialize, Deserialize)]
 pub(crate) struct StorageInfo {
     pub backend: String,
@@ -126,9 +115,3 @@ impl Storage for FSStorage {
         }
     }
 }
-
-pub trait ToWriter {
-    fn to_writer<W>(&self, writer: &mut W) -> Result<(), Error>
-    where
-        W: Write;
-}