[MRG] Bunch of Rust refactors

.github/workflows/rust.yml

@@ -95,11 +95,14 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install -e .
 
-      - name: Run tests
-        uses: actions-rs/cargo@v1
+      - uses: actions-rs/[email protected]
         with:
-          command: test
-          args: --no-fail-fast --all --all-features
+          crate: cargo-all-features
+          version: latest
+          use-tool-cache: true
+
+      - name: Run tests for all feature combinations
+        run: cargo test-all-features --no-fail-fast --all
 
   coverage:
     runs-on: ubuntu-latest
@@ -111,10 +114,15 @@ jobs:
           toolchain: stable
           override: true
 
-      - name: Coverage with tarpaulin
-        uses: actions-rs/tarpaulin@v0.1
+      - name: Install cargo-tarpaulin
+        uses: actions-rs/install@v0.1
         with:
-          args: '--all --all-features -- --test-threads 1'
+          crate: cargo-tarpaulin
+          version: latest
+          use-tool-cache: true
+
+      - name: Coverage with tarpaulin
+        run: cargo tarpaulin --all --all-features --timeout 600 --out Xml -- --test-threads 1
 
       - name: Upload Rust coverage to codecov
         uses: codecov/codecov-action@797e92895ec0eac368405352c0add9af878fe257
@@ -155,15 +163,27 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@master
+
       - uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
           target: wasm32-unknown-unknown
-      - uses: actions-rs/cargo@v1
+
+      - uses: actions-rs/[email protected]
         with:
-          command: install
-          args: --force wasm-pack --version 0.8.1
-      - name: run wasm-pack
+          crate: wasm-pack
+          version: latest
+
+      - name: Prepare node for running tests
+        uses: actions/setup-node@v1
+        with:
+          node-version: 12
+          registry-url: https://registry.npmjs.org/
+
+      - name: run wasm tests
+        run: wasm-pack test --node src/core
+
+      - name: run wasm-pack build
         run: wasm-pack build src/core -d ../../pkg
 
   wasm32-wasi:

include/sourmash.h

@@ -33,10 +33,13 @@ enum SourmashErrorCode {
   SOURMASH_ERROR_CODE_INVALID_PROT = 1102,
   SOURMASH_ERROR_CODE_INVALID_CODON_LENGTH = 1103,
   SOURMASH_ERROR_CODE_INVALID_HASH_FUNCTION = 1104,
+  SOURMASH_ERROR_CODE_READ_DATA = 1201,
+  SOURMASH_ERROR_CODE_STORAGE = 1202,
   SOURMASH_ERROR_CODE_IO = 100001,
   SOURMASH_ERROR_CODE_UTF8_ERROR = 100002,
   SOURMASH_ERROR_CODE_PARSE_INT = 100003,
   SOURMASH_ERROR_CODE_SERDE_ERROR = 100004,
+  SOURMASH_ERROR_CODE_NIFFLER_ERROR = 100005,
 };
 typedef uint32_t SourmashErrorCode;
 

src/core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sourmash"
-version = "0.8.0"
+version = "0.9.0"
 authors = ["Luiz Irber <[email protected]>"]
 description = "MinHash sketches for genomic data"
 repository = "https://github.com/dib-lab/sourmash"
@@ -21,46 +21,48 @@ bench = false
 from-finch = ["finch"]
 parallel = ["rayon"]
 
-#[build-dependencies]
-#cbindgen = "~0.14.2"
-
 [dependencies]
 backtrace = "=0.3.46" # later versions require rust 1.40
 byteorder = "1.3.4"
 cfg-if = "0.1.10"
-failure = "0.1.8"
-failure_derive = "0.1.8"
+failure = "0.1.8"  # can remove after .backtrace() is available in std::error::Error
 finch = { version = "0.3.0", optional = true }
 fixedbitset = "0.3.0"
 log = "0.4.8"
 md5 = "0.7.0"
 murmurhash3 = "0.0.5"
-niffler = { version = "2.2.0", default-features = false, features = ["gz"] }
+niffler = { version = "2.2.0", default-features = false, features = [ "gz" ] }
+nohash-hasher = "0.2.0"
 once_cell = "1.3.1"
 rayon = { version = "1.3.0", optional = true }
-serde = "1.0.110"
-serde_derive = "1.0.110"
+serde = { version = "1.0.110", features = ["derive"] }
 serde_json = "1.0.53"
 primal-check = "0.2.3"
-typed-builder = "0.4.1"  # Upgrade to 0.5.1 needs rust 1.37
+thiserror = "1.0"
+typed-builder = "0.6.0"
+getset = "0.1.1"
 
 [target.'cfg(all(target_arch = "wasm32", target_vendor="unknown"))'.dependencies.wasm-bindgen]
 version = "0.2.62"
 features = ["serde-serialize"]
 
+[target.'cfg(all(target_arch = "wasm32", target_vendor="unknown"))'.dev-dependencies]
+wasm-bindgen-test = "0.3.0"
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies.assert_cmd]
+version = "1.0.1"
+
 [dev-dependencies]
-assert_cmd = "1.0.1"
 assert_matches = "1.3.0"
 criterion = "0.3.2"
 predicates = "1.0.4"
-proptest = { version = "0.9.6", default-features = false, features = ["std"]}
+proptest = { version = "0.9.6", default-features = false, features = ["std"]}  # Upgrade to 0.10 requires rust 1.39
 rand = "0.7.3"
 tempfile = "3.1.0"
 
 [dev-dependencies.needletail]
-version = "0.3.2"
+version = "0.4.0"
 default-features = false
-#features = ["compression"]
 
 [[bench]]
 name = "index"

src/core/benches/compute.rs

@@ -4,7 +4,7 @@ extern crate criterion;
 use std::fs::File;
 use std::io::{Cursor, Read};
 
-use needletail::parse_sequence_reader;
+use needletail::parse_fastx_reader;
 use sourmash::cmd::ComputeParameters;
 use sourmash::signature::Signature;
 
@@ -35,59 +35,43 @@ fn add_sequence(c: &mut Criterion) {
         b.iter(|| {
             let fasta_data = Cursor::new(data_upper.clone());
             let mut sig = template_sig.clone();
-            parse_sequence_reader(
-                fasta_data,
-                |_| {},
-                |rec| {
-                    sig.add_sequence(&rec.seq, false).unwrap();
-                },
-            )
-            .unwrap();
+            let mut parser = parse_fastx_reader(fasta_data).unwrap();
+            while let Some(rec) = parser.next() {
+                sig.add_sequence(&rec.unwrap().seq(), false).unwrap();
+            }
         });
     });
 
     group.bench_function("lowercase", |b| {
         b.iter(|| {
             let fasta_data = Cursor::new(data_lower.clone());
             let mut sig = template_sig.clone();
-            parse_sequence_reader(
-                fasta_data,
-                |_| {},
-                |rec| {
-                    sig.add_sequence(&rec.seq, false).unwrap();
-                },
-            )
-            .unwrap();
+            let mut parser = parse_fastx_reader(fasta_data).unwrap();
+            while let Some(rec) = parser.next() {
+                sig.add_sequence(&rec.unwrap().seq(), false).unwrap();
+            }
         });
     });
 
     group.bench_function("invalid kmers", |b| {
         b.iter(|| {
             let fasta_data = Cursor::new(data_errors.clone());
             let mut sig = template_sig.clone();
-            parse_sequence_reader(
-                fasta_data,
-                |_| {},
-                |rec| {
-                    sig.add_sequence(&rec.seq, true).unwrap();
-                },
-            )
-            .unwrap();
+            let mut parser = parse_fastx_reader(fasta_data).unwrap();
+            while let Some(rec) = parser.next() {
+                sig.add_sequence(&rec.unwrap().seq(), true).unwrap();
+            }
         });
     });
 
     group.bench_function("force with valid kmers", |b| {
         b.iter(|| {
             let fasta_data = Cursor::new(data_upper.clone());
             let mut sig = template_sig.clone();
-            parse_sequence_reader(
-                fasta_data,
-                |_| {},
-                |rec| {
-                    sig.add_sequence(&rec.seq, true).unwrap();
-                },
-            )
-            .unwrap();
+            let mut parser = parse_fastx_reader(fasta_data).unwrap();
+            while let Some(rec) = parser.next() {
+                sig.add_sequence(&rec.unwrap().seq(), true).unwrap();
+            }
         });
     });
 }