Support Hypercore LTS Version 10

.github/workflows/ci.yml

@@ -0,0 +1,142 @@
+name: 'CI'
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+env:
+  RUST_BACKTRACE: 1
+  CARGO_TERM_COLOR: always
+
+jobs:
+  ci-pass:
+    name: CI is green
+    runs-on: ubuntu-latest
+    needs:
+      - test-linux
+      - test-windows
+      - test-macos
+      - build-extra
+      - lint
+    steps:
+      - run: exit 0
+
+  test-linux:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - uses: dtolnay/rust-toolchain@stable
+      with:
+        components: rustfmt
+    - name: Run tests
+      run: |
+        cargo check --no-default-features --features tokio
+        cargo check --no-default-features --features tokio,sparse
+        cargo check --no-default-features --features tokio,sparse,cache
+        cargo check --no-default-features --features async-std
+        cargo check --no-default-features --features async-std,sparse
+        cargo check --no-default-features --features async-std,sparse,cache
+        cargo test --no-default-features --features js_interop_tests,tokio
+        cargo test --no-default-features --features js_interop_tests,tokio,sparse
+        cargo test --no-default-features --features js_interop_tests,tokio,sparse,cache
+        cargo test --no-default-features --features js_interop_tests,async-std
+        cargo test --no-default-features --features js_interop_tests,async-std,sparse
+        cargo test --no-default-features --features js_interop_tests,async-std,sparse,cache
+        cargo test --benches --no-default-features --features tokio
+        cargo test --benches --no-default-features --features async-std
+
+  test-windows:
+    runs-on: windows-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt
+      - name: Run tests
+        run: |
+          cargo check --no-default-features --features tokio
+          cargo check --no-default-features --features tokio,sparse
+          cargo check --no-default-features --features tokio,sparse,cache
+          cargo check --no-default-features --features async-std
+          cargo check --no-default-features --features async-std,sparse
+          cargo check --no-default-features --features async-std,sparse,cache
+          cargo test --no-default-features --features tokio
+          cargo test --no-default-features --features tokio,sparse
+          cargo test --no-default-features --features tokio,sparse,cache
+          cargo test --no-default-features --features async-std
+          cargo test --no-default-features --features async-std,sparse
+          cargo test --no-default-features --features async-std,sparse,cache
+          cargo test --benches --no-default-features --features tokio
+          cargo test --benches --no-default-features --features async-std
+
+  test-macos:
+    runs-on: macos-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt
+      - name: Run tests
+        run: |
+          cargo check --no-default-features --features tokio
+          cargo check --no-default-features --features tokio,sparse
+          cargo check --no-default-features --features tokio,sparse,cache
+          cargo check --no-default-features --features async-std
+          cargo check --no-default-features --features async-std,sparse
+          cargo check --no-default-features --features async-std,sparse,cache
+          cargo test --no-default-features --features js_interop_tests,tokio
+          cargo test --no-default-features --features js_interop_tests,tokio,sparse
+          cargo test --no-default-features --features js_interop_tests,tokio,sparse,cache
+          cargo test --no-default-features --features js_interop_tests,async-std
+          cargo test --no-default-features --features js_interop_tests,async-std,sparse
+          cargo test --no-default-features --features js_interop_tests,async-std,sparse,cache
+          cargo test --benches --no-default-features --features tokio
+          cargo test --benches --no-default-features --features async-std
+
+  build-extra:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: wasm32-unknown-unknown
+      - name: Build WASM
+        run: |
+          cargo build --target=wasm32-unknown-unknown --no-default-features --features tokio
+          cargo build --target=wasm32-unknown-unknown --no-default-features --features async-std
+      - name: Build release
+        run: |
+          cargo build --release --no-default-features --features tokio
+          cargo build --release --no-default-features --features tokio,sparse
+          cargo build --release --no-default-features --features tokio,sparse,cache
+          cargo build --release --no-default-features --features async-std
+          cargo build --release --no-default-features --features async-std,sparse
+          cargo build --release --no-default-features --features async-std,sparse,cache
+      - name: Run examples
+        run: |
+          cargo run --no-default-features --features tokio --example disk 
+          cargo run --no-default-features --features async-std --example disk 
+          cargo run --no-default-features --features tokio --example memory 
+          cargo run --no-default-features --features async-std --example memory 
+          cargo run --no-default-features --features tokio --example replication 
+          cargo run --no-default-features --features async-std --example replication 
+
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - uses: dtolnay/rust-toolchain@stable
+      with:
+        components: rustfmt, clippy
+    - uses: actions-rs/clippy-check@v1
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Format check
+      run: |
+        cargo fmt -- --check

.gitignore

@@ -7,7 +7,9 @@ npm-debug.log*
 .nyc_output
 target/
 Cargo.lock
+package-lock.json
 
 my-first-dataset/
 feed.db/
 .vscode
+tests/js/work

Cargo.toml

@@ -1,12 +1,15 @@
 [package]
 name = "hypercore"
-version = "0.11.1-beta.10"
+version = "0.12.0-alpha.9"
 license = "MIT OR Apache-2.0"
 description = "Secure, distributed, append-only log"
 documentation = "https://docs.rs/hypercore"
 repository = "https://github.com/datrs/hypercore"
 readme = "README.md"
-authors = ["Yoshua Wuyts <[email protected]>"]
+authors = [
+  "Yoshua Wuyts <[email protected]>",
+  "Timo Tiuraniemi <[email protected]>"
+]
 keywords = ["dat", "p2p", "stream", "feed", "merkle"]
 categories = [
   "asynchronous",
@@ -15,33 +18,61 @@ categories = [
   "data-structures",
   "encoding",
 ]
-edition = "2018"
+edition = "2021"
 
 [dependencies]
-blake2-rfc = "0.2.18"
-byteorder = "1.3.4"
-ed25519-dalek = "1.0.1"
-anyhow = "1.0.26"
-flat-tree = "5.0.0"
-lazy_static = "1.4.0"
-memory-pager = "0.9.0"
-merkle-tree-stream =  "0.12.0"
-pretty-hash = "0.4.1"
-rand = "0.7.3"
-random-access-disk = "2.0.0"
-random-access-memory = "2.0.0"
-random-access-storage = "4.0.0"
-sha2 = "0.9.2"
-sleep-parser = "0.8.0"
-sparse-bitfield = "0.11.0"
-tree-index = "0.6.0"
-bitfield-rle = "0.2.0"
-futures = "0.3.4"
-async-std = "1.5.0"
+blake2 = "0.10"
+byteorder = "1"
+ed25519-dalek = { version = "2", features = ["rand_core"] }
+getrandom = { version = "0.2", features = ["js"] }
+thiserror = "1"
+tracing = "0.1"
+compact-encoding = "1"
+flat-tree = "6"
+merkle-tree-stream =  "0.12"
+pretty-hash = "0.4"
+rand = "0.8"
+random-access-memory = "3"
+random-access-storage = "5"
+sha2 = "0.10"
+futures = "0.3"
+crc32fast = "1"
+intmap = "2"
+moka = { version = "0.12", optional = true, features = ["sync"] }
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+random-access-disk = { version = "3", default-features = false }
 
 [dev-dependencies]
-quickcheck = "0.9.2"
+anyhow = "1.0.70"
+proptest = "1.1.0"
+proptest-derive = "0.2.0"
 data-encoding = "2.2.0"
 remove_dir_all = "0.7.0"
 tempfile = "3.1.0"
-async-std = { version = "1.5.0", features = ["attributes"] }
+async-std = { version = "1.12.0", features = ["attributes"] }
+tokio = { version = "1.27.0", default-features = false, features = ["macros", "rt", "rt-multi-thread"] }
+tokio-test = "0.4"
+sha2 = "0.10"
+criterion = { version = "0.4", features = ["async_std", "async_tokio"] }
+test-log = { version = "0.2.11", default-features = false, features = ["trace"] }
+tracing-subscriber = { version = "0.3.16", features = ["env-filter", "fmt"] }
+
+[features]
+default = ["async-std", "sparse"]
+sparse = ["random-access-disk/sparse"]
+tokio = ["random-access-disk/tokio"]
+async-std = ["random-access-disk/async-std"]
+cache = ["moka"]
+# Used only in interoperability tests under tests/js-interop which use the javascript version of hypercore
+# to verify that this crate works. To run them, use:
+# cargo test --features js-interop-tests
+js_interop_tests = []
+
+[[bench]]
+name = "memory"
+harness = false
+
+[[bench]]
+name = "disk"
+harness = false

README.md

@@ -1,53 +1,99 @@
-# hypercore
+# Hypercore
 [![crates.io version][1]][2] [![build status][3]][4]
 [![downloads][5]][6] [![docs.rs docs][7]][8]
 
-WIP. Secure, distributed, append-only log structure. Adapted from
-[mafintosh/hypercore](https://github.com/mafintosh/hypercore).
+Hypercore is a secure, distributed append-only log. This crate is a limited Rust
+port of the original Javascript
+[holepunchto/hypercore](https://github.com/holepunchto/hypercore). The goal is to
+maintain binary compatibility with the LTS version with regards to disk storage.
+
+See [hypercore-protocol-rs](https://github.com/datrs/hypercore-protocol-rs) for the
+corresponding wire protocol implementation.
 
 - [Documentation][8]
 - [Crates.io][2]
 
-**NOTE**: The master branch currently only works with the old hypercore version 9.
-For ongoing work to support the latest version 10 of hypercore [see the v10 branch](https://github.com/datrs/hypercore/tree/v10).
+## Features
+
+- [x] Create [in-memory](https://github.com/datrs/random-access-memory) and [disk](https://github.com/datrs/random-access-disk) hypercores
+- [x] Append to hypercore either a single entry or a batch of entries
+- [x] Get entries from hypercore
+- [x] Clear range from hypercore, with optional support for sparse files
+- [x] Support basic replication by creating proofs in a source hypercore and verifying and applying them to a destination hypercore
+- [x] Support `tokio` or `async-std` runtimes
+- [x] Support WASM for in-memory storage
+- [x] Test Javascript interoperability for supported features
+- [x] Add optional read cache
+- [ ] Support the new [manifest](https://github.com/holepunchto/hypercore/blob/main/lib/manifest.js) in the wire protocol to remain compatible with upcoming v11
+- [ ] Finalize documentation and release v1.0.0
 
 ## Usage
+
 ```rust
-let mut feed = hypercore::open("./feed.db").await?;
+// Create an in-memory hypercore using a builder
+let mut hypercore = HypercoreBuilder::new(Storage::new_memory().await.unwrap())
+    .build()
+    .await
+    .unwrap();
 
-feed.append(b"hello").await?;
-feed.append(b"world").await?;
+// Append entries to the log
+hypercore.append(b"Hello, ").await.unwrap();
+hypercore.append(b"world!").await.unwrap();
 
-assert_eq!(feed.get(0).await?, Some(b"hello".to_vec()));
-assert_eq!(feed.get(1).await?, Some(b"world".to_vec()));
+// Read entries from the log
+assert_eq!(hypercore.get(0).await.unwrap().unwrap(), b"Hello, ");
+assert_eq!(hypercore.get(1).await.unwrap().unwrap(), b"world!");
+```
+
+Find more examples in the [examples](./examples) folder, and/or run:
+
+```bash
+cargo run --example memory
+cargo run --example disk
+cargo run --example replication
 ```
 
 ## Installation
-```sh
-$ cargo add hypercore
+
+```bash
+cargo add hypercore
 ```
 
 ## Safety
-This crate uses ``#![deny(unsafe_code)]`` to ensure everything is implemented in
+
+This crate uses ``#![forbid(unsafe_code)]`` to ensure everythong is implemented in
 100% Safe Rust.
 
+## Development
+
+To test interoperability with Javascript, enable the `js_interop_tests` feature:
+
+```bash
+cargo test --features js_interop_tests
+```
+
+Run benches with:
+
+```bash
+cargo bench
+```
+
 ## Contributing
+
 Want to join us? Check out our ["Contributing" guide][contributing] and take a
 look at some of these issues:
 
 - [Issues labeled "good first issue"][good-first-issue]
 - [Issues labeled "help wanted"][help-wanted]
 
-## References
-- [github.com/mafintosh/hypercore](https://github.com/mafintosh/hypercore)
-
 ## License
+
 [MIT](./LICENSE-MIT) OR [Apache-2.0](./LICENSE-APACHE)
 
 [1]: https://img.shields.io/crates/v/hypercore.svg?style=flat-square
 [2]: https://crates.io/crates/hypercore
-[3]: https://img.shields.io/travis/datrs/hypercore/master.svg?style=flat-square
-[4]: https://travis-ci.org/datrs/hypercore
+[3]: https://github.com/datrs/hypercore/actions/workflows/ci.yml/badge.svg
+[4]: https://github.com/datrs/hypercore/actions
 [5]: https://img.shields.io/crates/d/hypercore.svg?style=flat-square
 [6]: https://crates.io/crates/hypercore
 [7]: https://img.shields.io/badge/docs-latest-blue.svg?style=flat-square