Merge branch 'master' into cherry-pick-all-0.4.x

.travis.yml

@@ -31,7 +31,10 @@ install:
 script:
   - if [[ $TRAVIS_RUST_VERSION == "stable" && $TRAVIS_OS_NAME == "linux" ]]; then cargo fmt -- --check; fi
   - if [[ $TRAVIS_RUST_VERSION == "stable" && $TRAVIS_OS_NAME == "linux" ]]; then cargo clippy -- -D clippy::all; fi
+  - if [[ $TRAVIS_RUST_VERSION == "stable" && $TRAVIS_OS_NAME == "linux" ]]; then cargo clippy --no-default-features --features prost-codec -- -D clippy::all; fi
   - cargo test --all -- --nocapture
+  # There is a bug in protobuf-build that cached size is not supported yet.
+  # TODO: - cargo test --no-default-features --features prost-codec -- --nocapture
   # Validate benches still work.
   - cargo bench --all -- --test
   # Because failpoints inject failure in code path, which will affect all concurrently running tests, Hence they need to be synchronized, which make tests slow.

Cargo.toml

@@ -16,21 +16,22 @@ edition = "2018"
 members = ["proto"]
 
 [features]
-default = []
+default = ["protobuf-codec"]
 # Enable failpoints
 failpoints = ["fail/failpoints"]
+protobuf-codec = ["raft-proto/protobuf-codec", "harness/protobuf-codec"]
+prost-codec = ["prost-derive", "bytes", "prost", "raft-proto/prost-codec", "harness/prost-codec"]
 
 # Make sure to synchronize updates with Harness.
 [dependencies]
 log = ">0.2"
 protobuf = "2"
-lazy_static = "1.3.0"
-prost = "0.5"
-prost-derive = "0.5"
-bytes = "0.4.11"
+prost = { version = "0.5", optional = true }
+prost-derive = { version = "0.5", optional = true }
+bytes = { version = "0.4.11", optional = true }
 slog = "2.2"
 quick-error = "1.2.2"
-raft-proto = { path = "proto" }
+raft-proto = { path = "proto", default-features = false }
 rand = "0.7.0"
 hashbrown = "0.5"
 fail = { version = "0.3", optional = true }
@@ -42,7 +43,7 @@ slog-envlogger = "2.1.0"
 [dev-dependencies]
 criterion = ">0.2.4"
 lazy_static = "1.0"
-harness = { path = "harness" }
+harness = { path = "harness", default-features = false }
 regex = "1.1"
 slog-async = "2.3.0"
 

README.md

@@ -31,6 +31,10 @@ A complete Raft model contains 4 essential parts:
 
 > Note: This Raft implementation in Rust includes the core Consensus Module only, not the other parts. The core Consensus Module in the Raft crate is customizable, flexible, and resilient. You can directly use the Raft crate, but you will need to build your own Log, State Machine and Transport components.
 
+## Using the raft crate
+
+You can use raft with either [rust-protobuf](https://github.com/pingcap/rust-protobuf) or [Prost](https://github.com/danburkert/prost) to encode/decode gRPC messages. We use rust-protobuf by default. To use Prost, build (or depend on) Raft using the `prost-codec` feature and without default features.
+
 ## Developing the Raft crate
 
 `Raft` is built using the latest version of `stable` Rust, using [the 2018 edition](https://doc.rust-lang.org/edition-guide/rust-2018/).

examples/five_mem_node/main.rs

@@ -16,7 +16,7 @@ use std::sync::{Arc, Mutex};
 use std::time::{Duration, Instant};
 use std::{str, thread};
 
-use prost::Message as ProstMsg;
+use protobuf::Message as PbMessage;
 use raft::eraftpb::ConfState;
 use raft::storage::MemStorage;
 use raft::{prelude::*, StateRole};
@@ -261,7 +261,7 @@ fn on_ready(
             if let EntryType::EntryConfChange = entry.get_entry_type() {
                 // For conf change messages, make them effective.
                 let mut cc = ConfChange::default();
-                ProstMsg::merge(&mut cc, &entry.data).unwrap();
+                cc.merge_from_bytes(&entry.data).unwrap();
                 let node_id = cc.node_id;
                 match cc.get_change_type() {
                     ConfChangeType::AddNode => raft_group.raft.add_node(node_id).unwrap(),

examples/single_mem_node/main.rs

@@ -171,7 +171,7 @@ fn on_ready(r: &mut RawNode<MemStorage>, cbs: &mut HashMap<u8, ProposeCallback>)
                 continue;
             }
 
-            if entry.entry_type() == EntryType::EntryNormal {
+            if entry.get_entry_type() == EntryType::EntryNormal {
                 if let Some(cb) = cbs.remove(entry.data.get(0).unwrap()) {
                     cb();
                 }

harness/Cargo.toml

@@ -11,9 +11,14 @@ description = "A testing harness for Raft."
 categories = []
 edition = "2018"
 
+[features]
+default = ["protobuf-codec"]
+protobuf-codec = ["raft/protobuf-codec"]
+prost-codec = ["raft/prost-codec"]
+
 # Make sure to synchronize updates with Raft.
 [dependencies]
-raft = { path = ".." }
+raft = { path = "..", default-features = false }
 rand = "0.7.0"
 slog = "2.2"
 slog-term = "2.4.0"

proto/Cargo.toml

@@ -13,15 +13,16 @@ categories = ["algorithms", "database-implementations"]
 build = "build.rs"
 
 [features]
-default = []
-gen = []
+default = ["protobuf-codec"]
+protobuf-codec = ["protobuf-build/protobuf-codec"]
+prost-codec = ["prost", "prost-derive", "bytes", "lazy_static", "protobuf-build/prost-codec"]
 
 [build-dependencies]
-protobuf-build = "0.6"
+protobuf-build = { git = "https://github.com/tikv/protobuf-build.git", rev = "54c089895e4bf42481c2af46ebf73295c0e5d6b9", default-features = false }
 
 [dependencies]
-bytes = "0.4.11"
-lazy_static = "1.3.0"
-prost = "0.5"
-prost-derive = "0.5"
+bytes = { version = "0.4.11", optional = true }
+lazy_static = { version = "1.3.0", optional = true }
+prost = { version = "0.5", optional = true }
+prost-derive = { version = "0.5", optional = true }
 protobuf = "2"

proto/build.rs

@@ -1,22 +1,11 @@
 // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0.
 
 use protobuf_build::*;
-use std::fs::{read_dir, File};
-use std::io::Write;
-use std::path::Path;
+use std::env;
+use std::fs::read_dir;
 
 fn main() {
-    // This build script creates files in the `src` directory. Since that is
-    // outside Cargo's OUT_DIR it will cause an error when this crate is used
-    // as a dependency. Therefore, the user must opt-in to regenerating the
-    // Rust files.
-    if !cfg!(feature = "gen") {
-        println!("cargo:rerun-if-changed=build.rs");
-        return;
-    }
-
-    check_protoc_version();
-
+    let out_dir = format!("{}/protos", env::var("OUT_DIR").unwrap());
     let file_names: Vec<_> = read_dir("proto")
         .expect("Couldn't read proto directory")
         .filter_map(|e| {
@@ -28,47 +17,9 @@ fn main() {
             }
         })
         .collect();
-
-    for f in &file_names {
-        println!("cargo:rerun-if-changed={}", f);
-    }
-
-    // Generate Prost output.
-    generate_prost_files(&file_names, "src/prost");
-    let mod_names = module_names_for_dir("src/prost");
-    generate_wrappers(
-        &mod_names
-            .iter()
-            .map(|m| format!("src/prost/{}.rs", m))
-            .collect::<Vec<_>>(),
-        "src/prost",
-        GenOpt::MUT | GenOpt::HAS | GenOpt::TAKE | GenOpt::CLEAR | GenOpt::MESSAGE,
+    generate_files(
+        &["include".to_owned(), "proto".to_owned()],
+        &file_names,
+        &out_dir,
     );
-    generate_prost_rs(&mod_names);
-}
-
-fn generate_prost_rs(mod_names: &[String]) {
-    let mut text = "#![allow(dead_code)]\n\
-                    #![allow(missing_docs)]\n\
-                    #![allow(clippy::all)]\n\n"
-        .to_owned();
-
-    for mod_name in mod_names {
-        text.push_str("pub mod ");
-        text.push_str(mod_name);
-        text.push_str("{\n");
-        text.push_str("include!(\"prost/");
-        text.push_str(mod_name);
-        text.push_str(".rs\");");
-        text.push_str("include!(\"prost/wrapper_");
-        text.push_str(mod_name);
-        text.push_str(".rs\");");
-        text.push_str("}\n\n");
-    }
-
-    let prost_rs = Path::new("src/prost.rs");
-    let mut lib = File::create(&prost_rs).expect("Could not create prost.rs");
-    lib.write_all(text.as_bytes())
-        .expect("Could not write prost.rs");
-    rustfmt(prost_rs);
 }

proto/src/lib.rs

@@ -1,8 +1,15 @@
 // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0.
 
-pub use crate::prost::eraftpb;
+pub use crate::protos::eraftpb;
 
-mod prost;
+#[allow(dead_code)]
+#[allow(unknown_lints)]
+#[allow(clippy::all)]
+#[allow(renamed_and_removed_lints)]
+#[allow(bare_trait_objects)]
+mod protos {
+    include!(concat!(env!("OUT_DIR"), "/protos/mod.rs"));
+}
 
 pub mod prelude {
     pub use crate::eraftpb::{