Optimize the performance of shadow service

Cargo.toml

@@ -29,7 +29,6 @@ path = "src/bin/shadow.rs"
 path = "src/lib.rs"
 
 [dependencies]
-async-std = "1.6.3"
 actix-web = "2.0.0"
 actix-rt = "1.0"
 blake2-rfc = "0.2.18"
@@ -42,6 +41,7 @@ libc = "0.2.71"
 log = "0.4.0"
 reqwest = { version = "0.10", features = ["json"] }
 rlp = "0.4.5"
+rocksdb = "0.15.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 structopt = "0.3.17"
@@ -51,7 +51,3 @@ uint = "0.8.3"
 package = "parity-scale-codec"
 version = "1.0.112"
 features = ["derive"]
-
-[dependencies.diesel]
-version = "1.4.4"
-features = ["r2d2", "sqlite"]

Dockerfile

@@ -1,31 +1,33 @@
-# Build Shadow in a stock rust builder container
-FROM rust:1.45.2-alpine as shadow
+FROM ubuntu:latest as builder
 ARG DEBIAN_FRONTEND=noninteractive
 ENV TZ=America/Los_Angeles
 COPY . shadow
-
-# Required dynamic libraries
-#
-# libdarwinia_shadow.so => /usr/local/lib/libdarwinia_shadow.so (0x7fd26af02000)
-# libssl.so.1.1 => /lib/libssl.so.1.1 (0x7fd26ae81000)
-# libcrypto.so.1.1 => /lib/libcrypto.so.1.1 (0x7fd26ac02000)
-# libsqlite3.so.0 => /usr/lib/libsqlite3.so.0 (0x7fd26ab1a000)
-# libc.musl-x86_64.so.1 => /lib/ld64.so.1 (0x7fd26bebb000)
-RUN apk add --no-cache gcc go openssl-dev sqlite-dev\
+RUN apt-get update && apt-get -y upgrade \
+    && apt-get -y install golang cargo libssl-dev clang-tools \
     && cd shadow \
     && cargo build --release -vv \
     && mkdir /target \
-    && cp target/release/shadow /target/ \
-    && cp /usr/lib/libsqlite3.so.0 /target/libsqlite3.so.0 \
-    && cp /usr/local/lib/libdarwinia_shadow.so /target/libdarwinia_shadow.so
+    && cp /shadow/target/release/shadow /target \
+    && cp /usr/local/lib/libdarwinia_shadow.so /target \
+    && cp /lib/x86_64-linux-gnu/libssl.so.1.1 /target \
+    && cp /lib/x86_64-linux-gnu/libcrypto.so.1.1 /target
+
+FROM ubuntu:latest
+#    linux-vdso.so.1 (0x00007fffdafe6000)
+#    libstdc++.so.6 => /lib/x86_64-linux-gnu/libstdc++.so.6 (0x00007f5221f0d000)
+#    libssl.so.1.1 => /lib/x86_64-linux-gnu/libssl.so.1.1 (0x00007f5221e7a000)
+#    libcrypto.so.1.1 => /lib/x86_64-linux-gnu/libcrypto.so.1.1 (0x00007f5221ba4000)
+#    libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f5221b9e000)
+#    libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f5221b7b000)
+#    libgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007f5221b60000)
+#    libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f522196c000)
+#    /lib64/ld-linux-x86-64.so.2 (0x00007f5223cb4000)
+#    libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f522181d000)
+COPY --from=builder /target /target
+RUN mv /target/shadow /usr/bin \
+    && mv /target/* /usr/lib \
+    && rm -rf /target \
+    && ldconfig
 
-# Pull Shadow into a second stage deploy alpine container
-FROM alpine:latest
-COPY --from=shadow /target /target
-RUN mv /target/shadow /usr/local/bin/shadow \
-    && mv /target/libsqlite3.so.0 /usr/lib/libsqlite3.so.0 \
-    && mv /target/libdarwinia_shadow.so /usr/local/lib/libdarwinia_shadow.so \
-    && cp /lib/libc.musl-x86_64.so.1 /lib/ld64.so.1 \
-    && rm -rf /target
-EXPOSE 3000
 ENTRYPOINT ["shadow"]
+

Dockerfiles/alpine.Dockerfile

@@ -0,0 +1,29 @@
+# Build Shadow in a stock rust builder container
+FROM rust:1.45.2-alpine as shadow
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=America/Los_Angeles
+COPY . shadow
+
+# Required dynamic libraries
+#
+# libdarwinia_shadow.so => /usr/local/lib/libdarwinia_shadow.so (0x7fd26af02000)
+# libssl.so.1.1 => /lib/libssl.so.1.1 (0x7fd26ae81000)
+# libcrypto.so.1.1 => /lib/libcrypto.so.1.1 (0x7fd26ac02000)
+# libsqlite3.so.0 => /usr/lib/libsqlite3.so.0 (0x7fd26ab1a000)
+# libc.musl-x86_64.so.1 => /lib/ld64.so.1 (0x7fd26bebb000)
+RUN apk add --no-cache gcc go openssl-dev llvm\
+    && cd shadow \
+    && cargo build --release -vv \
+    && mkdir /target \
+    && cp target/release/shadow /target/ \
+    && cp /usr/local/lib/libdarwinia_shadow.so /target/libdarwinia_shadow.so
+
+# Pull Shadow into a second stage deploy alpine container
+FROM alpine:latest
+COPY --from=shadow /target /target
+RUN mv /target/shadow /usr/local/bin/shadow \
+    && mv /target/libdarwinia_shadow.so /usr/local/lib/libdarwinia_shadow.so \
+    && cp /lib/libc.musl-x86_64.so.1 /lib/ld64.so.1 \
+    && rm -rf /target
+EXPOSE 3000
+ENTRYPOINT ["shadow"]

Dockerfiles/builder.Dockerfile

@@ -0,0 +1,33 @@
+FROM ubuntu:latest as builder
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=America/Los_Angeles
+COPY . shadow
+RUN apt-get update && apt-get -y upgrade \
+    && apt-get -y install golang cargo libssl-dev clang-tools \
+    && cd shadow \
+    && cargo build --release -vv \
+    && mkdir /target \
+    && cp /shadow/target/release/shadow /target \
+    && cp /usr/local/lib/libdarwinia_shadow.so /target \
+    && cp /lib/x86_64-linux-gnu/libssl.so.1.1 /target \
+    && cp /lib/x86_64-linux-gnu/libcrypto.so.1.1 /target
+
+# FROM debian:stretch-slim
+# #    linux-vdso.so.1 (0x00007fffdafe6000)
+# #    libstdc++.so.6 => /lib/x86_64-linux-gnu/libstdc++.so.6 (0x00007f5221f0d000)
+# #    libssl.so.1.1 => /lib/x86_64-linux-gnu/libssl.so.1.1 (0x00007f5221e7a000)
+# #    libcrypto.so.1.1 => /lib/x86_64-linux-gnu/libcrypto.so.1.1 (0x00007f5221ba4000)
+# #    libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f5221b9e000)
+# #    libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f5221b7b000)
+# #    libgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007f5221b60000)
+# #    libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f522196c000)
+# #    /lib64/ld-linux-x86-64.so.2 (0x00007f5223cb4000)
+# #    libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f522181d000)
+# COPY --from=builder /target /target
+# RUN mv /target/shadow /usr/bin \
+#     && mv /target/* /usr/local/lib \
+#     && rm -rf /target \
+#     && ldconfig
+# 
+# ENTRYPOINT ["shadow"]
+

Dockerfiles/release.Dockerfile

@@ -0,0 +1,33 @@
+# FROM ubuntu:latest as builder
+# ARG DEBIAN_FRONTEND=noninteractive
+# ENV TZ=America/Los_Angeles
+# COPY . shadow
+# RUN apt-get update && apt-get -y upgrade \
+#     && apt-get -y install golang cargo libssl-dev clang-tools \
+#     && cd shadow \
+#     && cargo build --release -vv \
+#     && mkdir /target \
+#     && cp /shadow/target/release/shadow /target \
+#     && cp /usr/local/lib/libdarwinia_shadow.so /target \
+#     && cp /lib/x86_64-linux-gnu/libssl.so.1.1 /target \
+#     && cp /lib/x86_64-linux-gnu/libcrypto.so.1.1 /target
+
+FROM debian:stretch-slim
+#    linux-vdso.so.1 (0x00007fffdafe6000)
+#    libstdc++.so.6 => /lib/x86_64-linux-gnu/libstdc++.so.6 (0x00007f5221f0d000)
+#    libssl.so.1.1 => /lib/x86_64-linux-gnu/libssl.so.1.1 (0x00007f5221e7a000)
+#    libcrypto.so.1.1 => /lib/x86_64-linux-gnu/libcrypto.so.1.1 (0x00007f5221ba4000)
+#    libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f5221b9e000)
+#    libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f5221b7b000)
+#    libgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007f5221b60000)
+#    libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f522196c000)
+#    /lib64/ld-linux-x86-64.so.2 (0x00007f5223cb4000)
+#    libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f522181d000)
+COPY --from=b:t /target /target
+# RUN mv /target/shadow /usr/bin \
+#     && mv /target/* /usr/local/lib \
+#     && rm -rf /target \
+#     && ldconfig
+# 
+# ENTRYPOINT ["shadow"]
+

build.rs

@@ -3,10 +3,48 @@ use std::{env, process::Command};
 fn main() {
     // Pre-check
     println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-changed=pkg/shadow/ffi/mod.go");
 
     // Declare build args
-    let mut dynamic = true;
+    let mut dynamic = match env::var("LIBRARY_TYPE") {
+        Ok(ty) => ty.to_lowercase() != "static",
+        Err(_) => true,
+    };
     let out_dir = env::var("OUT_DIR").unwrap();
+    go(&mut dynamic, &out_dir);
+
+    // Post-check
+    if dynamic {
+        println!("cargo:rustc-link-search=/usr/local/lib");
+        println!("cargo:rustc-link-lib=dylib=darwinia_shadow");
+    } else {
+        println!("cargo:rustc-link-search={}", out_dir);
+        println!("cargo:rustc-link-lib=static=darwinia_shadow");
+    }
+}
+
+/// Build golang library
+fn go(dynamic: &mut bool, out_dir: &str) {
+    if *dynamic {
+        if !Command::new("go")
+            .args(&gorgs(dynamic, out_dir))
+            .status()
+            .unwrap()
+            .success()
+        {
+            *dynamic = false;
+            go(dynamic, out_dir);
+        }
+    } else {
+        Command::new("go")
+            .args(&gorgs(dynamic, out_dir))
+            .status()
+            .unwrap();
+        println!("built static library at {}", out_dir);
+    }
+}
+
+fn gorgs(dynamic: &mut bool, out_dir: &str) -> Vec<String> {
     let staticlib = format!("{}/libdarwinia_shadow.a", out_dir);
     let dylib = format!(
         "/usr/local/lib/libdarwinia_shadow.{}",
@@ -20,39 +58,19 @@ fn main() {
         }
     );
 
-    let args = |dynamic: bool| {
-        vec![
-            "build",
-            "-o",
-            if dynamic { &dylib } else { &staticlib },
-            if dynamic {
-                "-buildmode=c-shared"
-            } else {
-                "-buildmode=c-archive"
-            },
-            "-v",
-            "pkg/shadow/ffi/mod.go",
-        ]
-    };
-
-    // Build the link library
-    if !Command::new("go")
-        .args(&args(dynamic))
-        .status()
-        .unwrap()
-        .success()
-    {
-        dynamic = false;
-        Command::new("go").args(&args(dynamic)).status().unwrap();
-        println!("built static library at {}", out_dir);
-    }
-
-    // Post-check
-    if dynamic {
-        println!("cargo:rustc-link-search=/usr/local/lib");
-        println!("cargo:rustc-link-lib=dylib=darwinia_shadow");
-    } else {
-        println!("cargo:rustc-link-search={}", out_dir);
-        println!("cargo:rustc-link-lib=static=darwinia_shadow");
-    }
+    vec![
+        "build",
+        "-o",
+        if *dynamic { &dylib } else { &staticlib },
+        if *dynamic {
+            "-buildmode=c-shared"
+        } else {
+            "-buildmode=c-archive"
+        },
+        "-v",
+        "pkg/shadow/ffi/mod.go",
+    ]
+    .iter()
+    .map(|s| s.to_string())
+    .collect()
 }

docker-compose.yml

@@ -5,7 +5,7 @@ services:
     command: --syncmode fast --http --http.addr 0.0.0.0 -http.vhosts *
     volumes:
       - data:/root
-  fetcher:
+  shadow:
     image: quay.io/darwinia-network/shadow:next
     command: run -v
     environment:

examples/bench.rs

@@ -0,0 +1,70 @@
+#[macro_use]
+extern crate log;
+use cmmr::MMR;
+use darwinia_shadow::{
+    chain::eth::EthHeaderRPCResp,
+    mmr::{helper, MergeHash, Store, H256},
+    ShadowShared,
+};
+use reqwest::Client;
+use rocksdb::IteratorMode;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+fn now() -> u128 {
+    let start = SystemTime::now();
+    let since_the_epoch = start
+        .duration_since(UNIX_EPOCH)
+        .expect("Time went backwards");
+    since_the_epoch.as_millis()
+}
+
+#[actix_rt::main]
+pub async fn main() {
+    std::env::set_var("RUST_LOG", "info,darwinia_shadow");
+    env_logger::init();
+
+    let shared = ShadowShared::new(None);
+    let mut mmr_size = shared.db.iterator(IteratorMode::Start).count() as u64;
+    let last_leaf = helper::mmr_size_to_last_leaf(mmr_size as i64);
+    let mut ptr = if last_leaf == 0 { 0 } else { last_leaf + 1 };
+
+    loop {
+        debug!("-{}-{}------------", ptr, mmr_size);
+        let a = now();
+        let mmr_size_new = push(&shared.store, &shared.client, ptr, mmr_size).await;
+
+        mmr_size = mmr_size_new;
+        ptr += 1;
+        debug!("total: {}", now() - a);
+    }
+}
+
+/// Push new header hash into storage
+pub async fn push(store: &Store, client: &Client, number: i64, mmr_size: u64) -> u64 {
+    let a = now();
+    let mut mmr = MMR::<_, MergeHash, _>::new(mmr_size, store);
+
+    let b = now();
+    debug!("mmr create  : {}", b - a);
+    let hash_from_ethereum = &EthHeaderRPCResp::get(&client, number as u64)
+        .await
+        .unwrap()
+        .result
+        .hash;
+
+    let c = now();
+    debug!("rpc call    : {}", c - b);
+    mmr.push(H256::from(hash_from_ethereum)).unwrap();
+
+    let d = now();
+    debug!("push to mmr : {}", d - c);
+
+    let mmr_size_new = mmr.mmr_size();
+    let e = now();
+    debug!("get new size: {}", e - d);
+    mmr.commit().expect("commit failed");
+
+    let f = now();
+    debug!("commit      : {}", f - e);
+    mmr_size_new
+}

examples/doctor.rs

@@ -1,14 +1,12 @@
 //! Mock the uncle block
 use cmmr::{leaf_index_to_mmr_size, leaf_index_to_pos, MMR};
 use darwinia_shadow::{
-    db::pool,
-    mmr::{MergeHash, Store, H256},
+    mmr::{MergeHash, H256},
+    ShadowShared,
 };
 
 fn main() {
-    let conn = pool::conn(None);
-    let store = Store::with(conn);
-
+    let store = ShadowShared::new(None).store;
     let mmr = MMR::<_, MergeHash, _>::new(leaf_index_to_mmr_size(9), &store);
     let root = mmr.get_root().expect("get root failed");
     // let root = H256::from("0xe28d7f650efb9cbaaca7f485d078c0f6b1104807a3a31f85fc1268b0673140ff");