Merge pull request #329 from CalvinNeo/v2/merge-7.1-freeze

Merge TiKV release-7.1(eaebf9a)

CONTRIBUTING.md

@@ -97,6 +97,20 @@ See the [style doc](https://github.com/rust-lang/fmt-rfcs/blob/master/guide/guid
 
 Please follow this style to make TiKV easy to review, maintain, and develop.
 
+### Run test in docker
+
+Alternatively, you can run test in a docker environment. Simply running the following command, it will build the pingcap/tikv_dev image and run the tikv unittests. And you may re-use the pingcap/tikv_dev image directly for ad-hoc test.
+
+```bash
+make docker_test
+```
+
+Note that you may find many messages below, which in fact are not errors. They're emitted by rustc or cargo.
+
+```bash
+<jemalloc>: Invalid conf pair: prof:true
+```
+
 ### Build issues
 
 To reduce compilation time and disk usage, TiKV builds do not include full debugging information by default &mdash; only tests package will have line debug info enabled. To change debuginfo, just precede build commands with `RUSTFLAGS=-Cdebuginfo=1` (for line numbers), or `RUSTFLAGS=-Cdebuginfo=2` (for full debuginfo). For example,

Cargo.toml

@@ -388,7 +388,7 @@ txn_types = { path = "components/txn_types" }
 grpcio = { version = "0.10.4", default-features = false, features = ["openssl-vendored", "protobuf-codec", "nightly"] }
 grpcio-health = { version = "0.10.4", default-features = false, features = ["protobuf-codec"] }
 tipb = { git = "https://github.com/pingcap/tipb.git" }
-kvproto = { git = "https://github.com/pingcap/kvproto.git" }
+kvproto = { git = "https://github.com/pingcap/kvproto.git", branch = "release-7.1" }
 yatp = { git = "https://github.com/tikv/yatp.git", branch = "master" }
 tokio-timer = { git = "https://github.com/tikv/tokio", branch = "tokio-timer-hotfix" }
 slog = { version = "2.3", features = ["max_level_trace", "release_max_level_debug"] }

Dockerfile.test

@@ -0,0 +1,57 @@
+# This Docker image contains a minimal build environment for TiKV
+#
+# It contains all the tools necessary to reproduce official production builds of TiKV
+
+# We need to use CentOS 7 because many of our users choose this as their deploy machine.
+# Since the glibc it uses (2.17) is from 2012 (https://sourceware.org/glibc/wiki/Glibc%20Timeline)
+# it is our lowest common denominator in terms of distro support.
+
+# Some commands in this script are structured in order to reduce the number of layers Docker
+# generates. Unfortunately Docker is limited to only 125 layers:
+# https://github.com/moby/moby/blob/a9507c6f76627fdc092edc542d5a7ef4a6df5eec/layer/layer.go#L50-L53
+
+# We require epel packages, so enable the fedora EPEL repo then install dependencies.
+# Install the system dependencies
+# Attempt to clean and rebuild the cache to avoid 404s
+
+# To avoid rebuilds we first install all Cargo dependencies
+
+
+# The prepare image avoid ruining the cache of the builder
+FROM centos:7.6.1810 as builder
+
+RUN yum install -y epel-release && \
+    yum clean all && \
+    yum makecache
+
+RUN yum install -y centos-release-scl && \
+    yum install -y \
+      devtoolset-8 \
+      perl cmake3 && \
+    yum clean all
+
+# CentOS gives cmake 3 a weird binary name, so we link it to something more normal
+# This is required by many build scripts, including ours.
+RUN ln -s /usr/bin/cmake3 /usr/bin/cmake
+ENV LIBRARY_PATH /usr/local/lib:$LIBRARY_PATH
+ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH
+
+# Install protoc
+RUN curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip"
+RUN unzip protoc-3.15.8-linux-x86_64.zip -d /usr/local/
+ENV PATH /usr/local/bin/:$PATH
+
+# Install Rustup
+RUN curl https://sh.rustup.rs -sSf | sh -s -- --no-modify-path --default-toolchain none -y
+ENV PATH /root/.cargo/bin/:$PATH
+
+# Install the Rust toolchain
+WORKDIR /tikv
+COPY rust-toolchain ./
+RUN rustup self update \
+  && rustup set profile minimal \
+  && rustup default $(cat "rust-toolchain")
+
+RUN cargo install cargo-nextest --locked
+
+ENTRYPOINT ["sh", "-c", "source /opt/rh/devtoolset-8/enable && \"$@\"", "-s"]

Makefile

@@ -137,6 +137,7 @@ export PROXY_BUILD_GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2> /dev
 
 export DOCKER_IMAGE_NAME ?= "pingcap/tikv"
 export DOCKER_IMAGE_TAG ?= "latest"
+export DEV_DOCKER_IMAGE_NAME ?= "pingcap/tikv_dev"
 
 # Turn on cargo pipelining to add more build parallelism. This has shown decent
 # speedups in TiKV.

cmd/tikv-ctl/src/executor.rs

@@ -1,13 +1,15 @@
 // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0.
 
 use std::{
-    borrow::ToOwned, cmp::Ordering, pin::Pin, str, string::ToString, sync::Arc, time::Duration, u64,
+    borrow::ToOwned, cmp::Ordering, path::Path, pin::Pin, str, string::ToString, sync::Arc,
+    time::Duration, u64,
 };
 
 use encryption_export::data_key_manager_from_config;
 use engine_rocks::util::{db_exist, new_engine_opt};
 use engine_traits::{
-    Engines, Error as EngineError, RaftEngine, ALL_CFS, CF_DEFAULT, CF_LOCK, CF_WRITE, DATA_CFS,
+    Engines, Error as EngineError, RaftEngine, TabletRegistry, ALL_CFS, CF_DEFAULT, CF_LOCK,
+    CF_WRITE, DATA_CFS,
 };
 use futures::{executor::block_on, future, stream, Stream, StreamExt, TryStreamExt};
 use grpcio::{ChannelBuilder, Environment};
@@ -25,12 +27,16 @@ use raft_log_engine::RaftLogEngine;
 use raftstore::store::{util::build_key_range, INIT_EPOCH_CONF_VER};
 use security::SecurityManager;
 use serde_json::json;
+use server::fatal;
+use slog_global::crit;
 use tikv::{
     config::{ConfigController, TikvConfig},
     server::{
-        debug::{BottommostLevelCompaction, Debugger, RegionInfo},
+        debug::{BottommostLevelCompaction, Debugger, DebuggerImpl, RegionInfo},
+        debug2::DebuggerImplV2,
         KvEngineFactoryBuilder,
     },
+    storage::config::EngineType,
 };
 use tikv_util::escape;
 
@@ -72,13 +78,10 @@ pub fn new_debug_executor(
     let factory = KvEngineFactoryBuilder::new(env.clone(), cfg, cache)
         .lite(true)
         .build();
-    let kv_db = match factory.create_shared_db(data_dir) {
-        Ok(db) => db,
-        Err(e) => handle_engine_error(e),
-    };
 
     let cfg_controller = ConfigController::default();
     if !cfg.raft_engine.enable {
+        assert_eq!(EngineType::RaftKv, cfg.storage.engine);
         let raft_db_opts = cfg.raftdb.build_opt(env, None);
         let raft_db_cf_opts = cfg.raftdb.build_cf_opts(factory.block_cache());
         let raft_path = cfg.infer_raft_db_path(Some(data_dir)).unwrap();
@@ -90,7 +93,13 @@ pub fn new_debug_executor(
             Ok(db) => db,
             Err(e) => handle_engine_error(e),
         };
-        let debugger = Debugger::new(Engines::new(kv_db, raft_db), cfg_controller);
+
+        let kv_db = match factory.create_shared_db(data_dir) {
+            Ok(db) => db,
+            Err(e) => handle_engine_error(e),
+        };
+
+        let debugger = DebuggerImpl::new(Engines::new(kv_db, raft_db), cfg_controller);
         Box::new(debugger) as Box<dyn DebugExecutor>
     } else {
         let mut config = cfg.raft_engine.config();
@@ -100,8 +109,24 @@ pub fn new_debug_executor(
             tikv_util::logger::exit_process_gracefully(-1);
         }
         let raft_db = RaftLogEngine::new(config, key_manager, None /* io_rate_limiter */).unwrap();
-        let debugger = Debugger::new(Engines::new(kv_db, raft_db), cfg_controller);
-        Box::new(debugger) as Box<dyn DebugExecutor>
+        match cfg.storage.engine {
+            EngineType::RaftKv => {
+                let kv_db = match factory.create_shared_db(data_dir) {
+                    Ok(db) => db,
+                    Err(e) => handle_engine_error(e),
+                };
+
+                let debugger = DebuggerImpl::new(Engines::new(kv_db, raft_db), cfg_controller);
+                Box::new(debugger) as Box<dyn DebugExecutor>
+            }
+            EngineType::RaftKv2 => {
+                let registry =
+                    TabletRegistry::new(Box::new(factory), Path::new(data_dir).join("tablets"))
+                        .unwrap_or_else(|e| fatal!("failed to create tablet registry {:?}", e));
+                let debugger = DebuggerImplV2::new(registry, raft_db, cfg_controller);
+                Box::new(debugger) as Box<dyn DebugExecutor>
+            }
+        }
     }
 }
 
@@ -869,11 +894,11 @@ impl DebugExecutor for DebugClient {
     }
 }
 
-impl<ER: RaftEngine> DebugExecutor for Debugger<ER> {
+impl<ER: RaftEngine> DebugExecutor for DebuggerImpl<ER> {
     fn check_local_mode(&self) {}
 
     fn get_all_regions_in_store(&self) -> Vec<u64> {
-        self.get_all_regions_in_store()
+        Debugger::get_all_regions_in_store(self)
             .unwrap_or_else(|e| perror_and_exit("Debugger::get_all_regions_in_store", e))
     }
 
@@ -929,7 +954,7 @@ impl<ER: RaftEngine> DebugExecutor for Debugger<ER> {
         threads: u32,
         bottommost: BottommostLevelCompaction,
     ) {
-        self.compact(db, cf, from, to, threads, bottommost)
+        Debugger::compact(self, db, cf, from, to, threads, bottommost)
             .unwrap_or_else(|e| perror_and_exit("Debugger::compact", e));
     }
 
@@ -973,7 +998,7 @@ impl<ER: RaftEngine> DebugExecutor for Debugger<ER> {
     }
 
     fn recover_all(&self, threads: usize, read_only: bool) {
-        Debugger::recover_all(self, threads, read_only)
+        DebuggerImpl::recover_all(self, threads, read_only)
             .unwrap_or_else(|e| perror_and_exit("Debugger::recover all", e));
     }
 
@@ -1117,3 +1142,129 @@ fn handle_engine_error(err: EngineError) -> ! {
 
     tikv_util::logger::exit_process_gracefully(-1);
 }
+
+impl<ER: RaftEngine> DebugExecutor for DebuggerImplV2<ER> {
+    fn check_local_mode(&self) {}
+
+    fn get_all_regions_in_store(&self) -> Vec<u64> {
+        Debugger::get_all_regions_in_store(self)
+            .unwrap_or_else(|e| perror_and_exit("Debugger::get_all_regions_in_store", e))
+    }
+
+    fn get_value_by_key(&self, cf: &str, key: Vec<u8>) -> Vec<u8> {
+        self.get(DbType::Kv, cf, &key)
+            .unwrap_or_else(|e| perror_and_exit("Debugger::get", e))
+    }
+
+    fn get_region_size(&self, region: u64, cfs: Vec<&str>) -> Vec<(String, usize)> {
+        self.region_size(region, cfs)
+            .unwrap_or_else(|e| perror_and_exit("Debugger::region_size", e))
+            .into_iter()
+            .map(|(cf, size)| (cf.to_owned(), size))
+            .collect()
+    }
+
+    fn get_region_info(&self, region: u64) -> RegionInfo {
+        self.region_info(region)
+            .unwrap_or_else(|e| perror_and_exit("Debugger::region_info", e))
+    }
+
+    fn get_raft_log(&self, region: u64, index: u64) -> Entry {
+        self.raft_log(region, index)
+            .unwrap_or_else(|e| perror_and_exit("Debugger::raft_log", e))
+    }
+
+    fn get_mvcc_infos(&self, from: Vec<u8>, to: Vec<u8>, limit: u64) -> MvccInfoStream {
+        let iter = self
+            .scan_mvcc(&from, &to, limit)
+            .unwrap_or_else(|e| perror_and_exit("Debugger::scan_mvcc", e));
+        let stream = stream::iter(iter).map_err(|e| e.to_string());
+        Box::pin(stream)
+    }
+
+    fn raw_scan_impl(&self, _from_key: &[u8], _end_key: &[u8], _limit: usize, _cf: &str) {
+        unimplemented!()
+    }
+
+    fn do_compaction(
+        &self,
+        db: DbType,
+        cf: &str,
+        from: &[u8],
+        to: &[u8],
+        threads: u32,
+        bottommost: BottommostLevelCompaction,
+    ) {
+        Debugger::compact(self, db, cf, from, to, threads, bottommost)
+            .unwrap_or_else(|e| perror_and_exit("Debugger::compact", e));
+    }
+
+    fn set_region_tombstone(&self, _regions: Vec<Region>) {
+        unimplemented!()
+    }
+
+    fn set_region_tombstone_by_id(&self, _regions: Vec<u64>) {
+        unimplemented!()
+    }
+
+    fn recover_regions(&self, _regions: Vec<Region>, _read_only: bool) {
+        unimplemented!()
+    }
+
+    fn recover_all(&self, _threads: usize, _read_only: bool) {
+        unimplemented!()
+    }
+
+    fn print_bad_regions(&self) {
+        unimplemented!()
+    }
+
+    fn remove_fail_stores(
+        &self,
+        _store_ids: Vec<u64>,
+        _region_ids: Option<Vec<u64>>,
+        _promote_learner: bool,
+    ) {
+        unimplemented!()
+    }
+
+    fn drop_unapplied_raftlog(&self, _region_ids: Option<Vec<u64>>) {
+        unimplemented!()
+    }
+
+    fn recreate_region(&self, _sec_mgr: Arc<SecurityManager>, _pd_cfg: &PdConfig, _region_id: u64) {
+        unimplemented!()
+    }
+
+    fn dump_metrics(&self, _tags: Vec<&str>) {
+        unimplemented!()
+    }
+
+    fn check_region_consistency(&self, _: u64) {
+        unimplemented!()
+    }
+
+    fn modify_tikv_config(&self, _config_name: &str, _config_value: &str) {
+        unimplemented!()
+    }
+
+    fn dump_region_properties(&self, _region_id: u64) {
+        unimplemented!()
+    }
+
+    fn dump_range_properties(&self, _start: Vec<u8>, _end: Vec<u8>) {
+        unimplemented!()
+    }
+
+    fn dump_store_info(&self) {
+        unimplemented!()
+    }
+
+    fn dump_cluster_info(&self) {
+        unimplemented!()
+    }
+
+    fn reset_to_version(&self, _version: u64) {
+        unimplemented!()
+    }
+}