From 5eae36f68f4e85e1c321e7785521f8d6a07081c8 Mon Sep 17 00:00:00 2001 From: Calvin Neo Date: Sat, 20 Apr 2024 00:10:34 +0800 Subject: [PATCH] Add log for current ReadIndex mechanism (#370) --- .../src/mock_cluster/v1/cluster.rs | 4 + .../mock_store/mock_engine_store_server.rs | 1 + .../proxy_ffi/src/read_index_helper.rs | 2 +- proxy_tests/proxy/shared/replica_read.rs | 284 ++++++++++++++++++ 4 files changed, 290 insertions(+), 1 deletion(-) diff --git a/proxy_components/mock-engine-store/src/mock_cluster/v1/cluster.rs b/proxy_components/mock-engine-store/src/mock_cluster/v1/cluster.rs index e2d62d4a9c6..62e03331a82 100644 --- a/proxy_components/mock-engine-store/src/mock_cluster/v1/cluster.rs +++ b/proxy_components/mock-engine-store/src/mock_cluster/v1/cluster.rs @@ -447,6 +447,10 @@ impl> Cluster { } } + pub fn get_router(&self, node_id: u64) -> Option> { + self.sim.rl().get_router(node_id) + } + fn valid_leader_id(&self, region_id: u64, leader_id: u64) -> bool { let store_ids = match self.voter_store_ids_of_region(region_id) { None => return false, diff --git a/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs b/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs index 969c28af033..1f769af8d17 100644 --- a/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs +++ b/proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs @@ -359,6 +359,7 @@ unsafe extern "C" fn ffi_release_pre_handled_snapshot( pub fn gen_engine_store_server_helper( wrap: Pin<&EngineStoreServerWrap>, ) -> EngineStoreServerHelper { + info!("mock gen_engine_store_server_helper"); EngineStoreServerHelper { magic_number: interfaces_ffi::RAFT_STORE_PROXY_MAGIC_NUMBER, version: interfaces_ffi::RAFT_STORE_PROXY_VERSION, diff --git a/proxy_components/proxy_ffi/src/read_index_helper.rs b/proxy_components/proxy_ffi/src/read_index_helper.rs index 604345c9bd4..02b2666c67d 100644 --- a/proxy_components/proxy_ffi/src/read_index_helper.rs +++ b/proxy_components/proxy_ffi/src/read_index_helper.rs @@ -82,7 +82,7 @@ fn into_read_index_response( resp } -fn gen_read_index_raft_cmd_req(req: &mut ReadIndexRequest) -> RaftCmdRequest { +pub fn gen_read_index_raft_cmd_req(req: &mut ReadIndexRequest) -> RaftCmdRequest { let region_id = req.get_context().get_region_id(); let mut cmd = RaftCmdRequest::default(); { diff --git a/proxy_tests/proxy/shared/replica_read.rs b/proxy_tests/proxy/shared/replica_read.rs index 1f991a6cf82..40c59967fbb 100644 --- a/proxy_tests/proxy/shared/replica_read.rs +++ b/proxy_tests/proxy/shared/replica_read.rs @@ -356,3 +356,287 @@ fn test_util() { } assert!(GC_MONITOR.valid_clean()); } + +use kvproto::{ + kvrpcpb::{Context, DiskFullOpt, KeyRange}, + raft_cmdpb::{CmdType, RaftCmdRequest, RaftRequestHeader, Request as RaftRequest}, + raft_serverpb::RaftMessage, +}; +use raftstore::{ + router::RaftStoreRouter, + store::{msg::Callback, RaftCmdExtraOpts, ReadIndexContext}, +}; +use tokio::sync::oneshot; +use txn_types::{Key, Lock, LockType, TimeStamp}; +use uuid::Uuid; + +use crate::utils::v1_server::{new_server_cluster, ChannelBuilder, Environment, TikvClient}; + +// https://github.com/tikv/tikv/issues/16823 +#[test] +fn test_raft_cmd_request_cant_advanve_max_ts() { + use kvproto::kvrpcpb::{ReadIndexRequest, ReadIndexResponse}; + + let mut cluster = new_server_cluster(0, 1); + cluster.run(); + + let cm = cluster.sim.read().unwrap().get_concurrency_manager(1); + + let region = cluster.get_region(b""); + let leader = region.get_peers()[0].clone(); + let addr = cluster.sim.rl().get_addr(leader.get_store_id()).to_owned(); + + let env = Arc::new(Environment::new(1)); + let channel = ChannelBuilder::new(env).connect(&addr); + let client = TikvClient::new(channel); + + let mut ctx = Context::default(); + let region_id = leader.get_id(); + ctx.set_region_id(leader.get_id()); + ctx.set_region_epoch(region.get_region_epoch().clone()); + ctx.set_peer(leader); + + let read_index = |ranges: &[(&[u8], &[u8])]| { + let start_ts = block_on(cluster.pd_client.get_tso()).unwrap(); + + let mut cmd = RaftCmdRequest::default(); + { + let mut header = RaftRequestHeader::default(); + let mut inner_req = RaftRequest::default(); + inner_req.set_cmd_type(CmdType::ReadIndex); + inner_req + .mut_read_index() + .set_start_ts(start_ts.into_inner()); + + let mut req = ReadIndexRequest::default(); + let start_ts = block_on(cluster.pd_client.get_tso()).unwrap(); + req.set_context(ctx.clone()); + req.set_start_ts(start_ts.into_inner()); + for &(start_key, end_key) in ranges { + let mut range = KeyRange::default(); + range.set_start_key(start_key.to_vec()); + range.set_end_key(end_key.to_vec()); + req.mut_ranges().push(range); + } + + header.set_region_id(region_id); + header.set_peer(req.get_context().get_peer().clone()); + header.set_region_epoch(req.get_context().get_region_epoch().clone()); + cmd.set_header(header); + cmd.set_requests(vec![inner_req].into()); + } + + let (result_tx, result_rx) = oneshot::channel(); + let router = cluster.get_router(1).unwrap(); + if let Err(e) = router.send_command( + cmd, + Callback::read(Box::new(move |resp| { + result_tx.send(resp.response).unwrap(); + })), + RaftCmdExtraOpts { + deadline: None, + disk_full_opt: DiskFullOpt::AllowedOnAlmostFull, + }, + ) { + panic!("router send msg failed, error: {}", e); + } + + let resp = block_on(result_rx).unwrap(); + (resp.get_responses()[0].get_read_index().clone(), start_ts) + }; + + // wait a while until the node updates its own max ts + std::thread::sleep(Duration::from_millis(300)); + + let prev_cm_max_ts = cm.max_ts(); + let (resp, start_ts) = read_index(&[(b"l", b"yz")]); + assert!(!resp.has_locked()); + // Actually not changed + assert_eq!(cm.max_ts(), prev_cm_max_ts); + assert_ne!(cm.max_ts(), start_ts); + cluster.shutdown(); + fail::remove("on_pre_write_apply_state") +} + +// https://github.com/tikv/tikv/pull/8669/files +#[test] +fn test_raft_cmd_request_learner_advanve_max_ts() { + use kvproto::kvrpcpb::{ReadIndexRequest, ReadIndexResponse}; + + let mut cluster = new_server_cluster(0, 2); + cluster.pd_client.disable_default_operator(); + let region_id = cluster.run_conf_change(); + let region = cluster.get_region(b""); + assert_eq!(region_id, 1); + assert_eq!(region.get_id(), 1); + let leader = region.get_peers()[0].clone(); + + fail::cfg("on_pre_write_apply_state", "return(true)").unwrap(); + let learner = new_learner_peer(2, 2); + cluster.pd_client.must_add_peer(1, learner.clone()); + + let cm = cluster.sim.read().unwrap().get_concurrency_manager(1); + let keys: Vec<_> = vec![b"k", b"l"] + .into_iter() + .map(|k| Key::from_raw(k)) + .collect(); + let guards = block_on(cm.lock_keys(keys.iter())); + let lock = Lock::new( + LockType::Put, + b"k".to_vec(), + 1.into(), + 20000, + None, + 1.into(), + 1, + 2.into(), + false, + ); + guards[0].with_lock(|l| *l = Some(lock.clone())); + + let addr = cluster.sim.rl().get_addr(learner.get_store_id()).to_owned(); + + let env = Arc::new(Environment::new(1)); + let channel = ChannelBuilder::new(env).connect(&addr); + let client = TikvClient::new(channel); + + // cluster.must_put(b"k", b"v"); + + let read_index = |ranges: &[(&[u8], &[u8])]| { + let start_ts = block_on(cluster.pd_client.get_tso()).unwrap(); + + // https://github.com/pingcap/tiflash/blob/14a127820d0530e496af624bb5b69acd48caf747/dbms/src/Storages/KVStore/Read/ReadIndex.cpp#L39 + let mut ctx = Context::default(); + let learner = learner.clone(); + ctx.set_region_id(region_id); + ctx.set_region_epoch(region.get_region_epoch().clone()); + ctx.set_peer(learner); + let mut read_index_request = ReadIndexRequest::default(); + read_index_request.set_context(ctx); + read_index_request.set_start_ts(start_ts.into_inner()); + for (s, e) in ranges { + let mut r = KeyRange::new(); + r.set_start_key(s.to_vec()); + r.set_end_key(e.to_vec()); + read_index_request.mut_ranges().push(r); + } + let mut cmd = + proxy_ffi::read_index_helper::gen_read_index_raft_cmd_req(&mut read_index_request); + + let (result_tx, result_rx) = oneshot::channel(); + let router = cluster.get_router(2).unwrap(); + if let Err(e) = router.send_command( + cmd, + Callback::read(Box::new(move |resp| { + result_tx.send(resp.response).unwrap(); + })), + RaftCmdExtraOpts { + deadline: None, + disk_full_opt: DiskFullOpt::AllowedOnAlmostFull, + }, + ) { + panic!("router send msg failed, error: {}", e); + } + + let resp = block_on(result_rx).unwrap(); + (resp.get_responses()[0].get_read_index().clone(), start_ts) + }; + + // wait a while until the node updates its own max ts + std::thread::sleep(Duration::from_millis(3000)); + + must_wait_until_cond_node( + &cluster.cluster_ext, + region_id, + None, + &|states: &States| -> bool { + states.in_disk_region_state.get_region().get_peers().len() == 2 + }, + ); + + let prev_cm_max_ts = cm.max_ts(); + let (resp, start_ts) = read_index(&[(b"l", b"yz")]); + assert!(!resp.has_locked()); + // Actually not changed + assert_ne!(cm.max_ts(), prev_cm_max_ts); + assert_eq!(cm.max_ts(), start_ts); + + // `gen_read_index_raft_cmd_req` can only handle one key-range + let (resp, start_ts) = read_index(&[(b"j", b"k0")]); + assert_eq!(resp.get_locked(), &lock.into_lock_info(b"k".to_vec())); + assert_eq!(cm.max_ts(), start_ts); + + drop(guards); + + let (resp, start_ts) = read_index(&[(b"a", b"z")]); + assert!(!resp.has_locked()); + assert_eq!(cm.max_ts(), start_ts); + cluster.shutdown(); + fail::remove("on_pre_write_apply_state") +} + +#[test] +fn test_raft_message_can_advanve_max_ts() { + use kvproto::raft_cmdpb::{ReadIndexRequest, ReadIndexResponse}; + let mut cluster = new_server_cluster(0, 1); + cluster.run(); + + let cm = cluster.sim.read().unwrap().get_concurrency_manager(1); + + let region = cluster.get_region(b""); + let leader = region.get_peers()[0].clone(); + let follower = new_learner_peer(2, 2); + let addr = cluster.sim.rl().get_addr(leader.get_store_id()).to_owned(); + + let env = Arc::new(Environment::new(1)); + let channel = ChannelBuilder::new(env).connect(&addr); + let client = TikvClient::new(channel); + + let region_id = leader.get_id(); + + let read_index = |ranges: &[(&[u8], &[u8])]| { + let mut m = raft::eraftpb::Message::default(); + m.set_msg_type(MessageType::MsgReadIndex); + let mut read_index_req = ReadIndexRequest::default(); + let start_ts = block_on(cluster.pd_client.get_tso()).unwrap(); + read_index_req.set_start_ts(start_ts.into_inner()); + for &(start_key, end_key) in ranges { + let mut range = KeyRange::default(); + range.set_start_key(start_key.to_vec()); + range.set_end_key(end_key.to_vec()); + read_index_req.mut_key_ranges().push(range); + } + + let rctx = ReadIndexContext { + id: Uuid::new_v4(), + request: Some(read_index_req), + locked: None, + }; + let mut e = raft::eraftpb::Entry::default(); + e.set_data(rctx.to_bytes().into()); + m.mut_entries().push(e); + m.set_from(2); + + let mut raft_msg = kvproto::raft_serverpb::RaftMessage::default(); + raft_msg.set_region_id(region.get_id()); + raft_msg.set_from_peer(follower); + raft_msg.set_to_peer(leader); + raft_msg.set_region_epoch(region.get_region_epoch().clone()); + raft_msg.set_message(m); + cluster.send_raft_msg(raft_msg).unwrap(); + + (ReadIndexResponse::default(), start_ts) + }; + + // wait a while until the node updates its own max ts + let prev_cm_max_ts = cm.max_ts(); + let (resp, start_ts) = read_index(&[(b"l", b"yz")]); + cluster.must_put(b"a", b"b"); + std::thread::sleep(Duration::from_millis(2000)); + // assert!(!resp.has_locked()); + // Actually not changed + assert_ne!(cm.max_ts(), prev_cm_max_ts); + assert_eq!(cm.max_ts(), start_ts); + cluster.shutdown(); + fail::remove("on_pre_write_apply_state") +}