From c9882b56ead07d3f045068557749619fcbfc38fb Mon Sep 17 00:00:00 2001 From: Tobias Grieger Date: Tue, 2 Nov 2021 09:17:24 +0100 Subject: [PATCH] kvserver: remove unused snapshot log entry code We used to have to send the raft log along with snapshots as a result of (in the olden days) requiring all replicas to agree on the truncated state. This hasn't been (generally) true as of v19.1 (#35701), though it was still a possibility until v22.1 (#58088). This commit removes the corresponding code. Release note: None --- pkg/kv/kvserver/raft.pb.go | 214 +++++++++------------- pkg/kv/kvserver/raft.proto | 7 +- pkg/kv/kvserver/replica_raftstorage.go | 73 +------- pkg/kv/kvserver/replica_sideload_test.go | 219 ----------------------- pkg/kv/kvserver/store_snapshot.go | 129 +------------ 5 files changed, 87 insertions(+), 555 deletions(-) diff --git a/pkg/kv/kvserver/raft.pb.go b/pkg/kv/kvserver/raft.pb.go index 5135d28b9ca3..8a643561373c 100644 --- a/pkg/kv/kvserver/raft.pb.go +++ b/pkg/kv/kvserver/raft.pb.go @@ -385,11 +385,7 @@ type SnapshotRequest struct { Header *SnapshotRequest_Header `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // A RocksDB BatchRepr. Multiple kv_batches may be sent across multiple request messages. KVBatch []byte `protobuf:"bytes,2,opt,name=kv_batch,json=kvBatch,proto3" json:"kv_batch,omitempty"` - // These are really raftpb.Entry, but we model them as raw bytes to avoid - // roundtripping through memory. They are separate from the kv_batch to - // allow flexibility in log implementations. - LogEntries [][]byte `protobuf:"bytes,3,rep,name=log_entries,json=logEntries,proto3" json:"log_entries,omitempty"` - Final bool `protobuf:"varint,4,opt,name=final,proto3" json:"final,omitempty"` + Final bool `protobuf:"varint,4,opt,name=final,proto3" json:"final,omitempty"` } func (m *SnapshotRequest) Reset() { *m = SnapshotRequest{} } @@ -570,87 +566,86 @@ func init() { func init() { proto.RegisterFile("kv/kvserver/raft.proto", fileDescriptor_acdcf79fd972c844) } var fileDescriptor_acdcf79fd972c844 = []byte{ - // 1280 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x57, 0xdf, 0x6e, 0x1a, 0x47, - 0x17, 0x67, 0xcd, 0x02, 0xcb, 0x60, 0xe2, 0xfd, 0xe6, 0xf3, 0xe7, 0x6f, 0x6b, 0xb5, 0xe0, 0xae, - 0xd2, 0xd6, 0xfd, 0xb7, 0x28, 0x4e, 0xda, 0x8b, 0xde, 0x54, 0x80, 0x89, 0x8c, 0x71, 0x6c, 0x32, - 0x60, 0x57, 0x6d, 0xd5, 0xae, 0x16, 0x18, 0x60, 0x05, 0xec, 0x6c, 0x66, 0x07, 0x5a, 0xf2, 0x14, - 0x95, 0xfa, 0x02, 0xbd, 0xeb, 0x53, 0xf4, 0xde, 0x97, 0xb9, 0xcc, 0x45, 0x85, 0x5a, 0x72, 0xdd, - 0x17, 0xc8, 0x55, 0x35, 0xb3, 0xb3, 0x40, 0x9c, 0x38, 0x0d, 0x52, 0x23, 0xb5, 0x57, 0x3e, 0x33, - 0x9c, 0xf3, 0xfb, 0xcd, 0x9c, 0xf3, 0x3b, 0x67, 0xd6, 0x60, 0x67, 0x30, 0x29, 0x0c, 0x26, 0x01, - 0xa6, 0x13, 0x4c, 0x0b, 0xd4, 0xe9, 0x32, 0xcb, 0xa7, 0x84, 0x11, 0xf8, 0xbf, 0x36, 0x69, 0x0f, - 0x28, 0x71, 0xda, 0x7d, 0x6b, 0x30, 0xb1, 0x22, 0x8f, 0xdd, 0x6d, 0xb1, 0xe5, 0xb7, 0x0a, 0x98, - 0x52, 0x42, 0x83, 0xd0, 0x79, 0x77, 0x27, 0xda, 0x1d, 0x61, 0xe6, 0x74, 0x1c, 0xe6, 0xc8, 0x7d, - 0x6b, 0x15, 0x7c, 0xe8, 0x4e, 0xb0, 0x87, 0x83, 0x60, 0x61, 0xf8, 0xad, 0x85, 0x29, 0xfd, 0xcd, - 0x55, 0xff, 0xc8, 0xf0, 0x5b, 0x85, 0x80, 0x39, 0x0c, 0x4b, 0x9f, 0x1c, 0x66, 0xed, 0x8e, 0x38, - 0x69, 0x61, 0x72, 0x5b, 0xfc, 0xf5, 0x5b, 0x2b, 0x07, 0xdf, 0xdd, 0xee, 0x91, 0x1e, 0x11, 0x66, - 0x81, 0x5b, 0xe1, 0xae, 0xf9, 0x87, 0x0a, 0xb2, 0xc8, 0xe9, 0xb2, 0x23, 0xec, 0x50, 0xd6, 0xc2, - 0x0e, 0x83, 0xdf, 0x02, 0x8d, 0x3a, 0x5e, 0x0f, 0xdb, 0x6e, 0xc7, 0x50, 0xf6, 0x94, 0x7d, 0xb5, - 0x54, 0x9e, 0xcf, 0xf2, 0x29, 0xc4, 0xf7, 0xaa, 0x87, 0x4f, 0x67, 0xf9, 0x3b, 0x3d, 0x97, 0xf5, - 0xc7, 0x2d, 0xab, 0x4d, 0x46, 0x85, 0x45, 0x32, 0x3a, 0xad, 0xa5, 0x5d, 0xf0, 0x07, 0xbd, 0x82, - 0xbc, 0xb9, 0x25, 0xe3, 0x50, 0x4a, 0x80, 0x56, 0x3b, 0x30, 0x00, 0x5b, 0x5d, 0x4a, 0x46, 0x36, - 0xc5, 0xfe, 0xd0, 0x6d, 0x3b, 0x9c, 0x66, 0x63, 0x4f, 0xd9, 0xcf, 0x96, 0x6a, 0xf3, 0x59, 0x3e, - 0x7b, 0x97, 0x92, 0x11, 0x0a, 0x7f, 0x11, 0x64, 0x9f, 0xae, 0x47, 0x16, 0x45, 0xa2, 0x6c, 0x77, - 0x05, 0xa8, 0x03, 0x47, 0x20, 0xcb, 0xc8, 0x2a, 0x65, 0x5c, 0x50, 0x56, 0xe7, 0xb3, 0x7c, 0xa6, - 0x49, 0xfe, 0x0e, 0xc2, 0x0c, 0x23, 0x4b, 0x3a, 0x08, 0x54, 0x86, 0xe9, 0xc8, 0x50, 0x79, 0xfe, - 0x90, 0xb0, 0xe1, 0x0e, 0x48, 0xb6, 0xc9, 0x68, 0xe4, 0x32, 0x23, 0x21, 0x76, 0xe5, 0x0a, 0x1a, - 0x20, 0xf5, 0x60, 0xec, 0xe2, 0xa0, 0x8d, 0x8d, 0xe4, 0x9e, 0xb2, 0xaf, 0xa1, 0x68, 0x09, 0x1f, - 0x82, 0x37, 0x87, 0x4e, 0xaf, 0xe7, 0x7a, 0x3d, 0xbb, 0x4b, 0x86, 0x43, 0xf2, 0x1d, 0xa6, 0x81, - 0x4d, 0x3c, 0x3b, 0x72, 0xd7, 0xf6, 0xe2, 0xfb, 0x99, 0x83, 0xdb, 0xd6, 0x0b, 0x15, 0x69, 0x2d, - 0x24, 0xb4, 0x94, 0x95, 0x75, 0x22, 0xcd, 0x92, 0x7a, 0x39, 0xcb, 0xc7, 0xd0, 0x1b, 0x12, 0xfe, - 0x6e, 0x84, 0x7e, 0xe6, 0xdd, 0x97, 0xdc, 0x75, 0xf0, 0xce, 0xcb, 0xb8, 0x6d, 0xa7, 0xdd, 0x1e, - 0x53, 0x87, 0x61, 0x03, 0x88, 0x33, 0xbf, 0x7d, 0x2d, 0x52, 0x51, 0x3a, 0x1e, 0xab, 0x5a, 0x4a, - 0xd7, 0xcc, 0x9f, 0x93, 0x00, 0x72, 0xbd, 0xdd, 0xc3, 0x41, 0xe0, 0xf4, 0x30, 0xc2, 0x0f, 0xc6, - 0x38, 0x78, 0xfd, 0xa2, 0xfb, 0x06, 0x6c, 0x85, 0xf8, 0x01, 0x73, 0x28, 0xb3, 0x07, 0x78, 0x6a, - 0x68, 0x7b, 0xca, 0xfe, 0x66, 0xe9, 0x93, 0xa7, 0xb3, 0xfc, 0xad, 0xf5, 0xb0, 0x6b, 0x78, 0x8a, - 0xb2, 0x02, 0xad, 0xc1, 0xc1, 0x6a, 0x78, 0x0a, 0xef, 0x81, 0xcd, 0x55, 0x4d, 0x0b, 0x41, 0x67, - 0x0e, 0x6e, 0xae, 0x54, 0xe6, 0x8a, 0x60, 0x0e, 0x71, 0xd0, 0xa6, 0xae, 0xcf, 0x08, 0x95, 0xa5, - 0xc8, 0xac, 0xe8, 0x15, 0x56, 0x01, 0x58, 0xaa, 0x55, 0x48, 0x75, 0x3d, 0xb0, 0xf4, 0x42, 0x8b, - 0xb0, 0x00, 0x52, 0xa3, 0x30, 0xd5, 0x42, 0x8c, 0x99, 0x83, 0x2d, 0x2b, 0x1c, 0x0d, 0x96, 0xac, - 0x80, 0x0c, 0x89, 0xbc, 0x56, 0xe5, 0x98, 0x58, 0x4f, 0x8e, 0xe9, 0x7f, 0x93, 0x1c, 0xe1, 0x31, - 0x00, 0xfd, 0x68, 0xe6, 0x05, 0x46, 0x52, 0x9c, 0xfd, 0xe6, 0x35, 0x67, 0x7f, 0x66, 0x40, 0xca, - 0xc3, 0xae, 0x44, 0xc3, 0x06, 0xd8, 0x5a, 0xac, 0x6c, 0x8a, 0x03, 0x3f, 0x30, 0x52, 0x6b, 0x03, - 0xde, 0x58, 0x40, 0x20, 0x8e, 0x60, 0x76, 0xc1, 0xff, 0x9f, 0x6f, 0x94, 0x92, 0xc3, 0xda, 0x7d, - 0x58, 0x03, 0x1a, 0x0d, 0xd7, 0x81, 0xa1, 0x08, 0xa2, 0xf7, 0x5f, 0x42, 0x74, 0x05, 0x21, 0x64, - 0x5b, 0x00, 0x98, 0x75, 0x60, 0x3c, 0xe3, 0x15, 0xf8, 0xc4, 0x0b, 0xf0, 0xb9, 0xe7, 0x12, 0x0f, - 0x5a, 0x20, 0x21, 0xde, 0x33, 0xd1, 0x93, 0x99, 0x03, 0xe3, 0x2a, 0x8b, 0xdf, 0xb2, 0x2a, 0xfc, - 0x77, 0x14, 0xba, 0x7d, 0xa6, 0x5e, 0xfe, 0x94, 0x57, 0xcc, 0x5f, 0x37, 0xc0, 0x7f, 0x5f, 0x00, - 0xf9, 0xda, 0x9b, 0xfc, 0x9f, 0xdb, 0x85, 0x35, 0x90, 0x18, 0xf3, 0x84, 0xca, 0x1e, 0x2c, 0xbc, - 0x4a, 0xb5, 0x56, 0xea, 0x20, 0x01, 0x43, 0x0c, 0xf3, 0x51, 0x12, 0x6c, 0x35, 0x3c, 0xc7, 0x0f, - 0xfa, 0x84, 0x45, 0xf3, 0xb3, 0x02, 0x92, 0x7d, 0xec, 0x74, 0x70, 0x54, 0xa9, 0x8f, 0xaf, 0x61, - 0xb8, 0x12, 0x67, 0x1d, 0x89, 0x20, 0x24, 0x83, 0xe1, 0xbb, 0x40, 0x1b, 0x4c, 0xec, 0x16, 0x17, - 0x99, 0xc8, 0xde, 0x66, 0x29, 0xc3, 0x2b, 0x54, 0xbb, 0x10, 0xba, 0x43, 0xa9, 0xc1, 0x24, 0x14, - 0x60, 0x1e, 0x64, 0x86, 0xa4, 0x67, 0x63, 0x8f, 0x51, 0x17, 0x07, 0x46, 0x7c, 0x2f, 0xbe, 0xbf, - 0x89, 0xc0, 0x90, 0xf4, 0x2a, 0xe1, 0x0e, 0xdc, 0x06, 0x89, 0xae, 0xeb, 0x39, 0x43, 0x71, 0x61, - 0x0d, 0x85, 0x8b, 0xdd, 0x1f, 0x55, 0x90, 0x0c, 0x19, 0x61, 0x15, 0x24, 0xc4, 0xc7, 0x8b, 0x18, - 0x32, 0xd7, 0x9f, 0x37, 0x60, 0x84, 0x3a, 0x3d, 0xbc, 0xcc, 0x72, 0x83, 0x07, 0x45, 0xf9, 0x10, - 0x08, 0xd0, 0x01, 0xdb, 0x7c, 0xa4, 0xd9, 0x72, 0x82, 0xd9, 0x52, 0xd9, 0xb2, 0xfc, 0x6b, 0x77, - 0x06, 0xa4, 0xcf, 0x3f, 0x4f, 0x6f, 0x01, 0x20, 0x9f, 0x0f, 0xf7, 0x21, 0x16, 0x52, 0x88, 0xa3, - 0x74, 0xf8, 0x04, 0xb8, 0x0f, 0x31, 0xef, 0x47, 0x9f, 0xba, 0x84, 0xba, 0x6c, 0x2a, 0xde, 0xf0, - 0x1b, 0xd7, 0x56, 0xf8, 0x6a, 0xfe, 0xeb, 0x32, 0x0c, 0x2d, 0x00, 0x38, 0x58, 0xc0, 0xf8, 0x88, - 0xea, 0x4d, 0x8d, 0xd4, 0x5a, 0x60, 0x0d, 0x19, 0x86, 0x16, 0x00, 0xf0, 0x73, 0xa0, 0xb2, 0xa9, - 0xcf, 0x67, 0x33, 0x07, 0xfa, 0xf0, 0x15, 0x81, 0x9a, 0x53, 0x1f, 0x23, 0x11, 0x08, 0xcf, 0xc1, - 0x7b, 0x1d, 0xec, 0x53, 0xdc, 0x76, 0x18, 0xee, 0xd8, 0x63, 0x4f, 0xb6, 0x03, 0x5f, 0x30, 0x3a, - 0xf6, 0x42, 0x2b, 0xac, 0xa4, 0x26, 0x4a, 0x7d, 0x73, 0xe9, 0x7e, 0xbe, 0xe2, 0xdd, 0x8c, 0x9c, - 0x45, 0x01, 0x8f, 0x55, 0x4d, 0xd1, 0x37, 0x8e, 0x55, 0x4d, 0xd5, 0x13, 0xe6, 0x1d, 0xa0, 0x45, - 0x69, 0x80, 0x19, 0x90, 0x3a, 0x3f, 0xad, 0x9d, 0x9e, 0x7d, 0x71, 0xaa, 0xc7, 0xe0, 0x26, 0xd0, - 0x50, 0xa5, 0x7c, 0x76, 0x51, 0x41, 0x5f, 0xea, 0x0a, 0xcc, 0x82, 0x34, 0xaa, 0x94, 0x8a, 0x27, - 0xc5, 0xd3, 0x72, 0x45, 0xdf, 0x30, 0x0d, 0xa0, 0x45, 0xf7, 0xe5, 0x8e, 0xb5, 0x0b, 0xbb, 0x54, - 0x6c, 0x96, 0x8f, 0xf4, 0x98, 0x79, 0x0b, 0xa8, 0xfc, 0x02, 0x70, 0x07, 0xc0, 0x8b, 0x6a, 0xd1, - 0x6e, 0x9c, 0x16, 0xeb, 0x8d, 0xa3, 0xb3, 0xa6, 0x7d, 0xff, 0xbc, 0x72, 0x5e, 0xd1, 0x63, 0x9c, - 0xa3, 0x7a, 0x5a, 0x6d, 0x56, 0x8b, 0x27, 0xba, 0x62, 0xaa, 0xda, 0x86, 0xbe, 0x61, 0xfe, 0xa2, - 0x00, 0x7d, 0x99, 0x04, 0x39, 0xae, 0xee, 0x82, 0x24, 0xbf, 0xd8, 0x38, 0x10, 0x3d, 0x75, 0xe3, - 0xc0, 0xfa, 0xcb, 0xec, 0x85, 0x81, 0x56, 0x43, 0x44, 0x21, 0x19, 0xcd, 0x5f, 0xd4, 0xe8, 0x09, - 0xe6, 0x92, 0x4c, 0x2f, 0xde, 0x5a, 0xb3, 0x08, 0x92, 0xa1, 0xef, 0x73, 0xf7, 0x2e, 0x96, 0xcb, - 0x95, 0x7a, 0xb3, 0x72, 0xa8, 0x2b, 0xfc, 0xa7, 0x62, 0xbd, 0x7e, 0x52, 0xad, 0x1c, 0xea, 0x1b, - 0x30, 0x0d, 0x12, 0x15, 0x84, 0xce, 0x90, 0x1e, 0x37, 0x79, 0xf2, 0xd4, 0x63, 0x55, 0x8b, 0xeb, - 0xaa, 0xf9, 0x35, 0xf8, 0x4f, 0x99, 0x78, 0xdd, 0x72, 0x9f, 0x4b, 0xb2, 0x4c, 0x3c, 0x86, 0xbf, - 0x67, 0xf0, 0x23, 0x00, 0xf8, 0x27, 0xa6, 0xe3, 0x75, 0xa2, 0x81, 0x9b, 0x2e, 0x65, 0xe7, 0xb3, - 0x7c, 0xba, 0x1c, 0xee, 0x56, 0x0f, 0x51, 0x5a, 0x3a, 0x54, 0x3b, 0xfc, 0x94, 0xbe, 0x33, 0x1d, - 0x12, 0x27, 0xfc, 0x1c, 0xdf, 0x44, 0xd1, 0xb2, 0xf4, 0xc1, 0xe5, 0xef, 0xb9, 0xd8, 0xe5, 0x3c, - 0xa7, 0x3c, 0x9a, 0xe7, 0x94, 0xc7, 0xf3, 0x9c, 0xf2, 0xdb, 0x3c, 0xa7, 0xfc, 0xf0, 0x24, 0x17, - 0x7b, 0xf4, 0x24, 0x17, 0x7b, 0xfc, 0x24, 0x17, 0xfb, 0x4a, 0x8b, 0x72, 0xd1, 0x4a, 0x8a, 0xff, - 0x2a, 0x6e, 0xff, 0x19, 0x00, 0x00, 0xff, 0xff, 0x38, 0x9a, 0x07, 0xc0, 0x3e, 0x0d, 0x00, 0x00, + // 1257 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x57, 0xcf, 0x73, 0x22, 0xc5, + 0x17, 0x67, 0x92, 0x01, 0x86, 0x26, 0x6c, 0xe6, 0xdb, 0xdf, 0x18, 0xc7, 0x94, 0x42, 0x9c, 0x5a, + 0x35, 0xfe, 0x1a, 0x6a, 0xb3, 0xab, 0x07, 0x2f, 0x16, 0x10, 0xb6, 0x42, 0xc8, 0x26, 0x6c, 0x43, + 0x62, 0xa9, 0xa5, 0x53, 0x03, 0x34, 0x30, 0x05, 0x4c, 0xcf, 0xf6, 0x34, 0x28, 0xfb, 0x57, 0x58, + 0xe5, 0x3f, 0xe0, 0xcd, 0xbf, 0xc0, 0xa3, 0xf7, 0x1c, 0xf7, 0xb8, 0x07, 0x8b, 0x52, 0xf6, 0xec, + 0x3f, 0xb0, 0x27, 0xab, 0x7b, 0x7a, 0x80, 0x4d, 0x36, 0xeb, 0x52, 0xe5, 0x56, 0xe9, 0x29, 0xaf, + 0x9b, 0xcf, 0xfb, 0xbc, 0xd7, 0xef, 0x7d, 0xfa, 0xf5, 0x04, 0x6c, 0xf7, 0xc7, 0xf9, 0xfe, 0x38, + 0xc0, 0x74, 0x8c, 0x69, 0x9e, 0x3a, 0x1d, 0x66, 0xf9, 0x94, 0x30, 0x02, 0x5f, 0x6b, 0x91, 0x56, + 0x9f, 0x12, 0xa7, 0xd5, 0xb3, 0xfa, 0x63, 0x2b, 0x42, 0xec, 0x6c, 0x89, 0x2d, 0xbf, 0x99, 0xc7, + 0x94, 0x12, 0x1a, 0x84, 0xe0, 0x9d, 0xed, 0x68, 0x77, 0x88, 0x99, 0xd3, 0x76, 0x98, 0x23, 0xf7, + 0xad, 0x65, 0xf2, 0x81, 0x3b, 0xc6, 0x1e, 0x0e, 0x82, 0xb9, 0xe1, 0x37, 0xe7, 0xa6, 0xc4, 0x9b, + 0xcb, 0xf8, 0xc8, 0xf0, 0x9b, 0xf9, 0x80, 0x39, 0x0c, 0x4b, 0x4c, 0x16, 0xb3, 0x56, 0x5b, 0x64, + 0x9a, 0x1f, 0xdf, 0x16, 0x7f, 0xfd, 0xe6, 0x52, 0xe2, 0x3b, 0x5b, 0x5d, 0xd2, 0x25, 0xc2, 0xcc, + 0x73, 0x2b, 0xdc, 0x35, 0xff, 0x54, 0x41, 0x06, 0x39, 0x1d, 0x76, 0x88, 0x1d, 0xca, 0x9a, 0xd8, + 0x61, 0xf0, 0x5b, 0xa0, 0x51, 0xc7, 0xeb, 0x62, 0xdb, 0x6d, 0x1b, 0xca, 0xae, 0xb2, 0xa7, 0x16, + 0x4b, 0xb3, 0x69, 0x2e, 0x89, 0xf8, 0x5e, 0xe5, 0xe0, 0xe9, 0x34, 0x77, 0xa7, 0xeb, 0xb2, 0xde, + 0xa8, 0x69, 0xb5, 0xc8, 0x30, 0x3f, 0x2f, 0x46, 0xbb, 0xb9, 0xb0, 0xf3, 0x7e, 0xbf, 0x9b, 0x97, + 0x27, 0xb7, 0xa4, 0x1f, 0x4a, 0x0a, 0xd2, 0x4a, 0x1b, 0x06, 0x60, 0xb3, 0x43, 0xc9, 0xd0, 0xa6, + 0xd8, 0x1f, 0xb8, 0x2d, 0x87, 0x87, 0x59, 0xdb, 0x55, 0xf6, 0x32, 0xc5, 0xea, 0x6c, 0x9a, 0xcb, + 0xdc, 0xa5, 0x64, 0x88, 0xc2, 0x5f, 0x44, 0xb0, 0x4f, 0x57, 0x0b, 0x16, 0x79, 0xa2, 0x4c, 0x67, + 0x89, 0xa8, 0x0d, 0x87, 0x20, 0xc3, 0xc8, 0x72, 0xc8, 0x75, 0x11, 0xb2, 0x32, 0x9b, 0xe6, 0xd2, + 0x0d, 0xf2, 0x4f, 0x04, 0x4c, 0x33, 0xb2, 0x08, 0x07, 0x81, 0xca, 0x30, 0x1d, 0x1a, 0x2a, 0xaf, + 0x1f, 0x12, 0x36, 0xdc, 0x06, 0x89, 0x16, 0x19, 0x0e, 0x5d, 0x66, 0xc4, 0xc5, 0xae, 0x5c, 0x41, + 0x03, 0x24, 0x1f, 0x8c, 0x5c, 0x1c, 0xb4, 0xb0, 0x91, 0xd8, 0x55, 0xf6, 0x34, 0x14, 0x2d, 0xe1, + 0x43, 0xf0, 0xe6, 0xc0, 0xe9, 0x76, 0x5d, 0xaf, 0x6b, 0x77, 0xc8, 0x60, 0x40, 0xbe, 0xc3, 0x34, + 0xb0, 0x89, 0x67, 0x47, 0x70, 0x6d, 0x77, 0x7d, 0x2f, 0xbd, 0x7f, 0xdb, 0x7a, 0xae, 0x22, 0xad, + 0xb9, 0x84, 0x16, 0xb2, 0xb2, 0x8e, 0xa5, 0x59, 0x54, 0x2f, 0xa6, 0xb9, 0x18, 0x7a, 0x43, 0xd2, + 0xdf, 0x8d, 0xd8, 0x4f, 0xbd, 0xfb, 0x32, 0x76, 0x0d, 0xbc, 0xf3, 0xa2, 0xd8, 0xb6, 0xd3, 0x6a, + 0x8d, 0xa8, 0xc3, 0xb0, 0x01, 0x44, 0xce, 0x6f, 0x5f, 0xcb, 0x54, 0x90, 0xc0, 0x23, 0x55, 0x4b, + 0xea, 0x9a, 0xf9, 0x73, 0x02, 0x40, 0xae, 0xb7, 0x7b, 0x38, 0x08, 0x9c, 0x2e, 0x46, 0xf8, 0xc1, + 0x08, 0x07, 0xaf, 0x5e, 0x74, 0xdf, 0x80, 0xcd, 0x90, 0x3f, 0x60, 0x0e, 0x65, 0x76, 0x1f, 0x4f, + 0x0c, 0x6d, 0x57, 0xd9, 0xdb, 0x28, 0x7e, 0xf2, 0x74, 0x9a, 0xbb, 0xb5, 0x1a, 0x77, 0x15, 0x4f, + 0x50, 0x46, 0xb0, 0xd5, 0x39, 0x59, 0x15, 0x4f, 0xe0, 0x3d, 0xb0, 0xb1, 0xac, 0x69, 0x21, 0xe8, + 0xf4, 0xfe, 0xcd, 0xa5, 0xce, 0x5c, 0x12, 0xcc, 0x01, 0x0e, 0x5a, 0xd4, 0xf5, 0x19, 0xa1, 0xb2, + 0x15, 0xe9, 0x25, 0xbd, 0xc2, 0x0a, 0x00, 0x0b, 0xb5, 0x0a, 0xa9, 0xae, 0x46, 0x96, 0x9a, 0x6b, + 0x11, 0xe6, 0x41, 0x72, 0x18, 0x96, 0x5a, 0x88, 0x31, 0xbd, 0xbf, 0x69, 0x85, 0xa3, 0xc1, 0x92, + 0x1d, 0x90, 0x2e, 0x11, 0x6a, 0x59, 0x8e, 0xf1, 0xd5, 0xe4, 0x98, 0xfa, 0x2f, 0xc9, 0x11, 0x1e, + 0x01, 0xd0, 0x8b, 0x66, 0x5e, 0x60, 0x24, 0x44, 0xee, 0x37, 0xaf, 0xc9, 0xfd, 0x99, 0x01, 0x29, + 0x93, 0x5d, 0xf2, 0x86, 0x75, 0xb0, 0x39, 0x5f, 0xd9, 0x14, 0x07, 0x7e, 0x60, 0x24, 0x57, 0x26, + 0xbc, 0x31, 0xa7, 0x40, 0x9c, 0xc1, 0xec, 0x80, 0xd7, 0xaf, 0x5e, 0x94, 0xa2, 0xc3, 0x5a, 0x3d, + 0x58, 0x05, 0x1a, 0x0d, 0xd7, 0x81, 0xa1, 0x88, 0x40, 0xef, 0xbf, 0x20, 0xd0, 0x25, 0x86, 0x30, + 0xda, 0x9c, 0xc0, 0xac, 0x01, 0xe3, 0x19, 0x54, 0xe0, 0x13, 0x2f, 0xc0, 0x67, 0x9e, 0x4b, 0x3c, + 0x68, 0x81, 0xb8, 0x78, 0xcf, 0xc4, 0x9d, 0x4c, 0xef, 0x1b, 0x97, 0xa3, 0xf8, 0x4d, 0xab, 0xcc, + 0x7f, 0x47, 0x21, 0xec, 0x33, 0xf5, 0xe2, 0xa7, 0x9c, 0x62, 0xfe, 0xb6, 0x06, 0xfe, 0xff, 0x1c, + 0xca, 0x57, 0x7e, 0xc9, 0xff, 0xbd, 0xb7, 0xb0, 0x0a, 0xe2, 0x23, 0x5e, 0x50, 0x79, 0x07, 0xf3, + 0x2f, 0xd3, 0xad, 0xa5, 0x3e, 0x48, 0xc2, 0x90, 0xc3, 0xfc, 0x25, 0x01, 0x36, 0xeb, 0x9e, 0xe3, + 0x07, 0x3d, 0xc2, 0xa2, 0xf9, 0x59, 0x06, 0x89, 0x1e, 0x76, 0xda, 0x38, 0xea, 0xd4, 0xc7, 0xd7, + 0x44, 0xb8, 0xe4, 0x67, 0x1d, 0x0a, 0x27, 0x24, 0x9d, 0xe1, 0xbb, 0x40, 0xeb, 0x8f, 0xed, 0x26, + 0x17, 0x99, 0xa8, 0xde, 0x46, 0x31, 0xcd, 0x3b, 0x54, 0x3d, 0x17, 0xba, 0x43, 0xc9, 0xfe, 0x38, + 0x14, 0xe0, 0x16, 0x88, 0x77, 0x5c, 0xcf, 0x19, 0x88, 0xf3, 0x68, 0x28, 0x5c, 0xec, 0xfc, 0xa8, + 0x82, 0x44, 0x48, 0x08, 0x2b, 0x20, 0x2e, 0xbe, 0x4d, 0xc4, 0x0c, 0xb9, 0x3e, 0x9d, 0x80, 0x11, + 0xea, 0x74, 0xf1, 0xa2, 0x88, 0x75, 0xee, 0x14, 0x1d, 0x57, 0x30, 0x40, 0x07, 0x6c, 0xf1, 0x89, + 0x65, 0xcb, 0x01, 0x65, 0x4b, 0xe1, 0xca, 0xee, 0xae, 0x2c, 0x7c, 0x48, 0xaf, 0xbe, 0x3e, 0x6f, + 0x01, 0x20, 0x5f, 0x07, 0xf7, 0x21, 0x16, 0x9d, 0x5e, 0x47, 0xa9, 0x70, 0xc2, 0xbb, 0x0f, 0x31, + 0xbf, 0x6e, 0x3e, 0x75, 0x09, 0x75, 0xd9, 0x44, 0x3c, 0xd1, 0x37, 0xae, 0x6d, 0xe0, 0xe5, 0xf2, + 0xd6, 0xa4, 0x1b, 0x9a, 0x13, 0x70, 0xb2, 0x80, 0xf1, 0x09, 0xd4, 0x9d, 0x18, 0xc9, 0x95, 0xc8, + 0xea, 0xd2, 0x0d, 0xcd, 0x09, 0xe0, 0xe7, 0x40, 0x65, 0x13, 0x9f, 0x8f, 0x5e, 0x4e, 0xf4, 0xe1, + 0x4b, 0x12, 0x35, 0x26, 0x3e, 0x46, 0xc2, 0x11, 0x9e, 0x81, 0xf7, 0xda, 0xd8, 0xa7, 0xb8, 0xe5, + 0x30, 0xdc, 0xb6, 0x47, 0x9e, 0x54, 0x3b, 0x5f, 0x30, 0x3a, 0xf2, 0x42, 0x2b, 0xec, 0xa4, 0x26, + 0x5a, 0x7d, 0x73, 0x01, 0x3f, 0x5b, 0x42, 0x37, 0x22, 0xb0, 0x68, 0xe0, 0x91, 0xaa, 0x29, 0xfa, + 0xda, 0x91, 0xaa, 0xa9, 0x7a, 0xdc, 0xbc, 0x03, 0xb4, 0xa8, 0x0c, 0x30, 0x0d, 0x92, 0x67, 0x27, + 0xd5, 0x93, 0xd3, 0x2f, 0x4e, 0xf4, 0x18, 0xdc, 0x00, 0x1a, 0x2a, 0x97, 0x4e, 0xcf, 0xcb, 0xe8, + 0x4b, 0x5d, 0x81, 0x19, 0x90, 0x42, 0xe5, 0x62, 0xe1, 0xb8, 0x70, 0x52, 0x2a, 0xeb, 0x6b, 0xa6, + 0x01, 0xb4, 0xe8, 0xbc, 0x1c, 0x58, 0x3d, 0xb7, 0x8b, 0x85, 0x46, 0xe9, 0x50, 0x8f, 0x99, 0xb7, + 0x80, 0xca, 0x0f, 0x00, 0xb7, 0x01, 0x3c, 0xaf, 0x14, 0xec, 0xfa, 0x49, 0xa1, 0x56, 0x3f, 0x3c, + 0x6d, 0xd8, 0xf7, 0xcf, 0xca, 0x67, 0x65, 0x3d, 0xc6, 0x63, 0x54, 0x4e, 0x2a, 0x8d, 0x4a, 0xe1, + 0x58, 0x57, 0x4c, 0x55, 0x5b, 0x13, 0x89, 0xac, 0xeb, 0xaa, 0xf9, 0xab, 0x02, 0xf4, 0x45, 0x29, + 0xe4, 0x4c, 0xba, 0x0b, 0x12, 0xfc, 0x78, 0xa3, 0x40, 0x5c, 0x9c, 0x1b, 0xfb, 0xd6, 0xdf, 0xd6, + 0x30, 0x74, 0xb4, 0xea, 0xc2, 0x0b, 0x49, 0x6f, 0xfe, 0x6c, 0x46, 0xef, 0x2c, 0x17, 0x66, 0x6a, + 0xfe, 0xa0, 0x9a, 0x05, 0x90, 0x08, 0xb1, 0x57, 0x4e, 0x5f, 0x28, 0x95, 0xca, 0xb5, 0x46, 0xf9, + 0x40, 0x57, 0xf8, 0x4f, 0x85, 0x5a, 0xed, 0xb8, 0x52, 0x3e, 0xd0, 0xd7, 0x60, 0x0a, 0xc4, 0xcb, + 0x08, 0x9d, 0x22, 0x7d, 0xdd, 0xe4, 0x25, 0x54, 0x65, 0xfe, 0x5f, 0x83, 0xff, 0x95, 0x88, 0xd7, + 0x29, 0xf5, 0xb8, 0x30, 0x4b, 0xc4, 0x63, 0xf8, 0x7b, 0x06, 0x3f, 0x02, 0x80, 0x7f, 0x47, 0x3a, + 0x5e, 0x3b, 0x9a, 0xaa, 0xa9, 0x62, 0x66, 0x36, 0xcd, 0xa5, 0x4a, 0xe1, 0x6e, 0xe5, 0x00, 0xa5, + 0x24, 0xa0, 0xd2, 0xe6, 0x59, 0xfa, 0xce, 0x64, 0x40, 0x9c, 0xf0, 0x9b, 0x7b, 0x03, 0x45, 0xcb, + 0xe2, 0x07, 0x17, 0x7f, 0x64, 0x63, 0x17, 0xb3, 0xac, 0xf2, 0x68, 0x96, 0x55, 0x1e, 0xcf, 0xb2, + 0xca, 0xef, 0xb3, 0xac, 0xf2, 0xc3, 0x93, 0x6c, 0xec, 0xd1, 0x93, 0x6c, 0xec, 0xf1, 0x93, 0x6c, + 0xec, 0x2b, 0x2d, 0xaa, 0x45, 0x33, 0x21, 0xfe, 0x75, 0xb8, 0xfd, 0x57, 0x00, 0x00, 0x00, 0xff, + 0xff, 0x6c, 0xef, 0x4b, 0x67, 0x23, 0x0d, 0x00, 0x00, } func (m *RaftHeartbeat) Marshal() (dAtA []byte, err error) { @@ -1022,15 +1017,6 @@ func (m *SnapshotRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x20 } - if len(m.LogEntries) > 0 { - for iNdEx := len(m.LogEntries) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.LogEntries[iNdEx]) - copy(dAtA[i:], m.LogEntries[iNdEx]) - i = encodeVarintRaft(dAtA, i, uint64(len(m.LogEntries[iNdEx]))) - i-- - dAtA[i] = 0x1a - } - } if len(m.KVBatch) > 0 { i -= len(m.KVBatch) copy(dAtA[i:], m.KVBatch) @@ -1351,12 +1337,6 @@ func (m *SnapshotRequest) Size() (n int) { if l > 0 { n += 1 + l + sovRaft(uint64(l)) } - if len(m.LogEntries) > 0 { - for _, b := range m.LogEntries { - l = len(b) - n += 1 + l + sovRaft(uint64(l)) - } - } if m.Final { n += 2 } @@ -2446,38 +2426,6 @@ func (m *SnapshotRequest) Unmarshal(dAtA []byte) error { m.KVBatch = []byte{} } iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field LogEntries", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRaft - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthRaft - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthRaft - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.LogEntries = append(m.LogEntries, make([]byte, postIndex-iNdEx)) - copy(m.LogEntries[len(m.LogEntries)-1], dAtA[iNdEx:postIndex]) - iNdEx = postIndex case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Final", wireType) diff --git a/pkg/kv/kvserver/raft.proto b/pkg/kv/kvserver/raft.proto index d74c92f26967..901068a10c8e 100644 --- a/pkg/kv/kvserver/raft.proto +++ b/pkg/kv/kvserver/raft.proto @@ -195,12 +195,9 @@ message SnapshotRequest { // A RocksDB BatchRepr. Multiple kv_batches may be sent across multiple request messages. bytes kv_batch = 2 [(gogoproto.customname) = "KVBatch"]; - // These are really raftpb.Entry, but we model them as raw bytes to avoid - // roundtripping through memory. They are separate from the kv_batch to - // allow flexibility in log implementations. - repeated bytes log_entries = 3; - bool final = 4; + + reserved 3; } message SnapshotResponse { diff --git a/pkg/kv/kvserver/replica_raftstorage.go b/pkg/kv/kvserver/replica_raftstorage.go index 8587db094378..aaa0e477aea8 100644 --- a/pkg/kv/kvserver/replica_raftstorage.go +++ b/pkg/kv/kvserver/replica_raftstorage.go @@ -14,7 +14,6 @@ import ( "context" "fmt" "math" - "strconv" "time" "github.com/cockroachdb/cockroach/pkg/keys" @@ -516,8 +515,6 @@ type IncomingSnapshot struct { SnapUUID uuid.UUID // The storage interface for the underlying SSTs. SSTStorageScratch *SSTSnapshotStorageScratch - // The Raft log entries for this snapshot. - LogEntries [][]byte // The replica state at the time the snapshot was generated (never nil). State *kvserverpb.ReplicaState snapType SnapshotRequest_Type @@ -852,29 +849,6 @@ func (r *Replica) applySnapshot( } // Update Raft entries. - var lastTerm uint64 - var raftLogSize int64 - if len(inSnap.LogEntries) > 0 { - logEntries := make([]raftpb.Entry, len(inSnap.LogEntries)) - for i, bytes := range inSnap.LogEntries { - if err := protoutil.Unmarshal(bytes, &logEntries[i]); err != nil { - return err - } - } - var sideloadedEntriesSize int64 - var err error - logEntries, sideloadedEntriesSize, err = r.maybeSideloadEntriesRaftMuLocked(ctx, logEntries) - if err != nil { - return err - } - raftLogSize += sideloadedEntriesSize - _, lastTerm, raftLogSize, err = r.append(ctx, &unreplicatedSST, 0, invalidLastTerm, raftLogSize, logEntries) - if err != nil { - return err - } - } else { - lastTerm = invalidLastTerm - } r.store.raftEntryCache.Drop(r.RangeID) if err := r.raftMu.stateLoader.SetRaftTruncatedState( @@ -899,26 +873,6 @@ func (r *Replica) applySnapshot( s.RaftAppliedIndex, nonemptySnap.Metadata.Index) } - if expLen := s.RaftAppliedIndex - s.TruncatedState.Index; expLen != uint64(len(inSnap.LogEntries)) { - entriesRange, err := extractRangeFromEntries(inSnap.LogEntries) - if err != nil { - return err - } - - tag := fmt.Sprintf("r%d_%s", r.RangeID, inSnap.SnapUUID.String()) - dir, err := r.store.checkpoint(ctx, tag) - if err != nil { - log.Warningf(ctx, "unable to create checkpoint %s: %+v", dir, err) - } else { - log.Warningf(ctx, "created checkpoint %s", dir) - } - - log.Fatalf(ctx, "missing log entries in snapshot (%s): got %d entries, expected %d "+ - "(TruncatedState.Index=%d, HardState=%s, LogEntries=%s)", - inSnap.String(), len(inSnap.LogEntries), expLen, s.TruncatedState.Index, - hs.String(), entriesRange) - } - // If we're subsuming a replica below, we don't have its last NextReplicaID, // nor can we obtain it. That's OK: we can just be conservative and use the // maximum possible replica ID. preDestroyRaftMuLocked will write a replica @@ -989,8 +943,8 @@ func (r *Replica) applySnapshot( // feelings about this ever change, we can add a LastIndex field to // raftpb.SnapshotMetadata. r.mu.lastIndex = s.RaftAppliedIndex - r.mu.lastTerm = lastTerm - r.mu.raftLogSize = raftLogSize + r.mu.lastTerm = invalidLastTerm + r.mu.raftLogSize = 0 // Update the store stats for the data in the snapshot. r.store.metrics.subtractMVCCStats(ctx, r.mu.tenantID, *r.mu.state.Stats) r.store.metrics.addMVCCStats(ctx, r.mu.tenantID, *s.Stats) @@ -1200,29 +1154,6 @@ func (r *Replica) clearSubsumedReplicaInMemoryData( return nil } -// extractRangeFromEntries returns a string representation of the range of -// marshaled list of raft log entries in the form of [first-index, last-index]. -// If the list is empty, "[n/a, n/a]" is returned instead. -func extractRangeFromEntries(logEntries [][]byte) (string, error) { - var firstIndex, lastIndex string - if len(logEntries) == 0 { - firstIndex = "n/a" - lastIndex = "n/a" - } else { - firstAndLastLogEntries := make([]raftpb.Entry, 2) - if err := protoutil.Unmarshal(logEntries[0], &firstAndLastLogEntries[0]); err != nil { - return "", err - } - if err := protoutil.Unmarshal(logEntries[len(logEntries)-1], &firstAndLastLogEntries[1]); err != nil { - return "", err - } - - firstIndex = strconv.FormatUint(firstAndLastLogEntries[0].Index, 10) - lastIndex = strconv.FormatUint(firstAndLastLogEntries[1].Index, 10) - } - return fmt.Sprintf("[%s, %s]", firstIndex, lastIndex), nil -} - type raftCommandEncodingVersion byte // Raft commands are encoded with a 1-byte version (currently 0 or 1), an 8-byte diff --git a/pkg/kv/kvserver/replica_sideload_test.go b/pkg/kv/kvserver/replica_sideload_test.go index 752d4e7ba403..8eaee788fcfe 100644 --- a/pkg/kv/kvserver/replica_sideload_test.go +++ b/pkg/kv/kvserver/replica_sideload_test.go @@ -13,7 +13,6 @@ import ( "bytes" "context" "fmt" - "io" "math" "math/rand" "path/filepath" @@ -28,7 +27,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverbase" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverpb" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/raftentry" - "github.com/cockroachdb/cockroach/pkg/kv/kvserver/stateloader" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage" @@ -668,33 +666,6 @@ func testRaftSSTableSideloadingProposal(t *testing.T, eng storage.Engine) { verifyLogSizeInSync(t, tc.repl) } -type mockSender struct { - logEntries [][]byte - done bool -} - -func (mr *mockSender) Send(req *SnapshotRequest) error { - if req.LogEntries != nil { - if mr.logEntries != nil { - return errors.New("already have log entries") - } - mr.logEntries = req.LogEntries - } - return nil -} - -func (mr *mockSender) Recv() (*SnapshotResponse, error) { - if mr.done { - return nil, io.EOF - } - status := SnapshotResponse_ACCEPTED - if len(mr.logEntries) > 0 { - status = SnapshotResponse_APPLIED - mr.done = true - } - return &SnapshotResponse{Status: status}, nil -} - func newOnDiskEngine(t *testing.T) (func(), storage.Engine) { dir, cleanup := testutils.TempDir(t) eng, err := storage.Open( @@ -707,196 +678,6 @@ func newOnDiskEngine(t *testing.T) (func(), storage.Engine) { return cleanup, eng } -// This test verifies that when a snapshot is sent, sideloaded proposals are -// inlined. -func TestRaftSSTableSideloadingSnapshot(t *testing.T) { - defer leaktest.AfterTest(t)() - defer log.Scope(t).Close(t) - defer SetMockAddSSTable()() - - ctx := context.Background() - tc := testContext{} - - cleanup, eng := newOnDiskEngine(t) - tc.engine = eng - defer cleanup() - defer eng.Close() - - stopper := stop.NewStopper() - defer stopper.Stop(ctx) - tc.Start(t, stopper) - - var ba roachpb.BatchRequest - ba.RangeID = tc.repl.RangeID - - // Disable log truncation as we want to be sure that we get to create - // snapshots that have our sideloaded proposal in them. - tc.store.SetRaftLogQueueActive(false) - - // Put a sideloaded proposal on the Range. - key, val := "don't", "care" - origSSTData, _ := MakeSSTable(key, val, hlc.Timestamp{}.Add(0, 1)) - { - - var addReq roachpb.AddSSTableRequest - addReq.Data = origSSTData - addReq.Key = roachpb.Key(key) - addReq.EndKey = addReq.Key.Next() - ba.Add(&addReq) - - _, pErr := tc.store.Send(ctx, ba) - if pErr != nil { - t.Fatal(pErr) - } - } - - // Run a happy case snapshot. Check that it properly inlines the payload in - // the contained log entries. - inlinedEntry := func() raftpb.Entry { - os, err := tc.repl.GetSnapshot(ctx, SnapshotRequest_VIA_SNAPSHOT_QUEUE, tc.store.StoreID()) - if err != nil { - t.Fatal(err) - } - defer os.Close() - - mockSender := &mockSender{} - if err := sendSnapshot( - ctx, - tc.store.cfg.Settings, - mockSender, - &fakeStorePool{}, - SnapshotRequest_Header{State: os.State, Priority: SnapshotRequest_RECOVERY}, - os, - tc.repl.store.Engine().NewBatch, - func() {}, - ); err != nil { - t.Fatal(err) - } - - var ent raftpb.Entry - var cmd kvserverpb.RaftCommand - var finalEnt raftpb.Entry - for _, entryBytes := range mockSender.logEntries { - if err := protoutil.Unmarshal(entryBytes, &ent); err != nil { - t.Fatal(err) - } - if sniffSideloadedRaftCommand(ent.Data) { - _, cmdBytes := DecodeRaftCommand(ent.Data) - if err := protoutil.Unmarshal(cmdBytes, &cmd); err != nil { - t.Fatal(err) - } - if as := cmd.ReplicatedEvalResult.AddSSTable; as == nil { - t.Fatalf("no AddSSTable found in sideloaded command %+v", cmd) - } else if len(as.Data) == 0 { - t.Fatalf("empty payload in sideloaded command: %+v", cmd) - } - finalEnt = ent - } - } - if finalEnt.Index == 0 { - t.Fatal("no sideloaded command found") - } - return finalEnt - }() - - sideloadedIndex := inlinedEntry.Index - - // This happens to be a good point in time to check the `entries()` method - // which has special handling to accommodate `term()`: when an empty - // sideload storage is passed in, `entries()` should not inline, and in turn - // also not populate the entries cache (since its contents must always be - // fully inlined). - func() { - tc.repl.raftMu.Lock() - defer tc.repl.raftMu.Unlock() - tc.repl.mu.Lock() - defer tc.repl.mu.Unlock() - for _, withSS := range []bool{false, true} { - tc.store.raftEntryCache.Clear(tc.repl.RangeID, sideloadedIndex+1) - - var ss SideloadStorage - if withSS { - ss = tc.repl.raftMu.sideloaded - } - rsl := stateloader.Make(tc.repl.RangeID) - entries, err := entries( - ctx, rsl, tc.store.Engine(), tc.repl.RangeID, tc.store.raftEntryCache, - ss, sideloadedIndex, sideloadedIndex+1, 1<<20, - ) - if err != nil { - t.Fatal(err) - } - if len(entries) != 1 { - t.Fatalf("no or too many entries returned from cache: %+v", entries) - } - ents, _, _, _ := tc.store.raftEntryCache.Scan(nil, tc.repl.RangeID, sideloadedIndex, sideloadedIndex+1, 1<<20) - if withSS { - // We passed the sideload storage, so we expect to get our - // inlined index back from the cache. - if len(ents) != 1 { - t.Fatalf("no or too many entries returned from cache: %+v", ents) - } - if err := entryEq(inlinedEntry, ents[0]); err != nil { - t.Fatalf("withSS=%t: %+v", withSS, err) - } - } else { - // Without sideload storage, expect the cache to remain - // unpopulated and the entry returned from entries() to not have - // been inlined. - if len(ents) != 0 { - t.Fatalf("expected no cached entries, but got %+v", ents) - } - if expErr, err := `ReplicatedEvalResult.AddSSTable.Data: \[\]uint8\[\d+\] != \[\]uint8\[0\]`, - entryEq(inlinedEntry, entries[0]); !testutils.IsError( - err, - expErr, - ) { - t.Fatalf("expected specific mismatch on `Data` field, but got %v\nwanted: %s", err, expErr) - } - } - } - }() - - // Now run a snapshot that will fail since it doesn't find one of its on-disk - // payloads. This can happen if the Raft log queue runs between the time the - // (engine) snapshot is taken and the log entries are actually read from the - // (engine) snapshot. We didn't run this before because we wanted the file - // to stay in sideloaded storage for the previous test. - func() { - failingOS, err := tc.repl.GetSnapshot(ctx, SnapshotRequest_VIA_SNAPSHOT_QUEUE, tc.store.StoreID()) - if err != nil { - t.Fatal(err) - } - defer failingOS.Close() - - // Remove the actual file. - tc.repl.raftMu.Lock() - if err := tc.repl.raftMu.sideloaded.Clear(ctx); err != nil { - tc.repl.raftMu.Unlock() - t.Fatal(err) - } - tc.repl.raftMu.Unlock() - // Additionally we need to clear out the entry from the cache because - // that would still save the day. - tc.store.raftEntryCache.Clear(tc.repl.RangeID, sideloadedIndex+1) - - mockSender := &mockSender{} - err = sendSnapshot( - ctx, - tc.store.cfg.Settings, - mockSender, - &fakeStorePool{}, - SnapshotRequest_Header{State: failingOS.State, Priority: SnapshotRequest_RECOVERY}, - failingOS, - tc.repl.store.Engine().NewBatch, - func() {}, - ) - if !errors.HasType(err, (*errMustRetrySnapshotDueToTruncation)(nil)) { - t.Fatal(err) - } - }() -} - func TestRaftSSTableSideloadingTruncation(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) diff --git a/pkg/kv/kvserver/store_snapshot.go b/pkg/kv/kvserver/store_snapshot.go index 4a269d5574aa..035a3251424d 100644 --- a/pkg/kv/kvserver/store_snapshot.go +++ b/pkg/kv/kvserver/store_snapshot.go @@ -28,7 +28,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/humanizeutil" "github.com/cockroachdb/cockroach/pkg/util/log" - "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/cockroachdb/errors" @@ -241,7 +240,6 @@ func (kvSS *kvBatchSnapshotStrategy) Receive( return noSnap, err } defer msstw.Close() - var logEntries [][]byte for { req, err := stream.Recv() @@ -272,9 +270,6 @@ func (kvSS *kvBatchSnapshotStrategy) Receive( } } } - if req.LogEntries != nil { - logEntries = append(logEntries, req.LogEntries...) - } if req.Final { // We finished receiving all batches and log entries. It's possible that // we did not receive any key-value pairs for some of the key ranges, but @@ -295,22 +290,11 @@ func (kvSS *kvBatchSnapshotStrategy) Receive( inSnap := IncomingSnapshot{ SnapUUID: snapUUID, SSTStorageScratch: kvSS.scratch, - LogEntries: logEntries, State: &header.State, snapType: header.Type, } - expLen := inSnap.State.RaftAppliedIndex - inSnap.State.TruncatedState.Index - if expLen != uint64(len(logEntries)) { - // We've received a botched snapshot. We could fatal right here but opt - // to warn loudly instead, and fatal when applying the snapshot - // (in Replica.applySnapshot) in order to capture replica hard state. - log.Warningf(ctx, - "missing log entries in snapshot (%s): got %d entries, expected %d", - inSnap.String(), len(logEntries), expLen) - } - - kvSS.status = fmt.Sprintf("log entries: %d, ssts: %d", len(logEntries), len(kvSS.scratch.SSTs())) + kvSS.status = fmt.Sprintf("ssts: %d", len(kvSS.scratch.SSTs())) return inSnap, nil } } @@ -375,105 +359,7 @@ func (kvSS *kvBatchSnapshotStrategy) Send( bytesSent += int64(b.Len()) } - // Iterate over the specified range of Raft entries and send them all out - // together. - rangeID := header.State.Desc.RangeID - firstIndex := header.State.TruncatedState.Index + 1 - endIndex := snap.RaftSnap.Metadata.Index + 1 - logEntries := make([]raftpb.Entry, 0, endIndex-firstIndex) - scanFunc := func(ent raftpb.Entry) error { - logEntries = append(logEntries, ent) - return nil - } - if err := iterateEntries(ctx, snap.EngineSnap, rangeID, firstIndex, endIndex, scanFunc); err != nil { - return 0, err - } - - // Inline the payloads for all sideloaded proposals. - // - // TODO(tschottdorf): could also send slim proposals and attach sideloaded - // SSTables directly to the snapshot. Probably the better long-term - // solution, but let's see if it ever becomes relevant. Snapshots with - // inlined proposals are hopefully the exception. - // - // TODO(tbg): this code is obsolete because as of the PR linked below, - // our snapshots will never contain log entries. Trim down this code. - // - // https://github.com/cockroachdb/cockroach/pull/70464 - { - for i, ent := range logEntries { - if !sniffSideloadedRaftCommand(ent.Data) { - continue - } - if err := snap.WithSideloaded(func(ss SideloadStorage) error { - newEnt, err := maybeInlineSideloadedRaftCommand( - ctx, rangeID, ent, ss, snap.RaftEntryCache, - ) - if err != nil { - return err - } - if newEnt != nil { - logEntries[i] = *newEnt - } - return nil - }); err != nil { - if errors.Is(err, errSideloadedFileNotFound) { - // We're creating the Raft snapshot based on a snapshot of - // the engine, but the Raft log may since have been - // truncated and corresponding on-disk sideloaded payloads - // unlinked. Luckily, we can just abort this snapshot; the - // caller can retry. - // - // TODO(tschottdorf): check how callers handle this. They - // should simply retry. In some scenarios, perhaps this can - // happen repeatedly and prevent a snapshot; not sending the - // log entries wouldn't help, though, and so we'd really - // need to make sure the entries are always here, for - // instance by pre-loading them into memory. Or we can make - // log truncation less aggressive about removing sideloaded - // files, by delaying trailing file deletion for a bit. - return 0, &errMustRetrySnapshotDueToTruncation{ - index: ent.Index, - term: ent.Term, - } - } - return 0, err - } - } - } - - // Marshal each of the log entries. - logEntriesRaw := make([][]byte, len(logEntries)) - for i := range logEntries { - entRaw, err := protoutil.Marshal(&logEntries[i]) - if err != nil { - return 0, err - } - logEntriesRaw[i] = entRaw - } - - // The difference between the snapshot index (applied index at the time of - // snapshot) and the truncated index should equal the number of log entries - // shipped over. - expLen := endIndex - firstIndex - if expLen != uint64(len(logEntries)) { - // We've generated a botched snapshot. We could fatal right here but opt - // to warn loudly instead, and fatal at the caller to capture a checkpoint - // of the underlying storage engine. - entriesRange, err := extractRangeFromEntries(logEntriesRaw) - if err != nil { - return 0, err - } - log.Warningf(ctx, "missing log entries in snapshot (%s): "+ - "got %d entries, expected %d (TruncatedState.Index=%d, LogEntries=%s)", - snap.String(), len(logEntries), expLen, snap.State.TruncatedState.Index, entriesRange) - return 0, errMalformedSnapshot - } - - kvSS.status = fmt.Sprintf("kv pairs: %d, log entries: %d", kvs, len(logEntries)) - if err := stream.Send(&SnapshotRequest{LogEntries: logEntriesRaw}); err != nil { - return 0, err - } + kvSS.status = fmt.Sprintf("kv pairs: %d", kvs) return bytesSent, nil } @@ -855,17 +741,6 @@ func snapshotRateLimit( } } -type errMustRetrySnapshotDueToTruncation struct { - index, term uint64 -} - -func (e *errMustRetrySnapshotDueToTruncation) Error() string { - return fmt.Sprintf( - "log truncation during snapshot removed sideloaded SSTable at index %d, term %d", - e.index, e.term, - ) -} - // SendEmptySnapshot creates an OutgoingSnapshot for the input range // descriptor and seeds it with an empty range. Then, it sends this // snapshot to the replica specified in the input.