From 09c91d8dc0d2f550b69fbe183399e98b1fd6d606 Mon Sep 17 00:00:00 2001 From: Nathan VanBenschoten Date: Wed, 21 Dec 2022 11:59:07 -0500 Subject: [PATCH] raft: store Vote in MsgStorageAppend, not *HardState This commit replaces the HardState field in Message with a Vote. For MsgStorageAppends, the term, vote, and commit fields will either all be set (to facilitate the construction of a HardState) if any of the fields have changed or will all be unset if none of the fields have changed. Signed-off-by: Nathan VanBenschoten --- raft.go | 12 +- raftpb/raft.pb.go | 198 ++++++++---------- raftpb/raft.proto | 16 +- raftpb/raft_test.go | 2 +- ...ction_env_handler_process_append_thread.go | 7 +- rawnode.go | 24 +-- testdata/async_storage_writes.txt | 108 +++++----- .../async_storage_writes_append_aba_race.txt | 54 ++--- util.go | 6 +- 9 files changed, 203 insertions(+), 224 deletions(-) diff --git a/raft.go b/raft.go index d43e917bf9c9..ee7859456ae6 100644 --- a/raft.go +++ b/raft.go @@ -167,12 +167,12 @@ type Config struct { // messages over the network) and those targeted at the same thread can't be // reordered. Messages to different targets can be processed in any order. // - // MsgStorageAppend carries Raft log entries to append, election votes to - // persist, and snapshots to apply. All writes performed in service of a - // MsgStorageAppend must be durable before response messages are delivered. - // However, if the MsgStorageAppend carries no response messages, durability - // is not required. The message assumes the role of the Entries, HardState, - // and Snapshot fields in Ready. + // MsgStorageAppend carries Raft log entries to append, election votes / + // term changes / updated commit indexes to persist, and snapshots to apply. + // All writes performed in service of a MsgStorageAppend must be durable + // before response messages are delivered. However, if the MsgStorageAppend + // carries no response messages, durability is not required. The message + // assumes the role of the Entries, HardState, and Snapshot fields in Ready. // // MsgStorageApply carries committed entries to apply. Writes performed in // service of a MsgStorageApply need not be durable before response messages diff --git a/raftpb/raft.pb.go b/raftpb/raft.pb.go index e1033908a1a3..148026397933 100644 --- a/raftpb/raft.pb.go +++ b/raftpb/raft.pb.go @@ -404,14 +404,19 @@ type Message struct { // entries from its leader as it already has an entry with term 5 at index 100. // (type=MsgStorageAppendResp,index=100,logTerm=5) means the local node wrote // entries up to index=100 in stable storage, and the term of the entry at index - // 100 was 5. + // 100 was 5. This doesn't always mean that the corresponding MsgStorageAppend + // message was the one that carried these entries, just that those entries were + // stable at the time of processing the corresponding MsgStorageAppend. LogTerm uint64 `protobuf:"varint,5,opt,name=logTerm" json:"logTerm"` Index uint64 `protobuf:"varint,6,opt,name=index" json:"index"` Entries []Entry `protobuf:"bytes,7,rep,name=entries" json:"entries"` Commit uint64 `protobuf:"varint,8,opt,name=commit" json:"commit"` - // hardState can be non-nil for MsgStorageAppend messages and is nil for all - // other message types. - HardState *HardState `protobuf:"bytes,13,opt,name=hardState" json:"hardState,omitempty"` + // (type=MsgStorageAppend,vote=5,term=10) means the local node is voting for + // peer 5 in term 10. For MsgStorageAppends, the term, vote, and commit fields + // will either all be set (to facilitate the construction of a HardState) if + // any of the fields have changed or will all be unset if none of the fields + // have changed. + Vote uint64 `protobuf:"varint,13,opt,name=vote" json:"vote"` // snapshot is non-nil and non-empty for MsgSnap messages and nil for all other // message types. However, peer nodes running older binary versions may send a // non-nil, empty value for the snapshot field of non-MsgSnap messages. Code @@ -721,76 +726,76 @@ func init() { func init() { proto.RegisterFile("raft.proto", fileDescriptor_b042552c306ae59b) } var fileDescriptor_b042552c306ae59b = []byte{ - // 1100 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x55, 0xcd, 0x6e, 0x23, 0x45, - 0x17, 0xed, 0x6e, 0x77, 0xfc, 0x73, 0xed, 0x38, 0x95, 0x8a, 0x27, 0x5f, 0x2b, 0x8a, 0x3c, 0xfe, - 0x3c, 0x83, 0xc6, 0x0a, 0x9a, 0x80, 0x8c, 0x40, 0x88, 0x5d, 0x7e, 0x46, 0x4a, 0x50, 0x1c, 0x06, - 0x27, 0x93, 0x05, 0x12, 0x8a, 0x2a, 0xee, 0x4a, 0xa7, 0xc1, 0xae, 0x6a, 0x55, 0x97, 0x43, 0xb2, - 0x41, 0x88, 0x17, 0x80, 0x25, 0x1b, 0xb6, 0xec, 0xe1, 0x29, 0xb2, 0xcc, 0x92, 0xd5, 0x88, 0x49, - 0x5e, 0x04, 0x55, 0x75, 0xf5, 0x8f, 0x9d, 0x68, 0x16, 0xec, 0xaa, 0xce, 0x39, 0x75, 0xeb, 0xdc, - 0x7b, 0xfb, 0x56, 0x03, 0x08, 0x72, 0x2e, 0x37, 0x23, 0xc1, 0x25, 0xc7, 0x65, 0xb5, 0x8e, 0xce, - 0xd6, 0x5a, 0x01, 0x0f, 0xb8, 0x86, 0x3e, 0x52, 0xab, 0x84, 0xed, 0xfe, 0x08, 0x0b, 0xaf, 0x98, - 0x14, 0xd7, 0xd8, 0x03, 0xf7, 0x98, 0x8a, 0x89, 0xe7, 0x74, 0xec, 0x9e, 0xbb, 0xed, 0xde, 0xbc, - 0x7d, 0x6a, 0x0d, 0x35, 0x82, 0xd7, 0x60, 0x61, 0x9f, 0xf9, 0xf4, 0xca, 0x2b, 0x15, 0xa8, 0x04, - 0xc2, 0x1f, 0x82, 0x7b, 0x7c, 0x1d, 0x51, 0xcf, 0xee, 0xd8, 0xbd, 0x66, 0x7f, 0x79, 0x33, 0xb9, - 0x6b, 0x53, 0x87, 0x54, 0x44, 0x16, 0xe8, 0x3a, 0xa2, 0x18, 0x83, 0xbb, 0x4b, 0x24, 0xf1, 0xdc, - 0x8e, 0xdd, 0x6b, 0x0c, 0xf5, 0xba, 0xfb, 0x93, 0x0d, 0xe8, 0x88, 0x91, 0x28, 0xbe, 0xe0, 0x72, - 0x40, 0x25, 0xf1, 0x89, 0x24, 0xf8, 0x33, 0x80, 0x11, 0x67, 0xe7, 0xa7, 0xb1, 0x24, 0x32, 0x89, - 0x5d, 0xcf, 0x63, 0xef, 0x70, 0x76, 0x7e, 0xa4, 0x08, 0x13, 0xbb, 0x36, 0x4a, 0x01, 0xe5, 0x34, - 0xd4, 0x4e, 0x8b, 0x49, 0x24, 0x90, 0xca, 0x4f, 0xaa, 0xfc, 0x8a, 0x49, 0x68, 0xa4, 0xfb, 0x0d, - 0x54, 0x53, 0x07, 0xca, 0xa2, 0x72, 0xa0, 0xef, 0x6c, 0x0c, 0xf5, 0x1a, 0x7f, 0x01, 0xd5, 0x89, - 0x71, 0xa6, 0x03, 0xd7, 0xfb, 0x5e, 0xea, 0x65, 0xde, 0xb9, 0x89, 0x9b, 0xe9, 0xbb, 0xbf, 0xb8, - 0x50, 0x19, 0xd0, 0x38, 0x26, 0x01, 0xc5, 0x2f, 0xc1, 0x95, 0x79, 0xad, 0x56, 0xd2, 0x18, 0x86, - 0x2e, 0x56, 0x4b, 0xc9, 0x70, 0x0b, 0x1c, 0xc9, 0x67, 0x32, 0x71, 0x24, 0x57, 0x69, 0x9c, 0x0b, - 0x3e, 0x97, 0x86, 0x42, 0xb2, 0x04, 0xdd, 0xf9, 0x04, 0x71, 0x1b, 0x2a, 0x63, 0x1e, 0xe8, 0xee, - 0x2e, 0x14, 0xc8, 0x14, 0xcc, 0xcb, 0x56, 0x7e, 0x58, 0xb6, 0x97, 0x50, 0xa1, 0x4c, 0x8a, 0x90, - 0xc6, 0x5e, 0xa5, 0x53, 0xea, 0xd5, 0xfb, 0x8b, 0x33, 0x3d, 0x4e, 0x43, 0x19, 0x0d, 0x5e, 0x87, - 0xf2, 0x88, 0x4f, 0x26, 0xa1, 0xf4, 0xaa, 0x85, 0x58, 0x06, 0xc3, 0x9f, 0x42, 0xed, 0x82, 0x08, - 0x5f, 0x37, 0xcb, 0x5b, 0x9c, 0x6d, 0xeb, 0x5e, 0x4a, 0xe8, 0x33, 0xf6, 0x30, 0x57, 0xe2, 0x3e, - 0x54, 0x63, 0x53, 0x68, 0xaf, 0xa6, 0x4f, 0xa1, 0xf9, 0x06, 0x98, 0x43, 0x99, 0x4e, 0x19, 0x11, - 0xf4, 0x3b, 0x3a, 0x92, 0x1e, 0x74, 0xec, 0x5e, 0x35, 0x35, 0x92, 0x60, 0xf8, 0x39, 0x40, 0xb2, - 0xda, 0x0b, 0x99, 0xf4, 0xea, 0x05, 0xab, 0x05, 0x5c, 0xd5, 0x6d, 0xc4, 0x99, 0xa4, 0x57, 0xd2, - 0x6b, 0xa8, 0xef, 0xc1, 0x5c, 0x92, 0x82, 0xf8, 0x13, 0xa8, 0x09, 0x1a, 0x47, 0x9c, 0xc5, 0x34, - 0xf6, 0x9a, 0xba, 0x3a, 0x4b, 0x73, 0x5d, 0x4d, 0xbf, 0xd1, 0x4c, 0xd7, 0xfd, 0x16, 0x6a, 0x59, - 0xaa, 0x59, 0xcf, 0xec, 0x07, 0x3d, 0xf3, 0xc0, 0xbd, 0xe4, 0x92, 0xce, 0x8e, 0xa3, 0x42, 0x0a, - 0x25, 0x2e, 0x3d, 0x2c, 0x71, 0xf7, 0x2f, 0x1b, 0x6a, 0xd9, 0x84, 0xe0, 0x55, 0x28, 0xab, 0x33, - 0x22, 0xf6, 0xec, 0x4e, 0xa9, 0xe7, 0x0e, 0xcd, 0x0e, 0xaf, 0x41, 0x75, 0x4c, 0x89, 0x60, 0x8a, - 0x71, 0x34, 0x93, 0xed, 0xf1, 0x0b, 0x58, 0x4a, 0x54, 0xa7, 0x7c, 0x2a, 0x03, 0x1e, 0xb2, 0xc0, - 0x2b, 0x69, 0x49, 0x33, 0x81, 0xbf, 0x32, 0x28, 0x7e, 0x06, 0x8b, 0xe9, 0xa1, 0x53, 0xa6, 0x8a, - 0xe4, 0x6a, 0x59, 0x23, 0x05, 0x0f, 0x55, 0x8d, 0x9e, 0x01, 0x90, 0xa9, 0xe4, 0xa7, 0x63, 0x4a, - 0x2e, 0xa9, 0xfe, 0xfc, 0xd2, 0x5e, 0xd4, 0x14, 0x7e, 0xa0, 0xe0, 0xee, 0xef, 0x36, 0x80, 0x32, - 0xbd, 0x73, 0x41, 0x58, 0x40, 0xf1, 0xc7, 0x66, 0x50, 0x1c, 0x3d, 0x28, 0xab, 0xc5, 0xc1, 0x4f, - 0x14, 0x0f, 0x66, 0xe5, 0x05, 0x54, 0x18, 0xf7, 0xe9, 0x69, 0xe8, 0x9b, 0xa2, 0x34, 0x15, 0x79, - 0xf7, 0xf6, 0x69, 0xf9, 0x90, 0xfb, 0x74, 0x7f, 0x77, 0x58, 0x56, 0xf4, 0xbe, 0x8f, 0xbd, 0xbc, - 0xa5, 0xc9, 0x2b, 0x94, 0x35, 0x73, 0x0d, 0x9c, 0xd0, 0x37, 0x8d, 0x00, 0x73, 0xda, 0xd9, 0xdf, - 0x1d, 0x3a, 0xa1, 0xdf, 0x9d, 0x00, 0xca, 0x2f, 0x3f, 0x0a, 0x59, 0x30, 0xce, 0x4d, 0xda, 0xff, - 0xc5, 0xa4, 0xf3, 0x3e, 0x93, 0xdd, 0x3f, 0x6c, 0x68, 0xe4, 0x71, 0x4e, 0xfa, 0x78, 0x1b, 0x40, - 0x0a, 0xc2, 0xe2, 0x50, 0x86, 0x9c, 0x99, 0x1b, 0xd7, 0x1f, 0xb9, 0x31, 0xd3, 0xa4, 0x1f, 0x73, - 0x7e, 0x0a, 0x7f, 0x0e, 0x95, 0x91, 0x56, 0x25, 0x1d, 0x2f, 0x3c, 0x62, 0xf3, 0xa9, 0xa5, 0x33, - 0x6d, 0xe4, 0xc5, 0x9a, 0x95, 0x66, 0x6a, 0xb6, 0xb1, 0x07, 0xb5, 0xec, 0xa5, 0xc7, 0x4b, 0x50, - 0xd7, 0x9b, 0x43, 0x2e, 0x26, 0x64, 0x8c, 0x2c, 0xbc, 0x02, 0x4b, 0x1a, 0xc8, 0xe3, 0x23, 0x1b, - 0x3f, 0x81, 0xe5, 0x39, 0xf0, 0xa4, 0x8f, 0x9c, 0x8d, 0x3f, 0x4b, 0x50, 0x2f, 0x3c, 0x84, 0x18, - 0xa0, 0x3c, 0x88, 0x83, 0xbd, 0x69, 0x84, 0x2c, 0x5c, 0x87, 0xca, 0x20, 0x0e, 0xb6, 0x29, 0x91, - 0xc8, 0x36, 0x9b, 0xd7, 0x82, 0x47, 0xc8, 0x31, 0xaa, 0xad, 0x28, 0x42, 0x25, 0xdc, 0x04, 0x48, - 0xd6, 0x43, 0x1a, 0x47, 0xc8, 0x35, 0xc2, 0x13, 0x2e, 0x29, 0x5a, 0x50, 0xde, 0xcc, 0x46, 0xb3, - 0x65, 0xc3, 0xaa, 0xd7, 0x03, 0x55, 0x30, 0x82, 0x86, 0xba, 0x8c, 0x12, 0x21, 0xcf, 0xd4, 0x2d, - 0x55, 0xdc, 0x02, 0x54, 0x44, 0xf4, 0xa1, 0x1a, 0xc6, 0xd0, 0x1c, 0xc4, 0xc1, 0x1b, 0x26, 0x28, - 0x19, 0x5d, 0x90, 0xb3, 0x31, 0x45, 0x80, 0x97, 0x61, 0xd1, 0x04, 0x52, 0x13, 0x37, 0x8d, 0x51, - 0xdd, 0xc8, 0x76, 0x2e, 0xe8, 0xe8, 0xfb, 0xaf, 0xa7, 0x5c, 0x4c, 0x27, 0xa8, 0xa1, 0xd2, 0x1e, - 0xc4, 0x81, 0x6e, 0xd0, 0x39, 0x15, 0x07, 0x94, 0xf8, 0x54, 0xa0, 0x45, 0x73, 0xfa, 0x38, 0x9c, - 0x50, 0x3e, 0x95, 0x87, 0xfc, 0x07, 0xd4, 0x34, 0x66, 0x86, 0x94, 0xf8, 0xfa, 0x0f, 0x8b, 0x96, - 0x8c, 0x99, 0x0c, 0xd1, 0x66, 0x90, 0xc9, 0xf7, 0xb5, 0xa0, 0x3a, 0xc5, 0x65, 0x73, 0xab, 0xd9, - 0x6b, 0x0d, 0x36, 0x27, 0x8f, 0x24, 0x17, 0x24, 0xa0, 0x5b, 0x51, 0x44, 0x99, 0x8f, 0x56, 0xb0, - 0x07, 0xad, 0x79, 0x54, 0xeb, 0x5b, 0xaa, 0x63, 0x33, 0xcc, 0xf8, 0x1a, 0x3d, 0xc1, 0xff, 0x83, - 0x95, 0x39, 0x50, 0xab, 0x57, 0x37, 0x7e, 0xb6, 0xa1, 0xf5, 0xd8, 0xc7, 0x87, 0xd7, 0xc1, 0x7b, - 0x0c, 0xdf, 0x9a, 0x4a, 0x8e, 0x2c, 0xfc, 0x01, 0xfc, 0xff, 0x31, 0xf6, 0x4b, 0x1e, 0x32, 0xb9, - 0x3f, 0x89, 0xc6, 0xe1, 0x28, 0x54, 0x8d, 0x7e, 0x9f, 0xec, 0xd5, 0x95, 0x91, 0x39, 0x1b, 0xd7, - 0xd0, 0x9c, 0x1d, 0x39, 0x55, 0xea, 0x1c, 0xd9, 0xf2, 0x7d, 0x35, 0x5c, 0xc8, 0x52, 0x59, 0xe7, - 0xf0, 0x90, 0x4e, 0xf8, 0x25, 0xd5, 0x8c, 0x3d, 0xcb, 0xbc, 0x89, 0x7c, 0x22, 0x13, 0xc6, 0x99, - 0x4d, 0x64, 0xcb, 0xf7, 0x0f, 0x92, 0x97, 0x4d, 0xb3, 0xa5, 0xed, 0xe7, 0x37, 0xef, 0xda, 0xd6, - 0xed, 0xbb, 0xb6, 0x75, 0x73, 0xd7, 0xb6, 0x6f, 0xef, 0xda, 0xf6, 0x3f, 0x77, 0x6d, 0xfb, 0xd7, - 0xfb, 0xb6, 0xf5, 0xdb, 0x7d, 0xdb, 0xba, 0xbd, 0x6f, 0x5b, 0x7f, 0xdf, 0xb7, 0xad, 0x7f, 0x03, - 0x00, 0x00, 0xff, 0xff, 0x17, 0x42, 0x8c, 0x08, 0x8b, 0x09, 0x00, 0x00, + // 1091 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x55, 0xcb, 0x6e, 0x23, 0x45, + 0x14, 0xed, 0x6e, 0x77, 0xfc, 0xb8, 0x76, 0x9c, 0x4a, 0xc5, 0x13, 0x5a, 0x51, 0xe4, 0x31, 0x9e, + 0x41, 0x63, 0x05, 0x4d, 0x40, 0x46, 0x42, 0x88, 0x5d, 0x1e, 0x23, 0x25, 0x28, 0x0e, 0x83, 0x93, + 0xc9, 0x02, 0x09, 0x45, 0x15, 0x77, 0xa5, 0xd3, 0x60, 0x57, 0xb5, 0xaa, 0xcb, 0x21, 0xd9, 0x20, + 0xc4, 0x17, 0xb0, 0x64, 0xc3, 0x96, 0x3d, 0x7c, 0x45, 0x96, 0x59, 0xb2, 0x1a, 0x31, 0xc9, 0x1f, + 0xf0, 0x05, 0xa8, 0xaa, 0xab, 0x1f, 0x76, 0xa2, 0x59, 0xb0, 0xab, 0x3a, 0xf7, 0xd4, 0xbd, 0xe7, + 0x9e, 0xdb, 0x55, 0x0d, 0x20, 0xc8, 0xb9, 0xdc, 0x8c, 0x04, 0x97, 0x1c, 0x97, 0xd5, 0x3a, 0x3a, + 0x5b, 0x6b, 0x05, 0x3c, 0xe0, 0x1a, 0xfa, 0x44, 0xad, 0x92, 0x68, 0xf7, 0x27, 0x58, 0x78, 0xc5, + 0xa4, 0xb8, 0xc6, 0x1e, 0xb8, 0xc7, 0x54, 0x4c, 0x3c, 0xa7, 0x63, 0xf7, 0xdc, 0x6d, 0xf7, 0xe6, + 0xed, 0x53, 0x6b, 0xa8, 0x11, 0xbc, 0x06, 0x0b, 0xfb, 0xcc, 0xa7, 0x57, 0x5e, 0xa9, 0x10, 0x4a, + 0x20, 0xfc, 0x31, 0xb8, 0xc7, 0xd7, 0x11, 0xf5, 0xec, 0x8e, 0xdd, 0x6b, 0xf6, 0x97, 0x37, 0x93, + 0x5a, 0x9b, 0x3a, 0xa5, 0x0a, 0x64, 0x89, 0xae, 0x23, 0x8a, 0x31, 0xb8, 0xbb, 0x44, 0x12, 0xcf, + 0xed, 0xd8, 0xbd, 0xc6, 0x50, 0xaf, 0xbb, 0x3f, 0xdb, 0x80, 0x8e, 0x18, 0x89, 0xe2, 0x0b, 0x2e, + 0x07, 0x54, 0x12, 0x9f, 0x48, 0x82, 0x3f, 0x07, 0x18, 0x71, 0x76, 0x7e, 0x1a, 0x4b, 0x22, 0x93, + 0xdc, 0xf5, 0x3c, 0xf7, 0x0e, 0x67, 0xe7, 0x47, 0x2a, 0x60, 0x72, 0xd7, 0x46, 0x29, 0xa0, 0x94, + 0x86, 0x5a, 0x69, 0xb1, 0x89, 0x04, 0x52, 0xfd, 0x49, 0xd5, 0x5f, 0xb1, 0x09, 0x8d, 0x74, 0xbf, + 0x85, 0x6a, 0xaa, 0x40, 0x49, 0x54, 0x0a, 0x74, 0xcd, 0xc6, 0x50, 0xaf, 0xf1, 0x97, 0x50, 0x9d, + 0x18, 0x65, 0x3a, 0x71, 0xbd, 0xef, 0xa5, 0x5a, 0xe6, 0x95, 0x9b, 0xbc, 0x19, 0xbf, 0xfb, 0x6f, + 0x09, 0x2a, 0x03, 0x1a, 0xc7, 0x24, 0xa0, 0xf8, 0x25, 0xb8, 0x32, 0xf7, 0x6a, 0x25, 0xcd, 0x61, + 0xc2, 0x45, 0xb7, 0x14, 0x0d, 0xb7, 0xc0, 0x91, 0x7c, 0xa6, 0x13, 0x47, 0x72, 0xd5, 0xc6, 0xb9, + 0xe0, 0x73, 0x6d, 0x28, 0x24, 0x6b, 0xd0, 0x9d, 0x6f, 0x10, 0xb7, 0xa1, 0x32, 0xe6, 0x81, 0x9e, + 0xee, 0x42, 0x21, 0x98, 0x82, 0xb9, 0x6d, 0xe5, 0x87, 0xb6, 0xbd, 0x84, 0x0a, 0x65, 0x52, 0x84, + 0x34, 0xf6, 0x2a, 0x9d, 0x52, 0xaf, 0xde, 0x5f, 0x9c, 0x99, 0x71, 0x9a, 0xca, 0x70, 0xf0, 0x3a, + 0x94, 0x47, 0x7c, 0x32, 0x09, 0xa5, 0x57, 0x2d, 0xe4, 0x32, 0x98, 0x92, 0x78, 0xc9, 0x25, 0xf5, + 0x16, 0x8b, 0x12, 0x15, 0x82, 0xfb, 0x50, 0x8d, 0x8d, 0x97, 0x5e, 0x4d, 0x7b, 0x8c, 0xe6, 0x3d, + 0xd6, 0x7c, 0x7b, 0x98, 0xf1, 0x54, 0x2d, 0x41, 0xbf, 0xa7, 0x23, 0xe9, 0x41, 0xc7, 0xee, 0x55, + 0xd3, 0x5a, 0x09, 0x86, 0x9f, 0x03, 0x24, 0xab, 0xbd, 0x90, 0x49, 0xaf, 0x5e, 0xa8, 0x58, 0xc0, + 0x95, 0x35, 0x23, 0xce, 0x24, 0xbd, 0x92, 0x5e, 0x43, 0x8d, 0xdc, 0x14, 0x49, 0x41, 0xfc, 0x19, + 0xd4, 0x04, 0x8d, 0x23, 0xce, 0x62, 0x1a, 0x7b, 0x4d, 0x6d, 0xc0, 0xd2, 0xdc, 0xe0, 0xd2, 0xcf, + 0x30, 0xe3, 0x75, 0xbf, 0x83, 0xda, 0x1e, 0x11, 0x7e, 0xf2, 0x4d, 0xa6, 0x63, 0xb1, 0x1f, 0x8c, + 0x25, 0x75, 0xc3, 0x79, 0xe0, 0x46, 0xee, 0x62, 0xe9, 0xa1, 0x8b, 0xdd, 0xbf, 0x6c, 0xa8, 0x65, + 0x97, 0x00, 0xaf, 0x42, 0x59, 0x9d, 0x11, 0xb1, 0x67, 0x77, 0x4a, 0x3d, 0x77, 0x68, 0x76, 0x78, + 0x0d, 0xaa, 0x63, 0x4a, 0x04, 0x53, 0x11, 0x47, 0x47, 0xb2, 0x3d, 0x7e, 0x01, 0x4b, 0x09, 0xeb, + 0x94, 0x4f, 0x65, 0xc0, 0x43, 0x16, 0x78, 0x25, 0x4d, 0x69, 0x26, 0xf0, 0xd7, 0x06, 0xc5, 0xcf, + 0x60, 0x31, 0x3d, 0x74, 0xca, 0x94, 0x49, 0xae, 0xa6, 0x35, 0x52, 0xf0, 0x50, 0x79, 0xf4, 0x0c, + 0x80, 0x4c, 0x25, 0x3f, 0x1d, 0x53, 0x72, 0x49, 0xf5, 0x17, 0x96, 0xce, 0xa2, 0xa6, 0xf0, 0x03, + 0x05, 0x77, 0x7f, 0xb7, 0x01, 0x94, 0xe8, 0x9d, 0x0b, 0xc2, 0x02, 0x8a, 0x3f, 0x35, 0x77, 0xc1, + 0xd1, 0x77, 0x61, 0xb5, 0x78, 0xb7, 0x13, 0xc6, 0x83, 0xeb, 0xf0, 0x02, 0x2a, 0x8c, 0xfb, 0xf4, + 0x34, 0xf4, 0x8d, 0x29, 0x4d, 0x15, 0xbc, 0x7b, 0xfb, 0xb4, 0x7c, 0xc8, 0x7d, 0xba, 0xbf, 0x3b, + 0x2c, 0xab, 0xf0, 0xbe, 0x8f, 0xbd, 0x7c, 0xa4, 0xc9, 0x43, 0x93, 0x0d, 0x73, 0x0d, 0x9c, 0xd0, + 0x37, 0x83, 0x00, 0x73, 0xda, 0xd9, 0xdf, 0x1d, 0x3a, 0xa1, 0xdf, 0x9d, 0x00, 0xca, 0x8b, 0x1f, + 0x85, 0x2c, 0x18, 0xe7, 0x22, 0xed, 0xff, 0x23, 0xd2, 0x79, 0x9f, 0xc8, 0xee, 0x1f, 0x36, 0x34, + 0xf2, 0x3c, 0x27, 0x7d, 0xbc, 0x0d, 0x20, 0x05, 0x61, 0x71, 0x28, 0x43, 0xce, 0x4c, 0xc5, 0xf5, + 0x47, 0x2a, 0x66, 0x9c, 0xf4, 0x63, 0xce, 0x4f, 0xe1, 0x2f, 0xa0, 0x32, 0xd2, 0xac, 0x64, 0xe2, + 0x85, 0x77, 0x6a, 0xbe, 0xb5, 0xf4, 0xda, 0x1a, 0x7a, 0xd1, 0xb3, 0xd2, 0x8c, 0x67, 0x1b, 0x7b, + 0x50, 0xcb, 0x1e, 0x73, 0xbc, 0x04, 0x75, 0xbd, 0x39, 0xe4, 0x62, 0x42, 0xc6, 0xc8, 0xc2, 0x2b, + 0xb0, 0xa4, 0x81, 0x3c, 0x3f, 0xb2, 0xf1, 0x13, 0x58, 0x9e, 0x03, 0x4f, 0xfa, 0xc8, 0xd9, 0xf8, + 0xb3, 0x04, 0xf5, 0xc2, 0x5b, 0x87, 0x01, 0xca, 0x83, 0x38, 0xd8, 0x9b, 0x46, 0xc8, 0xc2, 0x75, + 0xa8, 0x0c, 0xe2, 0x60, 0x9b, 0x12, 0x89, 0x6c, 0xb3, 0x79, 0x2d, 0x78, 0x84, 0x1c, 0xc3, 0xda, + 0x8a, 0x22, 0x54, 0xc2, 0x4d, 0x80, 0x64, 0x3d, 0xa4, 0x71, 0x84, 0x5c, 0x43, 0x3c, 0xe1, 0x92, + 0xa2, 0x05, 0xa5, 0xcd, 0x6c, 0x74, 0xb4, 0x6c, 0xa2, 0xea, 0xf5, 0x40, 0x15, 0x8c, 0xa0, 0xa1, + 0x8a, 0x51, 0x22, 0xe4, 0x99, 0xaa, 0x52, 0xc5, 0x2d, 0x40, 0x45, 0x44, 0x1f, 0xaa, 0x61, 0x0c, + 0xcd, 0x41, 0x1c, 0xbc, 0x61, 0x82, 0x92, 0xd1, 0x05, 0x39, 0x1b, 0x53, 0x04, 0x78, 0x19, 0x16, + 0x4d, 0x22, 0x75, 0xe3, 0xa6, 0x31, 0xaa, 0x1b, 0xda, 0xce, 0x05, 0x1d, 0xfd, 0xf0, 0xcd, 0x94, + 0x8b, 0xe9, 0x04, 0x35, 0x54, 0xdb, 0x83, 0x38, 0xd0, 0x03, 0x3a, 0xa7, 0xe2, 0x80, 0x12, 0x9f, + 0x0a, 0xb4, 0x68, 0x4e, 0x1f, 0x87, 0x13, 0xca, 0xa7, 0xf2, 0x90, 0xff, 0x88, 0x9a, 0x46, 0xcc, + 0x90, 0x12, 0x5f, 0xff, 0x44, 0xd1, 0x92, 0x11, 0x93, 0x21, 0x5a, 0x0c, 0x32, 0xfd, 0xbe, 0x16, + 0x54, 0xb7, 0xb8, 0x6c, 0xaa, 0x9a, 0xbd, 0xe6, 0x60, 0x73, 0xf2, 0x48, 0x72, 0x41, 0x02, 0xba, + 0x15, 0x45, 0x94, 0xf9, 0x68, 0x05, 0x7b, 0xd0, 0x9a, 0x47, 0x35, 0xbf, 0xa5, 0x26, 0x36, 0x13, + 0x19, 0x5f, 0xa3, 0x27, 0xf8, 0x03, 0x58, 0x99, 0x03, 0x35, 0x7b, 0x75, 0xe3, 0x17, 0x1b, 0x5a, + 0x8f, 0x7d, 0x7c, 0x78, 0x1d, 0xbc, 0xc7, 0xf0, 0xad, 0xa9, 0xe4, 0xc8, 0xc2, 0x1f, 0xc1, 0x87, + 0x8f, 0x45, 0xbf, 0xe2, 0x21, 0x93, 0xfb, 0x93, 0x68, 0x1c, 0x8e, 0x42, 0x35, 0xe8, 0xf7, 0xd1, + 0x5e, 0x5d, 0x19, 0x9a, 0xb3, 0x71, 0x0d, 0xcd, 0xd9, 0x2b, 0xa7, 0xac, 0xce, 0x91, 0x2d, 0xdf, + 0x57, 0x97, 0x0b, 0x59, 0xaa, 0xeb, 0x1c, 0x1e, 0xd2, 0x09, 0xbf, 0xa4, 0x3a, 0x62, 0xcf, 0x46, + 0xde, 0x44, 0x3e, 0x91, 0x49, 0xc4, 0x99, 0x6d, 0x64, 0xcb, 0xf7, 0x0f, 0x92, 0x97, 0x4d, 0x47, + 0x4b, 0xdb, 0xcf, 0x6f, 0xde, 0xb5, 0xad, 0xdb, 0x77, 0x6d, 0xeb, 0xe6, 0xae, 0x6d, 0xdf, 0xde, + 0xb5, 0xed, 0x7f, 0xee, 0xda, 0xf6, 0xaf, 0xf7, 0x6d, 0xeb, 0xb7, 0xfb, 0xb6, 0x75, 0x7b, 0xdf, + 0xb6, 0xfe, 0xbe, 0x6f, 0x5b, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0xe4, 0x8e, 0x4c, 0xc0, 0x6e, + 0x09, 0x00, 0x00, } func (m *Entry) Marshal() (dAtA []byte, err error) { @@ -945,18 +950,9 @@ func (m *Message) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x72 } } - if m.HardState != nil { - { - size, err := m.HardState.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintRaft(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x6a - } + i = encodeVarintRaft(dAtA, i, uint64(m.Vote)) + i-- + dAtA[i] = 0x68 if m.Context != nil { i -= len(m.Context) copy(dAtA[i:], m.Context) @@ -1315,10 +1311,7 @@ func (m *Message) Size() (n int) { l = len(m.Context) n += 1 + l + sovRaft(uint64(l)) } - if m.HardState != nil { - l = m.HardState.Size() - n += 1 + l + sovRaft(uint64(l)) - } + n += 1 + sovRaft(uint64(m.Vote)) if len(m.Responses) > 0 { for _, e := range m.Responses { l = e.Size() @@ -2108,10 +2101,10 @@ func (m *Message) Unmarshal(dAtA []byte) error { } iNdEx = postIndex case 13: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field HardState", wireType) + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Vote", wireType) } - var msglen int + m.Vote = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaft @@ -2121,28 +2114,11 @@ func (m *Message) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + m.Vote |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { - return ErrInvalidLengthRaft - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthRaft - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.HardState == nil { - m.HardState = &HardState{} - } - if err := m.HardState.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex case 14: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Responses", wireType) diff --git a/raftpb/raft.proto b/raftpb/raft.proto index 35c763f17eb6..a70f736a5992 100644 --- a/raftpb/raft.proto +++ b/raftpb/raft.proto @@ -13,14 +13,12 @@ option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.goproto_sizecache_all) = false; enum EntryType { - EntryNormal = 0; EntryConfChange = 1; // corresponds to pb.ConfChange EntryConfChangeV2 = 2; // corresponds to pb.ConfChangeV2 } message Entry { - optional uint64 Term = 2 [(gogoproto.nullable) = false]; // must be 64-bit aligned for atomic operations optional uint64 Index = 3 [(gogoproto.nullable) = false]; // must be 64-bit aligned for atomic operations optional EntryType Type = 1 [(gogoproto.nullable) = false]; @@ -81,14 +79,19 @@ message Message { // entries from its leader as it already has an entry with term 5 at index 100. // (type=MsgStorageAppendResp,index=100,logTerm=5) means the local node wrote // entries up to index=100 in stable storage, and the term of the entry at index - // 100 was 5. + // 100 was 5. This doesn't always mean that the corresponding MsgStorageAppend + // message was the one that carried these entries, just that those entries were + // stable at the time of processing the corresponding MsgStorageAppend. optional uint64 logTerm = 5 [(gogoproto.nullable) = false]; optional uint64 index = 6 [(gogoproto.nullable) = false]; repeated Entry entries = 7 [(gogoproto.nullable) = false]; optional uint64 commit = 8 [(gogoproto.nullable) = false]; - // hardState can be non-nil for MsgStorageAppend messages and is nil for all - // other message types. - optional HardState hardState = 13 [(gogoproto.nullable) = true]; + // (type=MsgStorageAppend,vote=5,term=10) means the local node is voting for + // peer 5 in term 10. For MsgStorageAppends, the term, vote, and commit fields + // will either all be set (to facilitate the construction of a HardState) if + // any of the fields have changed or will all be unset if none of the fields + // have changed. + optional uint64 vote = 13 [(gogoproto.nullable) = false]; // snapshot is non-nil and non-empty for MsgSnap messages and nil for all other // message types. However, peer nodes running older binary versions may send a // non-nil, empty value for the snapshot field of non-MsgSnap messages. Code @@ -204,7 +207,6 @@ message ConfChangeSingle { // // [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf message ConfChangeV2 { - optional ConfChangeTransition transition = 1 [(gogoproto.nullable) = false]; repeated ConfChangeSingle changes = 2 [(gogoproto.nullable) = false]; optional bytes context = 3; diff --git a/raftpb/raft_test.go b/raftpb/raft_test.go index 7f5be88b8d74..704ff94f4fc8 100644 --- a/raftpb/raft_test.go +++ b/raftpb/raft_test.go @@ -45,7 +45,7 @@ func TestProtoMemorySizes(t *testing.T) { assert(unsafe.Sizeof(s), if64Bit(144, 80), "Snapshot") var m Message - assert(unsafe.Sizeof(m), if64Bit(160, 108), "Message") + assert(unsafe.Sizeof(m), if64Bit(160, 112), "Message") var hs HardState assert(unsafe.Sizeof(hs), 24, "HardState") diff --git a/rafttest/interaction_env_handler_process_append_thread.go b/rafttest/interaction_env_handler_process_append_thread.go index 3f74988a8780..38c16f255873 100644 --- a/rafttest/interaction_env_handler_process_append_thread.go +++ b/rafttest/interaction_env_handler_process_append_thread.go @@ -56,9 +56,10 @@ func (env *InteractionEnv) ProcessAppendThread(idx int) error { m.Responses = nil env.Output.WriteString("Processing:\n") env.Output.WriteString(raft.DescribeMessage(m, defaultEntryFormatter) + "\n") - var st raftpb.HardState - if m.HardState != nil { - st = *m.HardState + st := raftpb.HardState{ + Term: m.Term, + Vote: m.Vote, + Commit: m.Commit, } var snap raftpb.Snapshot if m.Snapshot != nil { diff --git a/rawnode.go b/rawnode.go index 1e88a8011734..b46182868f8e 100644 --- a/rawnode.go +++ b/rawnode.go @@ -222,19 +222,19 @@ func newStorageAppendMsg(r *raft, rd Ready) pb.Message { Type: pb.MsgStorageAppend, To: LocalAppendThread, From: r.id, - Term: r.Term, Entries: rd.Entries, } if !IsEmptyHardState(rd.HardState) { - // TODO(nvanbenschoten): we could avoid this heap allocation by - // replacing the pb.Message.HardState field with a Vote uint64 field. We - // would then need to teach apps to construct a HardState from these - // three fields, or supply a function/method that does so. - // m.Term = rd.Term - // m.Vote = rd.Vote - // m.Commit = rd.Commit - hs := rd.HardState - m.HardState = &hs + // If the Ready includes a HardState update, assign each of its fields + // to the corresponding fields in the Message. This allows clients to + // reconstruct the HardState and save it to stable storage. + // + // If the Ready does not include a HardState update, make sure to not + // assign a value to any of the fields so that a HardState reconstructed + // from them will be empty (return true from raft.IsEmptyHardState). + m.Term = rd.Term + m.Vote = rd.Vote + m.Commit = rd.Commit } if !IsEmptySnap(rd.Snapshot) { snap := rd.Snapshot @@ -340,8 +340,8 @@ func newStorageAppendRespMsg(r *raft, rd Ready) pb.Message { // MsgStorageAppend that contained the last entry in the unstable slice carried // an earlier term and was dropped. // - // A MsgStorageAppend with a new HardState is emitted on each term change. This - // is the same condition that causes MsgStorageAppendResp messages with earlier + // A MsgStorageAppend with a new term is emitted on each term change. This is + // the same condition that causes MsgStorageAppendResp messages with earlier // terms to be ignored. As a result, we are guaranteed that, assuming a bounded // number of term changes, there will eventually be a MsgStorageAppendResp // message that is not ignored. This means that entries in the unstable log diff --git a/testdata/async_storage_writes.txt b/testdata/async_storage_writes.txt index f6a7ef8fc71d..52f37092f988 100644 --- a/testdata/async_storage_writes.txt +++ b/testdata/async_storage_writes.txt @@ -29,7 +29,7 @@ stabilize Messages: 1->2 MsgVote Term:1 Log:1/10 1->3 MsgVote Term:1 Log:1/10 - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:10 Responses:[1->1 MsgVoteResp Term:1 Log:0/0] + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 Responses:[1->1 MsgVoteResp Term:1 Log:0/0] > 2 receiving messages 1->2 MsgVote Term:1 Log:1/10 INFO 2 [term: 0] received a MsgVote message with higher term from 1 [term: 1] @@ -42,31 +42,31 @@ stabilize INFO 3 [logterm: 1, index: 10, vote: 0] cast MsgVote for 1 [logterm: 1, index: 10] at term 1 > 1 processing append thread Processing: - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:10 + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 Responses: 1->1 MsgVoteResp Term:1 Log:0/0 > 2 handling Ready Ready MustSync=true: HardState Term:1 Vote:1 Commit:10 Messages: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:10 Responses:[2->1 MsgVoteResp Term:1 Log:0/0] + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 Responses:[2->1 MsgVoteResp Term:1 Log:0/0] > 3 handling Ready Ready MustSync=true: HardState Term:1 Vote:1 Commit:10 Messages: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:10 Responses:[3->1 MsgVoteResp Term:1 Log:0/0] + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 Responses:[3->1 MsgVoteResp Term:1 Log:0/0] > 1 receiving messages 1->1 MsgVoteResp Term:1 Log:0/0 INFO 1 received MsgVoteResp from 1 at term 1 INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections > 2 processing append thread Processing: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:10 + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 Responses: 2->1 MsgVoteResp Term:1 Log:0/0 > 3 processing append thread Processing: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:10 + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:10 Vote:1 Responses: 3->1 MsgVoteResp Term:1 Log:0/0 > 1 receiving messages @@ -83,14 +83,14 @@ stabilize Messages: 1->2 MsgApp Term:1 Log:1/10 Commit:10 Entries:[1/11 EntryNormal ""] 1->3 MsgApp Term:1 Log:1/10 Commit:10 Entries:[1/11 EntryNormal ""] - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[1->1 MsgAppResp Term:1 Log:0/11, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/11] + 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[1->1 MsgAppResp Term:1 Log:0/11, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/11] > 2 receiving messages 1->2 MsgApp Term:1 Log:1/10 Commit:10 Entries:[1/11 EntryNormal ""] > 3 receiving messages 1->3 MsgApp Term:1 Log:1/10 Commit:10 Entries:[1/11 EntryNormal ""] > 1 processing append thread Processing: - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/11 EntryNormal ""] + 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses: 1->1 MsgAppResp Term:1 Log:0/11 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/11 @@ -100,26 +100,26 @@ stabilize Entries: 1/11 EntryNormal "" Messages: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[2->1 MsgAppResp Term:1 Log:0/11, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/11] + 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[2->1 MsgAppResp Term:1 Log:0/11, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/11] > 3 handling Ready Ready MustSync=true: Lead:1 State:StateFollower Entries: 1/11 EntryNormal "" Messages: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[3->1 MsgAppResp Term:1 Log:0/11, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/11] + 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[3->1 MsgAppResp Term:1 Log:0/11, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/11] > 1 receiving messages 1->1 MsgAppResp Term:1 Log:0/11 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/11 > 2 processing append thread Processing: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/11 EntryNormal ""] + 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses: 2->1 MsgAppResp Term:1 Log:0/11 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/11 > 3 processing append thread Processing: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/11 EntryNormal ""] + 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses: 3->1 MsgAppResp Term:1 Log:0/11 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/11 @@ -138,7 +138,7 @@ stabilize Messages: 1->2 MsgApp Term:1 Log:1/11 Commit:11 1->3 MsgApp Term:1 Log:1/11 Commit:11 - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:11 + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""]] > 2 receiving messages 1->2 MsgApp Term:1 Log:1/11 Commit:11 @@ -146,7 +146,7 @@ stabilize 1->3 MsgApp Term:1 Log:1/11 Commit:11 > 1 processing append thread Processing: - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:11 + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 Responses: > 1 processing apply thread Processing: @@ -159,7 +159,7 @@ stabilize CommittedEntries: 1/11 EntryNormal "" Messages: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:11 Responses:[2->1 MsgAppResp Term:1 Log:0/11] + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 Responses:[2->1 MsgAppResp Term:1 Log:0/11] 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""]] > 3 handling Ready Ready MustSync=false: @@ -167,18 +167,18 @@ stabilize CommittedEntries: 1/11 EntryNormal "" Messages: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:11 Responses:[3->1 MsgAppResp Term:1 Log:0/11] + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 Responses:[3->1 MsgAppResp Term:1 Log:0/11] 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] Responses:[ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""]] > 1 receiving messages ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] > 2 processing append thread Processing: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:11 + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 Responses: 2->1 MsgAppResp Term:1 Log:0/11 > 3 processing append thread Processing: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:11 + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:11 Vote:1 Responses: 3->1 MsgAppResp Term:1 Log:0/11 > 2 processing apply thread @@ -212,7 +212,7 @@ process-ready 1 2 3 Messages: 1->2 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1"] 1->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "prop_1"] - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[1->1 MsgAppResp Term:1 Log:0/12, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12] + 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[1->1 MsgAppResp Term:1 Log:0/12, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12] > 2 handling Ready > 3 handling Ready @@ -232,13 +232,13 @@ process-ready 1 2 3 Entries: 1/12 EntryNormal "prop_1" Messages: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[2->1 MsgAppResp Term:1 Log:0/12, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12] + 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[2->1 MsgAppResp Term:1 Log:0/12, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12] > 3 handling Ready Ready MustSync=true: Entries: 1/12 EntryNormal "prop_1" Messages: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[3->1 MsgAppResp Term:1 Log:0/12, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/12] + 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[3->1 MsgAppResp Term:1 Log:0/12, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/12] propose 1 prop_2 ---- @@ -253,7 +253,7 @@ process-ready 1 2 3 Messages: 1->2 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "prop_2"] 1->3 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "prop_2"] - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[1->1 MsgAppResp Term:1 Log:0/13, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/13] + 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[1->1 MsgAppResp Term:1 Log:0/13, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/13] > 2 handling Ready > 3 handling Ready @@ -273,31 +273,31 @@ process-ready 1 2 3 Entries: 1/13 EntryNormal "prop_2" Messages: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[2->1 MsgAppResp Term:1 Log:0/13, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/13] + 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[2->1 MsgAppResp Term:1 Log:0/13, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/13] > 3 handling Ready Ready MustSync=true: Entries: 1/13 EntryNormal "prop_2" Messages: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[3->1 MsgAppResp Term:1 Log:0/13, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/13] + 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[3->1 MsgAppResp Term:1 Log:0/13, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/13] process-append-thread 1 2 3 ---- > 1 processing append thread Processing: - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] + 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses: 1->1 MsgAppResp Term:1 Log:0/12 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12 > 2 processing append thread Processing: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] + 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses: 2->1 MsgAppResp Term:1 Log:0/12 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12 > 3 processing append thread Processing: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] + 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses: 3->1 MsgAppResp Term:1 Log:0/12 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/12 @@ -329,7 +329,7 @@ process-ready 1 2 3 1->3 MsgApp Term:1 Log:1/13 Commit:12 1->2 MsgApp Term:1 Log:1/13 Commit:12 Entries:[1/14 EntryNormal "prop_3"] 1->3 MsgApp Term:1 Log:1/13 Commit:12 Entries:[1/14 EntryNormal "prop_3"] - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] HardState: Term:1 Vote:1 Commit:12 Responses:[1->1 MsgAppResp Term:1 Log:0/14, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/14] + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[1->1 MsgAppResp Term:1 Log:0/14, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/14] 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"]] > 2 handling Ready @@ -355,7 +355,7 @@ process-ready 1 2 3 CommittedEntries: 1/12 EntryNormal "prop_1" Messages: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] HardState: Term:1 Vote:1 Commit:12 Responses:[2->1 MsgAppResp Term:1 Log:0/13, 2->1 MsgAppResp Term:1 Log:0/14, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/14] + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[2->1 MsgAppResp Term:1 Log:0/13, 2->1 MsgAppResp Term:1 Log:0/14, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/14] 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"]] > 3 handling Ready Ready MustSync=true: @@ -365,26 +365,26 @@ process-ready 1 2 3 CommittedEntries: 1/12 EntryNormal "prop_1" Messages: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] HardState: Term:1 Vote:1 Commit:12 Responses:[3->1 MsgAppResp Term:1 Log:0/13, 3->1 MsgAppResp Term:1 Log:0/14, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/14] + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[3->1 MsgAppResp Term:1 Log:0/13, 3->1 MsgAppResp Term:1 Log:0/14, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/14] 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"]] process-append-thread 1 2 3 ---- > 1 processing append thread Processing: - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] + 1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses: 1->1 MsgAppResp Term:1 Log:0/13 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/13 > 2 processing append thread Processing: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] + 2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses: 2->1 MsgAppResp Term:1 Log:0/13 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/13 > 3 processing append thread Processing: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] + 3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses: 3->1 MsgAppResp Term:1 Log:0/13 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/13 @@ -416,7 +416,7 @@ process-ready 1 2 3 1->3 MsgApp Term:1 Log:1/14 Commit:13 1->2 MsgApp Term:1 Log:1/14 Commit:13 Entries:[1/15 EntryNormal "prop_4"] 1->3 MsgApp Term:1 Log:1/14 Commit:13 Entries:[1/15 EntryNormal "prop_4"] - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] HardState: Term:1 Vote:1 Commit:13 Responses:[1->1 MsgAppResp Term:1 Log:0/15, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15] + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[1->1 MsgAppResp Term:1 Log:0/15, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15] 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"]] > 2 handling Ready @@ -442,7 +442,7 @@ process-ready 1 2 3 CommittedEntries: 1/13 EntryNormal "prop_2" Messages: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] HardState: Term:1 Vote:1 Commit:13 Responses:[2->1 MsgAppResp Term:1 Log:0/14, 2->1 MsgAppResp Term:1 Log:0/15, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15] + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[2->1 MsgAppResp Term:1 Log:0/14, 2->1 MsgAppResp Term:1 Log:0/15, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15] 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"]] > 3 handling Ready Ready MustSync=true: @@ -452,27 +452,27 @@ process-ready 1 2 3 CommittedEntries: 1/13 EntryNormal "prop_2" Messages: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] HardState: Term:1 Vote:1 Commit:13 Responses:[3->1 MsgAppResp Term:1 Log:0/14, 3->1 MsgAppResp Term:1 Log:0/15, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15] + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[3->1 MsgAppResp Term:1 Log:0/14, 3->1 MsgAppResp Term:1 Log:0/15, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15] 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"]] process-append-thread 1 2 3 ---- > 1 processing append thread Processing: - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] HardState: Term:1 Vote:1 Commit:12 + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses: 1->1 MsgAppResp Term:1 Log:0/14 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/14 > 2 processing append thread Processing: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] HardState: Term:1 Vote:1 Commit:12 + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses: 2->1 MsgAppResp Term:1 Log:0/13 2->1 MsgAppResp Term:1 Log:0/14 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/14 > 3 processing append thread Processing: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] HardState: Term:1 Vote:1 Commit:12 + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses: 3->1 MsgAppResp Term:1 Log:0/13 3->1 MsgAppResp Term:1 Log:0/14 @@ -520,7 +520,7 @@ process-ready 1 2 3 Messages: 1->2 MsgApp Term:1 Log:1/15 Commit:14 1->3 MsgApp Term:1 Log:1/15 Commit:14 - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:14 Responses:[AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15] + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses:[AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15] 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] Responses:[ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"]] > 2 handling Ready @@ -542,7 +542,7 @@ process-ready 1 2 3 CommittedEntries: 1/14 EntryNormal "prop_3" Messages: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:14 Responses:[2->1 MsgAppResp Term:1 Log:0/15, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15] + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses:[2->1 MsgAppResp Term:1 Log:0/15, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15] 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] Responses:[ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"]] > 3 handling Ready Ready MustSync=false: @@ -550,27 +550,27 @@ process-ready 1 2 3 CommittedEntries: 1/14 EntryNormal "prop_3" Messages: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:14 Responses:[3->1 MsgAppResp Term:1 Log:0/15, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15] + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses:[3->1 MsgAppResp Term:1 Log:0/15, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15] 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] Responses:[ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"]] process-append-thread 1 2 3 ---- > 1 processing append thread Processing: - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] HardState: Term:1 Vote:1 Commit:13 + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses: 1->1 MsgAppResp Term:1 Log:0/15 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15 > 2 processing append thread Processing: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] HardState: Term:1 Vote:1 Commit:13 + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses: 2->1 MsgAppResp Term:1 Log:0/14 2->1 MsgAppResp Term:1 Log:0/15 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15 > 3 processing append thread Processing: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] HardState: Term:1 Vote:1 Commit:13 + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses: 3->1 MsgAppResp Term:1 Log:0/14 3->1 MsgAppResp Term:1 Log:0/15 @@ -618,7 +618,7 @@ process-ready 1 2 3 Messages: 1->2 MsgApp Term:1 Log:1/15 Commit:15 1->3 MsgApp Term:1 Log:1/15 Commit:15 - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:15 + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] Responses:[ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"]] > 2 handling Ready @@ -640,7 +640,7 @@ process-ready 1 2 3 CommittedEntries: 1/15 EntryNormal "prop_4" Messages: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:15 Responses:[2->1 MsgAppResp Term:1 Log:0/15] + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses:[2->1 MsgAppResp Term:1 Log:0/15] 2->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] Responses:[ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"]] > 3 handling Ready Ready MustSync=false: @@ -648,20 +648,20 @@ process-ready 1 2 3 CommittedEntries: 1/15 EntryNormal "prop_4" Messages: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:15 Responses:[3->1 MsgAppResp Term:1 Log:0/15] + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses:[3->1 MsgAppResp Term:1 Log:0/15] 3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] Responses:[ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"]] process-append-thread 2 3 ---- > 2 processing append thread Processing: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:14 + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses: 2->1 MsgAppResp Term:1 Log:0/15 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15 > 3 processing append thread Processing: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:14 + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses: 3->1 MsgAppResp Term:1 Log:0/15 AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15 @@ -709,12 +709,12 @@ process-append-thread 2 3 ---- > 2 processing append thread Processing: - 2->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:15 + 2->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses: 2->1 MsgAppResp Term:1 Log:0/15 > 3 processing append thread Processing: - 3->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:15 + 3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses: 3->1 MsgAppResp Term:1 Log:0/15 @@ -757,11 +757,11 @@ stabilize ---- > 1 processing append thread Processing: - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:14 + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses: AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15 Processing: - 1->AppendThread MsgStorageAppend Term:1 Log:0/0 HardState: Term:1 Vote:1 Commit:15 + 1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses: > 1 receiving messages AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15 diff --git a/testdata/async_storage_writes_append_aba_race.txt b/testdata/async_storage_writes_append_aba_race.txt index 6593a46e7b0d..b1e82f1261fc 100644 --- a/testdata/async_storage_writes_append_aba_race.txt +++ b/testdata/async_storage_writes_append_aba_race.txt @@ -42,7 +42,7 @@ Messages: 2->5 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] 2->6 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] 2->7 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "init_prop"] -2->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses:[2->2 MsgAppResp Term:1 Log:0/12, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12] +2->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses:[2->2 MsgAppResp Term:1 Log:0/12, AppendThread->2 MsgStorageAppendResp Term:1 Log:1/12] deliver-msgs 1 drop=(3,4,5,6,7) ---- @@ -61,7 +61,7 @@ Ready MustSync=true: Entries: 1/12 EntryNormal "init_prop" Messages: -1->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses:[1->2 MsgAppResp Term:1 Log:0/12, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12] +1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses:[1->2 MsgAppResp Term:1 Log:0/12, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12] # Step 4: node 3 becomes the leader after getting a vote from nodes 4, 5, and 6. @@ -88,7 +88,7 @@ Messages: 3->5 MsgVote Term:2 Log:1/11 3->6 MsgVote Term:2 Log:1/11 3->7 MsgVote Term:2 Log:1/11 -3->AppendThread MsgStorageAppend Term:2 Log:0/0 HardState: Term:2 Vote:3 Commit:11 Responses:[3->3 MsgVoteResp Term:2 Log:0/0] +3->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses:[3->3 MsgVoteResp Term:2 Log:0/0] deliver-msgs 4 5 6 ---- @@ -112,40 +112,40 @@ process-ready 4 5 6 Lead:0 State:StateFollower HardState Term:2 Vote:3 Commit:11 Messages: - 4->AppendThread MsgStorageAppend Term:2 Log:0/0 HardState: Term:2 Vote:3 Commit:11 Responses:[4->3 MsgVoteResp Term:2 Log:0/0] + 4->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses:[4->3 MsgVoteResp Term:2 Log:0/0] > 5 handling Ready Ready MustSync=true: Lead:0 State:StateFollower HardState Term:2 Vote:3 Commit:11 Messages: - 5->AppendThread MsgStorageAppend Term:2 Log:0/0 HardState: Term:2 Vote:3 Commit:11 Responses:[5->3 MsgVoteResp Term:2 Log:0/0] + 5->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses:[5->3 MsgVoteResp Term:2 Log:0/0] > 6 handling Ready Ready MustSync=true: Lead:0 State:StateFollower HardState Term:2 Vote:3 Commit:11 Messages: - 6->AppendThread MsgStorageAppend Term:2 Log:0/0 HardState: Term:2 Vote:3 Commit:11 Responses:[6->3 MsgVoteResp Term:2 Log:0/0] + 6->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses:[6->3 MsgVoteResp Term:2 Log:0/0] process-append-thread 3 4 5 6 ---- > 3 processing append thread Processing: - 3->AppendThread MsgStorageAppend Term:2 Log:0/0 HardState: Term:2 Vote:3 Commit:11 + 3->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses: 3->3 MsgVoteResp Term:2 Log:0/0 > 4 processing append thread Processing: - 4->AppendThread MsgStorageAppend Term:2 Log:0/0 HardState: Term:2 Vote:3 Commit:11 + 4->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses: 4->3 MsgVoteResp Term:2 Log:0/0 > 5 processing append thread Processing: - 5->AppendThread MsgStorageAppend Term:2 Log:0/0 HardState: Term:2 Vote:3 Commit:11 + 5->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses: 5->3 MsgVoteResp Term:2 Log:0/0 > 6 processing append thread Processing: - 6->AppendThread MsgStorageAppend Term:2 Log:0/0 HardState: Term:2 Vote:3 Commit:11 + 6->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses: 6->3 MsgVoteResp Term:2 Log:0/0 @@ -183,7 +183,7 @@ Messages: 3->5 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] 3->6 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] 3->7 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] -3->AppendThread MsgStorageAppend Term:2 Log:0/0 Entries:[2/12 EntryNormal ""] Responses:[3->3 MsgAppResp Term:2 Log:0/12, AppendThread->3 MsgStorageAppendResp Term:2 Log:2/12] +3->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[2/12 EntryNormal ""] Responses:[3->3 MsgAppResp Term:2 Log:0/12, AppendThread->3 MsgStorageAppendResp Term:2 Log:2/12] deliver-msgs 1 drop=(2,4,5,6,7) ---- @@ -210,7 +210,7 @@ HardState Term:2 Commit:11 Entries: 2/12 EntryNormal "" Messages: -1->AppendThread MsgStorageAppend Term:2 Log:0/0 Entries:[2/12 EntryNormal ""] HardState: Term:2 Commit:11 Responses:[1->3 MsgVoteResp Term:2 Log:0/0 Rejected (Hint: 0), 1->3 MsgAppResp Term:2 Log:0/12, AppendThread->1 MsgStorageAppendResp Term:2 Log:2/12] +1->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Entries:[2/12 EntryNormal ""] Responses:[1->3 MsgVoteResp Term:2 Log:0/0 Rejected (Hint: 0), 1->3 MsgAppResp Term:2 Log:0/12, AppendThread->1 MsgStorageAppendResp Term:2 Log:2/12] # Step 6: node 3 crashes and node 4 becomes leader getting the vote from 5, 6, and 7. @@ -237,7 +237,7 @@ Messages: 4->5 MsgVote Term:3 Log:1/11 4->6 MsgVote Term:3 Log:1/11 4->7 MsgVote Term:3 Log:1/11 -4->AppendThread MsgStorageAppend Term:3 Log:0/0 HardState: Term:3 Vote:4 Commit:11 Responses:[4->4 MsgVoteResp Term:3 Log:0/0] +4->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses:[4->4 MsgVoteResp Term:3 Log:0/0] deliver-msgs 5 6 7 ---- @@ -260,39 +260,39 @@ process-ready 5 6 7 Ready MustSync=true: HardState Term:3 Vote:4 Commit:11 Messages: - 5->AppendThread MsgStorageAppend Term:3 Log:0/0 HardState: Term:3 Vote:4 Commit:11 Responses:[5->4 MsgVoteResp Term:3 Log:0/0] + 5->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses:[5->4 MsgVoteResp Term:3 Log:0/0] > 6 handling Ready Ready MustSync=true: HardState Term:3 Vote:4 Commit:11 Messages: - 6->AppendThread MsgStorageAppend Term:3 Log:0/0 HardState: Term:3 Vote:4 Commit:11 Responses:[6->4 MsgVoteResp Term:3 Log:0/0] + 6->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses:[6->4 MsgVoteResp Term:3 Log:0/0] > 7 handling Ready Ready MustSync=true: Lead:0 State:StateFollower HardState Term:3 Vote:4 Commit:11 Messages: - 7->AppendThread MsgStorageAppend Term:3 Log:0/0 HardState: Term:3 Vote:4 Commit:11 Responses:[7->4 MsgVoteResp Term:3 Log:0/0] + 7->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses:[7->4 MsgVoteResp Term:3 Log:0/0] process-append-thread 4 5 6 7 ---- > 4 processing append thread Processing: - 4->AppendThread MsgStorageAppend Term:3 Log:0/0 HardState: Term:3 Vote:4 Commit:11 + 4->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses: 4->4 MsgVoteResp Term:3 Log:0/0 > 5 processing append thread Processing: - 5->AppendThread MsgStorageAppend Term:3 Log:0/0 HardState: Term:3 Vote:4 Commit:11 + 5->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses: 5->4 MsgVoteResp Term:3 Log:0/0 > 6 processing append thread Processing: - 6->AppendThread MsgStorageAppend Term:3 Log:0/0 HardState: Term:3 Vote:4 Commit:11 + 6->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses: 6->4 MsgVoteResp Term:3 Log:0/0 > 7 processing append thread Processing: - 7->AppendThread MsgStorageAppend Term:3 Log:0/0 HardState: Term:3 Vote:4 Commit:11 + 7->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses: 7->4 MsgVoteResp Term:3 Log:0/0 @@ -325,7 +325,7 @@ Messages: 4->5 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] 4->6 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] 4->7 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] -4->AppendThread MsgStorageAppend Term:3 Log:0/0 Entries:[3/12 EntryNormal ""] Responses:[4->4 MsgAppResp Term:3 Log:0/12, AppendThread->4 MsgStorageAppendResp Term:3 Log:3/12] +4->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] Responses:[4->4 MsgAppResp Term:3 Log:0/12, AppendThread->4 MsgStorageAppendResp Term:3 Log:3/12] # Step 7: before the new entries reach node 1, it hears of the term change # through a heartbeat and persists the new term. Node 1 then receives these @@ -366,7 +366,7 @@ Lead:4 State:StateFollower HardState Term:3 Commit:11 Messages: 1->4 MsgHeartbeatResp Term:3 Log:0/0 -1->AppendThread MsgStorageAppend Term:3 Log:0/0 HardState: Term:3 Commit:11 Responses:[AppendThread->1 MsgStorageAppendResp Term:3 Log:2/12] +1->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Responses:[AppendThread->1 MsgStorageAppendResp Term:3 Log:2/12] deliver-msgs 4 ---- @@ -390,7 +390,7 @@ Ready MustSync=true: Entries: 3/12 EntryNormal "" Messages: -1->AppendThread MsgStorageAppend Term:3 Log:0/0 Entries:[3/12 EntryNormal ""] Responses:[1->4 MsgAppResp Term:3 Log:0/12, AppendThread->1 MsgStorageAppendResp Term:3 Log:3/12] +1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] Responses:[1->4 MsgAppResp Term:3 Log:0/12, AppendThread->1 MsgStorageAppendResp Term:3 Log:3/12] # Step 8: The asynchronous log appends from the first Ready complete and the # MsgStorageAppendResp is returned to the raft node state machine. A decision @@ -403,7 +403,7 @@ raft-log 1 process-append-thread 1 ---- Processing: -1->AppendThread MsgStorageAppend Term:1 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] +1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[1/12 EntryNormal "init_prop"] Responses: 1->2 MsgAppResp Term:1 Log:0/12 AppendThread->1 MsgStorageAppendResp Term:1 Log:1/12 @@ -429,7 +429,7 @@ INFO 1 [term: 3] ignored entry appends from a MsgStorageAppendResp message with process-append-thread 1 ---- Processing: -1->AppendThread MsgStorageAppend Term:2 Log:0/0 Entries:[2/12 EntryNormal ""] HardState: Term:2 Commit:11 +1->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Entries:[2/12 EntryNormal ""] Responses: 1->3 MsgVoteResp Term:2 Log:0/0 Rejected (Hint: 0) 1->3 MsgAppResp Term:2 Log:0/12 @@ -448,7 +448,7 @@ INFO 1 [term: 3] ignored entry appends from a MsgStorageAppendResp message with process-append-thread 1 ---- Processing: -1->AppendThread MsgStorageAppend Term:3 Log:0/0 HardState: Term:3 Commit:11 +1->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Responses: AppendThread->1 MsgStorageAppendResp Term:3 Log:2/12 @@ -465,7 +465,7 @@ INFO entry at (index,term)=(12,2) mismatched with entry at (12,3) in unstable lo process-append-thread 1 ---- Processing: -1->AppendThread MsgStorageAppend Term:3 Log:0/0 Entries:[3/12 EntryNormal ""] +1->AppendThread MsgStorageAppend Term:0 Log:0/0 Entries:[3/12 EntryNormal ""] Responses: 1->4 MsgAppResp Term:3 Log:0/12 AppendThread->1 MsgStorageAppendResp Term:3 Log:3/12 diff --git a/util.go b/util.go index 1bec53578800..6b68fe2d91cc 100644 --- a/util.go +++ b/util.go @@ -171,6 +171,9 @@ func DescribeMessage(m pb.Message, f EntryFormatter) string { if m.Commit != 0 { fmt.Fprintf(&buf, " Commit:%d", m.Commit) } + if m.Vote != 0 { + fmt.Fprintf(&buf, " Vote:%d", m.Vote) + } if len(m.Entries) > 0 { fmt.Fprint(&buf, " Entries:[") for i, e := range m.Entries { @@ -181,9 +184,6 @@ func DescribeMessage(m pb.Message, f EntryFormatter) string { } fmt.Fprint(&buf, "]") } - if m.HardState != nil { - fmt.Fprintf(&buf, " HardState: %s", DescribeHardState(*m.HardState)) - } if s := m.Snapshot; s != nil && !IsEmptySnap(*s) { fmt.Fprintf(&buf, " Snapshot: %s", DescribeSnapshot(*s)) }