diff --git a/cmake/common.cmake b/cmake/common.cmake index 4fc7ec0a51be..ac86014a369c 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -386,7 +386,7 @@ else() set(CONSENSUS_ARG "raft") endif() -if((NOT CMAKE_BUILD_TYPE STREQUAL "Debug") AND NOT PBFT) +if((NOT CMAKE_BUILD_TYPE STREQUAL "Debug") AND NOT SAN) set(WORKER_THREADS 2) else() set(WORKER_THREADS 0) diff --git a/src/consensus/pbft/libbyz/Checkpoint.h b/src/consensus/pbft/libbyz/Checkpoint.h index 51c41af24615..4d36bc25cf7d 100644 --- a/src/consensus/pbft/libbyz/Checkpoint.h +++ b/src/consensus/pbft/libbyz/Checkpoint.h @@ -37,6 +37,8 @@ class Checkpoint : public Message // Checkpoint messages // public: + Checkpoint(uint32_t msg_size = 0) : Message(msg_size) {} + Checkpoint(Seqno s, Digest& d, bool stable = false); // Effects: Creates a new signed Checkpoint message with sequence // number "s" and digest "d". "stable" should be true iff the checkpoint diff --git a/src/consensus/pbft/libbyz/Client_proxy.h b/src/consensus/pbft/libbyz/Client_proxy.h index 279d5c44cf24..5e599d51c1fb 100644 --- a/src/consensus/pbft/libbyz/Client_proxy.h +++ b/src/consensus/pbft/libbyz/Client_proxy.h @@ -189,7 +189,7 @@ bool ClientProxy::send_request( } Request_id rid = request_id_generator.next_rid(); - auto req = std::make_unique(rid); + auto req = std::make_unique(rid, -1); if (req == nullptr) { current_outstanding.fetch_sub(1); diff --git a/src/consensus/pbft/libbyz/Commit.h b/src/consensus/pbft/libbyz/Commit.h index befe72232734..b3c54749d4e4 100644 --- a/src/consensus/pbft/libbyz/Commit.h +++ b/src/consensus/pbft/libbyz/Commit.h @@ -33,6 +33,8 @@ class Commit : public Message // Commit messages // public: + Commit(uint32_t msg_size = 0) : Message(msg_size) {} + Commit(View v, Seqno s); // Effects: Creates a new Commit message with view number "v" // and sequence number "s". diff --git a/src/consensus/pbft/libbyz/Data.h b/src/consensus/pbft/libbyz/Data.h index 51c7ea734413..da1fea3c0d82 100644 --- a/src/consensus/pbft/libbyz/Data.h +++ b/src/consensus/pbft/libbyz/Data.h @@ -31,6 +31,8 @@ class Data : public Message // Data messages // public: + Data(uint32_t msg_size = 0) : Message(msg_size) {} + Data(size_t i, Seqno lm, char* data); // Effects: Creates a new Data message. i is the index of he data block, lm // is the last sequence number when it was modified, and data is a pointer to diff --git a/src/consensus/pbft/libbyz/Fetch.h b/src/consensus/pbft/libbyz/Fetch.h index 053363ecdb20..894e2696a773 100644 --- a/src/consensus/pbft/libbyz/Fetch.h +++ b/src/consensus/pbft/libbyz/Fetch.h @@ -40,6 +40,8 @@ class Fetch : public Message // Fetch messages // public: + Fetch(uint32_t msg_size = 0) : Message(msg_size) {} + Fetch( Request_id rid, Seqno lu, diff --git a/src/consensus/pbft/libbyz/Message.h b/src/consensus/pbft/libbyz/Message.h index edcb908b21ef..739c9e63c099 100644 --- a/src/consensus/pbft/libbyz/Message.h +++ b/src/consensus/pbft/libbyz/Message.h @@ -60,13 +60,14 @@ class Message // // Generic messages // -public: +protected: Message(unsigned sz = 0); // Effects: Creates an untagged Message object that can hold up // to "sz" bytes and holds zero bytes. Useful to create message // buffers to receive messages from the network. - ~Message(); +public: + virtual ~Message(); // Effects: Deallocates all storage associated with this message. void trim(); @@ -91,6 +92,18 @@ class Message int tag() const; // Effects: Fetches the message tag. + static int get_tag(const uint8_t* data) + { + Message_rep* m = (Message_rep*)data; + return m->tag; + } + + static int get_size(const uint8_t* data) + { + Message_rep* m = (Message_rep*)data; + return m->size; + } + bool has_tag(int t, int sz) const; // Effects: If message has tag "t", its size is greater than "sz", // its size less than or equal to "max_size", and its size is a @@ -103,10 +116,6 @@ class Message // Effects: Messages may be full or empty. Empty messages are just // digests of full messages. - // Message-specific heap management operators. - void* operator new(size_t s); - void operator delete(void* x, size_t s); - const char* stag(); // Effects: Returns a string with tag name @@ -186,21 +195,6 @@ inline bool Message::full() const return true; } -inline void* Message::operator new(size_t s) -{ - void* ret = malloc(s); - PBFT_ASSERT(ret != 0, "Ran out of memory\n"); - return ret; -} - -inline void Message::operator delete(void* x, size_t s) -{ - if (x != 0) - { - free(x); - } -} - inline int Message::msize() const { return (max_size >= 0) ? max_size : msg->size; diff --git a/src/consensus/pbft/libbyz/Meta_data.h b/src/consensus/pbft/libbyz/Meta_data.h index 36572882cf6d..7796442e43fb 100644 --- a/src/consensus/pbft/libbyz/Meta_data.h +++ b/src/consensus/pbft/libbyz/Meta_data.h @@ -50,6 +50,8 @@ class Meta_data : public Message // Meta_data messages // public: + Meta_data(uint32_t msg_size = 0) : Message(msg_size) {} + Meta_data(Request_id r, int l, size_t i, Seqno lu, Seqno lm, Digest& d); // Effects: Creates a new un-authenticated Meta_data message with no // subpartition information. diff --git a/src/consensus/pbft/libbyz/Meta_data_d.h b/src/consensus/pbft/libbyz/Meta_data_d.h index d561859b0d23..366f443067e5 100644 --- a/src/consensus/pbft/libbyz/Meta_data_d.h +++ b/src/consensus/pbft/libbyz/Meta_data_d.h @@ -46,6 +46,8 @@ class Meta_data_d : public Message // Meta_data_d messages // public: + Meta_data_d(uint32_t msg_size = 0) : Message(msg_size) {} + Meta_data_d(Request_id r, int l, size_t i, Seqno ls); // Effects: Creates a new un-authenticated Meta_data_d message with no // partition digests. diff --git a/src/consensus/pbft/libbyz/Network_open.h b/src/consensus/pbft/libbyz/Network_open.h index 2851e879f675..689d41acd30a 100644 --- a/src/consensus/pbft/libbyz/Network_open.h +++ b/src/consensus/pbft/libbyz/Network_open.h @@ -21,6 +21,8 @@ struct Network_open_rep : public Message_rep class Network_open : public Message { public: + Network_open(uint32_t msg_size = 0) : Message(msg_size) {} + Network_open(int id); int id() const; diff --git a/src/consensus/pbft/libbyz/New_principal.h b/src/consensus/pbft/libbyz/New_principal.h index a868c5096615..16659e148390 100644 --- a/src/consensus/pbft/libbyz/New_principal.h +++ b/src/consensus/pbft/libbyz/New_principal.h @@ -35,6 +35,8 @@ struct New_principal_rep : public Message_rep class New_principal : public Message { public: + New_principal(uint32_t msg_size = 0) : Message(msg_size) {} + New_principal( NodeId id, short port, diff --git a/src/consensus/pbft/libbyz/New_view.h b/src/consensus/pbft/libbyz/New_view.h index 832099e7b641..2c0e11fc1f69 100644 --- a/src/consensus/pbft/libbyz/New_view.h +++ b/src/consensus/pbft/libbyz/New_view.h @@ -67,6 +67,8 @@ class New_view : public Message // New_view messages // public: + New_view(uint32_t msg_size = 0) : Message(msg_size) {} + New_view(View v); // Effects: Creates a new (unsigned) New_view message with an empty // set of view change messages. diff --git a/src/consensus/pbft/libbyz/Pre_prepare.h b/src/consensus/pbft/libbyz/Pre_prepare.h index 0f90295d6430..4423d46561bb 100644 --- a/src/consensus/pbft/libbyz/Pre_prepare.h +++ b/src/consensus/pbft/libbyz/Pre_prepare.h @@ -61,6 +61,8 @@ class Pre_prepare : public Message // Pre_prepare messages // public: + Pre_prepare(uint32_t msg_size = 0) : Message(msg_size) {} + Pre_prepare(View v, Seqno s, Req_queue& reqs, size_t& requests_in_batch); // Effects: Creates a new signed Pre_prepare message with view // number "v", sequence number "s", the requests in "reqs" (up to a diff --git a/src/consensus/pbft/libbyz/Prepare.h b/src/consensus/pbft/libbyz/Prepare.h index bf613994d6a3..99ec2230ad30 100644 --- a/src/consensus/pbft/libbyz/Prepare.h +++ b/src/consensus/pbft/libbyz/Prepare.h @@ -50,6 +50,8 @@ class Prepare : public Message // Prepare messages // public: + Prepare(uint32_t msg_size = 0) : Message(msg_size) {} + Prepare( View v, Seqno s, Digest& d, Principal* dst = 0, bool is_signed = false); // Effects: Creates a new signed Prepare message with view number diff --git a/src/consensus/pbft/libbyz/Query_stable.h b/src/consensus/pbft/libbyz/Query_stable.h index 311d37b79276..ed37824394f4 100644 --- a/src/consensus/pbft/libbyz/Query_stable.h +++ b/src/consensus/pbft/libbyz/Query_stable.h @@ -32,6 +32,8 @@ class Query_stable : public Message // Query_stable messages // public: + Query_stable(uint32_t msg_size = 0) : Message(msg_size) {} + Query_stable(); // Effects: Creates a new authenticated Query_stable message. diff --git a/src/consensus/pbft/libbyz/Replica.cpp b/src/consensus/pbft/libbyz/Replica.cpp index 72f5be0ef5c0..ae39f430d26b 100644 --- a/src/consensus/pbft/libbyz/Replica.cpp +++ b/src/consensus/pbft/libbyz/Replica.cpp @@ -247,19 +247,107 @@ static void pre_verify_cb(std::unique_ptr> req) static uint64_t verification_thread = 0; +Message* Replica::create_message(const uint8_t* data, uint32_t size) +{ + uint64_t alloc_size = std::max(size, (uint32_t)Max_message_size); + + Message* m; + + switch (Message::get_tag(data)) + { + case Request_tag: + m = new Request(alloc_size); + break; + + case Reply_tag: + m = new Reply(alloc_size); + break; + + case Pre_prepare_tag: + m = new Pre_prepare(alloc_size); + break; + + case Prepare_tag: + m = new Prepare(alloc_size); + break; + + case Commit_tag: + m = new Commit(alloc_size); + break; + + case Checkpoint_tag: + m = new Checkpoint(alloc_size); + break; + +#ifndef USE_PKEY_VIEW_CHANGES + case View_change_ack_tag: + m = new View_change_ack(alloc_size); + break; +#endif + + case Status_tag: + m = new Status(alloc_size); + break; + + case Fetch_tag: + m = new Fetch(alloc_size); + break; + + case Query_stable_tag: + m = new Query_stable(alloc_size); + break; + + case Reply_stable_tag: + m = new Reply_stable(alloc_size); + break; + + case Meta_data_tag: + m = new Meta_data(alloc_size); + break; + + case Meta_data_d_tag: + m = new Meta_data_d(alloc_size); + break; + + case Data_tag: + m = new Data(alloc_size); + break; + + case View_change_tag: + m = new View_change(alloc_size); + break; + + case New_view_tag: + m = new New_view((uint32_t)alloc_size); + break; + + case New_principal_tag: + m = new New_principal(alloc_size); + break; + + case Network_open_tag: + m = new Network_open((uint32_t)alloc_size); + break; + + default: + // Unknown message type. + delete m; + } + + memcpy(m->contents(), data, size); + + return m; +} + void Replica::receive_message(const uint8_t* data, uint32_t size) { if (size > Max_message_size) { LOG_FAIL << "Received message size exceeds message: " << size << std::endl; } - uint64_t alloc_size = std::max(size, (uint32_t)Max_message_size); - Message* m = new Message(alloc_size); - + Message* m = create_message(data, size); uint32_t target_thread = 0; - memcpy(m->contents(), data, size); - if (enclave::ThreadMessaging::thread_count > 1 && m->tag() == Request_tag) { uint32_t num_worker_thread = enclave::ThreadMessaging::thread_count - 1; @@ -1168,8 +1256,7 @@ template std::unique_ptr Replica::create_message( const uint8_t* message_data, size_t data_size) { - Message* m = new Message(data_size); - std::copy(message_data, message_data + data_size, m->contents()); + Message* m = create_message(message_data, data_size); T* msg_type; T::convert(m, msg_type); return std::unique_ptr(msg_type); @@ -2776,7 +2863,7 @@ void Replica::handle(Reply_stable* m) LOG_INFO << "sending recovery request" << std::endl; // Send recovery request. START_CC(rr_time); - rr = new Request(new_rid()); + rr = new Request(new_rid(), -1); int len; char* buf = rr->store_command(len); diff --git a/src/consensus/pbft/libbyz/Replica.h b/src/consensus/pbft/libbyz/Replica.h index e50243ab482e..f7f026d86b36 100644 --- a/src/consensus/pbft/libbyz/Replica.h +++ b/src/consensus/pbft/libbyz/Replica.h @@ -185,6 +185,9 @@ class Replica : public Node, public IMessageReceiveBase // Effects: Use when messages are passed to Replica rather than replica // polling + static Message* create_message(const uint8_t* data, uint32_t size); + // Effects: Creates a new message from a buffer + bool compare_execution_results(const ByzInfo& info, Pre_prepare* pre_prepare); // Compare the merkle root and batch ctx between the pre-prepare and the // the corresponding fields in info after execution diff --git a/src/consensus/pbft/libbyz/Reply.cpp b/src/consensus/pbft/libbyz/Reply.cpp index e1a2ce566858..d7b69f7137a6 100644 --- a/src/consensus/pbft/libbyz/Reply.cpp +++ b/src/consensus/pbft/libbyz/Reply.cpp @@ -66,7 +66,7 @@ Reply::Reply( Reply* Reply::copy(int id) const { - Reply* ret = (Reply*)new Message(msg->size); + Reply* ret = (Reply*)new Reply(msg->size); memcpy(ret->msg, msg, msg->size); ret->rep().replica = id; return ret; diff --git a/src/consensus/pbft/libbyz/Reply.h b/src/consensus/pbft/libbyz/Reply.h index 6e019a316bf2..b4701a1bfa4d 100644 --- a/src/consensus/pbft/libbyz/Reply.h +++ b/src/consensus/pbft/libbyz/Reply.h @@ -44,6 +44,8 @@ class Reply : public Message public: Reply() : Message() {} + Reply(uint32_t msg_size) : Message(msg_size) {} + Reply(Reply_rep* r); Reply(View view, Request_id req, Seqno n, int replica, uint32_t reply_size); diff --git a/src/consensus/pbft/libbyz/Reply_stable.h b/src/consensus/pbft/libbyz/Reply_stable.h index b7aed42cd634..bfef638eda42 100644 --- a/src/consensus/pbft/libbyz/Reply_stable.h +++ b/src/consensus/pbft/libbyz/Reply_stable.h @@ -35,6 +35,8 @@ class Reply_stable : public Message // Reply_stable messages // public: + Reply_stable(uint32_t msg_size = 0) : Message(msg_size) {} + Reply_stable(Seqno lc, Seqno lp, int n, Principal* p); // Effects: Creates a new authenticated Reply_stable message with // last checkpoint "lc", last prepared request "lp", for a diff --git a/src/consensus/pbft/libbyz/Request.cpp b/src/consensus/pbft/libbyz/Request.cpp index 282ca4a7c94b..65a804fcd667 100644 --- a/src/consensus/pbft/libbyz/Request.cpp +++ b/src/consensus/pbft/libbyz/Request.cpp @@ -32,7 +32,7 @@ Request::Request(Request_id r, short rr) : Request* Request::clone() const { - Request* ret = (Request*)new Message(max_size); + Request* ret = (Request*)new Request(max_size); memcpy(ret->msg, msg, msg->size); return ret; } diff --git a/src/consensus/pbft/libbyz/Request.h b/src/consensus/pbft/libbyz/Request.h index 73ebf376ae56..d94218c2b70f 100644 --- a/src/consensus/pbft/libbyz/Request.h +++ b/src/consensus/pbft/libbyz/Request.h @@ -45,9 +45,10 @@ class Request : public Message // memory the user expects to be able to use.) // public: - Request() : Message() {} + // Request() : Message() {} + Request(uint32_t msg_size = 0) : Message(msg_size) {} - Request(Request_id r, short rr = -1); + Request(Request_id r, short rr); // Effects: Creates a new signed Request message with an empty // command and no authentication. The methods store_command and // authenticate should be used to finish message construction. diff --git a/src/consensus/pbft/libbyz/Status.h b/src/consensus/pbft/libbyz/Status.h index 8523fe699a71..508abbf8a167 100644 --- a/src/consensus/pbft/libbyz/Status.h +++ b/src/consensus/pbft/libbyz/Status.h @@ -71,6 +71,8 @@ class Status : public Message // Status messages // public: + Status(uint32_t msg_size = 0) : Message(msg_size) {} + Status(View v, Seqno ls, Seqno le, bool hnvi, bool hnvm); // Effects: Creates a new unauthenticated Status message. "v" // should be the sending replica's current view, "ls" should be the diff --git a/src/consensus/pbft/libbyz/View_change.h b/src/consensus/pbft/libbyz/View_change.h index bbb9cc7998c9..6889070c84ce 100644 --- a/src/consensus/pbft/libbyz/View_change.h +++ b/src/consensus/pbft/libbyz/View_change.h @@ -99,6 +99,8 @@ class View_change : public Message // View_change messages // public: + View_change(uint32_t msg_size = 0) : Message(msg_size) {} + View_change(View v, Seqno ls, int id); // Effects: Creates a new (unauthenticated) View_change message for // replica "id" in view "v". The message states that "ls" is the diff --git a/src/consensus/pbft/libbyz/View_change_ack.h b/src/consensus/pbft/libbyz/View_change_ack.h index 0f16086f872b..df468539d014 100644 --- a/src/consensus/pbft/libbyz/View_change_ack.h +++ b/src/consensus/pbft/libbyz/View_change_ack.h @@ -35,6 +35,8 @@ class View_change_ack : public Message // View_change_ack messages // public: + View_change_ack(uint32_t msg_size = 0) : Message(msg_size) {} + View_change_ack(View v, int id, int vcid, Digest const& vcd); // Effects: Creates a new authenticated View_change_ack message for // replica "id" stating that replica "vcid" sent out a view-change diff --git a/src/consensus/pbft/libbyz/libbyz.cpp b/src/consensus/pbft/libbyz/libbyz.cpp index b983bc88facf..36cdd2e9bdd1 100644 --- a/src/consensus/pbft/libbyz/libbyz.cpp +++ b/src/consensus/pbft/libbyz/libbyz.cpp @@ -33,7 +33,7 @@ void Byz_reset_client() int Byz_alloc_request(Byz_req* req, int size) { - Request* request = new Request((Request_id)0); + Request* request = new Request((Request_id)0, -1); if (request == 0) { return -1; diff --git a/src/consensus/pbft/libbyz/test/mocks/network_mock.h b/src/consensus/pbft/libbyz/test/mocks/network_mock.h index 664384da2053..c0fd12885111 100644 --- a/src/consensus/pbft/libbyz/test/mocks/network_mock.h +++ b/src/consensus/pbft/libbyz/test/mocks/network_mock.h @@ -24,8 +24,7 @@ class MockNetwork : public INetwork virtual Message* GetNextMessage() { - Message* m = new Message(Max_message_size); - return m; + throw std::logic_error("Should never be called"); } virtual bool has_messages(long to) diff --git a/src/consensus/pbft/libbyz/test/network_udp.cpp b/src/consensus/pbft/libbyz/test/network_udp.cpp index ed1740f82d6d..cffc977a246f 100644 --- a/src/consensus/pbft/libbyz/test/network_udp.cpp +++ b/src/consensus/pbft/libbyz/test/network_udp.cpp @@ -108,6 +108,9 @@ class UDPNetwork : public INetwork return error; } + std::unique_ptr buffer = + std::make_unique(Max_message_size); + virtual Message* GetNextMessage() { while (1) @@ -116,10 +119,13 @@ class UDPNetwork : public INetwork { ; } - Message* m = new Message(Max_message_size); - - int ret = recvfrom(sock, m->contents(), m->msize(), 0, 0, 0); - + int ret = recvfrom(sock, buffer.get(), Max_message_size, 0, 0, 0); + if (ret < sizeof(Message_rep)) + { + continue; + } + Message* m = + Replica::create_message(buffer.get(), Message::get_size(buffer.get())); if ( ret >= (int)sizeof(Message_rep) && ret >= (int)m->size() && Replica::pre_verify(m)) diff --git a/src/consensus/pbft/libbyz/test/network_udp_mt.cpp b/src/consensus/pbft/libbyz/test/network_udp_mt.cpp index 2856a7038699..94964ffcf1d7 100644 --- a/src/consensus/pbft/libbyz/test/network_udp_mt.cpp +++ b/src/consensus/pbft/libbyz/test/network_udp_mt.cpp @@ -455,29 +455,32 @@ Message* ReceiverThread::dequeue() void ReceiverThread::Work() { + std::unique_ptr buffer = + std::make_unique(Max_message_size); while (true) { - Message* m = new Message(Max_message_size); - if (m != nullptr) + int ret = + recvfrom(sock, buffer.get(), Message::get_size(buffer.get()), 0, 0, 0); + + if (ret < sizeof(Message_rep)) { - if (m->contents() != nullptr) - { - int ret = recvfrom(sock, m->contents(), m->msize(), 0, 0, 0); - - LOG_TRACE << " Received tag: " << m->tag() - << " at thread index: " << index << std::endl; - - if ( - ret >= (int)sizeof(Message_rep) && ret >= (int)m->size() && - Replica::pre_verify(m)) - { - queue(m); - continue; - } - } + continue; + } + Message* m = + Replica::create_message(buffer.get(), Message::get_size(buffer.get())); + + LOG_TRACE << " Received tag: " << m->tag() << " at thread index: " << index + << std::endl; - delete m; + if ( + ret >= (int)sizeof(Message_rep) && ret >= (int)m->size() && + Replica::pre_verify(m)) + { + queue(m); + continue; } + + delete m; } }