From 515cba3062cfe2f43f352b61015cc19a33d1e2ad Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Wed, 29 Jun 2022 10:55:10 +0000 Subject: [PATCH 1/5] Add current_service_create_version to service info table --- include/ccf/service/tables/service.h | 9 +++++++-- src/node/node_state.h | 5 ++++- src/node/rpc/node_call_types.h | 1 + src/node/rpc/node_frontend.h | 2 +- src/service/genesis_gen.h | 7 +++++-- tests/recovery.py | 5 +++++ 6 files changed, 23 insertions(+), 6 deletions(-) diff --git a/include/ccf/service/tables/service.h b/include/ccf/service/tables/service.h index 75436e66b6fd..46d5ba299bdc 100644 --- a/include/ccf/service/tables/service.h +++ b/include/ccf/service/tables/service.h @@ -29,15 +29,20 @@ namespace ccf crypto::Pem cert; /// Status of the service ServiceStatus status = ServiceStatus::OPENING; - /// Version of previous service identity (before the last recovery) + /// Version (seqno) of previous service identity (before the last recovery) std::optional previous_service_identity_version = std::nullopt; /// Number of disaster recoveries performed on this service std::optional recovery_count = std::nullopt; + /// Version (seqno) at which current service was created + std::optional current_service_create_version = std::nullopt; }; DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(ServiceInfo); DECLARE_JSON_REQUIRED_FIELDS(ServiceInfo, cert, status); DECLARE_JSON_OPTIONAL_FIELDS( - ServiceInfo, previous_service_identity_version, recovery_count); + ServiceInfo, + previous_service_identity_version, + recovery_count, + current_service_create_version); // As there is only one service active at a given time, it is stored in single // Value in the KV diff --git a/src/node/node_state.h b/src/node/node_state.h index 0faac4c1e7ad..41b721e32f93 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -1052,6 +1052,8 @@ namespace ccf last_recovered_term, last_recovered_signed_idx); + auto first_idx_in_new_service = last_recovered_signed_idx + 1; + auto tx = network.tables->create_read_only_tx(); if (network.consensus_type == ConsensusType::BFT) { @@ -1062,7 +1064,7 @@ namespace ccf open_frontend(ActorsType::members); } - network.ledger_secrets->init(last_recovered_signed_idx + 1); + network.ledger_secrets->init(first_idx_in_new_service); // Initialise snapshotter after public recovery snapshotter->init_after_public_recovery(); @@ -1951,6 +1953,7 @@ namespace ccf create_params.code_digest = node_code_id; create_params.node_info_network = config.network; create_params.node_data = config.node_data; + create_params.create_version = last_recovered_signed_idx + 1; const auto body = serdes::pack(create_params, serdes::Pack::Text); diff --git a/src/node/rpc/node_call_types.h b/src/node/rpc/node_call_types.h index 83017d18d7ee..368d44d846ad 100644 --- a/src/node/rpc/node_call_types.h +++ b/src/node/rpc/node_call_types.h @@ -62,6 +62,7 @@ namespace ccf CodeDigest code_digest; NodeInfoNetwork node_info_network; nlohmann::json node_data; + kv::Version create_version; // Only set on genesis transaction, but not on recovery std::optional genesis_info = std::nullopt; diff --git a/src/node/rpc/node_frontend.h b/src/node/rpc/node_frontend.h index 6baf798c70c1..379f2f3daf9e 100644 --- a/src/node/rpc/node_frontend.h +++ b/src/node/rpc/node_frontend.h @@ -1346,7 +1346,7 @@ namespace ccf "Service is already created."); } - g.create_service(in.service_cert, recovering); + g.create_service(in.service_cert, in.create_version, recovering); // Retire all nodes, in case there are any (i.e. post recovery) g.retire_active_nodes(); diff --git a/src/service/genesis_gen.h b/src/service/genesis_gen.h index 90e49d6f0b53..c8cc4f3a527a 100644 --- a/src/service/genesis_gen.h +++ b/src/service/genesis_gen.h @@ -288,7 +288,9 @@ namespace ccf // Service status should use a state machine, very much like NodeState. void create_service( - const crypto::Pem& service_cert, bool recovering = false) + const crypto::Pem& service_cert, + kv::Version create_version, + bool recovering = false) { auto service = tx.rw(tables.service); @@ -311,7 +313,8 @@ namespace ccf {service_cert, recovering ? ServiceStatus::RECOVERING : ServiceStatus::OPENING, recovering ? service->get_version_of_previous_write() : std::nullopt, - recovery_count}); + recovery_count, + create_version}); } bool is_service_created(const crypto::Pem& expected_service_cert) diff --git a/tests/recovery.py b/tests/recovery.py index 1f708a07215e..27f0f968c8cf 100644 --- a/tests/recovery.py +++ b/tests/recovery.py @@ -554,6 +554,11 @@ def run(args): ) as network: network.start_and_open(args) + network = test_recover_service(network, args, from_snapshot=False) + network = test_recover_service(network, args, from_snapshot=False) + + return network + if args.with_load: # See https://github.com/microsoft/CCF/issues/3788 for justification LOG.info("Loading service before recovery...") From 71602db76479408f958a162c1dc440018281bcc8 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Wed, 29 Jun 2022 13:02:08 +0000 Subject: [PATCH 2/5] Record TxID --- include/ccf/service/tables/service.h | 5 +++-- src/node/node_state.h | 30 ++++++++++++++++++---------- src/node/rpc/call_types.h | 1 + src/node/rpc/node_call_types.h | 2 +- src/node/rpc/node_frontend.h | 4 +++- src/node/rpc/serialization.h | 6 ++++-- src/service/genesis_gen.h | 4 ++-- tests/infra/consortium.py | 5 ++--- tests/lts_compatibility.py | 2 +- tests/recovery.py | 22 ++++++++++++++++---- 10 files changed, 54 insertions(+), 27 deletions(-) diff --git a/include/ccf/service/tables/service.h b/include/ccf/service/tables/service.h index 46d5ba299bdc..557612ed84a2 100644 --- a/include/ccf/service/tables/service.h +++ b/include/ccf/service/tables/service.h @@ -5,6 +5,7 @@ #include "ccf/crypto/pem.h" #include "ccf/ds/json.h" #include "ccf/service/map.h" +#include "ccf/tx_id.h" namespace ccf { @@ -34,7 +35,7 @@ namespace ccf /// Number of disaster recoveries performed on this service std::optional recovery_count = std::nullopt; /// Version (seqno) at which current service was created - std::optional current_service_create_version = std::nullopt; + std::optional current_service_create_txid = std::nullopt; }; DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(ServiceInfo); DECLARE_JSON_REQUIRED_FIELDS(ServiceInfo, cert, status); @@ -42,7 +43,7 @@ namespace ccf ServiceInfo, previous_service_identity_version, recovery_count, - current_service_create_version); + current_service_create_txid); // As there is only one service active at a given time, it is stored in single // Value in the KV diff --git a/src/node/node_state.h b/src/node/node_state.h index 41b721e32f93..c9957bf0b294 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -147,8 +147,12 @@ namespace ccf struct NodeStateMsg { - NodeStateMsg(NodeState& self_) : self(self_) {} + NodeStateMsg(NodeState& self_, View create_view_ = 0) : + self(self_), + create_view(create_view_) + {} NodeState& self; + View create_view; }; // @@ -399,7 +403,8 @@ namespace ccf // Become the primary and force replication consensus->force_become_primary(); - if (!create_and_send_boot_request(true /* Create new consortium */)) + if (!create_and_send_boot_request( + aft::starting_view_change, true /* Create new consortium */)) { throw std::runtime_error( "Genesis transaction could not be committed"); @@ -1037,7 +1042,7 @@ namespace ccf // When reaching the end of the public ledger, truncate to last signed // index const auto last_recovered_term = view_history.size(); - auto new_term = last_recovered_term + 2; + auto new_term = last_recovered_term + aft::starting_view_change; LOG_INFO_FMT("Setting term on public recovery store to {}", new_term); // Note: KV term must be set before the first Tx is committed @@ -1052,8 +1057,6 @@ namespace ccf last_recovered_term, last_recovered_signed_idx); - auto first_idx_in_new_service = last_recovered_signed_idx + 1; - auto tx = network.tables->create_read_only_tx(); if (network.consensus_type == ConsensusType::BFT) { @@ -1064,7 +1067,7 @@ namespace ccf open_frontend(ActorsType::members); } - network.ledger_secrets->init(first_idx_in_new_service); + network.ledger_secrets->init(last_recovered_signed_idx + 1); // Initialise snapshotter after public recovery snapshotter->init_after_public_recovery(); @@ -1109,6 +1112,7 @@ namespace ccf auto msg = std::make_unique>( [](std::unique_ptr> msg) { if (!msg->data.self.create_and_send_boot_request( + msg->data.create_view, false /* Restore consortium from ledger */)) { throw std::runtime_error( @@ -1116,7 +1120,8 @@ namespace ccf } msg->data.self.advance_part_of_public_network(); }, - *this); + *this, + new_term); threading::ThreadMessaging::thread_messaging.add_task( threading::get_current_thread_id(), std::move(msg)); } @@ -1920,7 +1925,8 @@ namespace ccf } } - std::vector serialize_create_request(bool create_consortium = true) + std::vector serialize_create_request( + View create_view, bool create_consortium = true) { CreateNetworkNodeToNode::In create_params; @@ -1953,7 +1959,7 @@ namespace ccf create_params.code_digest = node_code_id; create_params.node_info_network = config.network; create_params.node_data = config.node_data; - create_params.create_version = last_recovered_signed_idx + 1; + create_params.create_txid = {create_view, last_recovered_signed_idx + 1}; const auto body = serdes::pack(create_params, serdes::Pack::Text); @@ -2036,9 +2042,11 @@ namespace ccf return parse_create_response(response.value()); } - bool create_and_send_boot_request(bool create_consortium = true) + bool create_and_send_boot_request( + View create_view, bool create_consortium = true) { - return send_create_request(serialize_create_request(create_consortium)); + return send_create_request( + serialize_create_request(create_view, create_consortium)); } void backup_initiate_private_recovery() diff --git a/src/node/rpc/call_types.h b/src/node/rpc/call_types.h index 33fe840d5b0f..9703340e5cec 100644 --- a/src/node/rpc/call_types.h +++ b/src/node/rpc/call_types.h @@ -59,6 +59,7 @@ namespace ccf std::optional current_view; std::optional primary_id; size_t recovery_count; + std::optional current_service_create_txid; }; }; diff --git a/src/node/rpc/node_call_types.h b/src/node/rpc/node_call_types.h index 368d44d846ad..59c75468c99a 100644 --- a/src/node/rpc/node_call_types.h +++ b/src/node/rpc/node_call_types.h @@ -62,7 +62,7 @@ namespace ccf CodeDigest code_digest; NodeInfoNetwork node_info_network; nlohmann::json node_data; - kv::Version create_version; + ccf::TxID create_txid; // Only set on genesis transaction, but not on recovery std::optional genesis_info = std::nullopt; diff --git a/src/node/rpc/node_frontend.h b/src/node/rpc/node_frontend.h index 379f2f3daf9e..6490304bb2e5 100644 --- a/src/node/rpc/node_frontend.h +++ b/src/node/rpc/node_frontend.h @@ -809,6 +809,8 @@ namespace ccf out.service_status = service_value.status; out.service_certificate = service_value.cert; out.recovery_count = service_value.recovery_count.value_or(0); + out.current_service_create_txid = + service_value.current_service_create_txid; if (consensus != nullptr) { out.current_view = consensus->get_view(); @@ -1346,7 +1348,7 @@ namespace ccf "Service is already created."); } - g.create_service(in.service_cert, in.create_version, recovering); + g.create_service(in.service_cert, in.create_txid, recovering); // Retire all nodes, in case there are any (i.e. post recovery) g.retire_active_nodes(); diff --git a/src/node/rpc/serialization.h b/src/node/rpc/serialization.h index 4b8656e83206..2d6165a4a152 100644 --- a/src/node/rpc/serialization.h +++ b/src/node/rpc/serialization.h @@ -72,7 +72,8 @@ namespace ccf quote_info, public_encryption_key, code_digest, - node_info_network) + node_info_network, + create_txid) DECLARE_JSON_OPTIONAL_FIELDS( CreateNetworkNodeToNode::In, genesis_info, node_data) @@ -89,7 +90,8 @@ namespace ccf service_certificate, current_view, primary_id, - recovery_count) + recovery_count, + current_service_create_txid) DECLARE_JSON_TYPE(GetNode::NodeInfo) DECLARE_JSON_REQUIRED_FIELDS( diff --git a/src/service/genesis_gen.h b/src/service/genesis_gen.h index c8cc4f3a527a..e86d8dac592f 100644 --- a/src/service/genesis_gen.h +++ b/src/service/genesis_gen.h @@ -289,7 +289,7 @@ namespace ccf // Service status should use a state machine, very much like NodeState. void create_service( const crypto::Pem& service_cert, - kv::Version create_version, + ccf::TxID create_txid, bool recovering = false) { auto service = tx.rw(tables.service); @@ -314,7 +314,7 @@ namespace ccf recovering ? ServiceStatus::RECOVERING : ServiceStatus::OPENING, recovering ? service->get_version_of_previous_write() : std::nullopt, recovery_count, - create_version}); + create_txid}); } bool is_service_created(const crypto::Pem& expected_service_cert) diff --git a/tests/infra/consortium.py b/tests/infra/consortium.py index 218a6d45ed50..dec95e7ec49b 100644 --- a/tests/infra/consortium.py +++ b/tests/infra/consortium.py @@ -710,8 +710,7 @@ def check_for_service(self, remote_node, status, recovery_count=None): r = c.get("/node/network").body.json() current_status = r["service_status"] current_cert = r["service_certificate"] - # Note: to change once this is backported to 2.x - if remote_node.version_after("ccf-2.0.3"): + if remote_node.version_after("ccf-2.0.4"): current_recovery_count = r["recovery_count"] else: assert "recovery_count" not in r @@ -730,7 +729,7 @@ def check_for_service(self, remote_node, status, recovery_count=None): assert ( current_status == status.value ), f"Service status {current_status} (expected {status.value})" - if remote_node.version_after("ccf-2.0.3"): + if remote_node.version_after("ccf-2.0.4"): assert ( recovery_count is None or current_recovery_count == recovery_count ), f"Current recovery count {current_recovery_count} is not expected {recovery_count}" diff --git a/tests/lts_compatibility.py b/tests/lts_compatibility.py index ce387a020d12..5569da3efe07 100644 --- a/tests/lts_compatibility.py +++ b/tests/lts_compatibility.py @@ -479,7 +479,7 @@ def run_ledger_compatibility_since_first(args, local_branch, use_snapshot): network.recover( args, expected_recovery_count=1 - if not infra.node.version_after(previous_version, "ccf-2.0.3") + if not infra.node.version_after(previous_version, "ccf-2.0.4") else None, ) diff --git a/tests/recovery.py b/tests/recovery.py index 27f0f968c8cf..358301a0d541 100644 --- a/tests/recovery.py +++ b/tests/recovery.py @@ -51,6 +51,7 @@ def test_recover_service(network, args, from_snapshot=True): with old_primary.client() as c: r = c.get("/node/service/previous_identity") assert r.status_code in (200, 404), r.status_code + prev_view = c.get("/node/network").body.json()["current_view"] snapshots_dir = None if from_snapshot: @@ -99,6 +100,16 @@ def test_recover_service(network, args, from_snapshot=True): recovered_network.recover(args) + LOG.info("Check that new service view is as expected") + new_primary, _ = recovered_network.find_primary() + with new_primary.client() as c: + assert ( + ccf.tx_id.TxID.from_str( + c.get("/node/network").body.json()["current_service_create_txid"] + ).view + == prev_view + 2 + ) + return recovered_network @@ -553,11 +564,14 @@ def run(args): txs=txs, ) as network: network.start_and_open(args) + primary, _ = network.find_primary() - network = test_recover_service(network, args, from_snapshot=False) - network = test_recover_service(network, args, from_snapshot=False) - - return network + LOG.info("Check for well-known genesis service TxID") + with primary.client() as c: + r = c.get("/node/network").body.json() + assert ccf.tx_id.TxID.from_str( + r["current_service_create_txid"] + ) == ccf.tx_id.TxID(2, 1) if args.with_load: # See https://github.com/microsoft/CCF/issues/3788 for justification From 218adb82b616c62c4db725baf03fb3a147076302 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Wed, 29 Jun 2022 13:11:24 +0000 Subject: [PATCH 3/5] Schema --- .daily_canary | 2 +- doc/schemas/node_openapi.json | 8 ++++++-- include/ccf/service/tables/service.h | 2 +- src/node/rpc/node_frontend.h | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.daily_canary b/.daily_canary index 6682ca5343b4..0a8a9c540153 100644 --- a/.daily_canary +++ b/.daily_canary @@ -1 +1 @@ -Splicer!!!!! \ No newline at end of file +Splicer!!!!!!!! \ No newline at end of file diff --git a/doc/schemas/node_openapi.json b/doc/schemas/node_openapi.json index 8dc822a3944e..d47a74ddb246 100644 --- a/doc/schemas/node_openapi.json +++ b/doc/schemas/node_openapi.json @@ -249,6 +249,9 @@ }, "GetNetworkInfo__Out": { "properties": { + "current_service_create_txid": { + "$ref": "#/components/schemas/TransactionId" + }, "current_view": { "$ref": "#/components/schemas/uint64" }, @@ -270,7 +273,8 @@ "service_certificate", "current_view", "primary_id", - "recovery_count" + "recovery_count", + "current_service_create_txid" ], "type": "object" }, @@ -822,7 +826,7 @@ "info": { "description": "This API provides public, uncredentialed access to service and node state.", "title": "CCF Public Node API", - "version": "2.22.0" + "version": "2.23.0" }, "openapi": "3.0.0", "paths": { diff --git a/include/ccf/service/tables/service.h b/include/ccf/service/tables/service.h index 557612ed84a2..d983c337ac51 100644 --- a/include/ccf/service/tables/service.h +++ b/include/ccf/service/tables/service.h @@ -34,7 +34,7 @@ namespace ccf std::optional previous_service_identity_version = std::nullopt; /// Number of disaster recoveries performed on this service std::optional recovery_count = std::nullopt; - /// Version (seqno) at which current service was created + /// TxID at which current service was created std::optional current_service_create_txid = std::nullopt; }; DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(ServiceInfo); diff --git a/src/node/rpc/node_frontend.h b/src/node/rpc/node_frontend.h index 6490304bb2e5..d2911ee3d5e0 100644 --- a/src/node/rpc/node_frontend.h +++ b/src/node/rpc/node_frontend.h @@ -370,7 +370,7 @@ namespace ccf openapi_info.description = "This API provides public, uncredentialed access to service and node " "state."; - openapi_info.document_version = "2.22.0"; + openapi_info.document_version = "2.23.0"; } void init_handlers() override From 83929078a76af4976bb794f42c53b8a837c5e6c5 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Wed, 29 Jun 2022 13:11:47 +0000 Subject: [PATCH 4/5] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1472a999ea5..47fffbd28c3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `/node/version` now contains an `unsafe` flag reflecting the status of the build. - New per-interface configuration entries (`network.rpc_interfaces.http_configuration`) are added to let operators cap the maximum size of body, header value size and number of headers in client HTTP requests. The client session is automatically closed if the HTTP request exceeds one of these limits (#3941). - Added new `recovery_count` field to `GET /node/network` endpoint to track the number of disaster recovery procedures undergone by the service (#3982). +- Added new `current_service_create_txid` field to `GET /node/network` endpoint to indicate `TxID` at which current service was created (#3996). ### Changed From 628ca41e7099843ca6a3c40aa18efbb28f326b42 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Wed, 29 Jun 2022 13:48:20 +0000 Subject: [PATCH 5/5] Fix unit tests --- src/node/rpc/test/frontend_test.cpp | 2 +- src/node/rpc/test/node_frontend_test.cpp | 4 ++-- src/node/rpc/test/proposal_id_test.cpp | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/node/rpc/test/frontend_test.cpp b/src/node/rpc/test/frontend_test.cpp index 923fe299d57d..3b438dca4d79 100644 --- a/src/node/rpc/test/frontend_test.cpp +++ b/src/node/rpc/test/frontend_test.cpp @@ -500,7 +500,7 @@ void prepare_callers(NetworkState& network) init_network(network); GenesisGenerator g(network, tx); - g.create_service(network.identity->cert); + g.create_service(network.identity->cert, ccf::TxID{}); user_id = g.add_user({user_caller}); member_id = g.add_member(member_cert); invalid_member_id = g.add_member(invalid_caller); diff --git a/src/node/rpc/test/node_frontend_test.cpp b/src/node/rpc/test/node_frontend_test.cpp index aa4a938ddc09..79c970a6c72d 100644 --- a/src/node/rpc/test/node_frontend_test.cpp +++ b/src/node/rpc/test/node_frontend_test.cpp @@ -118,7 +118,7 @@ TEST_CASE("Add a node to an opening service") check_error_message(response, "No service is available to accept new node"); } - gen.create_service(network.identity->cert); + gen.create_service(network.identity->cert, ccf::TxID{}); REQUIRE(gen_tx.commit() == kv::CommitResult::SUCCESS); auto tx = network.tables->create_tx(); @@ -219,7 +219,7 @@ TEST_CASE("Add a node to an open service") network.ledger_secrets->set_secret( up_to_ledger_secret_seqno, make_ledger_secret()); - gen.create_service(network.identity->cert); + gen.create_service(network.identity->cert, ccf::TxID{}); gen.init_configuration({1}); gen.activate_member(gen.add_member( {member_cert, crypto::make_rsa_key_pair()->public_key_pem()})); diff --git a/src/node/rpc/test/proposal_id_test.cpp b/src/node/rpc/test/proposal_id_test.cpp index e6e4ee4ddc81..f2a1ac2d6018 100644 --- a/src/node/rpc/test/proposal_id_test.cpp +++ b/src/node/rpc/test/proposal_id_test.cpp @@ -24,7 +24,7 @@ DOCTEST_TEST_CASE("Unique proposal ids") init_network(network); auto gen_tx = network.tables->create_tx(); GenesisGenerator gen(network, gen_tx); - gen.create_service(network.identity->cert); + gen.create_service(network.identity->cert, ccf::TxID{}); const auto proposer_cert = get_cert(0, kp); const auto proposer_id = gen.add_member(proposer_cert); @@ -144,7 +144,7 @@ DOCTEST_TEST_CASE("Compaction conflict") network.tables->set_consensus(consensus); auto gen_tx = network.tables->create_tx(); GenesisGenerator gen(network, gen_tx); - gen.create_service(network.identity->cert); + gen.create_service(network.identity->cert, ccf::TxID{}); const auto proposer_cert = get_cert(0, kp); const auto proposer_id = gen.add_member(proposer_cert);