Multiple recoveries with key shares

CMakeLists.txt

@@ -464,7 +464,7 @@ if(BUILD_TESTS)
     NAME recovery_share_tests
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/recovery.py
     CONSENSUS raft
-    ADDITIONAL_ARGS --recovery 1 --use-shares
+    ADDITIONAL_ARGS --recovery 2 --use-shares
   )
 
   add_e2e_test(

src/node/ledgersecrets.h

@@ -140,6 +140,28 @@ namespace ccf
       return restored_versions;
     }
 
+    std::vector<kv::Version> restore(LedgerSecrets&& ledger_secrets)
+    {
+      std::vector<kv::Version> restored_versions;
+
+      for (auto it = ledger_secrets.secrets_map.begin();
+           it != ledger_secrets.secrets_map.end();)
+      {
+        auto it_ = ledger_secrets.secrets_map.extract(it++);
+        if (secrets_map.find(it_.key()) != secrets_map.end())
+        {
+          throw std::logic_error(fmt::format(
+            "Ledger secret at version {} cannot be restored as they already "
+            "exist",
+            it_.key()));
+        }
+        restored_versions.emplace_back(it_.key());
+        secrets_map.insert(std::move(it_));
+      }
+
+      return restored_versions;
+    }
+
     // Called during recovery to promote temporary secret created at startup (v
     // = 1) to new current secret at the latest signed version
     bool promote_secret(kv::Version old_v, kv::Version new_v)

src/node/nodestate.h

@@ -670,7 +670,9 @@ namespace ccf
       }
 
       if (result == kv::DeserialiseSuccess::PASS_SIGNATURE)
+      {
         recovery_store->compact(ledger_idx);
+      }
 
       if (recovery_store->current_version() == recovery_v)
       {
@@ -715,6 +717,8 @@ namespace ccf
       if (consensus->is_primary())
       {
         Store::Tx tx;
+        share_manager.update_key_share_info(tx);
+
         GenesisGenerator g(network, tx);
         if (!g.open_service())
         {
@@ -866,22 +870,22 @@ namespace ccf
     }
 
     bool finish_recovery_with_shares(
-      Store::Tx& tx, const LedgerSecret& ledger_secret)
+      Store::Tx& tx, const std::vector<kv::Version>& restored_versions)
     {
       std::lock_guard<SpinLock> guard(lock);
       sm.expect(State::partOfPublicNetwork);
 
-      // For now, this only supports one recovery
-
       LOG_INFO_FMT("Initiating end of recovery with shares (primary)");
 
       // Emit signature to certify transactions that happened on public
       // network
       history->emit_signature();
 
-      network.ledger_secrets->set_secret(0, ledger_secret.master);
-
-      broadcast_ledger_secret(tx, ledger_secret, 0, true);
+      for (auto const& v : restored_versions)
+      {
+        broadcast_ledger_secret(
+          tx, network.ledger_secrets->get_secret(v).value(), v, true);
+      }
 
       // Setup new temporary store and record current version/root
       setup_private_recovery_store();
@@ -1032,28 +1036,27 @@ namespace ccf
     {
       try
       {
-        share_manager.create(tx);
+        share_manager.update_key_share_info(tx);
       }
       catch (const std::logic_error& e)
       {
-        LOG_FAIL_FMT("Failed to create shares: {}", e.what());
+        LOG_FAIL_FMT("Failed to update key share info: {}", e.what());
         return false;
       }
       return true;
     }
 
-    bool combine_recovery_shares(
+    bool restore_ledger_secrets(
       Store::Tx& tx, const std::vector<SecretSharing::Share>& shares) override
     {
-      LedgerSecret restored_ledger_secret;
       try
       {
-        restored_ledger_secret = share_manager.restore(tx, shares);
-        finish_recovery_with_shares(tx, restored_ledger_secret);
+        finish_recovery_with_shares(
+          tx, share_manager.restore_key_share_info(tx, shares));
       }
       catch (const std::logic_error& e)
       {
-        LOG_FAIL_FMT("Failed to restore shares: {}", e.what());
+        LOG_FAIL_FMT("Failed to restore key share info: {}", e.what());
         return false;
       }
 
@@ -1409,8 +1412,10 @@ namespace ccf
                 if (!network.ledger_secrets->set_secret(
                       secret_version, plain_secret))
                 {
-                  throw std::logic_error(
-                    "Cannot set ledger secrets because they already exist");
+                  throw std::logic_error(fmt::format(
+                    "Cannot set ledger secrets at version {} because they "
+                    "already exist",
+                    secret_version));
                 }
               }
               else

src/node/notifier.h

@@ -25,7 +25,8 @@ namespace ccf
     {
       if (consensus == nullptr)
       {
-        LOG_FAIL_FMT("Unable to send notification - no consensus has been set");
+        LOG_DEBUG_FMT(
+          "Unable to send notification - no consensus has been set");
         return;
       }
 

src/node/rpc/memberfrontend.h

@@ -835,7 +835,7 @@ namespace ccf
         LOG_DEBUG_FMT(
           "Reached secret sharing threshold {}", pending_shares.size());
 
-        if (!node.combine_recovery_shares(args.tx, pending_shares))
+        if (!node.restore_ledger_secrets(args.tx, pending_shares))
         {
           pending_shares.clear();
           return make_error(

src/node/rpc/nodeinterface.h

@@ -28,7 +28,7 @@ namespace ccf
     virtual NodeId get_node_id() const = 0;
 
     virtual bool split_ledger_secrets(Store::Tx& tx) = 0;
-    virtual bool combine_recovery_shares(
+    virtual bool restore_ledger_secrets(
       Store::Tx& tx, const std::vector<SecretSharing::Share>& shares) = 0;
   };
 }

src/node/rpc/test/node_stub.h

@@ -94,7 +94,7 @@ namespace ccf
       return true;
     }
 
-    bool combine_recovery_shares(
+    bool restore_ledger_secrets(
       Store::Tx& tx, const std::vector<SecretSharing::Share>& shares) override
     {
       return true;

src/node/sharemanager.h

@@ -12,27 +12,80 @@
 #include "tls/25519.h"
 #include "tls/entropy.h"
 
+#include <nlohmann/json.hpp>
 #include <vector>
 
 namespace ccf
 {
-  struct LedgerSecretWrappingKey
+  class LedgerSecretsWrappingKey
   {
   private:
     static constexpr auto KZ_KEY_SIZE = crypto::GCM_SIZE_KEY;
-
-  public:
     std::vector<uint8_t> data; // Referred to as "kz" in TR
+    bool has_wrapped = false;
 
-    LedgerSecretWrappingKey() : data(tls::create_entropy()->random(KZ_KEY_SIZE))
+  public:
+    LedgerSecretsWrappingKey() :
+      data(tls::create_entropy()->random(KZ_KEY_SIZE))
     {}
 
     template <typename T>
-    LedgerSecretWrappingKey(const T& split_secret) :
+    LedgerSecretsWrappingKey(T&& split_secret) :
       data(
         std::make_move_iterator(split_secret.begin()),
         std::make_move_iterator(split_secret.begin() + split_secret.size()))
     {}
+
+    template <typename T>
+    T get_raw_data()
+    {
+      T ret;
+      std::copy_n(data.begin(), data.size(), ret.begin());
+      return ret;
+    }
+
+    std::vector<uint8_t> wrap(const LedgerSecrets& ledger_secrets)
+    {
+      if (has_wrapped)
+      {
+        throw std::logic_error(
+          "Ledger Secret wrapping key has already wrapped once");
+      }
+
+      auto serialised_ls = nlohmann::json::to_msgpack(ledger_secrets);
+      crypto::GcmCipher encrypted_ls(serialised_ls.size());
+
+      crypto::KeyAesGcm(data).encrypt(
+        encrypted_ls.hdr.get_iv(), // iv is always 0 here as the share wrapping
+                                   // key is never re-used for encryption
+        serialised_ls,
+        nullb,
+        encrypted_ls.cipher.data(),
+        encrypted_ls.hdr.tag);
+
+      has_wrapped = true;
+
+      return encrypted_ls.serialise();
+    }
+
+    LedgerSecrets unwrap(const std::vector<uint8_t>& encrypted_ledger_secrets)
+    {
+      crypto::GcmCipher encrypted_ls;
+      encrypted_ls.deserialise(encrypted_ledger_secrets);
+      std::vector<uint8_t> decrypted_ls(encrypted_ls.cipher.size());
+
+      if (!crypto::KeyAesGcm(data).decrypt(
+            encrypted_ls.hdr.get_iv(), // iv is 0
+            encrypted_ls.hdr.tag,
+            encrypted_ls.cipher,
+            nullb,
+            decrypted_ls.data()))
+      {
+        throw std::logic_error("Decryption of ledger secrets failed");
+      }
+
+      return nlohmann::json::from_msgpack(decrypted_ls);
+    }
   };
 
   class ShareManager
@@ -43,48 +96,32 @@ namespace ccf
   public:
     ShareManager(NetworkState& network_) : network(network_) {}
 
-    void create(Store::Tx& tx)
+    void update_key_share_info(Store::Tx& tx)
     {
-      // First, generated a fresh ledger secrets wrapping key and encrypt the
-      // current ledger secrets with it
-      auto ls_wrapping_key = LedgerSecretWrappingKey();
-
-      crypto::GcmCipher encrypted_ls(LedgerSecret::MASTER_KEY_SIZE);
-      crypto::KeyAesGcm(ls_wrapping_key.data)
-        .encrypt(
-          encrypted_ls.hdr
-            .get_iv(), // iv is always 0 here as the share wrapping
-                       // key is never re-used for encryption
-          network.ledger_secrets->get_secret(1)->master,
-          nullb,
-          encrypted_ls.cipher.data(),
-          encrypted_ls.hdr.tag);
-
-      // Then, split the ledger secrets wrapping key, allocating a share to each
-      // active member
-      SecretSharing::SplitSecret secret_to_split = {};
-      std::copy_n(
-        ls_wrapping_key.data.begin(),
-        ls_wrapping_key.data.size(),
-        secret_to_split.begin());
+      // First, generated a fresh ledger secrets wrapping key and wrap the
+      // ledger secrets with it. Then, split the ledger secrets wrapping key,
+      // allocating a new share for each active member. Finally, encrypt each
+      // share with the public key of each member and record it in the KV.
+      auto ls_wrapping_key = LedgerSecretsWrappingKey();
+      auto encrypted_ls = ls_wrapping_key.wrap(*network.ledger_secrets.get());
+
+      auto secret_to_split =
+        ls_wrapping_key.get_raw_data<SecretSharing::SplitSecret>();
 
       GenesisGenerator g(network, tx);
       auto active_members_info = g.get_active_members_keyshare();
 
-      // For now, the secret sharing threshold is set to the number of initial
+      // For now, the secret sharing threshold is set to the number of active
       // members
       size_t threshold = active_members_info.size();
       auto shares = SecretSharing::split(
         secret_to_split, active_members_info.size(), threshold);
 
-      // Finally, encrypt each share with the public key of each member, using a
-      // random nonce, and record in the KV
-      EncryptedSharesMap encrypted_shares;
-      auto nonce = tls::create_entropy()->random(crypto::Box::NONCE_SIZE);
-
       size_t share_index = 0;
+      EncryptedSharesMap encrypted_shares;
       for (auto const& [member_id, enc_pub_key] : active_members_info)
       {
+        auto nonce = tls::create_entropy()->random(crypto::Box::NONCE_SIZE);
         auto share_raw = std::vector<uint8_t>(
           shares[share_index].begin(), shares[share_index].end());
 
@@ -99,43 +136,30 @@ namespace ccf
         share_index++;
       }
 
-      g.add_key_share_info({encrypted_ls.serialise(), encrypted_shares});
+      g.add_key_share_info({encrypted_ls, encrypted_shares});
     }
 
     // For now, the shares are passed directly to this function. Shares should
     // be retrieved from the KV instead.
-    LedgerSecret restore(
+    std::vector<kv::Version> restore_key_share_info(
       Store::Tx& tx, const std::vector<SecretSharing::Share>& shares)
     {
       // First, re-assemble the ledger secrets wrapping key from the given
-      // shares
+      // shares. Then, unwrap and restore the ledger secrets.
       auto ls_wrapping_key =
-        LedgerSecretWrappingKey(SecretSharing::combine(shares, shares.size()));
+        LedgerSecretsWrappingKey(SecretSharing::combine(shares, shares.size()));
 
-      // Then, decrypt the ledger secrets
       auto shares_view = tx.get_view(network.shares);
       auto key_share_info = shares_view->get(0);
       if (!key_share_info.has_value())
       {
         throw std::logic_error("Failed to retrieve current key share info");
       }
 
-      std::vector<uint8_t> decrypted_ls(LedgerSecret::MASTER_KEY_SIZE);
-      crypto::GcmCipher encrypted_ls;
-      encrypted_ls.deserialise(key_share_info->encrypted_ledger_secret);
-
-      if (!crypto::KeyAesGcm(ls_wrapping_key.data)
-             .decrypt(
-               encrypted_ls.hdr.get_iv(), // iv is 0
-               encrypted_ls.hdr.tag,
-               encrypted_ls.cipher,
-               nullb,
-               decrypted_ls.data()))
-      {
-        throw std::logic_error("Decryption of ledger secrets failed");
-      }
+      auto restored_ls =
+        ls_wrapping_key.unwrap(key_share_info->encrypted_ledger_secret);
 
-      return LedgerSecret(std::move(decrypted_ls));
+      return network.ledger_secrets->restore(std::move(restored_ls));
     }
   };
 }

tests/infra/node.py

@@ -184,7 +184,7 @@ def _start(
         LOG.info("Remote {} started".format(self.node_id))
 
     def stop(self):
-        if self.remote:
+        if self.remote and self.network_state is not NodeNetworkState.stopped:
             errors = self.remote.stop()
             self.network_state = NodeNetworkState.stopped
             return errors

tests/recovery.py

@@ -27,6 +27,9 @@ def test(network, args, use_shares=False):
     ledger = primary.get_ledger()
     sealed_secrets = primary.get_sealed_secrets()
 
+    if use_shares:
+        network.consortium.store_current_network_encryption_key()
+
     recovered_network = infra.ccf.Network(
         network.hosts, args.binary_dir, args.debug_nodes, args.perf_nodes, network
     )
@@ -77,9 +80,6 @@ def run(args):
     ) as network:
         network.start_and_join(args)
 
-        if args.use_shares:
-            network.consortium.store_current_network_encryption_key()
-
         for recovery_idx in range(args.recovery):
             recovered_network = test(network, args, use_shares=args.use_shares)
             network.stop_all_nodes()