Create temporary ledger chunks while recovery is in progress

.daily_canary

@@ -1 +1 @@
-chirp tv
+chirp

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+## Changed
+
+- Failed recovery procedures no longer block subsequent recoveries: `.recovery` ledger files are now created while the recovery is in progress and ignored or deleted by nodes on startup (#3563).
+
 ## [2.0.0-rc1]
 
 ### Added

doc/operations/ledger_snapshot.rst

@@ -40,7 +40,10 @@ The listing below is an example of what a ledger directory may look like:
     -rw-rw-r-- 1 user user 1.1M Jan 31 14:00 ledger_92502-97520.committed
     -rw-rw-r-- 1 user user 553K Jan 31 14:00 ledger_97521 # File still in progress
 
-.. note:: On startup, a CCF node started with existing ledger files may suffix some of the file names with ``.corrupted`` if the ledger file cannot be parsed, depending on the sequence number the node will join from.
+.. note::
+
+    - On startup, a CCF node started with existing ledger files may suffix some of the file names with ``.corrupted`` if the ledger file cannot be parsed, depending on the sequence number the node will join from.
+    - While the :doc:`/operations/recovery` procedure is in progress, new ledger files are suffixed with ``.recovery``. These files are automatically renamed (i.e. recovery suffix removed) once the recovery procedure is complete. ``.recovery`` files are automatically discarded on node startup so that a failed recovery attempt does not prevent further recoveries.
 
 Snapshots
 ---------

include/ccf/ds/nonstd.h

@@ -160,4 +160,15 @@ namespace nonstd
 
     return s;
   }
+
+  static inline std::string remove_suffix(
+    const std::string& s, const std::string& suffix)
+  {
+    if (ends_with(s, suffix))
+    {
+      return s.substr(0, s.size() - suffix.size());
+    }
+
+    return s;
+  }
 }

python/ccf/ledger.py

@@ -37,6 +37,7 @@
 SERVICE_INFO_TABLE_NAME = "public:ccf.gov.service.info"
 
 COMMITTED_FILE_SUFFIX = ".committed"
+RECOVERY_FILE_SUFFIX = ".recovery"
 
 # Key used by CCF to record single-key tables
 WELL_KNOWN_SINGLETON_TABLE_KEY = bytes(bytearray(8))
@@ -110,6 +111,7 @@ def range_from_filename(filename: str) -> Tuple[int, Optional[int]]:
     elements = (
         os.path.basename(filename)
         .replace(COMMITTED_FILE_SUFFIX, "")
+        .replace(RECOVERY_FILE_SUFFIX, "")
         .replace("ledger_", "")
         .split("-")
     )
@@ -822,6 +824,7 @@ def __init__(
         self,
         directories: List[str],
         committed_only: bool = True,
+        read_recovery_files: bool = False,
         insecure_skip_verification: bool = False,
     ):
 
@@ -830,8 +833,17 @@ def __init__(
         ledger_files: List[str] = []
         for directory in directories:
             for path in os.listdir(directory):
-                if committed_only and not path.endswith(COMMITTED_FILE_SUFFIX):
+                sanitised_path = path
+                if path.endswith(RECOVERY_FILE_SUFFIX):
+                    sanitised_path = path[: -len(RECOVERY_FILE_SUFFIX)]
+                    if not read_recovery_files:
+                        continue
+
+                if committed_only and not sanitised_path.endswith(
+                    COMMITTED_FILE_SUFFIX
+                ):
                     continue
+
                 chunk = os.path.join(directory, path)
                 # The same ledger file may appear multiple times in different directories
                 # so ignore duplicates

src/consensus/aft/raft.h

@@ -345,7 +345,10 @@ namespace aft
     }
 
     void init_as_backup(
-      Index index, Term term, const std::vector<Index>& term_history) override
+      Index index,
+      Term term,
+      const std::vector<Index>& term_history,
+      Index recovery_start_index = 0) override
     {
       // This should only be called when the node resumes from a snapshot and
       // before it has received any append entries.
@@ -356,7 +359,7 @@ namespace aft
 
       state->view_history.initialise(term_history);
 
-      ledger->init(index);
+      ledger->init(index, recovery_start_index);
       snapshotter->set_last_snapshot_idx(index);
 
       become_aware_of_new_term(term);

src/consensus/aft/test/logging_stub.h

@@ -23,7 +23,7 @@ namespace aft
 
     LedgerStubProxy(const ccf::NodeId& id) : _id(id) {}
 
-    virtual void init(Index idx) {}
+    virtual void init(Index, Index) {}
 
     virtual void put_entry(
       const std::vector<uint8_t>& original,

src/consensus/ledger_enclave.h

@@ -121,7 +121,8 @@ namespace consensus
      */
     void truncate(Index idx)
     {
-      RINGBUFFER_WRITE_MESSAGE(consensus::ledger_truncate, to_host, idx);
+      RINGBUFFER_WRITE_MESSAGE(
+        consensus::ledger_truncate, to_host, idx, false /* no recovery */);
     }
 
     /**
@@ -138,10 +139,12 @@ namespace consensus
      * Initialise ledger at a given index (e.g. after a snapshot)
      *
      * @param idx Index to start ledger from
+     * @param recovery_start_idx Index at which the recovery starts
      */
-    void init(Index idx)
+    void init(Index idx = 0, Index recovery_start_idx = 0)
     {
-      RINGBUFFER_WRITE_MESSAGE(consensus::ledger_init, to_host, idx);
+      RINGBUFFER_WRITE_MESSAGE(
+        consensus::ledger_init, to_host, idx, recovery_start_idx);
     }
   };
 }

src/consensus/ledger_enclave_types.h

@@ -29,6 +29,7 @@ namespace consensus
     DEFINE_RINGBUFFER_MSG_TYPE(ledger_truncate),
     DEFINE_RINGBUFFER_MSG_TYPE(ledger_commit),
     DEFINE_RINGBUFFER_MSG_TYPE(ledger_init),
+    DEFINE_RINGBUFFER_MSG_TYPE(ledger_open),
 
     /// Create and commit a snapshot. Enclave -> Host
     DEFINE_RINGBUFFER_MSG_TYPE(snapshot),
@@ -53,15 +54,19 @@ DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(
   consensus::Index,
   consensus::LedgerRequestPurpose);
 
-DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(consensus::ledger_init, consensus::Index);
+DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(
+  consensus::ledger_init,
+  consensus::Index /* start idx */,
+  consensus::Index /* recovery start idx */);
 DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(
   consensus::ledger_append,
   bool /* committable */,
   bool /* force chunk */,
   std::vector<uint8_t>);
 DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(
-  consensus::ledger_truncate, consensus::Index);
+  consensus::ledger_truncate, consensus::Index, bool /* recovery mode */);
 DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(consensus::ledger_commit, consensus::Index);
+DECLARE_RINGBUFFER_MESSAGE_NO_PAYLOAD(consensus::ledger_open);
 DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(
   consensus::snapshot,
   consensus::Index /* snapshot idx */,

src/ds/files.h

@@ -12,8 +12,13 @@
 #include <string>
 #include <vector>
 
+#define FMT_HEADER_ONLY
+#include <fmt/format.h>
+
 namespace files
 {
+  namespace fs = std::filesystem;
+
   /**
    * @brief Checks if a path exists
    *
@@ -122,4 +127,15 @@ namespace files
   {
     return dump(std::vector<uint8_t>(data.begin(), data.end()), file);
   }
+
+  void rename(const fs::path& src, const fs::path& dst)
+  {
+    std::error_code ec;
+    fs::rename(src, dst, ec);
+    if (ec)
+    {
+      throw std::logic_error(fmt::format(
+        "Could not rename file {} to {}: {}", src, dst, ec.message()));
+    }
+  }
 }