change(state): Only do continuous format checks in CI (#7627)

* Only do continuous format checks if enabled by a config

* Run continuous format checks in CI

Cargo.lock

@@ -5740,6 +5740,7 @@ dependencies = [
  "hex",
  "hex-literal",
  "howudoin",
+ "humantime-serde",
  "indexmap 2.0.0",
  "insta",
  "itertools 0.11.0",

zebra-state/Cargo.toml

@@ -49,6 +49,7 @@ dirs = "5.0.1"
 futures = "0.3.28"
 hex = "0.4.3"
 hex-literal = "0.4.1"
+humantime-serde = "1.1.1"
 indexmap = "2.0.0"
 itertools = "0.11.0"
 lazy_static = "1.4.0"

zebra-state/src/config.rs

@@ -4,6 +4,7 @@ use std::{
     fs::{self, canonicalize, remove_dir_all, DirEntry, ReadDir},
     io::ErrorKind,
     path::{Path, PathBuf},
+    time::Duration,
 };
 
 use semver::Version;
@@ -83,18 +84,28 @@ pub struct Config {
     /// [`cache_dir`]: struct.Config.html#structfield.cache_dir
     pub ephemeral: bool,
 
-    /// Commit blocks to the finalized state up to this height, then exit Zebra.
-    ///
-    /// Set to `None` by default: Zebra continues syncing indefinitely.
-    pub debug_stop_at_height: Option<u32>,
-
     /// Whether to delete the old database directories when present.
     ///
     /// Set to `true` by default. If this is set to `false`,
     /// no check for old database versions will be made and nothing will be
     /// deleted.
     pub delete_old_database: bool,
 
+    // Debug configs
+    //
+    /// Commit blocks to the finalized state up to this height, then exit Zebra.
+    ///
+    /// Set to `None` by default: Zebra continues syncing indefinitely.
+    pub debug_stop_at_height: Option<u32>,
+
+    /// While Zebra is running, check state validity this often.
+    ///
+    /// Set to `None` by default: Zebra only checks state format validity on startup and shutdown.
+    #[serde(with = "humantime_serde")]
+    pub debug_validity_check_interval: Option<Duration>,
+
+    // Elasticsearch configs
+    //
     #[cfg(feature = "elasticsearch")]
     /// The elasticsearch database url.
     pub elasticsearch_url: String,
@@ -162,8 +173,9 @@ impl Default for Config {
         Self {
             cache_dir,
             ephemeral: false,
-            debug_stop_at_height: None,
             delete_old_database: true,
+            debug_stop_at_height: None,
+            debug_validity_check_interval: None,
             #[cfg(feature = "elasticsearch")]
             elasticsearch_url: "https://localhost:9200".to_string(),
             #[cfg(feature = "elasticsearch")]

zebra-state/src/constants.rs

@@ -1,7 +1,5 @@
 //! Constants that impact state behaviour.
 
-use std::time::Duration;
-
 use lazy_static::lazy_static;
 use regex::Regex;
 use semver::Version;
@@ -69,12 +67,6 @@ pub fn latest_version_for_adding_subtrees() -> Version {
 /// Use [`Config::version_file_path()`] to get the path to this file.
 pub(crate) const DATABASE_FORMAT_VERSION_FILE_NAME: &str = "version";
 
-/// The interval between database format checks for newly added blocks.
-///
-/// This should be short enough that format bugs cause CI test failures,
-/// but long enough that it doesn't impact performance.
-pub(crate) const DATABASE_FORMAT_CHECK_INTERVAL: Duration = Duration::from_secs(5 * 60);
-
 /// The maximum number of blocks to check for NU5 transactions,
 /// before we assume we are on a pre-NU5 legacy chain.
 ///

zebra-state/src/service/finalized_state/disk_format/upgrade.rs

@@ -2,7 +2,6 @@
 
 use std::{
     cmp::Ordering,
-    convert::Infallible,
     sync::{mpsc, Arc},
     thread::{self, JoinHandle},
 };
@@ -23,9 +22,7 @@ use DbFormatChange::*;
 
 use crate::{
     config::write_database_format_version_to_disk,
-    constants::{
-        latest_version_for_adding_subtrees, DATABASE_FORMAT_CHECK_INTERVAL, DATABASE_FORMAT_VERSION,
-    },
+    constants::{latest_version_for_adding_subtrees, DATABASE_FORMAT_VERSION},
     database_format_version_in_code, database_format_version_on_disk,
     service::finalized_state::{DiskWriteBatch, ZebraDb},
     Config,
@@ -94,11 +91,11 @@ pub enum DbFormatChange {
 #[derive(Clone, Debug)]
 pub struct DbFormatChangeThreadHandle {
     /// A handle to the format change/check thread.
-    /// This thread continues running so it can perform periodic format checks.
+    /// If configured, this thread continues running so it can perform periodic format checks.
     ///
     /// Panics from this thread are propagated into Zebra's state service.
     /// The task returns an error if the upgrade was cancelled by a shutdown.
-    update_task: Option<Arc<JoinHandle<Result<Infallible, CancelFormatChange>>>>,
+    update_task: Option<Arc<JoinHandle<Result<(), CancelFormatChange>>>>,
 
     /// A channel that tells the running format thread to finish early.
     cancel_handle: mpsc::SyncSender<CancelFormatChange>,
@@ -258,18 +255,19 @@ impl DbFormatChange {
         handle
     }
 
-    /// Run the initial format change or check to the database, then periodically check the format
-    /// of newly added blocks matches the current format.
+    /// Run the initial format change or check to the database. Under the default runtime config,
+    /// this method returns after the format change or check.
     ///
-    /// This method runs until it is cancelled or panics.
+    /// But if runtime validity checks are enabled, this method periodically checks the format of
+    /// newly added blocks matches the current format. It will run until it is cancelled or panics.
     fn format_change_run_loop(
         self,
         config: Config,
         network: Network,
         initial_tip_height: Option<Height>,
         upgrade_db: ZebraDb,
         cancel_receiver: mpsc::Receiver<CancelFormatChange>,
-    ) -> Result<Infallible, CancelFormatChange> {
+    ) -> Result<(), CancelFormatChange> {
         self.run_format_change_or_check(
             &config,
             network,
@@ -278,11 +276,15 @@ impl DbFormatChange {
             &cancel_receiver,
         )?;
 
+        let Some(debug_validity_check_interval) = config.debug_validity_check_interval else {
+            return Ok(());
+        };
+
         loop {
             // We've just run a format check, so sleep first, then run another one.
             // But return early if there is a cancel signal.
             if !matches!(
-                cancel_receiver.recv_timeout(DATABASE_FORMAT_CHECK_INTERVAL),
+                cancel_receiver.recv_timeout(debug_validity_check_interval),
                 Err(mpsc::RecvTimeoutError::Timeout)
             ) {
                 return Err(CancelFormatChange);

zebrad/tests/common/cached_state.rs

@@ -30,9 +30,15 @@ use crate::common::{
     test_type::TestType,
 };
 
-/// Path to a directory containing a cached Zebra state.
+/// The environmental variable that holds the path to a directory containing a cached Zebra state.
 pub const ZEBRA_CACHED_STATE_DIR: &str = "ZEBRA_CACHED_STATE_DIR";
 
+/// In integration tests, the interval between database format checks for newly added blocks.
+///
+/// This should be short enough that format bugs cause CI test failures,
+/// but long enough that it doesn't impact performance.
+pub const DATABASE_FORMAT_CHECK_INTERVAL: Duration = Duration::from_secs(5 * 60);
+
 /// Type alias for a boxed state service.
 pub type BoxStateService =
     BoxService<zebra_state::Request, zebra_state::Response, zebra_state::BoxError>;

zebrad/tests/common/config.rs

@@ -21,6 +21,8 @@ use zebrad::{
     config::ZebradConfig,
 };
 
+use crate::common::cached_state::DATABASE_FORMAT_CHECK_INTERVAL;
+
 /// Returns a config with:
 /// - a Zcash listener on an unused port on IPv4 localhost, and
 /// - an ephemeral state,
@@ -61,9 +63,12 @@ pub fn default_test_config() -> Result<ZebradConfig> {
         ..tracing::Config::default()
     };
 
+    let mut state = zebra_state::Config::ephemeral();
+    state.debug_validity_check_interval = Some(DATABASE_FORMAT_CHECK_INTERVAL);
+
     let config = ZebradConfig {
         network,
-        state: zebra_state::Config::ephemeral(),
+        state,
         sync,
         mempool,
         consensus,