Merge branch 'main' into check-contextual-validity-request

.github/workflows/continous-delivery.yml

@@ -120,6 +120,8 @@ jobs:
         run: |
           gcloud compute instance-templates create-with-container zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
           --boot-disk-type=pd-ssd \
+          --image-project=cos-cloud \
+          --image-family=cos-stable \
           --container-image ${{ env.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} \
           --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=300GB,type=pd-ssd \
           --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
@@ -196,6 +198,8 @@ jobs:
           gcloud compute instances create-with-container "zebrad-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 300GB \
           --boot-disk-type=pd-ssd \
+          --image-project=cos-cloud \
+          --image-family=cos-stable \
           --container-stdin \
           --container-tty \
           --container-image ${{ env.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} \

.github/workflows/continous-integration-docker.patch.yml

@@ -90,6 +90,12 @@ jobs:
     steps:
       - run: 'echo "No build required"'
 
+  get-block-template-test:
+    name: get block template / Run get-block-template test
+    runs-on: ubuntu-latest
+    steps:
+      - run: 'echo "No build required"'
+
   submit-block-test:
     name: submit block / Run submit-block test
     runs-on: ubuntu-latest

.github/workflows/continous-integration-docker.yml

@@ -581,6 +581,31 @@ jobs:
       zebra_state_dir: 'zebrad-cache'
       lwd_state_dir: 'lwd-cache'
 
+  # Test that Zebra can handle a getblocktemplate RPC call, using a cached Zebra tip state
+  #
+  # Runs:
+  # - after every PR is merged to `main`
+  # - on every PR update
+  #
+  # If the state version has changed, waits for the new cached states to be created.
+  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
+  get-block-template-test:
+    name: get block template
+    needs: test-full-sync
+    uses: ./.github/workflows/deploy-gcp-tests.yml
+    if: ${{ !cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' && github.event.inputs.run-lwd-send-tx != 'true' }}
+    with:
+      app_name: zebrad
+      test_id: get-block-template
+      test_description: Test getblocktemplate RPC method via Zebra's rpc server
+      test_variables: '-e TEST_GET_BLOCK_TEMPLATE=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
+      needs_zebra_state: true
+      needs_lwd_state: false
+      saves_to_disk: false
+      disk_suffix: tip
+      root_state_path: '/var/cache'
+      zebra_state_dir: 'zebrad-cache'
+
   # Test that Zebra can handle a submit block RPC call, using a cached Zebra tip state
   #
   # Runs:

.github/workflows/deploy-gcp-tests.yml

@@ -144,9 +144,10 @@ jobs:
           gcloud compute instances create-with-container "${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 300GB \
           --boot-disk-type pd-ssd \
+          --image-project=cos-cloud \
+          --image-family=cos-stable \
           --create-disk name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",device-name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",size=300GB,type=pd-ssd \
-          --container-image gcr.io/google-containers/busybox \
-          --container-restart-policy=never \
+          --container-image=gcr.io/google-containers/busybox \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \
           --metadata=google-monitoring-enabled=TRUE,google-logging-enabled=TRUE,enable-oslogin=TRUE \
@@ -363,9 +364,10 @@ jobs:
           gcloud compute instances create-with-container "${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 300GB \
           --boot-disk-type pd-ssd \
+          --image-project=cos-cloud \
+          --image-family=cos-stable \
           --create-disk image=${{ env.CACHED_DISK_NAME }},name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",device-name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",size=300GB,type=pd-ssd \
-          --container-image gcr.io/google-containers/busybox \
-          --container-restart-policy=never \
+          --container-image=gcr.io/google-containers/busybox \
           --machine-type ${{ env.MACHINE_TYPE }} \
           --scopes cloud-platform \
           --metadata=google-monitoring-enabled=TRUE,google-logging-enabled=TRUE,enable-oslogin=TRUE \

.github/workflows/lint.yml

@@ -173,7 +173,7 @@ jobs:
     if: ${{ needs.changed-files.outputs.workflows == 'true' }}
     steps:
       - uses: actions/[email protected]
-      - uses: reviewdog/[email protected].0
+      - uses: reviewdog/[email protected].1
         with:
           level: warning
           fail_on_error: false

.github/workflows/zcashd-manual-deploy.yml

@@ -55,6 +55,8 @@ jobs:
           gcloud compute instance-templates create-with-container zcashd-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
           --boot-disk-size 10GB \
           --boot-disk-type=pd-ssd \
+          --image-project=cos-cloud \
+          --image-family=cos-stable \
           --container-stdin \
           --container-tty \
           --container-image electriccoinco/zcashd \

README.md

@@ -242,6 +242,8 @@ There are a few bugs in Zebra that we're still working on fixing:
 - Experimental Tor support is disabled until [`arti-client` upgrades to `x25519-dalek` 2.0.0 or later](https://github.com/ZcashFoundation/zebra/issues/5492)
   - This happens due to a Rust dependency conflict, which can only be resolved by changing the dependencies of `x25519-dalek`
 
+- Output of `help`, `--help` flag, and usage of invalid commands or options are inconsistent. Reports of these issues can be found [here](https://github.com/ZcashFoundation/zebra/issues/5502) and are planned to be fixed in the context of [upgrading Abscissa](https://github.com/ZcashFoundation/zebra/issues/5502).
+
 ## Future Work
 
 Performance and Reliability:

docker/entrypoint.sh

@@ -78,6 +78,10 @@ case "$1" in
             ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
             ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory")
             cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored sending_transactions_using_lightwalletd
+        elif [[ "$TEST_GET_BLOCK_TEMPLATE" -eq "1" ]]; then
+            # Starting with a cached Zebra tip, test getting a block template from Zebra's RPC server.
+            ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
+            cargo test --locked --release --features getblocktemplate-rpcs --package zebrad --test acceptance -- --nocapture --include-ignored get_block_template
         elif [[ "$TEST_SUBMIT_BLOCK" -eq "1" ]]; then
             # Starting with a cached Zebra tip, test sending a block to Zebra's RPC port.
             ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")

zebra-chain/Cargo.toml

@@ -76,7 +76,7 @@ tracing = "0.1.37"
 # Serialization
 hex = { version = "0.4.3", features = ["serde"] }
 serde = { version = "1.0.147", features = ["serde_derive", "rc"] }
-serde_with = "2.0.1"
+serde_with = "2.1.0"
 serde-big-array = "0.4.1"
 
 # Processing

zebra-chain/src/chain_tip.rs

@@ -58,6 +58,28 @@ pub trait ChainTip {
 
         Some(estimator.estimate_height_at(now))
     }
+
+    /// Return an estimate of how many blocks there are ahead of Zebra's best chain tip
+    /// until the network chain tip, and Zebra's best chain tip height.
+    ///
+    /// The estimate is calculated based on the current local time, the block time of the best tip
+    /// and the height of the best tip.
+    ///
+    /// This estimate may be negative if the current local time is behind the chain tip block's timestamp.
+    fn estimate_distance_to_network_chain_tip(
+        &self,
+        network: Network,
+    ) -> Option<(i32, block::Height)> {
+        let (current_height, current_block_time) = self.best_tip_height_and_block_time()?;
+
+        let estimator =
+            NetworkChainTipHeightEstimator::new(current_block_time, current_height, network);
+
+        Some((
+            estimator.estimate_height_at(Utc::now()) - current_height,
+            current_height,
+        ))
+    }
 }
 
 /// A chain tip that is always empty.

zebra-chain/src/chain_tip/mock.rs

@@ -5,7 +5,7 @@ use std::sync::Arc;
 use chrono::{DateTime, Utc};
 use tokio::sync::watch;
 
-use crate::{block, chain_tip::ChainTip, transaction};
+use crate::{block, chain_tip::ChainTip, parameters::Network, transaction};
 
 /// A sender to sets the values read by a [`MockChainTip`].
 pub struct MockChainTipSender {
@@ -17,6 +17,9 @@ pub struct MockChainTipSender {
 
     /// A sender that sets the `best_tip_block_time` of a [`MockChainTip`].
     best_tip_block_time: watch::Sender<Option<DateTime<Utc>>>,
+
+    /// A sender that sets the `estimate_distance_to_network_chain_tip` of a [`MockChainTip`].
+    estimated_distance_to_network_chain_tip: watch::Sender<Option<i32>>,
 }
 
 /// A mock [`ChainTip`] implementation that allows setting the `best_tip_height` externally.
@@ -30,6 +33,9 @@ pub struct MockChainTip {
 
     /// A mocked `best_tip_height` value set by the [`MockChainTipSender`].
     best_tip_block_time: watch::Receiver<Option<DateTime<Utc>>>,
+
+    /// A mocked `estimate_distance_to_network_chain_tip` value set by the [`MockChainTipSender`].
+    estimated_distance_to_network_chain_tip: watch::Receiver<Option<i32>>,
 }
 
 impl MockChainTip {
@@ -43,17 +49,21 @@ impl MockChainTip {
         let (height_sender, height_receiver) = watch::channel(None);
         let (hash_sender, hash_receiver) = watch::channel(None);
         let (time_sender, time_receiver) = watch::channel(None);
+        let (estimated_distance_to_tip_sender, estimated_distance_to_tip_receiver) =
+            watch::channel(None);
 
         let mock_chain_tip = MockChainTip {
             best_tip_height: height_receiver,
             best_tip_hash: hash_receiver,
             best_tip_block_time: time_receiver,
+            estimated_distance_to_network_chain_tip: estimated_distance_to_tip_receiver,
         };
 
         let mock_chain_tip_sender = MockChainTipSender {
             best_tip_height: height_sender,
             best_tip_hash: hash_sender,
             best_tip_block_time: time_sender,
+            estimated_distance_to_network_chain_tip: estimated_distance_to_tip_sender,
         };
 
         (mock_chain_tip, mock_chain_tip_sender)
@@ -90,6 +100,18 @@ impl ChainTip for MockChainTip {
     fn best_tip_mined_transaction_ids(&self) -> Arc<[transaction::Hash]> {
         unreachable!("Method not used in tests");
     }
+
+    fn estimate_distance_to_network_chain_tip(
+        &self,
+        _network: Network,
+    ) -> Option<(i32, block::Height)> {
+        self.estimated_distance_to_network_chain_tip
+            .borrow()
+            .and_then(|estimated_distance| {
+                self.best_tip_height()
+                    .map(|tip_height| (estimated_distance, tip_height))
+            })
+    }
 }
 
 impl MockChainTipSender {
@@ -113,4 +135,11 @@ impl MockChainTipSender {
             .send(block_time.into())
             .expect("attempt to send a best tip block time to a dropped `MockChainTip`");
     }
+
+    /// Send a new estimated distance to network chain tip to the [`MockChainTip`].
+    pub fn send_estimated_distance_to_network_chain_tip(&self, distance: impl Into<Option<i32>>) {
+        self.estimated_distance_to_network_chain_tip
+            .send(distance.into())
+            .expect("attempt to send a best tip height to a dropped `MockChainTip`");
+    }
 }

zebra-rpc/Cargo.toml

@@ -65,7 +65,7 @@ zebra-script = { path = "../zebra-script" }
 zebra-state = { path = "../zebra-state" }
 
 [dev-dependencies]
-insta = { version = "1.21.0", features = ["redactions", "json"] }
+insta = { version = "1.21.1", features = ["redactions", "json"] }
 proptest = "0.10.1"
 proptest-derive = "0.3.0"
 thiserror = "1.0.37"