Merge branch 'main' into add-coinbase

.github/workflows/continous-integration-docker.yml

@@ -187,6 +187,8 @@ jobs:
   #
   # - We run all the tests behind the `getblocktemplate-rpcs` feature as a separated step.
   # - We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests.
+  #
+  # TODO: turn this test and the getblocktemplate test into a matrix, so the jobs use exactly the same diagnostics settings
   test-all:
     name: Test all
     runs-on: ubuntu-latest
@@ -198,13 +200,21 @@ jobs:
         with:
           short-length: 7
 
-      - name: Run all zebrad tests
+      # Run unit and basic acceptance tests, only showing command output if the test fails.
+      #
+      # If some tests hang, add "-- --nocapture" for just that test, or for all the tests.
+      - name: Run basic zebrad tests
         run: |
           docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
-          docker run --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "lightwalletd-grpc-tests" --workspace -- --nocapture --include-ignored
+          docker run --name zebrad-tests --tty ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "lightwalletd-grpc-tests" --workspace
 
-  # zebrad tests without cached state with `getblocktemplate-rpcs` feature
+      # Run ignored by default acceptance tests, only showing command output if the test fails.
+      - name: Run ignored zebrad tests
+        run: |
+          docker run --name zebrad-tests-ignored --tty ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "lightwalletd-grpc-tests" --workspace -- --ignored --nocapture
 
+  # zebrad tests without cached state with `getblocktemplate-rpcs` feature
+  #
   # Same as above but we run all the tests behind the `getblocktemplate-rpcs` feature.
   test-all-getblocktemplate-rpcs:
     name: Test all with getblocktemplate-rpcs feature
@@ -217,10 +227,15 @@ jobs:
         with:
           short-length: 7
 
-      - name: Run all zebrad tests
+      - name: Run basic zebrad tests
+        run: |
+          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
+          docker run --name zebrad-tests --tty ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "lightwalletd-grpc-tests getblocktemplate-rpcs" --workspace
+
+      - name: Run ignored zebrad tests
         run: |
           docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
-          docker run --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "lightwalletd-grpc-tests getblocktemplate-rpcs" --workspace -- --nocapture --include-ignored
+          docker run --name zebrad-tests-ignored --tty ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "lightwalletd-grpc-tests getblocktemplate-rpcs" --workspace -- --ignored --nocapture
 
   # Run state tests with fake activation heights.
   #

.github/workflows/continous-integration-os.yml

@@ -162,11 +162,14 @@ jobs:
         working-directory: ./zebra-consensus
         run: cargo run --example download-params
 
+      # Run unit and basic acceptance tests, only showing command output if the test fails.
+      #
+      # If some tests hang, add "-- --nocapture" for just that test, or for all the tests.
       - name: Run tests${{ matrix.features }}
         uses: actions-rs/[email protected]
         with:
           command: test
-          args: ${{ matrix.features }} --verbose --workspace -- --nocapture
+          args: ${{ matrix.features }} --verbose --workspace
 
       # Explicitly run any tests that are usually #[ignored]
 

book/src/dev/zebra-dependencies-for-audit.md

@@ -88,7 +88,6 @@ Click the triangle for details:
 | [ripemd](https://github.com/RustCrypto/hashes) | 0.1.3 | no audits, but seems widely used
 | [secp256k1](https://github.com/rust-bitcoin/rust-secp256k1/) | 0.21.3 | no audits, but seems widely used
 | [subtle](https://github.com/dalek-cryptography/subtle) | [2.4.1](https://github.com/dalek-cryptography/subtle/releases/tag/2.4.1) | no audits, but seems widely used
-| [fpe](https://github.com/str4d/fpe) | 0.5.1 |  I think it's not being used yet
 | [group](https://github.com/zkcrypto/group) | [0.12.0](https://github.com/zkcrypto/group/releases/tag/0.12.0) | no audits but it's just traits, seems widely used
 | [x25519-dalek](https://github.com/dalek-cryptography/x25519-dalek) | [1.2.0](https://github.com/dalek-cryptography/x25519-dalek/releases/tag/1.2.0) | no audits, but seems widely used
 | [jubjub](https://github.com/zkcrypto/jubjub) | [0.9.0](https://github.com/zkcrypto/jubjub/releases/tag/0.9.0) | not sure if were covered by ECC audits. Seem widely used.

zebra-chain/Cargo.toml

@@ -25,7 +25,6 @@ bls12_381 = "0.7.0"
 bs58 = { version = "0.4.0", features = ["check"] }
 byteorder = "1.4.3"
 equihash = "0.2.0"
-fpe = "0.5.1"
 group = "0.12.0"
 incrementalmerkletree = "0.3.0"
 jubjub = "0.9.0"

zebra-state/src/service.rs

@@ -609,14 +609,17 @@ impl StateService {
             .contains(&prepared.hash)
         {
             let (rsp_tx, rsp_rx) = oneshot::channel();
-            let _ = rsp_tx.send(Err("block already sent to be committed to the state".into()));
+            let _ = rsp_tx.send(Err(
+                "block has already been sent to be committed to the state".into(),
+            ));
             return rsp_rx;
         }
 
         if self.read_service.db.contains_height(prepared.height) {
             let (rsp_tx, rsp_rx) = oneshot::channel();
             let _ = rsp_tx.send(Err(
-                "block height is already committed to the finalized state".into(),
+                "block height is in the finalized state: block is already committed to the state"
+                    .into(),
             ));
             return rsp_rx;
         }

zebrad/src/components/sync.rs

@@ -1126,9 +1126,12 @@ where
             BlockDownloadVerifyError::Invalid {
                 error: VerifyChainError::Block(VerifyBlockError::Commit(ref source)),
                 ..
-            } if format!("{source:?}").contains("block is already committed to the state") => {
+            } if format!("{source:?}").contains("block is already committed to the state")
+                || format!("{source:?}")
+                    .contains("block has already been sent to be committed to the state") =>
+            {
                 // TODO: improve this by checking the type (#2908)
-                debug!(error = ?e, "block is already committed, possibly from a previous sync run, continuing");
+                debug!(error = ?e, "block is already committed or pending a commit, possibly from a previous sync run, continuing");
                 false
             }
             BlockDownloadVerifyError::DownloadFailed { ref error, .. }
@@ -1158,6 +1161,7 @@ where
                 if err_str.contains("AlreadyVerified")
                     || err_str.contains("AlreadyInChain")
                     || err_str.contains("block is already committed to the state")
+                    || err_str.contains("block has already been sent to be committed to the state")
                     || err_str.contains("NotFound")
                 {
                     error!(?e,