[CI] Adjust the replay verify job (#5849)

* [replay-verify] Add adjust the CI task

* fixup! [replay-verify] Add adjust the CI task

.github/actions/replay-verify/action.yaml

@@ -24,7 +24,7 @@ runs:
       shell: bash
       run: |
         set -o nounset -o errexit -o pipefail
-        N=48
+        N=32
         PER_PARTITION=$(( ($LATEST_VERSION - $HISTORY_START) / $N ))
 
         for n in `seq 1 $N`
@@ -39,8 +39,9 @@ runs:
 
           echo [main process] spawning $NAME
           unbuffer target/release/replay-verify \
-          --concurrent-downloads 4 \
-          --replay-concurrency-level 4 \
+          --txns-to-skip $TXNS_TO_SKIP \
+          --concurrent-downloads 2 \
+          --replay-concurrency-level 2 \
           --metadata-cache-dir ./$NAME/metadata-cache \
           --target-db-dir ./$NAME/db \
           --start-version $START \

.github/actions/verify-modules/action.yaml

@@ -0,0 +1,35 @@
+runs:
+  using: composite
+  steps:
+    - uses: ./.github/actions/rust-setup
+    - name: install dependencies
+      shell: bash
+      run: |
+        echo "${HOME}/bin/" >> $GITHUB_PATH # default INSTALL_DIR to path
+        scripts/dev_setup.sh -b -i awscli # install individual tool awscli in batch mode
+        sudo apt-get update -y && sudo apt-get install -y expect
+    - name: Build CLI binaries in release mode
+      shell: bash
+      run: cargo build --release -p aptos-backup-cli --bin replay-verify --bin db-backup
+    - name: query latest version in backup, at the same time, pre-heat metadata cache
+      shell: bash
+      run: |
+        set -o nounset -o errexit -o pipefail
+        target/release/db-backup one-shot query backup-storage-state \
+        --metadata-cache-dir ./metadata-cache \
+        command-adapter --config terraform/helm/fullnode/files/backup/$CONFIG_TEMPLATE_NAME \
+        | echo LATEST_VERSION=`grep -Po '(?<=latest_transaction_version: )\d+'` \
+        | tee -a $GITHUB_ENV
+    - name: run verify-modules in parallel
+      shell: bash
+      run: |
+          mkdir local
+          cp -r metadata-cache local/
+          unbuffer target/release/replay-verify \
+          --validate-modules \
+          --concurrent-downloads 16 \
+          --replay-concurrency-level 4 \
+          --metadata-cache-dir ./local/metadata-cache \
+          --target-db-dir ./local/db \
+          --start-version $LATEST_VERSION \
+          command-adapter --config terraform/helm/fullnode/files/backup/s3-public.yaml 2>&1 \

.github/workflows/replay-verify-mainnet.yaml

@@ -16,7 +16,8 @@ on:
 env:
   BUCKET: aptos-mainnet-backup-backup-831a69a8
   SUB_DIR: e1
-  HISTORY_START: 32000000
+  HISTORY_START: 0
+  TXNS_TO_SKIP: 12253479 12277499
   CONFIG_TEMPLATE_NAME: s3-public.yaml
 
 jobs:

.github/workflows/replay-verify-testnet.yaml

@@ -16,7 +16,8 @@ on:
 env:
   BUCKET: aptos-testnet-backup-2223d95b
   SUB_DIR: e1
-  HISTORY_START: 350000000
+  HISTORY_START: 350000000 # TODO: We need an exhaustive list of txns_to_skip before we can set this to 0.
+  TXNS_TO_SKIP: 46874937 151020059
   CONFIG_TEMPLATE_NAME: s3-public.yaml
 
 jobs:

.github/workflows/verify-on-chain-modules-mainnet.yaml

@@ -0,0 +1,31 @@
+# This defines a workflow to verify all modules that have been published on chain with the latest aptos node software.
+# In order to trigger it go to the Actions Tab of the Repo, click "Verify Modules" and then "Run Workflow".
+
+name: "Verify Modules"
+on:
+  # Allow triggering manually
+  workflow_dispatch:
+    inputs:
+      GIT_SHA:
+        required: false
+        type: string
+        description: The git SHA1 to test. If not specified, Forge will check the latest commits on the current branch
+  pull_request:
+    paths:
+      - ".github/workflows/nightly-replay-verify.yaml"
+
+env:
+  BUCKET:  aptos-mainnet-backup-backup-831a69a8
+  SUB_DIR: e1
+  CONFIG_TEMPLATE_NAME: s3-public.yaml
+
+jobs:
+  replay-transactions:
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # pin@v3
+        with:
+          ref: ${{ inputs.GIT_SHA }}
+          fetch-depth: 0 # get all the history because cargo xtest --change-since origin/main requires it.
+      - uses: ./.github/actions/verify-modules

.github/workflows/verify-on-chain-modules-testnet.yaml

@@ -0,0 +1,31 @@
+# This defines a workflow to verify all modules that have been published on chain with the latest aptos node software.
+# In order to trigger it go to the Actions Tab of the Repo, click "Verify Modules" and then "Run Workflow".
+
+name: "Verify Modules"
+on:
+  # Allow triggering manually
+  workflow_dispatch:
+    inputs:
+      GIT_SHA:
+        required: false
+        type: string
+        description: The git SHA1 to test. If not specified, Forge will check the latest commits on the current branch
+  pull_request:
+    paths:
+      - ".github/workflows/nightly-replay-verify.yaml"
+
+env:
+  BUCKET: aptos-testnet-backup-2223d95b
+  SUB_DIR: e1
+  CONFIG_TEMPLATE_NAME: s3-public.yaml
+
+jobs:
+  replay-transactions:
+    timeout-minutes: 720
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # pin@v3
+        with:
+          ref: ${{ inputs.GIT_SHA }}
+          fetch-depth: 0 # get all the history because cargo xtest --change-since origin/main requires it.
+      - uses: ./.github/actions/verify-modules

execution/executor/src/chunk_executor.rs

@@ -15,7 +15,7 @@ use crate::{
         APTOS_EXECUTOR_EXECUTE_CHUNK_SECONDS, APTOS_EXECUTOR_VM_EXECUTE_CHUNK_SECONDS,
     },
 };
-use anyhow::{anyhow, Result};
+use anyhow::Result;
 use aptos_executor_types::{
     ChunkCommitNotification, ChunkExecutorTrait, ExecutedChunk, TransactionReplayer,
 };
@@ -328,23 +328,15 @@ impl<V: VMExecutor> TransactionReplayer for ChunkExecutorInner<V> {
         events: Vec<Vec<ContractEvent>>,
         txns_to_skip: Arc<BTreeSet<Version>>,
     ) -> Result<()> {
-        let current_begin_version = {
-            self.commit_queue
-                .lock()
-                .persisted_and_latest_view()
-                .1
-                .version()
-                .ok_or_else(|| anyhow!("Current version is not available"))?
-        };
-
-        let mut offset = current_begin_version;
+        let (_parent_view, latest_view) = self.commit_queue.lock().persisted_and_latest_view();
+        let first_version = latest_view.num_transactions() as Version;
+        let mut offset = first_version;
         let total_length = transactions.len();
 
-        for version in txns_to_skip
-            .range(current_begin_version + 1..current_begin_version + total_length as u64 + 1)
-        {
-            let remaining = transactions.split_off((version - offset) as usize);
-            let remaining_info = transaction_infos.split_off((version - offset) as usize);
+        for version in txns_to_skip.range(first_version..first_version + total_length as u64) {
+            let version = *version;
+            let remaining = transactions.split_off((version - offset + 1) as usize);
+            let remaining_info = transaction_infos.split_off((version - offset + 1) as usize);
             let txn_to_skip = transactions.pop().unwrap();
             let txn_info = transaction_infos.pop().unwrap();
 
@@ -353,8 +345,8 @@ impl<V: VMExecutor> TransactionReplayer for ChunkExecutorInner<V> {
             self.apply_transaction_and_output(
                 txn_to_skip,
                 TransactionOutput::new(
-                    writesets[(version - current_begin_version - 1) as usize].clone(),
-                    events[(version - current_begin_version - 1) as usize].clone(),
+                    writesets[(version - first_version) as usize].clone(),
+                    events[(version - first_version) as usize].clone(),
                     txn_info.gas_used(),
                     TransactionStatus::Keep(txn_info.status().clone()),
                 ),
@@ -363,7 +355,7 @@ impl<V: VMExecutor> TransactionReplayer for ChunkExecutorInner<V> {
 
             transactions = remaining;
             transaction_infos = remaining_info;
-            offset = version + 1;
+            offset = version;
         }
         self.replay_impl(transactions, transaction_infos)
     }