diff --git a/.github/workflows/replay-verify.yaml b/.github/workflows/replay-verify.yaml index 4fbc6a03ca8f38..8539a5f8037092 100644 --- a/.github/workflows/replay-verify.yaml +++ b/.github/workflows/replay-verify.yaml @@ -52,7 +52,7 @@ jobs: runs-on: ubuntu-latest steps: # checkout the repo first, so check-aptos-core can use it and cancel the workflow if necessary - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ./.github/actions/check-aptos-core with: cancel-workflow: ${{ github.event_name == 'schedule' }} # Cancel the workflow if it is scheduled on a fork diff --git a/.github/workflows/workflow-run-replay-verify.yaml b/.github/workflows/workflow-run-replay-verify.yaml index 7046c16df0731c..635acebaf97006 100644 --- a/.github/workflows/workflow-run-replay-verify.yaml +++ b/.github/workflows/workflow-run-replay-verify.yaml @@ -86,7 +86,7 @@ on: jobs: prepare: - runs-on: "runs-on,cpu=64,family=c7,hdd=500,image=aptos-ubuntu-x64,spot=false" + runs-on: ${{ inputs.RUNS_ON }} outputs: jobs_ids: ${{ steps.gen-jobs.outputs.job_ids }} steps: @@ -108,7 +108,7 @@ jobs: - name: Prepare for build if not cached if: steps.cache-aptos-debugger-binary.outputs.cache-hit != 'true' - uses: aptos-labs/aptos-core/.github/actions/rust-setup@main + uses: aptos-debugger-ee4ecab92c937d27426acce2c8e9e3da88f94c53 #aptos-labs/aptos-core/.github/actions/rust-setup@main with: GIT_CREDENTIALS: ${{ inputs.GIT_CREDENTIALS }} @@ -171,7 +171,7 @@ jobs: strategy: fail-fast: false matrix: - job_id: ${{ fromJson(steps.prepare.outputs.job_ids) }} + job_id: ${{ fromJson(needs.prepare.outputs.job_ids) }} steps: - name: Load cached aptos-debugger binary and replay_verify.py script uses: actions/cache/restore@v4 @@ -179,7 +179,7 @@ jobs: path: | aptos-debugger testsuite/replay_verify.py - key: aptos-debugger-${{ inputs.GIT_SHA || github.sha }} + key: aptos-debugger-ee4ecab92c937d27426acce2c8e9e3da88f94c53 #aptos-debugger-${{ inputs.GIT_SHA || github.sha }} fail-on-cache-miss: true - name: Load cached backup storage metadata cache dir diff --git a/storage/db-tool/src/gen_replay_verify_jobs.rs b/storage/db-tool/src/gen_replay_verify_jobs.rs index 62c3f047c0e074..3429e2d9ee4a9c 100644 --- a/storage/db-tool/src/gen_replay_verify_jobs.rs +++ b/storage/db-tool/src/gen_replay_verify_jobs.rs @@ -12,7 +12,7 @@ use aptos_backup_cli::{ use aptos_logger::info; use aptos_types::transaction::Version; use clap::Parser; -use itertools::{zip_eq, Itertools}; +use itertools::Itertools; use std::{ io::Write, iter::{once, zip}, @@ -50,14 +50,14 @@ pub struct Opt { value_delimiter = ' ' )] ranges_to_skip: Vec, - #[clap(long, help = "Output job ranges to json files, evenly distributed.")] - output_json_files: Vec, + #[clap(long, help = "Maximum ranges per job.")] + max_ranges_per_job: u64, + #[clap(long, help = "Output json file containing the jobs.")] + output_json_file: PathBuf, } impl Opt { pub async fn run(self) -> anyhow::Result<()> { - assert!(!self.output_json_files.is_empty()); - let storage = self.storage.init_storage().await?; let metadata_view = sync_and_load( &self.metadata_cache_opt, @@ -185,24 +185,21 @@ impl Opt { job_ranges.len() ); - let mut outputs = vec![vec![]; self.output_json_files.len()]; + let num_jobs = (job_ranges.len() as f32 / self.max_ranges_per_job as f32).ceil() as usize; + let mut jobs = vec![vec![]; num_jobs]; + let mut job_idx = -1; - zip(job_ranges, (0..self.output_json_files.len()).cycle()).for_each( + zip(job_ranges, (0..num_jobs).cycle()).for_each( |((partial, first, last, desc), output_idx)| { job_idx += 1; let suffix = if partial { "-partial" } else { "" }; let job = format!("{output_idx}-{job_idx}{suffix} {first} {last} {desc}"); - outputs[output_idx].push(job); + jobs[output_idx].push(job); }, ); - zip_eq(self.output_json_files.iter(), outputs.into_iter()).try_for_each( - |(path, jobs)| { - info!("Writing to {:?}", path); - info!("{}", serde_json::to_string_pretty(&jobs)?); - std::fs::File::create(path)?.write_all(&serde_json::to_vec(&jobs)?) - }, - )?; + info!("{}", serde_json::to_string_pretty(&jobs)?); + std::fs::File::create(self.output_json_file)?.write_all(&serde_json::to_vec(&jobs)?)?; Ok(()) }